diff --git a/external/basis_universal/.gitrepo b/external/basis_universal/.gitrepo index 8519ab5814..560308621c 100644 --- a/external/basis_universal/.gitrepo +++ b/external/basis_universal/.gitrepo @@ -5,8 +5,8 @@ ; [subrepo] remote = https://github.com/KhronosGroup/basis_universal.git - branch = cmake_fixes - commit = daf79c6ee7343f547d29c36d180331a73300607b - parent = d5787860a2a47e8ecc79f437566c0d62cc5cebee + branch = set_fp_options + commit = c40de6605612385cc39a6ce433343ee3d7e8b3fd + parent = 15e03d47b67cb1d670571897ab8453223f53189d method = merge cmdver = 0.4.9 diff --git a/external/basis_universal/CMakeLists.txt b/external/basis_universal/CMakeLists.txt index 3b7d698b39..39e4d072e8 100644 --- a/external/basis_universal/CMakeLists.txt +++ b/external/basis_universal/CMakeLists.txt @@ -4,21 +4,39 @@ cmake_minimum_required(VERSION 3.20) if (NOT CMAKE_OSX_DEPLOYMENT_TARGET) # Needed otherwise Xcode builds with the default installed SDK which can often be - # more recent than the macOS version being used. + # more recent than the macOS version being used. Must be before project. set(CMAKE_OSX_DEPLOYMENT_TARGET "11.0" CACHE STRING "macOS Deployment Target") endif() project(basisu C CXX) -option(BASISU_TOOL "Include basisu tool in build" TRUE) -option(BASISU_EXAMPLES "Include examples in build" TRUE) +# pybind11: allow old Python finder modules without complaining +if (POLICY CMP0148) + cmake_policy(SET CMP0148 OLD) +endif() -option(BASISU_STATIC "static linking" TRUE) -option(BASISU_SAN "sanitize" FALSE) +if (CMAKE_SYSTEM_NAME STREQUAL "WASI") + set(BASISU_BUILD_WASM TRUE) +else() + set(BASISU_BUILD_WASM FALSE) +endif() + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) -# Using a generator expression here prevents multi-config generators (VS, Xcode, Ninja Multi-Config) -# from appending a per-configuration subdirectory. NOTE: This means the output could be overwritten -# by a subsequent build for a different configuration. +option(BASISU_STATIC "static linking" TRUE) # NO-OP. basisu_encoder always built as a static library. +option(BASISU_SAN "sanitize" FALSE) +option(BASISU_EXAMPLES "build examples" TRUE) +option(BASISU_TOOL "build basisu tool" TRUE) +option(BASISU_WASM_THREADING "Enable WASI threading support" OFF) +option(BASISU_BUILD_PYTHON "Build native Python module via pybind11" OFF) + +# Using a generator expression here prevents multi-config generators (VS, Xcode, +# Ninja Multi-Config) from appending a per-configuration subdirectory. This is +# a MUCH simpler alternative to specifying RUNTIME_OUTPUT_DIRECTORY_ +# for each target and each config. +# NOTE: This means the output will be overwritten by a subsequent build for a +# different but that is true of the alternative too. set(CMAKE_RUNTIME_OUTPUT_DIRECTORY $<1:${CMAKE_CURRENT_SOURCE_DIR}/bin>) # For MSVC builds default to SSE enabled, and determine if it's a 64-bit (-A x64) vs. 32-bit (-A Win32) build. @@ -39,14 +57,58 @@ endif() option(BASISU_ZSTD "ZSTD support for KTX2 transcoding/encoding" TRUE) option(BASISU_OPENCL "OpenCL support in encoder" FALSE) -message("Initial BASISU_BUILD_X64=${BASISU_BUILD_X64}") +# Old option to new (BASISU_ prefixed) automatic remapping +foreach(pair + "STATIC;BASISU_STATIC" + "SAN;BASISU_SAN" + "EXAMPLES;BASISU_EXAMPLES" + "WASM_THREADING;BASISU_WASM_THREADING" + "BUILD_PYTHON;BASISU_BUILD_PYTHON" + "BUILD_X64;BASISU_BUILD_X64" + "SSE;BASISU_SSE" + "ZSTD;BASISU_ZSTD" + "OPENCL;BASISU_OPENCL" +) + list(GET pair 0 OLD) + list(GET pair 1 NEW) + + if(DEFINED ${OLD}) + message(WARNING "[BASISU] Legacy option '${OLD}' is deprecated. Use '${NEW}' instead.") + set(${NEW} "${${OLD}}" CACHE BOOL "" FORCE) + endif() +endforeach() + +if (BASISU_BUILD_WASM) + message(STATUS "Configuring for WASM (WASI-SDK)") + + # WASM is always 32-bit + set(BASISU_BUILD_X64 OFF CACHE BOOL "" FORCE) + + # WASM cannot use SSE + set(BASISU_SSE OFF CACHE BOOL "" FORCE) + + # WASM cannot use OpenCL + set(BASISU_OPENCL OFF CACHE BOOL "" FORCE) + + # WASM cannot use dynamic linking + # TODO: Fix this untrue statement. basisu_encoder always built as a static library. + set(BASISU_STATIC OFF CACHE BOOL "" FORCE) + + # WASM cannot use sanitizers + set(BASISU_SAN OFF CACHE BOOL "" FORCE) +endif() + message("Initial CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}") +message("Initial BASISU_BUILD_X64=${BASISU_BUILD_X64}") +message("Initial BASISU_BUILD_WASM=${BASISU_BUILD_WASM}") +message("Initial BASISU_WASM_THREADING=${BASISU_WASM_THREADING}") +message("Initial BASISU_BUILD_PYTHON=${BASISU_BUILD_PYTHON}") message("Initial BASISU_SSE=${BASISU_SSE}") message("Initial BASISU_ZSTD=${BASISU_ZSTD}") message("Initial BASISU_OPENCL=${BASISU_OPENCL}") message("Initial BASISU_SAN=${BASISU_SAN}") -message("initial BASISU_TOOL=${BASISU_TOOL}") -message("initial BASISU_EXAMPLES=${BASISU_EXAMPLES}") +message("Initial BASISU_EXAMPLES=${BASISU_EXAMPLES}") +message("Initial BASISU_TOOL=${BASISU_TOOL}") if(MINGW) # Check if the Threads package is provided; if using Mingw it MIGHT be @@ -88,7 +150,67 @@ else() message("Zstandard disabled") endif() -if (NOT MSVC) +####################################################################### +# Specify floating point handling +# +# For deterministic results specify FP compile options for uniform +# handling of floating point operations across platforms and compilers + +# Compiler info query helpers + +# On CMake 3.25 or older CXX_COMPILER_FRONTEND_VARIANT is not always set +if(CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "") + set(CMAKE_CXX_COMPILER_FRONTEND_VARIANT "${CMAKE_CXX_COMPILER_ID}") +endif() + +#cmake_print_variables( +# CMAKE_CXX_COMPILER_ID +# CMAKE_CXX_COMPILER_VERSION +# CMAKE_CXX_COMPILER_FRONTEND_VARIANT +#) + +# Compiler accepts MSVC-style command line options +set(is_msvc_fe "$") +# Compiler accepts GNU-style command line options +set(is_gnu_fe1 "$") +# Compiler accepts AppleClang-style command line options, which is also GNU-style +set(is_gnu_fe2 "$") +# Compiler accepts GNU-style command line options +set(is_gnu_fe "$") + +# Compiler is Visual Studio cl.exe +set(is_msvccl "$>") +# Compiler is Visual Studio clangcl.exe +set(is_clangcl "$>") +# Compiler is upstream clang with the standard frontend +set(is_clang "$>") + +# Genex debug helper. Note that CXX_COMPILER_ID can only be used in +# compilation targets so won't work in a custom target. +#if(NOT TARGET debug_isgnufe1) +# add_custom_target(debug_isgnufe1 +# COMMAND ${CMAKE_COMMAND} -E echo "is_gnufe1_exp = $" +# ) +#endif() + +# On MSVC and CLangCL, set /fp:strict and /fp:precise, on clang +# -ffp-model=precise and on GNU CC -ffp-contract=off. These result in +# precise semantics always being used and fused multiply-add never +# being used, +add_compile_options( + $<$,19.30>>:/fp:strict> + $<$,19.30>>:/fp:precise> + $<${is_clangcl}:/fp:precise> + $<$,14.0.0>>:-Xclang$-ffp-contract=off> + $<$,10.0.0>>:-ffp-model=precise> + $<${is_gnu_fe}:-ffp-contract=off> + # Hide noise from clang and clangcl 20 warning the 2nd fp option changes + # one of the settings made the first. + $<$,20.0.0>>:-Wno-overriding-option> +) +####################################################################### + +if (NOT MSVC AND NOT BASISU_BUILD_WASM) add_compile_options($<$:-g>) # If you want to set an optimization option for non-debug too, use this instead. #add_compile_options($,-g,-O3>) @@ -121,6 +243,17 @@ if (NOT MSVC) if (EMSCRIPTEN) add_link_options("SHELL:-s ALLOW_MEMORY_GROWTH=1") endif() +elseif (BASISU_BUILD_WASM) + # _WASI_EMULATED_PROCESS_CLOCKS/-lwasi-emulated-process-clocks is only for ZStd + add_compile_definitions(_WASI_EMULATED_PROCESS_CLOCKS) + add_link_options(-lwasi-emulated-process-clocks) + + add_compile_options(-fno-strict-aliasing -fvisibility=hidden -Wall -Wextra -Wno-unknown-warning-option) + + add_compile_options($,-g,-O2>) + + # We need a few MB of stack - don't skip this or WASMTime will silently allow the stack to grow into the heap or static memory, causing corruption. + add_link_options(-Wl,--stack-first -Wl,-z,stack-size=8388608) else() add_compile_options("$<$:-Wno-unused-variable;-Wno-unused-function>") endif() @@ -147,12 +280,11 @@ set(ENCODER_LIB_SRC_LIST encoder/basisu_uastc_hdr_4x4_enc.cpp encoder/basisu_astc_hdr_6x6_enc.cpp encoder/basisu_astc_hdr_common.cpp + encoder/basisu_astc_ldr_common.cpp + encoder/basisu_astc_ldr_encode.cpp encoder/3rdparty/android_astc_decomp.cpp encoder/3rdparty/tinyexr.cpp transcoder/basisu_transcoder.cpp -) - -set(ENCODER_LIB_HDR_LIST encoder/basisu_astc_hdr_6x6_enc.h encoder/basisu_astc_hdr_common.h encoder/basisu_backend.h @@ -175,6 +307,8 @@ set(ENCODER_LIB_HDR_LIST encoder/basisu_ssim.h encoder/basisu_uastc_enc.h encoder/basisu_uastc_hdr_4x4_enc.h + encoder/basisu_astc_ldr_common.h + encoder/basisu_astc_ldr_encode.h encoder/cppspmd_flow.h encoder/cppspmd_math_declares.h encoder/cppspmd_math.h @@ -193,24 +327,23 @@ set(ENCODER_LIB_HDR_LIST transcoder/basisu_transcoder_internal.h transcoder/basisu_transcoder_uastc.h transcoder/basisu_transcoder.h + transcoder/basisu_idct.h transcoder/basisu.h + zstd/zstd.h ) if (BASISU_ZSTD) set(ENCODER_LIB_SRC_LIST ${ENCODER_LIB_SRC_LIST} zstd/zstd.c) - set(ENCODER_LIB_HDR_LIST ${ENCODER_LIB_HDR_LIST} zstd/zstd.h) endif() # Create the static library -add_library(basisu_encoder STATIC ${ENCODER_LIB_SRC_LIST} ${ENCODER_LIB_HDR_LIST}) +add_library(basisu_encoder STATIC ${ENCODER_LIB_SRC_LIST}) target_include_directories(basisu_encoder INTERFACE $ $ # So KTX-Software can use it. ) -# PUBLIC so it will be exported to dependent programs. -target_compile_features(basisu_encoder PUBLIC cxx_std_17) if (EMSCRIPTEN) target_compile_definitions(basisu_encoder PUBLIC BASISU_SUPPORT_SSE=0) @@ -218,16 +351,22 @@ else() target_compile_definitions(basisu_encoder PUBLIC BASISU_SUPPORT_SSE=$,1,0> ) + target_compile_definitions(basisu_encoder PUBLIC + BASISU_WASI_THREADS=$,1,0> + ) target_compile_options(basisu_encoder PRIVATE "$<$,$>:-msse4.1>" ) + target_compile_options(basisu_encoder PUBLIC + "$<$:-fno-exceptions -fno-rtti>" + ) endif() target_compile_definitions(basisu_encoder PRIVATE "BASISD_SUPPORT_KTX2_ZSTD=$,1,0>") if (BASISU_OPENCL) # basisu uses this to confirm the library has been compiled with OpenCL support hence PUBLIC. target_compile_definitions(basisu_encoder PUBLIC BASISU_SUPPORT_OPENCL=1) - if (NOT WIN32) # True when the target system is Windows. + if (NOT WIN32) # True when the target system is not Windows. # For Non-Windows builds, use the system OpenCL headers/libs, if cmake found them. target_include_directories(basisu_encoder PRIVATE ${OpenCL_INCLUDE_DIRS}) target_link_libraries(basisu_encoder PRIVATE ${OpenCL_LIBRARIES}) @@ -244,7 +383,7 @@ else() target_compile_definitions(basisu_encoder PUBLIC BASISU_SUPPORT_OPENCL=0) endif() -if (NOT MSVC) +if (NOT MSVC AND NOT BASISU_BUILD_WASM) # Only link 'm' on non-Windows platforms (Linux, macOS) if (NOT WIN32) target_link_libraries(basisu_encoder INTERFACE m) @@ -259,30 +398,57 @@ if (NOT MSVC) endif() endif() -macro(set_common_executable_properties target) +macro(set_common_executable_properties target link_encoder) #if (MSVC) target_sources(${target} PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/basisu.manifest") #endif() - target_link_libraries(${target} PRIVATE basisu_encoder) + if (${link_encoder}) + target_link_libraries(${target} PRIVATE basisu_encoder) + endif() if (NOT BASISU_STATIC AND NOT EMSCRIPTEN AND NOT WIN32) target_link_options(${target} PUBLIC -Wl,-rpath .) endif() + if (BASISU_BUILD_WASM) + # Add proper suffix + set_target_properties(${target} PROPERTIES SUFFIX ".wasm") + if (BASISU_WASM_THREADING) + set_target_properties(${target} PROPERTIES OUTPUT_NAME "$_mt") + else() + set_target_properties(${target} PROPERTIES OUTPUT_NAME "$_st") + endif() + # 256 MB initial, 3.5 GB max safe defaults for BasisU + target_link_options(${target} PRIVATE + -Wl,--initial-memory=268435456 + -Wl,--max-memory=3758096384 + ) + endif() endmacro() if (BASISU_TOOL) - # Create the basisu executable and link against the static library + # Create the basisu executable add_executable(basisu basisu_tool.cpp) - set_common_executable_properties(basisu) + set_common_executable_properties(basisu TRUE) endif() if (BASISU_EXAMPLES) - # Create the new example executable and link against the static library + # Create the example executables add_executable(examples example/example.cpp) - set_common_executable_properties(examples) + set_common_executable_properties(examples TRUE) + + add_executable(example_capi example_capi/example_capi.c encoder/basisu_wasm_api.cpp encoder/basisu_wasm_transcoder_api.cpp) + set_common_executable_properties(example_capi TRUE) + + add_executable(example_transcoding example_transcoding/example_transcoding.cpp example_transcoding/utils.cpp zstd/zstddeclib.c transcoder/basisu_transcoder.cpp) + set_common_executable_properties(example_transcoding FALSE) + # As target is not linked with basisu_encoder, these values won't be imported. + target_compile_definitions(example_transcoding PRIVATE "BASISD_SUPPORT_KTX2_ZSTD=$,1,0>") + target_compile_options(example_transcoding PUBLIC + "$<$:-fno-exceptions -fno-rtti>" + ) endif() if (BASISU_TOOL AND NOT EMSCRIPTEN) - if (UNIX) + if (UNIX AND NOT BASISU_BUILD_WASM) if (CMAKE_BUILD_TYPE STREQUAL "Release") if (APPLE) add_custom_command(TARGET basisu POST_BUILD COMMAND strip -X -x ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/basisu) @@ -294,3 +460,135 @@ if (BASISU_TOOL AND NOT EMSCRIPTEN) endif() endif() endif() + +# ------------------------------------------------------------ +# Build WASM WASI API module (single or multi-threaded) +# ------------------------------------------------------------ +if (BASISU_BUILD_WASM) + set(BASISU_WASM_API_SRC + ${CMAKE_CURRENT_SOURCE_DIR}/encoder/basisu_wasm_api.cpp + ) + + # Select output name based on threading flag + if (BASISU_WASM_THREADING) + set(BASISU_WASM_OUTPUT_NAME "basisu_module_mt") + else() + set(BASISU_WASM_OUTPUT_NAME "basisu_module_st") + endif() + + add_executable(${BASISU_WASM_OUTPUT_NAME} ${BASISU_WASM_API_SRC}) + target_link_libraries(${BASISU_WASM_OUTPUT_NAME} PRIVATE basisu_encoder) + + set_target_properties(${BASISU_WASM_OUTPUT_NAME} PROPERTIES SUFFIX ".wasm") + + # Common WASM options + target_link_options(${BASISU_WASM_OUTPUT_NAME} PRIVATE + -Wl,--initial-memory=268435456 + -Wl,--max-memory=3758096384 + -Wl,--stack-first + -Wl,-z,stack-size=8388608 + ) + + set_target_properties(${BASISU_WASM_OUTPUT_NAME} PROPERTIES + RUNTIME_OUTPUT_DIRECTORY $<1:${CMAKE_CURRENT_SOURCE_DIR}/python/basisu_py/wasm> + ) + + # Threading options + if (BASISU_WASM_THREADING) + target_compile_options(${BASISU_WASM_OUTPUT_NAME} PRIVATE + -pthread + -matomics + ) + target_link_options(${BASISU_WASM_OUTPUT_NAME} PRIVATE + -pthread + -Wl,--shared-memory + -Wl,--export-memory + ) + + target_compile_definitions(${BASISU_WASM_OUTPUT_NAME} PRIVATE BASISU_WASI_THREADS=1) + endif() +endif() + +if (BASISU_BUILD_WASM) + # Select output name based on threading flag + if (BASISU_WASM_THREADING) + set(BASISU_TRANSCODER_WASM_OUTPUT_NAME "basisu_transcoder_module_mt") + else() + set(BASISU_TRANSCODER_WASM_OUTPUT_NAME "basisu_transcoder_module_st") + endif() + + add_executable(${BASISU_TRANSCODER_WASM_OUTPUT_NAME} + ${CMAKE_CURRENT_SOURCE_DIR}/encoder/basisu_wasm_transcoder_api.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/transcoder/basisu_transcoder.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/zstd/zstddeclib.c) + + set_target_properties(${BASISU_TRANSCODER_WASM_OUTPUT_NAME} PROPERTIES SUFFIX ".wasm") + + target_link_options(${BASISU_TRANSCODER_WASM_OUTPUT_NAME} PRIVATE + -Wl,--initial-memory=16777216 + -Wl,--stack-first + -Wl,-z,stack-size=4194304 + ) + target_compile_options(${BASISU_TRANSCODER_WASM_OUTPUT_NAME} PRIVATE -fno-exceptions -fno-rtti) + + set_target_properties(${BASISU_TRANSCODER_WASM_OUTPUT_NAME} PROPERTIES + RUNTIME_OUTPUT_DIRECTORY $<1:${CMAKE_CURRENT_SOURCE_DIR}/python/basisu_py/wasm> + ) +endif() + + +# ------------------------------------------------------------ +# Optional: Build native Python modules (pybind11) +# ------------------------------------------------------------ +if (BASISU_BUILD_PYTHON AND NOT BASISU_BUILD_WASM) + find_package(pybind11 CONFIG REQUIRED) + + message(STATUS "Building pybind11 Python extension: basisu_python") + + pybind11_add_module(basisu_python + python/basisu_encoder_pybind11.cpp + encoder/basisu_wasm_api.cpp + ) + + # Ensure PIC ONLY for this target + set_property(TARGET basisu_python PROPERTY POSITION_INDEPENDENT_CODE ON) + + target_link_libraries(basisu_python PRIVATE basisu_encoder) + + # Put basisu_python so into python/basisu_py + set_target_properties(basisu_python PROPERTIES + LIBRARY_OUTPUT_DIRECTORY $<1:${CMAKE_CURRENT_SOURCE_DIR}/python/basisu_py> + PREFIX "" # Required by Python + OUTPUT_NAME "basisu_python" # Just to be explicit + ) + + if (MSVC) + set_target_properties(basisu_python PROPERTIES SUFFIX ".pyd") + endif() +endif() + +if (BASISU_BUILD_PYTHON AND NOT BASISU_BUILD_WASM) + find_package(pybind11 CONFIG REQUIRED) + + message(STATUS "Building pybind11 Python extension: basisu_transcoder_python") + + pybind11_add_module(basisu_transcoder_python + python/basisu_transcoder_pybind11.cpp + encoder/basisu_wasm_transcoder_api.cpp + transcoder/basisu_transcoder.cpp + zstd/zstddeclib.c + ) + + # Ensure PIC ONLY for this target + set_property(TARGET basisu_transcoder_python PROPERTY POSITION_INDEPENDENT_CODE ON) + + set_target_properties(basisu_transcoder_python PROPERTIES + LIBRARY_OUTPUT_DIRECTORY $<1:${CMAKE_CURRENT_SOURCE_DIR}/python/basisu_py> + PREFIX "" + OUTPUT_NAME "basisu_transcoder_python" + ) + + if (MSVC) + set_target_properties(basisu_transcoder_python PROPERTIES SUFFIX ".pyd") + endif() +endif() diff --git a/external/basis_universal/LICENSE b/external/basis_universal/LICENSE index 30e4b202d8..94ea880430 100644 --- a/external/basis_universal/LICENSE +++ b/external/basis_universal/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2019-2025 Binomial LLC + Copyright 2019-2026 Binomial LLC Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/external/basis_universal/README.md b/external/basis_universal/README.md index 9a85a8b51a..485e2fd194 100644 --- a/external/basis_universal/README.md +++ b/external/basis_universal/README.md @@ -1,46 +1,55 @@ -# basis_universal -An LDR/HDR portable GPU compressed texture transcoding system. +# basis_universal v2.0 +An LDR/HDR portable GPU supercompressed texture transcoding system. -[![Build status](https://ci.appveyor.com/api/projects/status/87eb0o96pjho4sh0?svg=true)](https://ci.appveyor.com/project/BinomialLLC/basis-universal) +[![Build status](https://img.shields.io/appveyor/build/BinomialLLC/basis-universal/master.svg)](https://ci.appveyor.com/project/BinomialLLC/basis-universal) ---- Intro ----- -Basis Universal is an open source [supercompressed](http://gamma.cs.unc.edu/GST/gst.pdf) LDR/HDR GPU compressed texture interchange system from Binomial LLC that supports two intermediate file formats: the [.KTX2 open standard from the Khronos Group](https://registry.khronos.org/KTX/specs/2.0/ktxspec.v2.html), and our own ".basis" file format. These file formats support rapid transcoding to virtually any compressed [GPU texture format](https://en.wikipedia.org/wiki/Texture_compression) released in the past ~25 years. +Basis Universal v2.0 is an open source [supercompressed](http://gamma.cs.unc.edu/GST/gst.pdf) LDR/HDR GPU compressed texture interchange system from Binomial LLC that supports two intermediate file formats: the [.KTX2 open standard from the Khronos Group](https://registry.khronos.org/KTX/specs/2.0/ktxspec.v2.html), and our own ".basis" file format. These file formats support rapid transcoding to virtually any compressed [GPU texture format](https://en.wikipedia.org/wiki/Texture_compression) released over the past quarter century. -Our overall goal with this project is to simplify the encoding and efficient distribution of *portable* LDR and HDR GPU texture, image, and short texture video content in a way that is compatible with any GPU or rendering/graphics API. +Our overall goal is to simplify the encoding and efficient distribution of *portable* LDR and HDR GPU texture, image, and short [texture video](https://github.com/BinomialLLC/basis_universal/wiki/Encoding-ETC1S-Texture-Video-Tips) content in a way that is compatible with any GPU or rendering/graphics API. -The system supports five modes: ETC1S, UASTC LDR 4x4, UASTC HDR 4x4, UASTC HDR 6x6 (with or without RDO), or UASTC HDR 6x6 Intermediate ("GPU Photo"). The C/C++ encoder and transcoder libaries can be compiled to native code or WebAssembly, and all encoder/transcoder features can be accessed from Javascript via a C++ wrapper library which optionally supports [WASM multithreading](https://web.dev/articles/webassembly-threads) for fast encoding in the browser. ETC1S transcoding (which uses no SIMD code) is slightly faster than libjpeg when compiled to native code. +The system supports seven modes (or codecs). In the order they were implemented: +1. **ETC1S**: A supercompressed subset of ETC1 designed for very fast transcoding to other LDR texture formats, low/medium quality but high compression, slightly faster transcoding to other LDR texture formats vs. libjpeg. +2. **UASTC LDR 4x4 (with or without RDO)**: Custom ASTC 4x4-like format designed for very fast transcoding to other LDR texture formats, high quality +3. **UASTC HDR 4x4**: Standard ASTC HDR 4x4 texture data, but constrained for very fast transcoding to BC6H +4. **ASTC HDR 6x6 (with or without RDO)**: Standard ASTC HDR 6x6 +5. **UASTC HDR 6x6 Intermediate ("GPU Photo HDR")**: Supercompressed ASTC HDR 6x6 +6. **ASTC LDR 4x4-12x12 (all 14 standard ASTC block sizes, with or without basic windowed RDO)**: Standard ASTC LDR 4x4-12x12 +7. **XUASTC LDR 4x4-12x12 (all 14 standard ASTC block sizes, "GPU Photo LDR/SDR")**: Latent space supercompressed ASTC LDR with Weight Grid DCT ([Discrete Cosine Transform](https://grokipedia.com/page/Discrete_cosine_transform)) for very high quality, extreme bitrate scalability, optional adaptive deblocking, three entropy coding profiles (Zstd, arithmetic or hybrid). See [JPEG for ASTC](https://github.com/BinomialLLC/basis_universal/wiki/JPEG-for-ASTC), and the [ASTC and XUASTC LDR Usage Guide](https://github.com/BinomialLLC/basis_universal/wiki/ASTC-and-XUASTC-LDR-Usage-Guide). + +The C/C++ encoder and transcoder libraries can be compiled to native code or WebAssembly (web or WASI), and all encoder/transcoder features can be accessed from JavaScript via a C++ wrapper library which optionally supports [WASM multithreading](https://web.dev/articles/webassembly-threads) for fast encoding in the browser. [WASM WASI](https://wasi.dev/) builds, for the command line tool and the encoder/transcoder as a WASI module using a pure C API, are also supported. + +Full Python support for encoding/transcoding is now available, supporting native or WASM modules, but is still in the early stages of development. Links ----- +- [Wiki/Specifications](https://github.com/BinomialLLC/basis_universal/wiki) - [Release Notes](https://github.com/BinomialLLC/basis_universal/wiki/Release-Notes) +- [Live Compression/Transcoding Testbed](https://subquantumtech.com/xu/ktx2_encode_test/) +- [Live WebGL Examples](https://subquantumtech.com/xu/) +- [JavaScript API/WASM/WebGL info](https://github.com/BinomialLLC/basis_universal/tree/master/webgl) +- [XUASTC LDR Specification](https://github.com/BinomialLLC/basis_universal/wiki/XUASTC-LDR-Specification-v1.0) -- [Live Compression/Transcoding Testbed](https://subquantumtech.com/bu_6x6/ktx2_encode_test/) - -- [Live WebGL Examples](https://subquantumtech.com/bu_6x6/) - -- [Javascript API/WASM/WebGL info](https://github.com/BinomialLLC/basis_universal/tree/master/webgl) +### UASTC HDR 4x4/6x6 Specific Links: - [UASTC HDR 4x4 Example Images](https://github.com/BinomialLLC/basis_universal/wiki/UASTC-HDR-Examples) - -### UASTC 6x6 HDR Specific Links: - - [UASTC HDR 6x6 Example Images](https://github.com/BinomialLLC/basis_universal/wiki/ASTC-HDR-6x6-Example-Images) +- [UASTC HDR 6x6 Support Notes](https://github.com/BinomialLLC/basis_universal/wiki/UASTC-HDR-6x6-Support-Notes) +- [Quick comparison of ARM's astcenc HDR 6x6 encoder vs. ours](https://github.com/richgel999/junkdrawer/wiki/ASTC-6x6-HDR:-astcenc-%E2%80%90thorough-%E2%80%90exhaustive-vs.-basis-universal-comp_level-3) -- [UASTC HDR 6x6 Support Nodes](https://github.com/BinomialLLC/basis_universal/wiki/UASTC-HDR-6x6-Support-Notes) - -- [Quick comparison of ARM's astcenc HDR 6x6 encoder vs. ours](https://github.com/richgel999/junkdrawer/wiki/ASTC-6x6-HDR:-astcenc-%E2%80%90thorough-%E2%80%90exhausive-vs.-basis-universal-comp_level-3) +---- Supported LDR GPU Texture Formats --------------------------------- -ETC1S and UASTC LDR 4x4 files can be transcoded to: - +ETC1S, UASTC LDR 4x4, XUASTC LDR 4x4-12x12 and ASTC LDR 4x4-12x12 files can be transcoded to: - ASTC LDR 4x4 L/LA/RGB/RGBA 8bpp +- ASTC LDR 4x4-12x12 (XUASTC/ASTC), 0.89-8bpp - BC1-5 RGB/RGBA/X/XY - BC7 RGB/RGBA - ETC1 RGB, ETC2 RGBA, and ETC2 EAC R11/RG11 @@ -51,39 +60,61 @@ ETC1S and UASTC LDR 4x4 files can be transcoded to: Supported HDR GPU Texture Formats --------------------------------- -UASTC HDR 4x4 and UASTC HDR 6x6 files can be transcoded to: +UASTC HDR 4x4, ASTC HDR 6x6, and UASTC HDR 6x6 files can be transcoded to: - ASTC HDR 4x4 (8bpp, UASTC HDR 4x4 only) - ASTC HDR 6x6 RGB (3.56bpp, ASTC HDR 6x6 or UASTC HDR 6x6 intermediate only) - BC6H RGB (8bpp, either UASTC HDR 4x4 or UASTC HDR 6x6) - Uncompressed HDR raster image formats: RGB_16F/RGBA_16F (half float/FP16 RGB, 48 or 64bpp), or 32-bit/pixel shared exponent [RGB_9E5](https://registry.khronos.org/OpenGL/extensions/EXT/EXT_texture_shared_exponent.txt) -Supported Texture Compression Modes ------------------------------------ +---- + +Supported Texture Compression/Supercompression Modes +---------------------------------------------------- -1. [ETC1S](https://github.com/BinomialLLC/basis_universal/wiki/.basis-File-Format-and-ETC1S-Texture-Video-Specification): A roughly .3-3bpp low to medium quality supercompressed mode based off a subset of [ETC1](https://en.wikipedia.org/wiki/Ericsson_Texture_Compression) called "ETC1S". This mode supports variable quality vs. file size levels (like JPEG), alpha channels, built-in compression, and texture arrays optionally compressed as a video sequence using skip blocks ([Conditional Replenishment](https://en.wikipedia.org/wiki/MPEG-1)). This mode can be rapidly transcoded to all of the supported LDR texture formats. +1. **[ETC1S](https://github.com/BinomialLLC/basis_universal/wiki/.basis-File-Format-and-ETC1S-Texture-Video-Specification)**: A roughly .3-3bpp low to medium quality supercompressed mode based on a subset of [ETC1](https://en.wikipedia.org/wiki/Ericsson_Texture_Compression) called "ETC1S". This mode supports variable quality vs. file size levels (like JPEG), alpha channels, built-in compression, and texture arrays optionally compressed as a video sequence using skip blocks ([Conditional Replenishment](https://en.wikipedia.org/wiki/MPEG-1)). This mode can be rapidly transcoded to all of the supported LDR texture formats. -2. [UASTC LDR 4x4](https://github.com/BinomialLLC/basis_universal/wiki/UASTC-LDR-4x4-Texture-Specification): An 8 bits/pixel LDR high quality mode. UASTC LDR is a 19 mode subset of the standard [ASTC LDR](https://en.wikipedia.org/wiki/Adaptive_scalable_texture_compression) 4x4 (8bpp) texture format, but with a custom block format containing transcoding hints. Transcoding UASTC LDR to ASTC LDR and BC7 are particularly fast and simple, because UASTC LDR is a common subset of both BC7 and ASTC. The transcoders for the other texture formats are accelerated by several format-specific hint bits present in each UASTC LDR block. +2. **[UASTC LDR 4x4](https://github.com/BinomialLLC/basis_universal/wiki/UASTC-LDR-4x4-Texture-Specification)**: An 8 bits/pixel LDR high quality mode. UASTC LDR is a 19 mode subset of the standard [ASTC LDR](https://en.wikipedia.org/wiki/Adaptive_scalable_texture_compression) 4x4 (8bpp) texture format, but with a custom block format containing transcoding hints. Transcoding UASTC LDR to ASTC LDR and BC7 is particularly fast and simple, because UASTC LDR is a common subset of both BC7 and ASTC. The transcoders for the other texture formats are accelerated by several format-specific hint bits present in each UASTC LDR block. -This mode supports an optional [Rate-Distortion Optimizated (RDO)](https://en.wikipedia.org/wiki/Rate%E2%80%93distortion_optimization) post-process stage that conditions the encoded UASTC LDR texture data in the .KTX2/.basis file so it can be more effectively LZ compressed. More details [here](https://github.com/BinomialLLC/basis_universal/wiki/UASTC-implementation-details). +This mode supports an optional [Rate-Distortion Optimized (RDO)](https://en.wikipedia.org/wiki/Rate%E2%80%93distortion_optimization) post-process stage that conditions the encoded UASTC LDR texture data in the .KTX2/.basis file so it can be more effectively LZ compressed. More details [here](https://github.com/BinomialLLC/basis_universal/wiki/UASTC-implementation-details). Here is the [UASTC LDR 4x4 specification document](https://github.com/BinomialLLC/basis_universal/wiki/UASTC-LDR-4x4-Texture-Specification). -3. [UASTC HDR 4x4](https://github.com/BinomialLLC/basis_universal/wiki/UASTC-HDR-4x4-Texture-Specification-v1.0): An 8 bits/pixel HDR high quality mode. This is a 24 mode subset of the standard [ASTC HDR](https://en.wikipedia.org/wiki/Adaptive_scalable_texture_compression) 4x4 (8bpp) texture format. It's designed to be high quality, supporting the 27 partition patterns in common between BC6H and ASTC, and fast to transcode with very little loss (typically a fraction of a dB PSNR) to the BC6H HDR texture format. Notably, **UASTC HDR 4x4 data is 100% standard ASTC texture data**, so no transcoding at all is required on devices or API's supporting ASTC HDR. This mode can also be transcoded to various 32-64bpp uncompressed HDR texture/image formats. +3. **[UASTC HDR 4x4](https://github.com/BinomialLLC/basis_universal/wiki/UASTC-HDR-4x4-Texture-Specification-v1.0)**: An 8 bits/pixel HDR high quality mode. This is a 24 mode subset of the standard [ASTC HDR](https://en.wikipedia.org/wiki/Adaptive_scalable_texture_compression) 4x4 (8bpp) texture format. It's designed to be high quality, supporting the 27 partition patterns in common between BC6H and ASTC, and fast to transcode with very little loss (typically a fraction of a dB PSNR) to the BC6H HDR texture format. Notably, **UASTC HDR 4x4 data is 100% standard ASTC texture data**, so no transcoding at all is required on devices or APIs that support ASTC HDR. This mode can also be transcoded to various 32-64bpp uncompressed HDR texture/image formats. Here is the [UASTC HDR 4x4 specification document](https://github.com/BinomialLLC/basis_universal/wiki/UASTC-HDR-4x4-Texture-Specification-v1.0), and here are some compressed [example images](https://github.com/BinomialLLC/basis_universal/wiki/UASTC-HDR-Examples). -4. UASTC HDR 6x6 or RDO UASTC HDR 6x6: A 3.56 bits/pixel (or less with RDO+Zstd) HDR high quality mode. Just like mode #3, **UASTC HDR 6x6 data is 100% standard ASTC texture data**. Here's a [page with details](https://github.com/BinomialLLC/basis_universal/wiki/UASTC-HDR-6x6-Support-Notes). The current encoder supports weight grid upsampling, 1-3 subsets, single or dual planes, CEM's 7 and 11, and all unique ASTC partition patterns. +4. **ASTC HDR 6x6 or RDO ASTC HDR 6x6**: A 3.56 bits/pixel (or less with RDO+Zstd) HDR high quality mode. Just like mode #3, **ASTC HDR 6x6 data is 100% standard ASTC texture data**. Here's a [page with details](https://github.com/BinomialLLC/basis_universal/wiki/UASTC-HDR-6x6-Support-Notes). The current encoder supports weight grid upsampling, 1-3 subsets, single or dual planes, CEM's 7 and 11, and all unique ASTC partition patterns. + +The ASTC HDR decoder, used in the transcoder module, supports the entire ASTC HDR format. + +5. **UASTC HDR 6x6 Intermediate ("GPU Photo HDR")**: A custom compressed intermediate format that can be rapidly transcoded to ASTC HDR 6x6, BC6H, and various uncompressed HDR formats. The custom compressed file format is [described here](https://github.com/BinomialLLC/basis_universal/wiki/UASTC-HDR-6x6-Intermediate-File-Format-(Basis-GPU-Photo-6x6)). The format supports 75 unique ASTC configurations, weight grid upsampling, 1-3 subsets, single or dual planes, CEM's 7 and 11, and all unique ASTC partition patterns. One of the first HDR GPU texture codecs supporting the [delta E ITP (ICtCp) colorspace metric](https://www.portrait.com/resource-center/about-deltae-e/) and perceptual saliency maps. -5. UASTC HDR 6x6 Intermediate ("GPU Photo"): A custom compressed intermediate format that can be rapidly transcoded to ASTC HDR 6x6, BC6H, and various uncompressed HDR formats. The custom compressed file format is [described here](https://github.com/BinomialLLC/basis_universal/wiki/UASTC-HDR-6x6-Intermediate-File-Format-(Basis-GPU-Photo-6x6)). The format supports 75 unique ASTC configurations, weight grid upsampling, 1-3 subsets, single or dual planes, CEM's 7 and 11, and all unique ASTC partition patterns. +6. **Standard ASTC LDR-4x4-12x12**. Supports all standard 14 ASTC block sizes. Transcodable from any ASTC block size to any other supported LDR texture format with adaptive deblocking, including BC7 using the [bc7f "one-shot" analytical BC7 encoder](https://github.com/BinomialLLC/basis_universal/wiki/Transcoder-Internals-%E2%80%90-Analytical-Real%E2%80%90Time-Encoders) (supporting all BC7 modes/features) and ETC1 (using etc1f, which also supports the entire ETC1 format). -Notes: -- Modes #3 and #4 output 100% standard or plain ASTC texture data (with or without RDO), like any other ASTC encoder. The .KTX2 files are just plain textures. -- The other modes (#1, #2, #5) output compressed data in various custom formats, which our transcoder library can convert in real-time to various GPU texture or pixel formats. -- Modes #4 and #5 internally use the same unified HDR 6x6 encoder. +The ASTC LDR decoder, used in the transcoder module, supports the entire standard ASTC LDR format (i.e. not just ASTC texture blocks generated using our encoder). The ASTC LDR transcoder can transcode any block size ASTC (4x4 - 12x12) to the other LDR texture formats. + +7. **XUASTC LDR 4x4-12x12 ("GPU Photo LDR/SDR")**: Supercompressed ASTC with **Weight Grid DCT**, supporting all 14 standard ASTC block sizes, with adaptive deblocking when transcoding to other texture/pixel formats. Bitrates range from approximately 0.3–5.7 bpp, depending on content, profile, block size, windowed RDO, and Weight Grid DCT quality settings. Typical XUASTC LDR 4×4 (**8 bpp in memory**) transmission/on-disk bitrate with Weight Grid DCT (where it is least effective) is **1.15–3.5 bpp (typical ≈2.25 bpp)**, with larger block sizes achieving even lower usable bitrates, down to approximately 0.3 bpp. Like ASTC LDR, the XUASTC LDR transcoder can transcode any block size ASTC (4x4 - 12x12) to the other LDR texture formats, but with additional block-size specific optimizations. + +Supports three profiles: context-based range/arithmetic coding (for higher compression ratios), Zstd (for faster and simpler transcoding), or a hybrid profile using both approaches. Transcodable to all other supported LDR texture formats, including fully featured (all 8 modes, all dual-plane channel configurations, all mode settings) BC7. Certain common block sizes (4×4, 6×6, and 8×6) have specializations for particularly fast transcoding directly to BC7, bypassing analytical BC7 encoding (using [bc7f](https://github.com/BinomialLLC/basis_universal/wiki/Transcoder-Internals-%E2%80%90-Analytical-Real%E2%80%90Time-Encoders)) entirely for the most common ASTC configurations (solid color and single-subset CEMs). + +Weight Grid DCT can be disabled; however, supercompression remains available with optional, configurable windowed RDO. Compatible with all major image and texture content types, including photographic images, lightmaps, albedo/specular textures, various types of normal maps, luminance-only maps, and geospatial mapping signals. + +Supports adaptive deblocking when transcoding from larger block sizes; this can be disabled using a transcoder flag. + +One interesting use of XUASTC LDR which works with any of the 14 block sizes: the efficient distribution of texture content compressed to very low bitrates vs. older systems, resulting in game-changing download time reductions. Using the larger XUASTC block sizes (beyond 6x6) with Weight Grid DCT and adaptive deblocking, **any developer can now distribute texture and image content destined for BC7 at .35-1.5 bpp**, and **cache the transcoded BC7 data on a modern Gen 4 or 5 (10+ GB/sec.) SSD**. + +XUASTC LDR supports the following ASTC configurations: L/LA/RGB/RGBA CEMs; base+scale or RGB/RGBA direct; base+offset CEMs; Blue Contraction encoding; 1–3 subsets; all partition patterns; and single- or dual-plane modes. Here is the [XUASTC LDR specification](https://github.com/BinomialLLC/basis_universal/wiki/XUASTC-LDR-Specification-v1.0). Also see the [ASTC and XUASTC LDR Usage Guide](https://github.com/BinomialLLC/basis_universal/wiki/ASTC-and-XUASTC-LDR-Usage-Guide). + +Notes: +- Mode #1 (ETC1S) has special support and optimizations for basic temporal supercompression ([texture video](https://github.com/BinomialLLC/basis_universal/wiki/Encoding-ETC1S-Texture-Video-Tips)). +- Modes #3 (UASTC HDR 4x4) and #4 (RDO ASTC HDR 6x6), and #6 (ASTC LDR 4x4-12x12) output 100% standard ASTC texture data (with or without RDO), like any other ASTC encoder. The .KTX2 files are just plain textures. +- The other modes (#1, #2, #5, #7) output compressed data in various custom supercompressed formats, which our transcoder library can convert in real-time to various GPU texture or pixel formats. +- Modes #4 (ASTC HDR 6x6) and #5 (UASTC HDR 6x6) internally use the same unified ASTC HDR 6x6 encoder. +- Modes #6 (ASTC LDR 4x4-12x12) and #7 (XUASTC LDR 4x4-12x12) internally use the same unified ASTC LDR ASTC encoder. ### Other Features -Both .basis and .KTX2 files support mipmap levels, texture arrays, cubemaps, cubemap arrays, and texture video, in all five modes. Additionally, .basis files support non-uniform texture arrays, where each image in the file can have a different resolution or number of mipmap levels. +Both .basis and .KTX2 files support mipmap levels, texture arrays, cubemaps, cubemap arrays, and texture video, in all modes. Additionally, .basis files support non-uniform texture arrays, where each image in the file can have a different resolution or number of mipmap levels. In ETC1S mode, the compressor is able to exploit color and pattern correlations across all the images in the entire file using global endpoint/selector codebooks, so multiple images with mipmaps can be stored efficiently in a single file. The ETC1S mode also supports skip blocks (Conditional Replenishment) for short video sequences, to prevent sending blocks which haven't changed relative to the previous frame. @@ -91,18 +122,53 @@ The LDR image formats supported for reading are .PNG, [.DDS with mipmaps](https: The system now supports loading basic 2D .DDS files with optional mipmaps, but the .DDS file must be in one of the supported uncompressed formats: 24bpp RGB, 32bpp RGBA/BGRA, half-float RGBA, or float RGBA. Using .DDS files allows the user to control exactly how the mipmaps are generated before compression. -Building --------- +---- + +Running the Precompiled WASM WASI Executables +--------------------------------------------- + +There are precompiled, secure, cross-platform .WASM WASI executables checked into the `bin` directory: `basisu_mt.wasm` (multithreaded) and `basisu_st.wasm` (single threaded). Quick testing - ETC1S/UASTC LDR 4x4 (all platforms) - multithreaded and single threaded, using [wasmtime](https://wasmtime.dev/): + +``` +cd bin +wasmtime run --dir=. --dir=../test_files --wasm threads=yes --wasi threads=yes ./basisu_mt.wasm -test +wasmtime run --dir=. --dir=../test_files ./basisu_st.wasm -test +``` + +See the `runwt.sh`, `runwt.bat`, `runw.sh`, or `runw.bat` scripts for examples on how to run the WASM executables using wasmtime. Windows example for XUASTC LDR 6x6 compression using the arithmetic profile, with Weight Grid DCT level 70: + +``` +cd bin +runwt.bat ../test_files/tough.png -xuastc_ldr_6x6 -quality 70 -xuastc_arith +runwt.bat tough.ktx2 +``` + +Linux/macOS: + +``` +cd bin +chmod +x runwt.sh +./runwt.sh ../test_files/tough.png -xuastc_ldr_6x6 -quality 70 -xuastc_arith +./runwt.sh tough.ktx2 +``` + +The [wasmer](https://wasmer.io/) runtime should work too, but we haven't tested it yet. + +---- + +Building (Native) +----------------- The encoding library and command line tool have no required 3rd party dependencies that are not already in the repo itself. The transcoder is a single .cpp source file (in `transcoder/basisu_transcoder.cpp`) which has no 3rd party dependencies. We build and test under: -- Windows x86/x64 using Visual Studio 2019/2022, MSVC or clang -- Windows ARM using Visual Studio 2022 ARM v17.13.0 or later -- Mac OSX (M1) with clang v15.0 -- Ubuntu Linux with gcc v11.4 or clang v14 -- Arch Linux ARM, on a [Pinebook Pro](https://pine64.org/devices/pinebook_pro/), with gcc v12.1. +- Windows x86/x64 using Visual Studio 2026, MSVC or clang +- Windows ARM using Visual Studio 2022 ARM 17.13.0 +- Ubuntu Linux 24.04.3 LTS (noble) with gcc 13.3.0 or clang 18.1.3 +- macOS (M1) with clang 16.0.0 +- Arch Linux ARM, on a [Pinebook Pro](https://pine64.org/devices/pinebook_pro/), with gcc 12.1. - Ubuntu Linux 24.04 on RISC-V (Orange PI RV2) +- cmake: 3.28.3, emcc: 4.0.19 Under Windows with Visual Studio you can use the included `basisu.sln` file. Alternatively, you can use cmake to create new VS solution/project files. @@ -115,9 +181,34 @@ cmake .. make ``` -To build with SSE 4.1 support on x86/x64 systems (encoding is roughly 15-30% faster), add `-DSSE=TRUE` to the cmake command line. Add `-DOPENCL=TRUE` to build with (optional) OpenCL support. Use `-DCMAKE_BUILD_TYPE=Debug` to build in debug. To build 32-bit executables, add `-DBUILD_X64=FALSE`. +To build with SSE 4.1 support on x86/x64 systems (ETC1S encoding is roughly 15-30% faster), add `-DBASISU_SSE=TRUE` to the cmake command line. Add `-DBASISU_OPENCL=TRUE` to build with (optional) OpenCL support. Use `-DCMAKE_BUILD_TYPE=Debug` to build in debug. To build 32-bit executables, add `-DBASISU_BUILD_X64=FALSE`. + +After building, the native command line tool used to create, validate, and transcode/unpack .KTX2/.basis files is `bin/basisu`. -After building, the native command line tool used to create, validate, and transcode/unpack .basis/.KTX2 files is `bin/basisu`. +Building (WASM WASI) +-------------------- + +To build the WASM WASI executables, you will need the [WASM WASI SDK](https://github.com/WebAssembly/wasi-sdk) installed. The `WASI_SDK_PATH` environment variable must be set to the correct path where the SDK is installed. + +Multithreaded: +``` +mkdir build_wasm_mt +cd build_wasm_mt +cmake -DCMAKE_TOOLCHAIN_FILE=$WASI_SDK_PATH/share/cmake/wasi-sdk-pthread.cmake -DCMAKE_BUILD_TYPE=Release -DBASISU_WASM_THREADING=ON .. +make +``` + +Single threaded: +``` +mkdir build_wasm_st +cd build_wasm_st +cmake -DCMAKE_TOOLCHAIN_FILE=$WASI_SDK_PATH/share/cmake/wasi-sdk.cmake -DCMAKE_BUILD_TYPE=Release -DBASISU_WASM_THREADING=OFF .. +make +``` + +The WASM WASI executables will be placed in the `bin` directory. These platform-independent executables are fully functional, and can be executed using a WASM WASI runtime such as [wasmtime](https://github.com/bytecodealliance/wasmtime). + +---- ### Testing the Codec @@ -129,26 +220,60 @@ basisu -test basisu -test_hdr_4x4 basisu -test_hdr_6x6 basisu -test_hdr_6x6i +basisu -test_xuastc_ldr ``` -To test the codec in OpenCL mode (must have OpenCL libs/headers/drivers installed and have compiled OpenCL support in by running cmake with `-DOPENCL=TRUE`): +To test the codec in OpenCL mode (must have OpenCL libs/headers/drivers installed and have compiled OpenCL support in by running cmake with `-DBASISU_OPENCL=TRUE`): ``` basisu -test -opencl ``` +---- + Compressing and Unpacking .KTX2/.basis Files -------------------------------------------- -- To compress an LDR sRGB PNG/QOI/TGA/JPEG/DDS image to an ETC1S .KTX2 file, at quality level 255 (the highest): +- To compress an LDR sRGB PNG/QOI/TGA/JPEG/DDS image to a supercompressed XUASTC LDR 6x6 .KTX2 file, at quality level 75 (**valid quality levels 1-100, where higher values=higher quality**), effort level 4 (**valid effort levels 0-10, higher values=slower compression, default effort is 3**): + +`basisu -xuastc_ldr_6x6 -quality 75 -effort 4 x.png` + +An alias for `-xuastc_ldr_6x6` is `-ldr_6x6i` (where 'i'="intermediate"). All **[14 standard ASTC block sizes](https://developer.nvidia.com/astc-texture-compression-for-game-assets) are supported, from 4x4-12x12**: 4x4, 5x4, 5x5, 6x5, 6x6, 8x5, 8x6, 10x5, 10x6, 8x8, 10x8, 10x10, 12x10 and 12x12. The **XUASTC LDR to BC7 transcoder has special optimizations for several common block sizes: 4x4, 6x6 and 8x6**. When transcoding XUASTC LDR at these particular block sizes, most XUASTC blocks are *directly* transcoded to BC7, skipping the real-time analytical bc7f encoding step. + +More XUASTC LDR specific options (many of these also apply to standard ASTC - see our [ASTC/XUASTC Usage Guide](https://github.com/BinomialLLC/basis_universal/wiki/ASTC-and-XUASTC-LDR-Usage-Guide)): + + - The options `-xuastc_arith`, `-xuastc_zstd` (the default), and `-xuastc_hybrid` control the **XUASTC LDR profile used**. The arithmetic profile trades off transcoding throughput for roughly 5-18% better compression vs. the Zstd profile, and the hybrid profile is a balance between the two. + + - `-ts` or `-srgb` enables the **sRGB profile (the default)**, and `-tl` or `-linear` **enables the linear profile**. Ideally this setting will match how the ASTC texture is sampled by the GPU. Use linear on normal maps. + + - `-weights X Y Z W` sets the unsigned integer **channel error weights**, used to favor certain channels during compression. + + - Another set of XUASTC specific options overrides the **windowed RDO behavior** (windowed or bounded RDO is a separate and optional perceptual optimization vs. Weight Grid DCT): `-xy` enables and `-xyd` disables windowed RDO. By default, if Weight Grid DCT is not enabled (i.e. `-quality` isn't specified, or is set to 100), windowed RDO is disabled. Windowed RDO is automatically enabled if the quality level is less than 100, unless `-xyd` is specified. Also see the tool's [help text](https://github.com/BinomialLLC/basis_universal/blob/master/cmd_help/cmd_help.txt) for additional windowed RDO options: `-ls_min_psnr`, `-ls_min_alpha_psnr`, `-ls_thresh_psnr`, `-ls_thresh_alpha_psnr`, etc. + + - `-xs` disables 2-3 subset usage, and `-xp` disables dual plane usage (slightly higher compression, faster direct transcoding to BC7 will occur more often) + - `-higher_quality_transcoding`: Permits slower but higher quality transcoding + - `-no_deblocking`: Disables adaptive deblocking on ASTC block sizes > 8x6 (faster) + - `-force_deblocking`: Always use adaptive deblocking filter, even for block sizes <= 8x6 (slower) + - `-stronger_deblocking`: Use stronger deblocking when it's enabled (same performance) + - `-fast_xuastc_ldr_bc7_transcoding` and `-no_fast_xuastc_ldr_bc7_transcoding`: Controls faster direct XUASTC->BC7 transcoding (defaults to enabled, which is slightly lower quality) + +- To compress an LDR sRGB image to a standard ASTC LDR 6x6 .KTX2 file, using effort level 4 (valid effort levels 0-10): + +`basisu -astc_ldr_6x6 -effort 4 x.png` + +An alias for `-astc_ldr_6x6` is `-ldr_6x6`. -`basisu -q 255 x.png` +Just like XUASTC LDR, all 14 standard ASTC block sizes are supported, from 4x4-12x12. Internally the XUASTC LDR encoder is used, but standard ASTC block data is output, instead of supercompressed XUASTC LDR. Most XUASTC LDR options also work in ASTC LDR mode. -- For a linear LDR image, in ETC1S mode, at default quality (128): +- To compress an LDR sRGB image to an ETC1S .KTX2 file, at quality level 100 (the highest): + +`basisu -quality 100 x.png` + +- For a linear LDR image, in ETC1S mode, at default quality (`-quality 50`, or the older `-q 128`): `basisu -linear x.png` -- To compress to UASTC LDR, which is much higher quality than ETC1S: +- To compress to UASTC LDR 4x4, which is much higher quality than ETC1S, but lower maximum quality vs. ASTC/XUASTC LDR 4x4: `basisu -uastc x.png` @@ -159,36 +284,34 @@ Compressing and Unpacking .KTX2/.basis Files - To compress an HDR 6x6 file: ``` -basisu -hdr_6x6 x.exr -basisu -hdr_6x6 -lambda 500 x.exr +basisu -hdr_6x6 x.exr +basisu -hdr_6x6 -lambda 500 x.exr basisu -hdr_6x6_level 5 -lambda 500 x.exr ``` - To compress an HDR 6x6 file using the compressed intermediate format for smaller files: ``` -basisu -hdr_6x6i x.exr -basisu -hdr_6x6i -lambda 500 x.exr +basisu -hdr_6x6i x.exr +basisu -hdr_6x6i -lambda 500 x.exr basisu -hdr_6x6i_level 5 -lambda 500 x.exr ``` -Note the .EXR reader we're using is [TinyEXR's](https://github.com/syoyo/tinyexr), which doesn't support all possible .EXR compression modes. Tools like [ImageMagick](https://imagemagick.org/) can be used to create .EXR files that TinyEXR can read. - -Alternatively, LDR images (such as .PNG) can be compressed to an HDR format by specifying `-hdr`, `-hdr_6x6`, or `-hdr_6x6i`. By default LDR images, when compressed to an HDR format, are first upconverted to HDR by converting them from sRGB to linear light and scaled to 100 [nits](https://en.wikipedia.org/wiki/Candela_per_square_metre) (candelas per square meter). The sRGB conversion step can be disabled by specifying `-hdr_ldr_no_srgb_to_linear`, and the normalized RGB linear light to nit multiplier can be changed by specifying `-hdr_ldr_upconversion_nit_multiplier X`. +Note the unified `-quality` and `-effort` options work in HDR, too. These examples use the older non-unified options, which allow more direct/precise control. -Note: If you're compressing LDR/SDR image files to an HDR format, the codec's default behavior is to convert the 8-bit image data to linear light (by undoing the sRGB transfer function). It then multiplies the linear light RGB values by the LDR->HDR upconversion multiplier, which is in [nits (candela per sq. meter)](https://en.wikipedia.org/wiki/Candela_per_square_metre). In previous versions of the codec, this multiplier was effectively 1 nit, but it now defaults to 100 nits in all modes. (The typical luminance of LDR monitors is 80-100 nits.) To change this, use the "-hdr_ldr_upconversion_nit_multiplier X" command line option. (This is done because the HDR 6x6 codecs function internally in the [ICtCp HDR colorspace](https://en.wikipedia.org/wiki/ICtCp). LDR/SDR images must be upconverted to linear light HDR images scaled to a proper max. luminance based off how the image data will be displayed on actual SDR/HDR monitors.) +Be aware that the .EXR reader we use is [TinyEXR's](https://github.com/syoyo/tinyexr), which doesn't support all possible .EXR compression modes. Tools like [ImageMagick](https://imagemagick.org/) can be used to create .EXR files that TinyEXR can read. -### Some Useful Command Line Options +Alternatively, LDR images (such as .PNG) can be compressed to an HDR format by specifying `-hdr`, `-hdr_6x6`, or `-hdr_6x6i`. By default LDR images, when compressed to an HDR format, are first upconverted to HDR by converting them from sRGB to linear light and scaled to 100 [nits - candela per sq. meter, cd/m²](https://grokipedia.com/page/Candela_per_square_metre). The sRGB conversion step can be disabled by specifying `-hdr_ldr_no_srgb_to_linear`, and the normalized RGB linear light to nit multiplier can be changed by specifying `-hdr_ldr_upconversion_nit_multiplier X`. -- `-fastest` (which is equivalent to `-uastc_level 0`) puts the UASTC LDR/HDR encoders in their fastest (but lower quality) modes. +Note: If you're compressing LDR/SDR image files to an HDR format, the codec's default behavior is to convert the 8-bit image data to linear light (by undoing the sRGB transfer function). It then multiplies the linear light RGB values by the LDR->HDR upconversion multiplier, which is in nits. In previous versions of the codec, this multiplier was effectively 1 nit, but it now defaults to 100 nits in all modes. (The typical luminance of LDR monitors is 80-100 nits.) To change this, use the "-hdr_ldr_upconversion_nit_multiplier X" command line option. (This is done because the HDR 6x6 codecs function internally in the [ICtCp HDR colorspace](https://en.wikipedia.org/wiki/ICtCp). LDR/SDR images must be upconverted to linear light HDR images scaled to a proper max. luminance based on how the image data will be displayed on actual SDR/HDR monitors.) -- `-slower` puts the UASTC LDR/HDR encoders in higher quality but slower modes (equivalent to `-uastc_level 3`). The default level is 1, and the highest is 4 (which is quite slow). +### Some Useful Command Line Options -- `-q X`, where X ranges from [1,255], controls the ETC1S mode's quality vs. file size tradeoff level. 255 is the highest quality, and the default is 128. +- All codecs now support simple unified "quality" and "effort" settings. `-effort X` [0,10] controls how much of the search space (and how slowly) compression proceeds, and `-quality X` [1,100] controls the quality vs. bitrate tradeoff. Internally these settings will be mapped to each codec's specific configuration settings. Almost all the older settings still work, however. Previously, `-q X`, where X ranged from [1,255], controlled the ETC1S quality setting. This option is still available, but `-quality` is preferred now. - `-debug` causes the encoder to print internal and developer-oriented verbose debug information. -- `-stats` to see various quality (PSNR) statistics. +- `-stats` to see various quality (PSNR) statistics. - `-linear`: ETC1S defaults to sRGB colorspace metrics, UASTC LDR currently always uses linear metrics, and UASTC HDR defaults to weighted RGB metrics (with 2,3,1 weights). If the input is a normal map, or some other type of non-sRGB (non-photographic) texture content, be sure to use `-linear` to avoid extra unnecessary artifacts. (Angular normal map metrics for UASTC LDR/HDR are definitely doable and on our TODO list.) @@ -205,9 +328,9 @@ More Example Command Lines `-uastc_rdo_l X` controls the RDO ([Rate-Distortion Optimization](https://en.wikipedia.org/wiki/Rate%E2%80%93distortion_optimization)) quality setting. The lower this value, the higher the quality, but the larger the compressed file size. Good values to try are between .2-3.0. The default is 1.0. -- To add automatically generated mipmaps to a ETC1S .KTX2 file, at a higher than default quality level (which ranges from [1,255]): +- To add automatically generated mipmaps to an ETC1S .KTX2 file: -`basisu -mipmap -q 200 x.png` +`basisu -mipmap -quality 75 x.png` There are several mipmap options to change the filter kernel, the filter colorspace for the RGB channels (linear vs. sRGB), the smallest mipmap dimension, etc. The tool also supports generating cubemap files, 2D/cubemap texture arrays, etc. To bypass the automatic mipmap generator, you can create LDR or HDR uncompressed [.DDS texture files](https://learn.microsoft.com/en-us/windows/win32/direct3ddds/dx-graphics-dds-pguide) and feed them to the compressor. @@ -215,9 +338,9 @@ There are several mipmap options to change the filter kernel, the filter colorsp `basisu -comp_level 2 x.png` -On some rare images (ones with blue sky gradients come to bind), you may need to increase the ETC1S `-comp_level` setting, which ranges from 1,6. This controls the amount of overall effort the encoder uses to optimize the ETC1S codebooks and the compressed data stream. Higher comp_level's are *significantly* slower. +On some rare images (ones with blue sky gradients come to mind), you may need to increase the ETC1S `-comp_level` setting, which ranges from 1 to 6. This controls the amount of overall effort the encoder uses to optimize the ETC1S codebooks and the compressed data stream. Higher -comp_level's are *significantly* slower. -- To manually set the ETC1S codebook sizes (instead of using -q), with a higher codebook generation level (this is useful with texture video): +- To manually set the ETC1S codebook sizes (instead of using -quality, or the older -q options), with a higher codebook generation level (this is useful with texture video): `basisu x.png -comp_level 2 -max_endpoints 16128 -max_selectors 16128` @@ -238,31 +361,70 @@ See the help text for a complete listing of the tool's command line options. The Unpacking .KTX2/.basis files to .PNG/.EXR/.KTX/.DDS files --------------------------------------------------------- -You can either use the command line tool or [call the transcoder directly](https://github.com/BinomialLLC/basis_universal/wiki/How-to-Use-and-Configure-the-Transcoder) from JavaScript or C/C++ code to decompress .KTX2/.basis files to GPU texture data or uncompressed image data. To unpack a .KTX2 or.basis file to multiple .png/.exr/.ktx/.dds files: +You can either use the command line tool or [call the transcoder directly](https://github.com/BinomialLLC/basis_universal/wiki/How-to-Use-and-Configure-the-Transcoder) from JavaScript or C/C++ code to decompress .KTX2/.basis files to GPU texture data or uncompressed image data. To unpack a .KTX2 or .basis file to multiple .png/.exr/.ktx/.dds files: `basisu x.ktx2` -Use the `-no_ktx` and `-etc1_only`/`-format_only` options to unpack to less files. +Use the `-no_ktx` and `-etc1_only`/`-format_only` options to unpack to less files. + +`-info` and `-validate` will just display file information and not output any files. + +The written mipmapped, cubemap, or texture array .KTX/.DDS files will be in a wide variety of compressed GPU texture formats (PVRTC1 4bpp, ETC1-2, BC1-5, BC7, etc.), and to our knowledge there is unfortunately (as of 2024) still no single .KTX or .DDS viewer tool that correctly and reliably supports every GPU texture format that we support. BC1-5 and BC7 files are viewable using AMD's Compressonator, ETC1/2 using Mali's Texture Compression Tool, and PVRTC1 using Imagination Tech's PVRTexTool. [RenderDoc](https://renderdoc.org/) has a useful texture file viewer for many formats. The macOS Finder supports previewing .EXR and .KTX files in various GPU formats. The Windows 11 Explorer can preview .DDS files. The [online OpenHDR Viewer](https://viewer.openhdr.org/) is useful for viewing .EXR/.HDR image files. + +---- + +Python Support +-------------- + +All key encoder and all transcoder functionality is now available from Python, but this is still in the early stages of development. See the README files in the python directory for how to build the native SO's/PYD's. The Python support module supports both native and WASM modules, which is used as a fallback if native libraries can't be loaded. Python support has been tested under Ubuntu Linux and Windows 11 so far. -`-info` and `-validate` will just display file information and not output any files. +Example: +``` +cd python +python3 -m tests.test_backend_loading +========== BACKEND LOADING TEST ========== + +Testing native backend... +[Encoder] Using native backend + [OK] Native backend loaded +Hello from basisu_wasm_api.cpp version 200 + Native get_version() ? 200 + Native alloc() returned ptr = 190977024 + Native free() OK + [OK] Native basic operations working. + +Testing WASM backend... +[WASM Encoder] Loaded: /mnt/c/dev/xuastc4/python/basisu_py/wasm/basisu_module_st.wasm +[Encoder] Using WASM backend + [OK] WASM backend loaded +Hello from basisu_wasm_api.cpp version 200 + WASM get_version() ? 200 + WASM alloc() returned ptr = 26920160 + WASM free() OK + [OK] WASM basic operations working. + +========== DONE ========== +``` -The written mipmapped, cubemap, or texture array .KTX/.DDS files will be in a wide variety of compressed GPU texture formats (PVRTC1 4bpp, ETC1-2, BC1-5, BC7, etc.), and to our knowledge there is unfortunately (as of 2024) still no single .KTX or .DDS viewer tool that correctly and reliably supports every GPU texture format that we support. BC1-5 and BC7 files are viewable using AMD's Compressonator, ETC1/2 using Mali's Texture Compression Tool, and PVRTC1 using Imagination Tech's PVRTexTool. [RenderDoc](https://renderdoc.org/) has a useful texture file viewer for many formats. The Mac OSX Finder supports previewing .EXR and .KTX files in various GPU formats. The Windows 11 Explorer can preview .DDS files. The [online OpenHDR Viewer](https://viewer.openhdr.org/) is useful for viewing .EXR/.HDR image files. +---- WebGL Examples -------------- -The 'WebGL' directory contains several simple WebGL demos that use the transcoder and compressor compiled to [WASM](https://webassembly.org/) with [emscripten](https://emscripten.org/). These demos are online [here](https://subquantumtech.com/uastchdr2/). See more details in the readme file [here](webgl/README.md). +The 'WebGL' directory contains several simple WebGL demos that use the transcoder and compressor compiled to [WASM](https://webassembly.org/) with [Emscripten](https://emscripten.org/). These demos are online [here](https://subquantumtech.com/xu/). See more details in the readme file [here](webgl/README.md). ![Screenshot of 'texture' example running in a browser.](webgl/texture_test/preview.png) ![Screenshot of 'gltf' example running in a browser.](webgl/gltf/preview.png) ![Screenshot of 'encode_test' example running in a browser.](webgl/ktx2_encode_test/preview.png) -Building the WASM Modules with [Emscripten](https://emscripten.org/) +---- + +Building the WASM Modules with [Emscripten](https://emscripten.org/) -------------------------------------------------------------------- -Both the transcoder and encoder may be compiled using emscripten to WebAssembly and used on the web. A set of JavaScript wrappers to the codec, written in C++ with emscripten extensions, is located in `webgl/transcoding/basis_wrappers.cpp`. The JavaScript wrapper supports nearly all features and modes, including texture video. See the README.md and CMakeLists.txt files in `webgl/transcoder` and `webgl/encoder`. +Both the transcoder and encoder may be compiled using Emscripten to WebAssembly and used on the web. A set of JavaScript wrappers to the codec, written in C++ with Emscripten extensions, is located in [`webgl/transcoding/basis_wrappers.cpp`](https://github.com/BinomialLLC/basis_universal/blob/master/webgl/transcoder/basis_wrappers.cpp). The JavaScript wrapper supports nearly all features and modes, including texture video. See the [README.md](https://github.com/BinomialLLC/basis_universal/tree/master/webgl) and CMakeLists.txt files in `webgl/transcoder` and `webgl/encoder`. -To build the WASM transcoder, after installing emscripten: +To build the WASM transcoder, after installing Emscripten: ``` cd webgl/transcoder/build @@ -278,17 +440,16 @@ emcmake cmake .. make ``` -There are two simple encoding/transcoding web demos, located in `webgl/ktx2_encode_test` and `webgl/texture_test`, that show how to use the encoder's and transcoder's Javascript wrapper API's. +There are several simple encoding/transcoding web demos, located in `webgl/ktx2_encode_test` and `webgl/texture_test`, that show how to use the encoder's and transcoder's JavaScript wrapper APIs. They are [live on the web here](https://subquantumtech.com/xu/). + +---- Low-level C++ Encoder/Transcoder API Examples --------------------------------------------- -Some simple examples showing how to directly call the C++ encoder and transcoder library API's are in [`example/examples.cpp`](https://github.com/BinomialLLC/basis_universal/blob/master/example/example.cpp). +Some simple examples showing how to directly call the C++ encoder and transcoder library APIs are in [`example/example.cpp`](https://github.com/BinomialLLC/basis_universal/blob/master/example/example.cpp). -ETC1S Texture Video Tips ------------------------- - -See the wiki [here](https://github.com/BinomialLLC/basis_universal/wiki/Encoding-ETC1S-Texture-Video-Tips). +---- Installation using the vcpkg dependency manager ----------------------------------------------- @@ -303,10 +464,26 @@ You can download and install Basis Universal using the [vcpkg](https://github.co The Basis Universal port in vcpkg is kept up to date by Microsoft team members and community contributors. If the version is out of date, please [create an issue or pull request](https://github.com/Microsoft/vcpkg) on the vcpkg repository. (9/10/2024: UASTC HDR support is not available here yet.) +--- + +Project Policies +---------------- + +See our wiki page: [Project Policies: PRs, compiler warnings, release cadence etc.](https://github.com/BinomialLLC/basis_universal/wiki/Project-Policies:-PR's,-compiler-warnings,-release-cadence,-etc.). + +---- + +KTX2 Support Status +------------------- + +Note as of Jan. 2026 we are working with Khronos on the exact details of how we embed UASTC HDR 6x6 intermediate and XUASTC LDR supercompressed texture data into the KTX2 file format. We expect some KTX2 file format changes to be merged in within a month or so. .basis files shouldn't be impacted. + +---- + License ------- -The transcoder and core encoder libraries are Apache 2.0. The transcoder utilizes no 3rd party libraries or dependencies. See [LICENSE](https://github.com/BinomialLLC/basis_universal/blob/master/LICENSE). +The transcoder and core encoder libraries are Apache 2.0. The transcoder utilizes no 3rd party libraries or dependencies, other than Zstd (which is optional but limits the transcoder to non-Zstd utilizing codecs). See [LICENSE](https://github.com/BinomialLLC/basis_universal/blob/master/LICENSE). The encoder library is Apache 2.0, but it utilizes some open source 3rd party modules (in 'encoder/3rdparty' and in the 'Zstd' directory) to load [.QOI](https://qoiformat.org/), [.DDS](https://github.com/DeanoC/tiny_dds), [.EXR](https://github.com/syoyo/tinyexr) images, to handle [Zstd](https://github.com/facebook/zstd) compression, and to unpack ASTC texture blocks. See the [LICENSES](https://github.com/BinomialLLC/basis_universal/tree/master/LICENSES) and [.reuse](https://github.com/BinomialLLC/basis_universal/blob/master/.reuse/dep5) folders. @@ -319,19 +496,13 @@ checking tool (https://reuse.software/). See the `.reuse` subdirectory. External Tool Links ------------------- -[Online .EXR HDR Image File Viewer](https://viewer.openhdr.org/) - -[Windows HDR + WCG Image Viewer](https://13thsymphony.github.io/hdrimageviewer/) - A true HDR image viewer for Windows. Also see [the github repo](https://github.com/13thsymphony/HDRImageViewer). - -[RenderDoc](https://renderdoc.org/) - -[AMD Compressonator](https://gpuopen.com/gaming-product/compressonator/) - -[Microsoft's DirectXTex](https://github.com/microsoft/DirectXTex) - -[PVRTexTool](https://www.imgtec.com/developers/powervr-sdk-tools/pvrtextool/) - -[Mali Texture Compression Tool](https://community.arm.com/support-forums/f/graphics-gaming-and-vr-forum/52390/announcement-mali-texture-compression-tool-end-of-life) - Now deprecated +- [Online .EXR HDR Image File Viewer](https://viewer.openhdr.org/) +- [Windows HDR + WCG Image Viewer](https://13thsymphony.github.io/hdrimageviewer/) - A true HDR image viewer for Windows. Also see [the github repo](https://github.com/13thsymphony/HDRImageViewer). +- [RenderDoc](https://renderdoc.org/) +- [AMD Compressonator](https://gpuopen.com/gaming-product/compressonator/) +- [Microsoft's DirectXTex](https://github.com/microsoft/DirectXTex) +- [PVRTexTool](https://www.imgtec.com/developers/powervr-sdk-tools/pvrtextool/) +- [Mali Texture Compression Tool](https://community.arm.com/support-forums/f/graphics-gaming-and-vr-forum/52390/announcement-mali-texture-compression-tool-end-of-life) - Now deprecated For more useful links, papers, and tools/libraries, see the end of the [UASTC HDR texture specification](https://github.com/BinomialLLC/basis_universal/wiki/UASTC-HDR-Texture-Specification-v1.0). diff --git a/external/basis_universal/all_builds.py b/external/basis_universal/all_builds.py new file mode 100644 index 0000000000..005c73e7ea --- /dev/null +++ b/external/basis_universal/all_builds.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +import subprocess +import shutil +import os +import sys + +# ------------------------------------------------------------------- +# CONFIGURATION - Easily add new build directories and options. +# ------------------------------------------------------------------- +BUILD_CONFIGS = { + "build_python": ["cmake", "-DBASISU_SSE=1 -DBASISU_BUILD_PYTHON=ON", ".."], + "build_wasm_mt": ["cmake", "-DCMAKE_TOOLCHAIN_FILE=$WASI_SDK_PATH/share/cmake/wasi-sdk-pthread.cmake -DCMAKE_BUILD_TYPE=Release -DBASISU_WASM_THREADING=ON", ".."], + "build_wasm_st": ["cmake", "-DCMAKE_TOOLCHAIN_FILE=$WASI_SDK_PATH/share/cmake/wasi-sdk.cmake -DCMAKE_BUILD_TYPE=Release -DBASISU_WASM_THREADING=OFF", ".."], + "build_native": ["cmake", "-DBASISU_SSE=1", ".."] +} +# ------------------------------------------------------------------- + + +def log(msg): + print(f"[INFO] {msg}") + + +def run(cmd, work_dir): + """ + Execute a shell command after changing the working directory. + Always restore the original directory, even on exceptions. + """ + + if isinstance(cmd, list): + cmd = " ".join(cmd) + + original_dir = os.getcwd() + + log(f"Preparing to run command:\n CMD: {cmd}\n IN: {work_dir}") + print(f"[INFO] Current working directory before change: {original_dir}") + + try: + os.chdir(work_dir) + print(f"[INFO] Changed working directory to: {os.getcwd()}") + + log(f"Running command: {cmd}") + subprocess.check_call(cmd, shell=True) + + except subprocess.CalledProcessError: + log(f"ERROR: Command failed: {cmd}") + raise + + finally: + # Always restore the directory + os.chdir(original_dir) + print(f"[INFO] Restored working directory to: {original_dir}") + + +def clean_build_dirs(): + log("Cleaning all build directories...") + for build_dir in BUILD_CONFIGS: + if os.path.isdir(build_dir): + log(f"Deleting directory: {build_dir}") + shutil.rmtree(build_dir) + else: + log(f"Directory not found, skipping: {build_dir}") + log("Clean complete.\n") + + +def create_dir(path): + if not os.path.isdir(path): + log(f"Creating directory: {path}") + os.makedirs(path) + else: + log(f"Directory already exists: {path}") + + +def perform_builds(): + for build_dir, cmake_cmd in BUILD_CONFIGS.items(): + log(f"Starting build in: {build_dir}") + + create_dir(build_dir) + + # Run CMake inside the directory + log(f"Executing CMake for {build_dir}") + run(cmake_cmd, work_dir=build_dir) + + # Run Make inside the directory + log(f"Running make for {build_dir}") + run("make", work_dir=build_dir) + + log(f"Finished build for {build_dir}\n") + + +def main(): + if "--clean" in sys.argv: + clean_build_dirs() + + perform_builds() + log("SUCCESS\n") + + +if __name__ == "__main__": + main() diff --git a/external/basis_universal/appveyor.yml b/external/basis_universal/appveyor.yml index 9e35a55971..ad9be959a2 100644 --- a/external/basis_universal/appveyor.yml +++ b/external/basis_universal/appveyor.yml @@ -11,19 +11,51 @@ environment: APPVEYOR_YML_DISABLE_PS_LINUX: true build_script: + # ============================ + # Windows (PowerShell) + # ============================ - ps: | New-Item -Path . -Name "build" -ItemType "directory" cd build + cmake --version + cmake ../ -DCMAKE_BUILD_TYPE:STRING="$env:CONFIGURATION" + if ($LASTEXITCODE -ne 0) { + Write-Host "ERROR: CMake configuration failed" + exit $LASTEXITCODE + } + cmake --build . --config $env:CONFIGURATION + if ($LASTEXITCODE -ne 0) { + Write-Host "ERROR: Build failed" + exit $LASTEXITCODE + } + cd ../ + + + # ============================ + # Linux + macOS (sh) + # ============================ - sh: | mkdir build cd build + cmake --version + cmake ../ -DCMAKE_BUILD_TYPE:STRING="${CONFIGURATION}" + if [ $? -ne 0 ]; then + echo "ERROR: CMake configuration failed" + exit 1 + fi + cmake --build . --config ${CONFIGURATION} + if [ $? -ne 0 ]; then + echo "ERROR: Build failed" + exit 1 + fi + cd ../ artifacts: @@ -32,4 +64,4 @@ artifacts: # MacOS - path: bin/basisu # Windows - - path: bin\$(configuration)\basisu.exe + - path: bin\$(CONFIGURATION)\basisu.exe diff --git a/external/basis_universal/basisu.sln b/external/basis_universal/basisu.sln index 2309ae59ab..566f85c43f 100644 --- a/external/basis_universal/basisu.sln +++ b/external/basis_universal/basisu.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.8.34322.80 +# Visual Studio Version 18 +VisualStudioVersion = 18.0.11222.15 MinimumVisualStudioVersion = 10.0.40219.1 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "basisu", "basisu.vcxproj", "{59586A07-8E7E-411D-BC3D-387E039AA423}" EndProject @@ -9,6 +9,10 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "example", "example\example. EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "encoder_lib", "encoder_lib\encoder_lib.vcxproj", "{97C34996-F458-4030-A402-B32C581872F1}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "example_transcoding", "example_transcoding\example_transcoding.vcxproj", "{13333092-FCFE-4D74-8E76-F10C6037593C}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "example_capi", "example_capi\example_capi.vcxproj", "{BE889347-E4FD-47DD-BBF4-81F98FAA8BA9}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|ARM64EC = Debug|ARM64EC @@ -55,6 +59,30 @@ Global {97C34996-F458-4030-A402-B32C581872F1}.Release|x64.Build.0 = Release|x64 {97C34996-F458-4030-A402-B32C581872F1}.Release|x86.ActiveCfg = Release|Win32 {97C34996-F458-4030-A402-B32C581872F1}.Release|x86.Build.0 = Release|Win32 + {13333092-FCFE-4D74-8E76-F10C6037593C}.Debug|ARM64EC.ActiveCfg = Debug|ARM64EC + {13333092-FCFE-4D74-8E76-F10C6037593C}.Debug|ARM64EC.Build.0 = Debug|ARM64EC + {13333092-FCFE-4D74-8E76-F10C6037593C}.Debug|x64.ActiveCfg = Debug|x64 + {13333092-FCFE-4D74-8E76-F10C6037593C}.Debug|x64.Build.0 = Debug|x64 + {13333092-FCFE-4D74-8E76-F10C6037593C}.Debug|x86.ActiveCfg = Debug|Win32 + {13333092-FCFE-4D74-8E76-F10C6037593C}.Debug|x86.Build.0 = Debug|Win32 + {13333092-FCFE-4D74-8E76-F10C6037593C}.Release|ARM64EC.ActiveCfg = Release|ARM64EC + {13333092-FCFE-4D74-8E76-F10C6037593C}.Release|ARM64EC.Build.0 = Release|ARM64EC + {13333092-FCFE-4D74-8E76-F10C6037593C}.Release|x64.ActiveCfg = Release|x64 + {13333092-FCFE-4D74-8E76-F10C6037593C}.Release|x64.Build.0 = Release|x64 + {13333092-FCFE-4D74-8E76-F10C6037593C}.Release|x86.ActiveCfg = Release|Win32 + {13333092-FCFE-4D74-8E76-F10C6037593C}.Release|x86.Build.0 = Release|Win32 + {BE889347-E4FD-47DD-BBF4-81F98FAA8BA9}.Debug|ARM64EC.ActiveCfg = Debug|ARM64EC + {BE889347-E4FD-47DD-BBF4-81F98FAA8BA9}.Debug|ARM64EC.Build.0 = Debug|ARM64EC + {BE889347-E4FD-47DD-BBF4-81F98FAA8BA9}.Debug|x64.ActiveCfg = Debug|x64 + {BE889347-E4FD-47DD-BBF4-81F98FAA8BA9}.Debug|x64.Build.0 = Debug|x64 + {BE889347-E4FD-47DD-BBF4-81F98FAA8BA9}.Debug|x86.ActiveCfg = Debug|Win32 + {BE889347-E4FD-47DD-BBF4-81F98FAA8BA9}.Debug|x86.Build.0 = Debug|Win32 + {BE889347-E4FD-47DD-BBF4-81F98FAA8BA9}.Release|ARM64EC.ActiveCfg = Release|ARM64EC + {BE889347-E4FD-47DD-BBF4-81F98FAA8BA9}.Release|ARM64EC.Build.0 = Release|ARM64EC + {BE889347-E4FD-47DD-BBF4-81F98FAA8BA9}.Release|x64.ActiveCfg = Release|x64 + {BE889347-E4FD-47DD-BBF4-81F98FAA8BA9}.Release|x64.Build.0 = Release|x64 + {BE889347-E4FD-47DD-BBF4-81F98FAA8BA9}.Release|x86.ActiveCfg = Release|Win32 + {BE889347-E4FD-47DD-BBF4-81F98FAA8BA9}.Release|x86.Build.0 = Release|Win32 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/external/basis_universal/basisu.vcxproj b/external/basis_universal/basisu.vcxproj index fce24bde5f..1e52b46772 100644 --- a/external/basis_universal/basisu.vcxproj +++ b/external/basis_universal/basisu.vcxproj @@ -1,4 +1,4 @@ - + @@ -36,40 +36,40 @@ Application true MultiByte - v143 + v145 Application false true MultiByte - v143 + v145 Application true Unicode - v143 + v145 Application true Unicode - v143 + v145 Application false true Unicode - v143 + v145 Application false true Unicode - v143 + v145 @@ -129,7 +129,7 @@ true OpenCL _MBCS;%(PreprocessorDefinitions);BASISU_SUPPORT_SSE=1;BASISU_SUPPORT_OPENCL=1;_HAS_EXCEPTIONS=0 - StreamingSIMDExtensions2 + AdvancedVectorExtensions stdcpp17 @@ -147,9 +147,9 @@ true true _MBCS;%(PreprocessorDefinitions);BASISU_SUPPORT_SSE=1;BASISU_SUPPORT_OPENCL=1; - StreamingSIMDExtensions2 - Level4 + AdvancedVectorExtensions stdcpp17 + Level4 Console @@ -186,7 +186,7 @@ NDEBUG;_HAS_EXCEPTIONS=0;%(PreprocessorDefinitions);BASISU_SUPPORT_SSE=1;BASISU_SUPPORT_OPENCL=1 false AnySuitable - StreamingSIMDExtensions2 + AdvancedVectorExtensions Precise false true @@ -217,7 +217,7 @@ Precise true Speed - StreamingSIMDExtensions2 + AdvancedVectorExtensions false stdcpp17 @@ -266,7 +266,10 @@ {97c34996-f458-4030-a402-b32c581872f1} + + + - + \ No newline at end of file diff --git a/external/basis_universal/basisu.vcxproj.filters b/external/basis_universal/basisu.vcxproj.filters index a772c989f3..5281d4839a 100644 --- a/external/basis_universal/basisu.vcxproj.filters +++ b/external/basis_universal/basisu.vcxproj.filters @@ -1,4 +1,4 @@ - + @@ -6,4 +6,7 @@ - + + + + \ No newline at end of file diff --git a/external/basis_universal/basisu_tool.cpp b/external/basis_universal/basisu_tool.cpp index 4c63f461dd..65034c02bc 100644 --- a/external/basis_universal/basisu_tool.cpp +++ b/external/basis_universal/basisu_tool.cpp @@ -1,4 +1,4 @@ -// basisu_tool.cpp +// basisu_tool.cpp // Copyright (C) 2019-2025 Binomial LLC. All Rights Reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. #if _MSC_VER -// For sprintf(), strcpy() +// For sprintf(), strcpy() #define _CRT_SECURE_NO_WARNINGS (1) #pragma warning(disable:4505) // unreferenced function with internal linkage has been removed #pragma warning(disable:4189) // local variable is initialized but not referenced @@ -30,6 +30,7 @@ #include "transcoder/basisu_transcoder.h" #include "encoder/basisu_ssim.h" #include "encoder/basisu_opencl.h" +#include "encoder/basisu_astc_ldr_common.h" #define MINIZ_HEADER_FILE_ONLY #define MINIZ_NO_ZLIB_COMPATIBLE_NAMES @@ -44,6 +45,7 @@ #ifdef _WIN32 #define WIN32_LEAN_AND_MEAN #include +// Work around Win11 debug console bug. DO NOT SHIP SET TO 1. #define CLEAR_WIN32_CONSOLE 0 #endif @@ -55,7 +57,21 @@ using namespace basisu; using namespace buminiz; -#define BASISU_TOOL_VERSION "1.60.0" +#define BASISU_TOOL_VERSION "2.00.0" + +#if defined(DEBUG) +#pragma message("DEBUG defined") +#endif + +#if defined(_DEBUG) +#pragma message("_DEBUG defined") +#endif + +#ifndef NDEBUG +#if !defined(DEBUG) && !defined(_DEBUG) +#pragma message("NDEBUG is NOT defined, but DEBUG or _DEBUG are also NOT defined, which isn't ideal as extra debug assertion checks will not be compiled.") +#endif +#endif // Define to lower the -test and -test_hdr tolerances //#define USE_TIGHTER_TEST_TOLERANCES @@ -79,6 +95,7 @@ enum tool_mode cTestHDR_4x4, cTestHDR_6x6, cTestHDR_6x6i, + cTestXUASTCLDR, cCLBench, cSplitImage, cCombineImages, @@ -89,197 +106,8 @@ static void print_usage() { printf("\nUsage: basisu filename [filename ...] \n"); - puts("\n" - "The default processing mode is compression of one or more .PNG/.BMP/.TGA/.JPG/.QOI/.DDS/.EXR/.HDR files to a LDR or HDR .KTX2 file. Alternate modes:\n" - " -unpack: Use transcoder to unpack a .basis/.KTX2 file to one or more .KTX/.PNG files\n" - " -validate: Validate and display information about a .basis/.KTX2 file\n" - " -info: Display high-level information about a .basis/.KTX2 file\n" - " -compare: Compare two LDR PNG/BMP/TGA/JPG/QOI images specified with -file, output PSNR and SSIM statistics and RGB/A delta images\n" - " -compare_hdr: Compare two HDR .EXR/.HDR images specified with -file, output PSNR statistics and RGB delta images\n" - " -tonemap: Tonemap an HDR or EXR image to PNG at multiple exposures, use -file to specify filename\n" - " -version: Print version and exit\n" - "\n" - "--- Notes:\n" - "\nUnless an explicit mode is specified, if one or more files have the .basis or .KTX2 extension this tool defaults to unpack mode.\n" - "\nBy default, the compressor assumes the input is in the sRGB colorspace (like photos/albedo textures).\n" - "If the input is NOT sRGB (like a normal map), be sure to specify -linear for less artifacts. Depending on the content type, some experimentation may be needed.\n" - "\n" - "The TinyEXR library is used to read .EXR images. This library does not support all .EXR compression methods. For unsupported images, you can use ImageMagick to convert them to uncompressed .EXR.\n" - "\n" - "For .DDS source files: Mipmapped or not mipmapped 2D textures (but not cubemaps) are supported. Only uncompressed 32-bit RGBA/BGRA, half float RGBA, or float RGBA .DDS files are supported. In -tex_array mode, if a .DDS file is specified, all source files must be in .DDS format.\n" - "\n" - "Filenames prefixed with a @ symbol are read as filename listing files. Listing text files specify which actual filenames to process (one filename per line).\n" - "\n" - "--- Texture Mode Options:\n" - " -etc1s: Encode to ETC1S LDR (the default for SDR/LDR inputs). Roughly .8-2.5 bpp.\n" - " -uastc: Encode to UASTC LDR 4x4. Roughly 5-8 bpp.\n" - " -hdr/-hdr_4x4: Encode input as UASTC HDR 4x4 (the default if any input file has the .EXR or .HDR extension, or if any .DDS file is HDR). Roughly 5-8 bpp.\n" - " -hdr_6x6: Encode input as RDO or highest quality UASTC HDR 6x6. Use -lambda X (try 100-20000 or higher) option to enable RDO UASTC HDR 6x6, where x controls the quality vs. size tradeoff. Roughly 1.2-3.2 bpp.\n" - " -hdr_6x6i: Encode input as UASTC HDR 6x6 intermediate. Use -lambda X (try 100-20000 or higher) option to enable RDO UASTC HDR 6x6, where x controls the quality vs. size tradeoff. Roughly 1-3.2 bpp.\n" - "\n" - "--- Options:\n" - " -ktx2: Write .KTX2 files (the default). By default, UASTC LDR/HDR 4x4 and ASTC 6x6 files will be compressed using Zstandard unless -ktx2_no_zstandard is specified.\n" - " -basis: Write .basis files instead of .KTX2 files (the previous default).\n" - " -file filename.png/tga/jpg/qoi/exr/hdr: Input image filename, multiple images are OK, use -file X for each input filename (prefixing input filenames with -file is optional)\n" - " -alpha_file filename.png/tga/jpg/qoi: Input alpha image filename, multiple images are OK, use -file X for each input filename (must be paired with -file), images converted to REC709 grayscale and used as input alpha\n" - " -output_file filename: Output .basis/.KTX2 filename\n" - " -output_path: Output .basis/.KTX2 files to specified directory.\n" - " -debug or -verbose: Enable codec debug print to stdout (slightly slower).\n" - " -debug_images: Enable codec debug images (much slower).\n" - " -stats: Compute and display image quality metrics (slightly to much slower).\n" - " -individual: Process input images individually and output multiple .basis/.KTX2 files (not as a texture array - this is now the default as of v1.16)\n" - "\n" - " -fastest: Set UASTC LDR 4x4 and HDR 4x4/6x6 to fastest but lowest quality encoding mode (same as -uastc_level 0 or -hdr_6x6_level 0)\n" - " -slower: Set UASTC LDR 4x4 and HDR 4x4/6x6 to slower but a higher quality encoding mode (same as -uastc_level 3 or -hdr_6x6_level 5)\n" - " -parallel: Compress multiple textures simumtanously (one per thread), instead of one at a time. Compatible with OpenCL mode. This is much faster, but in OpenCL mode the driver is pushed harder, and the CLI output will be jumbled.\n" - " -linear: Use linear colorspace metrics (instead of the default sRGB or scaled RGB for HDR), and by default linear (not sRGB) mipmap filtering.\n" - " -tex_type <2d, 2darray, 3d, video, cubemap>: Set Basis file header's texture type field. Cubemap arrays require multiples of 6 images, in X+, X-, Y+, Y-, Z+, Z- order, each image must be the same resolutions.\n" - " 2d=arbitrary 2D images, 2darray=2D array, 3D=volume texture slices, video=video frames, cubemap=array of faces. For 2darray/3d/cubemaps/video, each source image's dimensions and # of mipmap levels must be the same.\n" - " For video, the .basis file will be written with the first frame being an I-Frame, and subsequent frames being P-Frames (using conditional replenishment). Playback must always occur in order from first to last image.\n" - " -cubemap: same as -tex_type cubemap\n" - " -tex_array: Process input images as a single texture array and write a single .basis/.KTX2 file (the former default before v1.16)\n" - " -fuzz_testing: Use with -validate: Disables CRC16 validation of file contents before transcoding\n" - " -multifile_printf: printf() format strint to use to compose multiple filenames\n" - " -multifile_first: The index of the first file to process, default is 0 (must specify -multifile_printf and -multifile_num)\n" - " -multifile_num: The total number of files to process.\n" - " -opencl: Enable OpenCL usage (currently only accelerates ETC1S encoding)\n" - " -opencl_serialize: Serialize all calls to the OpenCL driver (to work around buggy drivers, only useful with -parallel)\n" - "\n" - "--- ETC1S specific options (-etc1s - the LDR/SDR default):\n" - " -q X: Set ETC1S quality level, 1-255, default is 128, lower=better compression/lower quality/faster, higher=less compression/higher quality/slower, default is 128. For even higher quality, use -max_endpoints/-max_selectors.\n" - " -comp_level X: Set ETC1S encoding speed vs. quality tradeoff. Range is 0-6, default is 1. Higher values=MUCH slower, but slightly higher quality. Higher levels intended for videos. Use -q first!\n" - " -max_endpoints X: ETC1S: Manually set the max number of color endpoint clusters from 1-16128, use instead of -q\n" - " -max_selectors X: ETC1S: Manually set the max number of color selector clusters from 1-16128, use instead of -q\n" - "\n" - "--- UASTC LDR/HDR 4x4 specific options (-uastc):\n" - " -uastc: Enable UASTC LDR 4x4 texture mode, instead of the default ETC1S mode. Significantly higher texture quality, but much larger (~8bpp) files. (Note that UASTC .basis files must be losslessly compressed by the user.)\n" - " -uastc_level: Set UASTC LDR/HDR 4x4 encoding level. LDR Range is [0,4], default is 2, higher=slower but higher quality. 0=fastest/lowest quality, 3=slowest practical option, 4=impractically slow/highest achievable quality\n" - " UASTC HDR 4x4 range is [0,4] - higher=slower but higher quality. HDR 4x4 default level=1.\n" - " -uastc_rdo_l X: Enable UASTC LDR 4x4 RDO post-processing and set UASTC LDR 4x4 RDO quality scalar (lambda) to X. Lower values=higher quality/larger LZ\n" - " compressed files, higher values=lower quality/smaller LZ compressed files. Good range to try is [.25-10].\n" - " Note: Previous versons used the -uastc_rdo_q option, which was removed because the RDO algorithm was changed.\n" - " -uastc_rdo_d X: Set UASTC LDR 4x4 RDO dictionary size in bytes. Default is 4096, max is 65536. Lower values=faster, but less compression.\n" - " -uastc_rdo_b X: Set UASTC LDR 4x4 RDO max smooth block error scale. Range is [1,300]. Default is 10.0, 1.0=disabled. Larger values suppress more artifacts (and allocate more bits) on smooth blocks.\n" - " -uastc_rdo_s X: Set UASTC LDR 4x4 RDO max smooth block standard deviation. Range is [.01,65536]. Default is 18.0. Larger values expand the range of blocks considered smooth.\n" - " -uastc_rdo_f: Don't favor simpler UASTC LDR 4x4 modes in RDO mode.\n" - " -uastc_rdo_m: Disable RDO multithreading (slightly higher compression, deterministic).\n" - "\n" - "--- UASTC HDR 4x4 specific options (-hdr or -hdr_4x4 - the HDR default):\n" - " -uastc_level X: Sets the UASTC HDR 4x4 compressor's level. Valid range is [0,4] - higher=slower but higher quality. HDR default=1.\n" - " Level 0=fastest/lowest quality, 3=highest practical setting, 4=exhaustive\n" - " -hdr_uber_mode: Allow the UASTC HDR 4x4 encoder to try varying the CEM 11 selectors more for slightly higher quality (slower). This may negatively impact BC6H quality, however.\n" - " -hdr_ultra_quant: UASTC HDR 4x4: Try to find better quantized CEM 7/11 endpoint values (slower).\n" - " -hdr_favor_astc: UASTC HDR 4x4: By default the UASTC HDR 4x4 encoder tries to strike a balance or even slightly favor BC6H quality. If this option is specified, ASTC HDR 4x4 quality is favored instead.\n" - "\n" - "--- UASTC HDR 6x6 specific options (-hdr_6x6 or -hdr_6x6i):\n" - " -lambda X: Enables rate distortion optimization (RDO). The higher this value, the lower the quality, but the smaller the file size. Try 100-20000, or higher values on some images.\n" - " -hdr_6x6_level X: Sets the codec to 6x6 HDR mode (same as -hdr_6x6) and controls encoder performance vs. max quality tradeoff. X may range from [0,12]. Default level is 2. Higher values result in better quality but slower encoding. Values above 10 are extremely slow.\n" - " -hdr_6x6i_level X: Sets the codec to 6x6 HDR intermediate mode (same as -hdr_6x6i) and controls encoder performance vs. max quality tradeoff. X may range from [0,12]. Default level is 2.\n" - " -rec_2020: The input image's gamut is Rec. 2020 vs. the default Rec. 709 - for accurate colorspace error calculations.\n" - " -hdr_6x6_jnd X, -hdr_6x6_extra_pats, -hdr_6x6_brute_force_pats, -hdr_6x6_comp_levels X Y or -hdr_6x6i_comp_levels X Y: Low-level control over the encoder's configuration.\n" - "\n" - "--- SDR/LDR->HDR upconversion options (only used when encoding to HDR formats from an LDR/SDR source image):\n" - " -hdr_ldr_no_srgb_to_linear: If specified, LDR images will NOT be converted to normalized linear light (via a sRGB->Linear conversion) during SDR->HDR upconversion before compressing as HDR.\n" - " -hdr_ldr_upconversion_nit_multiplier X: Specify how many nits (candelas per sq. meter) LDR/SDR images are converted to after converting to linear light. Default is 100 nits. Note: Previous builds used 1 nit.\n" - "\n" - "--- More options:\n" - " -test: Run an automated LDR ETC1S/UASTC encoding and transcoding test. Returns EXIT_FAILURE if any failures\n" - " -test_hdr_4x4/-test_hdr_6x6/-test_hdr_6x6i: Run automated UASTC HDR encoding and transcoding tests. Returns EXIT_FAILURE if any failures\n" - " -test_dir: Optional directory of test files. Defaults to \"../test_files\".\n" - " -y_flip: Flip input images vertically before compression\n" - " -normal_map: Tunes codec parameters for better quality on normal maps (linear colorspace metrics, linear mipmap filtering, no selector RDO, no sRGB)\n" - " -no_alpha: Always output non-alpha basis files, even if one or more inputs has alpha\n" - " -force_alpha: Always output alpha basis files, even if no inputs has alpha\n" - " -separate_rg_to_color_alpha: Separate input R and G channels to RGB and A (for tangent space XY normal maps)\n" - " -swizzle rgba: Specify swizzle for the 4 input color channels using r, g, b and a (the -separate_rg_to_color_alpha flag is equivalent to rrrg)\n" - " -renorm: Renormalize each input image before any further processing/compression\n" - " -no_multithreading: Disable multithreading\n" - " -max_threads X: Use at most X threads total when multithreading is enabled (this includes the main thread)\n" - " -no_ktx: Disable KTX writing when unpacking (faster, less output files)\n" - " -ktx_only: Only write KTX files when unpacking (faster, less output files)\n" - " -write_out: Write 3dfx OUT files when unpacking FXT1 textures\n" - " -format_only: Only unpack the specified format, by its numeric code.\n" - " -etc1_only: Only unpack to ETC1, skipping the other texture formats during -unpack\n" - " -disable_hierarchical_endpoint_codebooks: Disable hierarchical endpoint codebook usage, slower but higher quality on some compression levels\n" - " -compare_ssim: Compute and display SSIM of image comparison (slow)\n" - " -compare_plot: Display histogram plots in -compare mode\n" - " -bench: UASTC benchmark mode, for development only\n" - " -resample X Y: Resample all input textures to XxY pixels using a box filter\n" - " -resample_factor X: Resample all input textures by scale factor X using a box filter\n" - " -no_sse: Forbid all SSE instruction set usage\n" - " -validate_etc1s: Validate internal ETC1S compressor's data structures during compression (slower, intended for development).\n" - " -ktx2_animdata_duration X: Set KTX2animData duration field to integer value X (only valid/useful for -tex_type video, default is 1)\n" - " -ktx2_animdata_timescale X: Set KTX2animData timescale field to integer value X (only valid/useful for -tex_type video, default is 15)\n" - " -ktx2_animdata_loopcount X: Set KTX2animData loopcount field to integer value X (only valid/useful for -tex_type video, default is 0)\n" - " -framerate X: Set framerate in .basis header to X/frames sec.\n" - " -ktx2_no_zstandard: Don't compress UASTC texture data using Zstandard -- store it uncompressed instead.\n" - " -ktx2_zstandard_level X: Set ZStandard compression level to X (see Zstandard documentation, default level is 6)\n" - " -tonemap_dither: Dither tonemapper's 8-bit/component output by adding a small amount of white noise, only used with -tonemap mode\n" - "\n" - "--- Mipmap generation options:\n" - " -mipmap: Generate mipmaps for each source image\n" - " -mip_srgb: Convert image to linear before filtering, then back to sRGB\n" - " -mip_linear: Keep image in linear light during mipmap filtering (i.e. do not convert to/from sRGB for filtering purposes)\n" - " -mip_scale X: Set mipmap filter kernel's scale, lower=sharper, higher=more blurry, default is 1.0\n" - " -mip_filter X: Set mipmap filter kernel, default is kaiser, filters: box, tent, bell, blackman, catmullrom, mitchell, etc.\n" - " -mip_renorm: Renormalize normal map to unit length vectors after filtering\n" - " -mip_clamp: Use clamp addressing on borders, instead of wrapping\n" - " -mip_fast: Use faster mipmap generation (resample from previous mip, not always first/largest mip level). The default (as of 1/2021)\n" - " -mip_slow: Always resample each mipmap level starting from the largest mipmap. Higher quality, but slower. Opposite of -mip_fast. Was the prior default before 1/2021.\n" - " -mip_smallest X: Set smallest pixel dimension for generated mipmaps, default is 1 pixel\n" - " By default, textures will be converted from sRGB to linear light before mipmap filtering, then back to sRGB (for the RGB color channels) unless -linear is specified.\n" - " You can override this behavior with -mip_srgb/-mip_linear.\n" - "\n" - "--- ETC1S backend endpoint/selector RDO codec options:\n" - " -no_selector_rdo: Disable backend's selector rate distortion optimizations (slightly faster, less noisy output, but lower quality per output bit)\n" - " -selector_rdo_thresh X: Set selector RDO quality threshold, default is 1.25, lower is higher quality but less quality per output bit (try 1.0-3.0)\n" - " -no_endpoint_rdo: Disable backend's endpoint rate distortion optimizations (slightly faster, less noisy output, but lower quality per output bit)\n" - " -endpoint_rdo_thresh X: Set endpoint RDO quality threshold, default is 1.5, lower is higher quality but less quality per output bit (try 1.0-3.0)\n" - "\n" - "--- Set various fields in the Basis file header:\n" - " -userdata0 X: Set 32-bit userdata0 field in Basis file header to X (X is a signed 32-bit int)\n" - " -userdata1 X: Set 32-bit userdata1 field in Basis file header to X (X is a signed 32-bit int)\n" - "\n" - "--- Example LDR ETC1S/UASTC LDR 4x4 command lines:\n" - " basisu x.png : Compress sRGB image x.png to x.ktx2 using default settings (multiple filenames OK, use -tex_array if you want a tex array vs. multiple output files)\n" - " basisu -basis x.qoi : Compress sRGB image x.qoi to x.basis (supports 24-bit or 32-bit .QOI files)\n" - " basisu x.ktx2 : Unpack x.basis to PNG/KTX files (multiple filenames OK)\n" - " basisu x.basis : Unpack x.basis to PNG/KTX files (multiple filenames OK)\n" - " basisu -uastc x.png -uastc_rdo_l 2.0 -ktx2 -stats : Compress to a UASTC .KTX2 file with RDO (rate distortion optimization) to reduce .KTX2 compressed file size\n" - " basisu -file x.png -mipmap -y_flip : Compress a mipmapped x.ktx2 file from an sRGB image named x.png, Y flip each source image\n" - " basisu -validate -file x.basis : Validate x.basis (check header, check file CRC's, attempt to transcode all slices)\n" - " basisu -unpack -file x.basis : Validates, transcodes and unpacks x.basis to mipmapped .KTX and RGB/A .PNG files (transcodes to all supported GPU texture formats)\n" - " basisu -q 255 -file x.png -mipmap -debug -stats : Compress sRGB x.png to x.ktx2 at quality level 255 with compressor debug output/statistics\n" - " basisu -linear -max_endpoints 16128 -max_selectors 16128 -file x.png : Compress non-sRGB x.png to x.ktx2 using the largest supported manually specified codebook sizes\n" - " basisu -basis -comp_level 2 -max_selectors 8192 -max_endpoints 8192 -tex_type video -framerate 20 -multifile_printf \"x%02u.png\" -multifile_first 1 -multifile_num 20 : Compress a 20 sRGB source image video sequence (x01.png, x02.png, x03.png, etc.) to x01.basis\n" - "\n" - "--- Example UASTC HDR 4x4 command lines:\n" - " basisu x.exr : Compress a HDR .EXR (or .HDR) image to a UASTC HDR 4x4 .KTX2 file. LDR/SDR images will be upconverted to linear light HDR before compression. See HDR upconversion options, above.\n" - " basisu -hdr_4x4 x.exr : Compress a HDR .EXR image to a UASTC HDR 4x4 .KTX2 file.\n" - " basisu x.hdr -uastc_level 0 : Compress a HDR .hdr image to a UASTC HDR 4x4 .KTX2 file, fastest encoding but lowest quality\n" - " basisu -hdr x.png : Compress a LDR .PNG image to UASTC HDR 4x4 (image is converted from sRGB to linear light first, use -hdr_ldr_no_srgb_to_linear to disable)\n" - " basisu x.hdr -uastc_level 3 : Compress a HDR .hdr image to UASTC HDR 4x4 at higher quality (-uastc_level 4 is highest quality, but very slow encoding)\n" - " basisu x.hdr -uastc_level 3 -mipmap -basis -stats -debug -debug_images : Compress a HDR .hdr image to UASTC HDR 4x4, .basis output file, at higher quality, generate mipmaps, output statistics and debug information, and write tone mapped debug images\n" - " basisu x.hdr -stats -hdr_favor_astc -hdr_uber_mode -uastc_level 4 : Highest achievable ASTC HDR 4x4 quality (very slow encoding, BC6H quality is traded off)\n" - "\n--- Example UASTC HDR 6x6 command lines:\n" - " basisu -hdr_6x6 x.exr : Compress a HDR .EXR (or .HDR) image to a UASTC HDR 6x6 .KTX2 file. LDR/SDR images will be upconverted to linear light HDR before compression. See HDR upconversion options, above.\n" - " basisu -lambda 1000 -hdr_6x6 x.exr : Compress a HDR .EXR (or .HDR) image to a UASTC HDR 6x6 .KTX2 file with rate-distortion optimization (RDO), at lambda level 1000.\n" - " basisu -hdr_6x6i x.exr : Compress a HDR .EXR image to a compressed intermediate format UASTC HDR 6x6 .KTX2 file.\n" - " basisu -lambda 1000 -hdr_6x6i x.exr : Compress a HDR .EXR image to a compressed intermediate format UASTC HDR 6x6 .KTX2 file with rate-distortion optimization (RDO), at lambda level 1000.\n" - "\n" - "--- Video notes: For video use, it's recommended to encode on a machine with many cores. Use -comp_level 2 or higher for better codebook\n" - "generation, specify very large codebooks using -max_endpoints and -max_selectors, and reduce the default endpoint RDO threshold\n" - "(-endpoint_rdo_thresh) to around 1.25. Videos may have mipmaps and alpha channels. Videos must always be played back by the transcoder\n" - "in first to last image order.\n" - "Video files currently use I-Frames on the first image, and P-Frames using conditional replenishment on subsequent frames.\n" - "\nETC1S Compression level (-comp_level X) details. This controls the ETC1S speed vs. quality trandeoff. (Use -q to control the quality vs. compressed size tradeoff.):\n" - " Level 0: Fastest, but has marginal quality and can be brittle on complex images. Avg. Y dB: 35.45\n" - " Level 1: Hierarchical codebook searching, faster ETC1S encoding. 36.87 dB, ~1.4x slower vs. level 0. (This is the default setting.)\n" - " Level 2: Use this or higher for video. Hierarchical codebook searching. 36.87 dB, ~1.4x slower vs. level 0. (This is the v1.12's default setting.)\n" - " Level 3: Full codebook searching. 37.13 dB, ~1.8x slower vs. level 0. (Equivalent the the initial release's default settings.)\n" - " Level 4: Hierarchical codebook searching, codebook k-means iterations. 37.15 dB, ~4x slower vs. level 0\n" - " Level 5: Full codebook searching, codebook k-means iterations. 37.41 dB, ~5.5x slower vs. level 0.\n" - " Level 6: Full codebook searching, twice as many codebook k-means iterations, best ETC1 endpoint opt. 37.43 dB, ~12x slower vs. level 0\n" + puts( +#include "basisu_tool_help.h" ); } @@ -327,7 +155,7 @@ static bool load_listing_file(const std::string &f, basisu::vector { if (read_filename[0] == ' ') read_filename.erase(0, 1); - else + else break; } @@ -336,7 +164,7 @@ static bool load_listing_file(const std::string &f, basisu::vector const char c = read_filename.back(); if ((c == ' ') || (c == '\n') || (c == '\r')) read_filename.erase(read_filename.size() - 1, 1); - else + else break; } @@ -360,9 +188,347 @@ class command_line_params #define REMAINING_ARGS_CHECK(n) if (num_remaining_args < (n)) { error_printf("Error: Expected %u values to follow %s!\n", n, pArg); return false; } + bool check_for_general_options(const char** arg_v, const char* pArg, int arg_index, const int num_remaining_args, int& arg_count) + { + BASISU_NOTE_UNUSED(arg_v); + BASISU_NOTE_UNUSED(arg_index); + BASISU_NOTE_UNUSED(num_remaining_args); + BASISU_NOTE_UNUSED(arg_count); + + if (strcasecmp(pArg, "-wasi_threads") == 0) + { + REMAINING_ARGS_CHECK(1); + int num_threads = atoi(arg_v[arg_index + 1]); + if ((num_threads < 0) || (num_threads > 256)) + { + error_printf("Invalid number of threads\n"); + exit(EXIT_FAILURE); + } + set_num_wasi_threads(num_threads); + arg_count++; + return true; + } + else if (strcasecmp(pArg, "-higher_quality_transcoding") == 0) + { + m_higher_quality_transcoding = true; + return true; + } + else if (strcasecmp(pArg, "-no_fast_xuastc_ldr_bc7_transcoding") == 0) + { + m_xuastc_ldr_disable_bc7_transcoding = true; + return true; + } + else if (strcasecmp(pArg, "-fast_xuastc_ldr_bc7_transcoding") == 0) + { + m_xuastc_ldr_disable_bc7_transcoding = false; + return true; + } + else if (strcasecmp(pArg, "-no_etc1s_chroma_filtering") == 0) + { + m_no_etc1s_transcoding_chroma_filtering = true; + return true; + } + else if (strcasecmp(pArg, "-force_deblocking") == 0) + { + m_force_deblocking = true; + return true; + } + else if ((strcasecmp(pArg, "-disable_deblocking") == 0) || (strcasecmp(pArg, "-no_deblocking") == 0)) + { + m_disable_deblocking = true; + return true; + } + else if (strcasecmp(pArg, "-stronger_deblocking") == 0) + { + m_stronger_deblocking = true; + return true; + } + + return false; + } + + bool check_for_xuastc_options(const char** arg_v, const char* pArg, int arg_index, const int num_remaining_args, int& arg_count) + { + // New unified -quality level which works across all codecs + if (strcasecmp(pArg, "-quality") == 0) + { + REMAINING_ARGS_CHECK(1); + m_quality_level = clamp(atoi(arg_v[arg_index + 1]), 0, 100); + arg_count++; + return true; + } + // New unified -effort level, which works across all codecs + else if (strcasecmp(pArg, "-effort") == 0) + { + REMAINING_ARGS_CHECK(1); + m_effort_level = clamp(atoi(arg_v[arg_index + 1]), 0, 10); + //m_comp_params.m_xuastc_ldr_effort_level = atoi(arg_v[arg_index + 1]); + arg_count++; + return true; + } + else if (strcasecmp(pArg, "-xuastc_blurring") == 0) // experimental, not recommended, very slow + { + m_comp_params.m_xuastc_ldr_blurring = true; + return true; + } + else if (strcasecmp(pArg, "-weights") == 0) + { + REMAINING_ARGS_CHECK(4); + m_comp_params.m_xuastc_ldr_channel_weights[0] = (uint32_t)clamp((float)atof(arg_v[arg_index + 1]), 0.0f, 1024.0f); + m_comp_params.m_xuastc_ldr_channel_weights[1] = (uint32_t)clamp((float)atof(arg_v[arg_index + 2]), 0.0f, 1024.0f); + m_comp_params.m_xuastc_ldr_channel_weights[2] = (uint32_t)clamp((float)atof(arg_v[arg_index + 3]), 0.0f, 1024.0f); + m_comp_params.m_xuastc_ldr_channel_weights[3] = (uint32_t)clamp((float)atof(arg_v[arg_index + 4]), 0.0f, 1024.0f); + arg_count += 4; + return true; + } + else if (strcasecmp(pArg, "-ls_min_psnr") == 0) + { + REMAINING_ARGS_CHECK(1); + m_comp_params.m_ls_min_psnr = (float)atof(arg_v[arg_index + 1]); + arg_count++; + return true; + } + else if (strcasecmp(pArg, "-ls_min_alpha_psnr") == 0) + { + REMAINING_ARGS_CHECK(1); + m_comp_params.m_ls_min_alpha_psnr = (float)atof(arg_v[arg_index + 1]); + arg_count++; + return true; + } + else if (strcasecmp(pArg, "-ls_thresh_psnr") == 0) + { + REMAINING_ARGS_CHECK(1); + m_comp_params.m_ls_thresh_psnr = (float)atof(arg_v[arg_index + 1]); + arg_count++; + return true; + } + else if (strcasecmp(pArg, "-ls_thresh_alpha_psnr") == 0) + { + REMAINING_ARGS_CHECK(1); + m_comp_params.m_ls_thresh_alpha_psnr = (float)atof(arg_v[arg_index + 1]); + arg_count++; + return true; + } + else if (strcasecmp(pArg, "-ls_thresh_edge_psnr") == 0) + { + REMAINING_ARGS_CHECK(1); + m_comp_params.m_ls_thresh_edge_psnr = (float)atof(arg_v[arg_index + 1]); + arg_count++; + return true; + } + else if (strcasecmp(pArg, "-ls_thresh_edge_alpha_psnr") == 0) + { + REMAINING_ARGS_CHECK(1); + m_comp_params.m_ls_thresh_edge_alpha_psnr = (float)atof(arg_v[arg_index + 1]); + arg_count++; + return true; + } + else if (strcasecmp(pArg, "-xuastc_arith") == 0) + { + m_comp_params.m_xuastc_ldr_syntax = (int)basist::astc_ldr_t::xuastc_ldr_syntax::cFullArith; + return true; + } + else if (strcasecmp(pArg, "-xuastc_zstd") == 0) + { + m_comp_params.m_xuastc_ldr_syntax = (int)basist::astc_ldr_t::xuastc_ldr_syntax::cFullZStd; + return true; + } + else if (strcasecmp(pArg, "-xuastc_hybrid") == 0) + { + m_comp_params.m_xuastc_ldr_syntax = (int)basist::astc_ldr_t::xuastc_ldr_syntax::cHybridArithZStd; + return true; + } + else if (strcasecmp(pArg, "-xy") == 0) + { + m_comp_params.m_xuastc_ldr_use_lossy_supercompression = true; + return true; + } + else if (strcasecmp(pArg, "-xyd") == 0) + { + m_comp_params.m_xuastc_ldr_use_lossy_supercompression = false; + return true; + } + else if (strcasecmp(pArg, "-xs") == 0) + { + m_comp_params.m_xuastc_ldr_force_disable_subsets = true; + return true; + } + else if (strcasecmp(pArg, "-xsu") == 0) + { + m_comp_params.m_xuastc_ldr_force_disable_subsets = false; + return true; + } + else if (strcasecmp(pArg, "-xp") == 0) + { + m_comp_params.m_xuastc_ldr_force_disable_rgb_dual_plane = true; + return true; + } + else if (strcasecmp(pArg, "-xpu") == 0) + { + m_comp_params.m_xuastc_ldr_force_disable_rgb_dual_plane = false; + return true; + } + else if (strcasecmp(pArg, "-ts") == 0) + { + m_comp_params.m_perceptual = true; + m_comp_params.m_ktx2_and_basis_srgb_transfer_function = true; + return true; + } + else if (strcasecmp(pArg, "-tl") == 0) + { + m_comp_params.m_perceptual = false; + m_comp_params.m_ktx2_and_basis_srgb_transfer_function = false; + return true; + } + // Supercompressed XUASTC LDR 4x4-12x12 + else if ((strcasecmp(pArg, "-ldr_4x4i") == 0) || (strcasecmp(pArg, "-xuastc_ldr_4x4") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cXUASTC_LDR_4x4); + return true; + } + else if ((strcasecmp(pArg, "-ldr_5x4i") == 0) || (strcasecmp(pArg, "-xuastc_ldr_5x4") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cXUASTC_LDR_5x4); + return true; + } + else if ((strcasecmp(pArg, "-ldr_5x5i") == 0) || (strcasecmp(pArg, "-xuastc_ldr_5x5") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cXUASTC_LDR_5x5); + return true; + } + else if ((strcasecmp(pArg, "-ldr_6x5i") == 0) || (strcasecmp(pArg, "-xuastc_ldr_6x5") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cXUASTC_LDR_6x5); + return true; + } + else if ((strcasecmp(pArg, "-ldr_6x6i") == 0) || (strcasecmp(pArg, "-xuastc_ldr_6x6") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cXUASTC_LDR_6x6); + return true; + } + else if ((strcasecmp(pArg, "-ldr_8x5i") == 0) || (strcasecmp(pArg, "-xuastc_ldr_8x5") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cXUASTC_LDR_8x5); + return true; + } + else if ((strcasecmp(pArg, "-ldr_8x6i") == 0) || (strcasecmp(pArg, "-xuastc_ldr_8x6") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cXUASTC_LDR_8x6); + return true; + } + else if ((strcasecmp(pArg, "-ldr_10x5i") == 0) || (strcasecmp(pArg, "-xuastc_ldr_10x5") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cXUASTC_LDR_10x5); + return true; + } + else if ((strcasecmp(pArg, "-ldr_10x6i") == 0) || (strcasecmp(pArg, "-xuastc_ldr_10x6") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cXUASTC_LDR_10x6); + return true; + } + else if ((strcasecmp(pArg, "-ldr_8x8i") == 0) || (strcasecmp(pArg, "-xuastc_ldr_8x8") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cXUASTC_LDR_8x8); + return true; + } + else if ((strcasecmp(pArg, "-ldr_10x8i") == 0) || (strcasecmp(pArg, "-xuastc_ldr_10x8") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cXUASTC_LDR_10x8); + return true; + } + else if ((strcasecmp(pArg, "-ldr_10x10i") == 0) || (strcasecmp(pArg, "-xuastc_ldr_10x10") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cXUASTC_LDR_10x10); + return true; + } + else if ((strcasecmp(pArg, "-ldr_12x10i") == 0) || (strcasecmp(pArg, "-xuastc_ldr_12x10") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cXUASTC_LDR_12x10); + return true; + } + else if ((strcasecmp(pArg, "-ldr_12x12i") == 0) || (strcasecmp(pArg, "-xuastc_ldr_12x12") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cXUASTC_LDR_12x12); + return true; + } + // Plain ASTC LDR 4x4-12x12 + else if ((strcasecmp(pArg, "-ldr_4x4") == 0) || (strcasecmp(pArg, "-astc_ldr_4x4") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_LDR_4x4); + return true; + } + else if ((strcasecmp(pArg, "-ldr_5x4") == 0) || (strcasecmp(pArg, "-astc_ldr_5x4") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_LDR_5x4); + return true; + } + else if ((strcasecmp(pArg, "-ldr_5x5") == 0) || (strcasecmp(pArg, "-astc_ldr_5x5") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_LDR_5x5); + return true; + } + else if ((strcasecmp(pArg, "-ldr_6x5") == 0) || (strcasecmp(pArg, "-astc_ldr_6x5") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_LDR_6x5); + return true; + } + else if ((strcasecmp(pArg, "-ldr_6x6") == 0) || (strcasecmp(pArg, "-astc_ldr_6x6") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_LDR_6x6); + return true; + } + else if ((strcasecmp(pArg, "-ldr_8x5") == 0) || (strcasecmp(pArg, "-astc_ldr_8x5") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_LDR_8x5); + return true; + } + else if ((strcasecmp(pArg, "-ldr_8x6") == 0) || (strcasecmp(pArg, "-astc_ldr_8x6") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_LDR_8x6); + return true; + } + else if ((strcasecmp(pArg, "-ldr_10x5") == 0) || (strcasecmp(pArg, "-astc_ldr_10x5") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_LDR_10x5); + return true; + } + else if ((strcasecmp(pArg, "-ldr_10x6") == 0) || (strcasecmp(pArg, "-astc_ldr_10x6") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_LDR_10x6); + return true; + } + else if ((strcasecmp(pArg, "-ldr_8x8") == 0) || (strcasecmp(pArg, "-astc_ldr_8x8") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_LDR_8x8); + return true; + } + else if ((strcasecmp(pArg, "-ldr_10x8") == 0) || (strcasecmp(pArg, "-astc_ldr_10x8") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_LDR_10x8); + return true; + } + else if ((strcasecmp(pArg, "-ldr_10x10") == 0) || (strcasecmp(pArg, "-astc_ldr_10x10") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_LDR_10x10); + return true; + } + else if ((strcasecmp(pArg, "-ldr_12x10") == 0) || (strcasecmp(pArg, "-astc_ldr_12x10") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_LDR_12x10); + return true; + } + else if ((strcasecmp(pArg, "-ldr_12x12") == 0) || (strcasecmp(pArg, "-astc_ldr_12x12") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_LDR_12x12); + return true; + } + + return false; + } + bool check_for_hdr_options(const char** arg_v, const char* pArg, int arg_index, const int num_remaining_args, int& arg_count) { - if ((strcasecmp(pArg, "-hdr") == 0) || (strcasecmp(pArg, "-hdr_4x4") == 0)) + if ((strcasecmp(pArg, "-hdr") == 0) || (strcasecmp(pArg, "-hdr_4x4") == 0) || (strcasecmp(pArg, "-uastc_hdr_4x4") == 0)) { m_comp_params.set_format_mode(basist::basis_tex_format::cUASTC_HDR_4x4); return true; @@ -372,16 +538,16 @@ class command_line_params m_comp_params.m_astc_hdr_6x6_options.m_rec2020_bt2100_color_gamut = true; return true; } - else if (strcasecmp(pArg, "-hdr_6x6") == 0) + else if ((strcasecmp(pArg, "-hdr_6x6") == 0) || (strcasecmp(pArg, "-astc_hdr_6x6") == 0)) { // max quality (if -lambda=0) or RDO UASTC HDR 6x6 m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_HDR_6x6); return true; } - else if (strcasecmp(pArg, "-hdr_6x6i") == 0) + else if ((strcasecmp(pArg, "-hdr_6x6i") == 0) || (strcasecmp(pArg, "-uastc_hdr_6x6") == 0)) { // intermediate format UASTC HDR 6x6 - m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE); + m_comp_params.set_format_mode(basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE); return true; } else if (strcasecmp(pArg, "-lambda") == 0) @@ -401,6 +567,8 @@ class command_line_params m_comp_params.m_rdo_uastc_ldr_4x4_quality_scalar = (float)atof(arg_v[arg_index + 1]); m_comp_params.m_rdo_uastc_ldr_4x4 = true; + m_used_old_style_codec_config_param = true; + arg_count++; return true; } @@ -418,6 +586,9 @@ class command_line_params const int level = atoi(arg_v[arg_index + 1]); m_comp_params.m_astc_hdr_6x6_options.set_user_level(level); m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_HDR_6x6); + + m_used_old_style_codec_config_param = true; + arg_count++; return true; } @@ -426,7 +597,10 @@ class command_line_params REMAINING_ARGS_CHECK(1); const int level = atoi(arg_v[arg_index + 1]); m_comp_params.m_astc_hdr_6x6_options.set_user_level(level); - m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE); + m_comp_params.set_format_mode(basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE); + + m_used_old_style_codec_config_param = true; + arg_count++; return true; } @@ -444,16 +618,19 @@ class command_line_params { REMAINING_ARGS_CHECK(2); + // Intended for low-level/development/testing const int lo_level = clamp(atoi(arg_v[arg_index + 1]), 0, astc_6x6_hdr::ASTC_HDR_6X6_MAX_COMP_LEVEL); const int hi_level = clamp(atoi(arg_v[arg_index + 2]), 0, astc_6x6_hdr::ASTC_HDR_6X6_MAX_COMP_LEVEL); m_comp_params.m_astc_hdr_6x6_options.m_master_comp_level = minimum(lo_level, hi_level); m_comp_params.m_astc_hdr_6x6_options.m_highest_comp_level = maximum(lo_level, hi_level); - + if (strcasecmp(pArg, "-hdr_6x6_comp_levels") == 0) m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_HDR_6x6); else - m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE); + m_comp_params.set_format_mode(basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE); + + m_used_old_style_codec_config_param = true; arg_count += 2; return true; @@ -514,43 +691,230 @@ class command_line_params return false; } -public: - command_line_params() : - m_mode(cDefault), - m_ktx2_mode(true), - m_ktx2_zstandard(true), - m_ktx2_zstandard_level(6), - m_ktx2_animdata_duration(1), - m_ktx2_animdata_timescale(15), - m_ktx2_animdata_loopcount(0), - m_format_only(-1), - m_multifile_first(0), - m_multifile_num(0), - m_max_threads(1024), // surely this is high enough - m_individual(true), - m_no_ktx(false), - m_ktx_only(false), - m_write_out(false), - m_etc1_only(false), - m_fuzz_testing(false), - m_compare_ssim(false), - m_compare_plot(false), - m_parallel_compression(false), - m_tonemap_dither_flag(false) + // ETC1S or UASTC LDR 4x4 specific options + bool check_for_etc1s_or_uastc_options(const char** arg_v, const char* pArg, int arg_index, const int num_remaining_args, int& arg_count) { - m_comp_params.m_compression_level = basisu::maximum(0, BASISU_DEFAULT_COMPRESSION_LEVEL - 1); + if (strcasecmp(pArg, "-etc1s") == 0) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cETC1S); + return true; + } + else if ((strcasecmp(pArg, "-uastc") == 0) || (strcasecmp(pArg, "-uastc_ldr") == 0) || (strcasecmp(pArg, "-uastc_ldr_4x4") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cUASTC_LDR_4x4); + return true; + } + else if (strcasecmp(pArg, "-uastc_level") == 0) + { + REMAINING_ARGS_CHECK(1); - m_comp_params.m_uastc_hdr_4x4_options.set_quality_level(uastc_hdr_4x4_codec_options::cDefaultLevel); + int uastc_level = atoi(arg_v[arg_index + 1]); - m_test_file_dir = "../test_files"; - } + uastc_level = clamp(uastc_level, 0, TOTAL_PACK_UASTC_LEVELS - 1); - bool parse(int arg_c, const char **arg_v) - { - int arg_index = 1; - while (arg_index < arg_c) - { - const char *pArg = arg_v[arg_index]; + static_assert(TOTAL_PACK_UASTC_LEVELS == 5, "TOTAL_PACK_UASTC_LEVELS==5"); + static const uint32_t s_level_flags[TOTAL_PACK_UASTC_LEVELS] = { cPackUASTCLevelFastest, cPackUASTCLevelFaster, cPackUASTCLevelDefault, cPackUASTCLevelSlower, cPackUASTCLevelVerySlow }; + + m_comp_params.m_pack_uastc_ldr_4x4_flags &= ~cPackUASTCLevelMask; + m_comp_params.m_pack_uastc_ldr_4x4_flags |= s_level_flags[uastc_level]; + + m_comp_params.m_uastc_hdr_4x4_options.set_quality_level(uastc_level); + + m_used_old_style_codec_config_param = true; + + arg_count++; + return true; + } + else if (strcasecmp(pArg, "-uastc_rdo_l") == 0) + { + REMAINING_ARGS_CHECK(1); + m_comp_params.m_rdo_uastc_ldr_4x4_quality_scalar = (float)atof(arg_v[arg_index + 1]); + m_comp_params.m_rdo_uastc_ldr_4x4 = true; + + m_used_old_style_codec_config_param = true; + + arg_count++; + return true; + } + else if (strcasecmp(pArg, "-uastc_rdo_d") == 0) + { + REMAINING_ARGS_CHECK(1); + m_comp_params.m_rdo_uastc_ldr_4x4_dict_size = atoi(arg_v[arg_index + 1]); + arg_count++; + return true; + } + else if (strcasecmp(pArg, "-uastc_rdo_b") == 0) + { + REMAINING_ARGS_CHECK(1); + m_comp_params.m_rdo_uastc_ldr_4x4_max_smooth_block_error_scale = (float)atof(arg_v[arg_index + 1]); + arg_count++; + return true; + } + else if (strcasecmp(pArg, "-uastc_rdo_s") == 0) + { + REMAINING_ARGS_CHECK(1); + m_comp_params.m_rdo_uastc_ldr_4x4_smooth_block_max_std_dev = (float)atof(arg_v[arg_index + 1]); + arg_count++; + return true; + } + else if (strcasecmp(pArg, "-uastc_rdo_f") == 0) + { + m_comp_params.m_rdo_uastc_ldr_4x4_favor_simpler_modes_in_rdo_mode = false; + return true; + } + else if (strcasecmp(pArg, "-uastc_rdo_m") == 0) + { + m_comp_params.m_rdo_uastc_ldr_4x4_multithreading = false; + return true; + } + else if (strcasecmp(pArg, "-validate_etc1s") == 0) + { + m_comp_params.m_validate_etc1s = true; + return true; + } + else if (strcasecmp(pArg, "-comp_level") == 0) + { + REMAINING_ARGS_CHECK(1); + m_comp_params.m_etc1s_compression_level = atoi(arg_v[arg_index + 1]); + + m_used_old_style_codec_config_param = true; + + arg_count++; + return true; + } + else if (strcasecmp(pArg, "-max_endpoints") == 0) + { + REMAINING_ARGS_CHECK(1); + m_comp_params.m_etc1s_max_endpoint_clusters = clamp(atoi(arg_v[arg_index + 1]), 1, BASISU_MAX_ENDPOINT_CLUSTERS); + + m_used_old_style_codec_config_param = true; + + arg_count++; + return true; + } + else if (strcasecmp(pArg, "-max_selectors") == 0) + { + REMAINING_ARGS_CHECK(1); + m_comp_params.m_etc1s_max_selector_clusters = clamp(atoi(arg_v[arg_index + 1]), 1, BASISU_MAX_SELECTOR_CLUSTERS); + + m_used_old_style_codec_config_param = true; + + arg_count++; + return true; + } +#if 0 + else if (strcasecmp(pArg, "-gen_global_codebooks") == 0) + { + // TODO + } +#endif + else if (strcasecmp(pArg, "-use_global_codebooks") == 0) + { + REMAINING_ARGS_CHECK(1); + m_etc1s_use_global_codebooks_file = arg_v[arg_index + 1]; + arg_count++; + return true; + } + else if (strcasecmp(pArg, "-etc1_only") == 0) + { + m_etc1_only = true; + m_unpack_format_only = (int)basist::transcoder_texture_format::cTFETC1_RGB; + return true; + } + else if (strcasecmp(pArg, "-disable_hierarchical_endpoint_codebooks") == 0) + { + m_comp_params.m_disable_hierarchical_endpoint_codebooks = true; + return true; + } + else if (strcasecmp(pArg, "-q") == 0) // old-style -q, prefer -quality instead + { + REMAINING_ARGS_CHECK(1); + m_comp_params.m_quality_level = clamp(atoi(arg_v[arg_index + 1]), BASISU_QUALITY_MIN, BASISU_QUALITY_MAX); + + m_used_old_style_codec_config_param = true; + + arg_count++; + return true; + } + else if (strcasecmp(pArg, "-no_selector_rdo") == 0) + { + m_comp_params.m_no_selector_rdo = true; + return true; + } + else if (strcasecmp(pArg, "-selector_rdo_thresh") == 0) + { + REMAINING_ARGS_CHECK(1); + m_comp_params.m_selector_rdo_thresh = (float)atof(arg_v[arg_index + 1]); + arg_count++; + return true; + } + else if (strcasecmp(pArg, "-no_endpoint_rdo") == 0) + { + m_comp_params.m_no_endpoint_rdo = true; + return true; + } + else if (strcasecmp(pArg, "-endpoint_rdo_thresh") == 0) + { + REMAINING_ARGS_CHECK(1); + m_comp_params.m_endpoint_rdo_thresh = (float)atof(arg_v[arg_index + 1]); + arg_count++; + return true; + } + + return false; + } + +public: + command_line_params() : + m_mode(cDefault), + m_ktx2_mode(true), + m_ktx2_zstandard(true), + m_ktx2_zstandard_level(6), + m_ktx2_animdata_duration(1), + m_ktx2_animdata_timescale(15), + m_ktx2_animdata_loopcount(0), + m_unpack_format_only(-1), + m_multifile_first(0), + m_multifile_num(0), + m_max_threads(1024), // surely this is high enough + m_individual(true), + m_no_ktx(false), + m_ktx_only(false), + m_write_out(false), + m_etc1_only(false), + m_fuzz_testing(false), + m_compare_ssim(false), + m_compare_plot(false), + m_parallel_compression(false), + m_tonemap_dither_flag(false), + m_xuastc_ldr_disable_bc7_transcoding(false), + m_no_etc1s_transcoding_chroma_filtering(false), + m_higher_quality_transcoding(false), + m_force_deblocking(false), + m_disable_deblocking(false), + m_stronger_deblocking(false), + m_effort_level(-1), + m_quality_level(-1), + m_used_old_style_codec_config_param(false) + { + // This command line tool defaults to ETC1S level 1, not 2 which is the API default (for backwards compat). + m_comp_params.m_etc1s_compression_level = maximum((int)BASISU_DEFAULT_ETC1S_COMPRESSION_LEVEL - 1, 0); + + m_comp_params.m_uastc_hdr_4x4_options.set_quality_level(uastc_hdr_4x4_codec_options::cDefaultLevel); + + // Default to sRGB colorspace metrics/transfer functions (independent of the code defaults). + m_comp_params.m_perceptual = true; + m_comp_params.m_ktx2_and_basis_srgb_transfer_function = true; + + m_test_file_dir = "../test_files"; + } + + bool parse(int arg_c, const char **arg_v) + { + int arg_index = 1; + while (arg_index < arg_c) + { + const char *pArg = arg_v[arg_index]; const int num_remaining_args = arg_c - (arg_index + 1); int arg_count = 1; @@ -559,8 +923,20 @@ class command_line_params print_usage(); exit(EXIT_SUCCESS); } - - if (strcasecmp(pArg, "-ktx2") == 0) + + if (check_for_etc1s_or_uastc_options(arg_v, pArg, arg_index, num_remaining_args, arg_count)) + { + } + else if (check_for_hdr_options(arg_v, pArg, arg_index, num_remaining_args, arg_count)) + { + } + else if (check_for_xuastc_options(arg_v, pArg, arg_index, num_remaining_args, arg_count)) + { + } + else if (check_for_general_options(arg_v, pArg, arg_index, num_remaining_args, arg_count)) + { + } + else if (strcasecmp(pArg, "-ktx2") == 0) { m_ktx2_mode = true; } @@ -626,6 +1002,8 @@ class command_line_params m_mode = cCompSize; else if ((strcasecmp(pArg, "-test") == 0) || (strcasecmp(pArg, "-test_ldr") == 0)) m_mode = cTestLDR; + else if ((strcasecmp(pArg, "-test_xuastc") == 0) || (strcasecmp(pArg, "-test_xuastc_ldr") == 0)) + m_mode = cTestXUASTCLDR; else if (strcasecmp(pArg, "-test_hdr_4x4") == 0) m_mode = cTestHDR_4x4; else if (strcasecmp(pArg, "-test_hdr_6x6") == 0) @@ -646,7 +1024,7 @@ class command_line_params g_cpu_supports_sse41 = false; #endif } - else if (strcasecmp(pArg, "-no_status_output") == 0) + else if ((strcasecmp(pArg, "-no_status_output") == 0) || (strcasecmp(pArg, "-quiet") == 0)) { m_comp_params.m_status_output = false; } @@ -680,50 +1058,6 @@ class command_line_params m_multifile_num = atoi(arg_v[arg_index + 1]); arg_count++; } - else if (strcasecmp(pArg, "-uastc") == 0) - { - m_comp_params.set_format_mode(basist::basis_tex_format::cUASTC4x4); - } - else if (strcasecmp(pArg, "-etc1s") == 0) - { - m_comp_params.set_format_mode(basist::basis_tex_format::cETC1S); - } - else if (strcasecmp(pArg, "-fastest") == 0) - { - m_comp_params.m_pack_uastc_ldr_4x4_flags &= ~cPackUASTCLevelMask; - m_comp_params.m_pack_uastc_ldr_4x4_flags |= cPackUASTCLevelFastest; - - m_comp_params.m_uastc_hdr_4x4_options.set_quality_level(0); - - m_comp_params.m_astc_hdr_6x6_options.set_user_level(0); - } - else if (strcasecmp(pArg, "-slower") == 0) - { - m_comp_params.m_pack_uastc_ldr_4x4_flags &= ~cPackUASTCLevelMask; - m_comp_params.m_pack_uastc_ldr_4x4_flags |= cPackUASTCLevelSlower; - - m_comp_params.m_uastc_hdr_4x4_options.set_quality_level(3); - - m_comp_params.m_astc_hdr_6x6_options.set_user_level(5); - } - else if (strcasecmp(pArg, "-uastc_level") == 0) - { - REMAINING_ARGS_CHECK(1); - - int uastc_level = atoi(arg_v[arg_index + 1]); - - uastc_level = clamp(uastc_level, 0, TOTAL_PACK_UASTC_LEVELS - 1); - - static_assert(TOTAL_PACK_UASTC_LEVELS == 5, "TOTAL_PACK_UASTC_LEVELS==5"); - static const uint32_t s_level_flags[TOTAL_PACK_UASTC_LEVELS] = { cPackUASTCLevelFastest, cPackUASTCLevelFaster, cPackUASTCLevelDefault, cPackUASTCLevelSlower, cPackUASTCLevelVerySlow }; - - m_comp_params.m_pack_uastc_ldr_4x4_flags &= ~cPackUASTCLevelMask; - m_comp_params.m_pack_uastc_ldr_4x4_flags |= s_level_flags[uastc_level]; - - m_comp_params.m_uastc_hdr_4x4_options.set_quality_level(uastc_level); - - arg_count++; - } else if (strcasecmp(pArg, "-resample") == 0) { REMAINING_ARGS_CHECK(2); @@ -737,44 +1071,15 @@ class command_line_params m_comp_params.m_resample_factor = (float)atof(arg_v[arg_index + 1]); arg_count++; } - else if (strcasecmp(pArg, "-uastc_rdo_l") == 0) - { - REMAINING_ARGS_CHECK(1); - m_comp_params.m_rdo_uastc_ldr_4x4_quality_scalar = (float)atof(arg_v[arg_index + 1]); - m_comp_params.m_rdo_uastc_ldr_4x4 = true; - arg_count++; - } - else if (strcasecmp(pArg, "-uastc_rdo_d") == 0) - { - REMAINING_ARGS_CHECK(1); - m_comp_params.m_rdo_uastc_ldr_4x4_dict_size = atoi(arg_v[arg_index + 1]); - arg_count++; - } - else if (strcasecmp(pArg, "-uastc_rdo_b") == 0) - { - REMAINING_ARGS_CHECK(1); - m_comp_params.m_rdo_uastc_ldr_4x4_max_smooth_block_error_scale = (float)atof(arg_v[arg_index + 1]); - arg_count++; - } - else if (strcasecmp(pArg, "-uastc_rdo_s") == 0) - { - REMAINING_ARGS_CHECK(1); - m_comp_params.m_rdo_uastc_ldr_4x4_smooth_block_max_std_dev = (float)atof(arg_v[arg_index + 1]); - arg_count++; - } - else if (strcasecmp(pArg, "-uastc_rdo_f") == 0) - m_comp_params.m_rdo_uastc_ldr_4x4_favor_simpler_modes_in_rdo_mode = false; - else if (strcasecmp(pArg, "-uastc_rdo_m") == 0) - m_comp_params.m_rdo_uastc_ldr_4x4_multithreading = false; else if (strcasecmp(pArg, "-linear") == 0) + { m_comp_params.m_perceptual = false; + m_comp_params.m_ktx2_and_basis_srgb_transfer_function = false; + } else if (strcasecmp(pArg, "-srgb") == 0) - m_comp_params.m_perceptual = true; - else if (strcasecmp(pArg, "-q") == 0) { - REMAINING_ARGS_CHECK(1); - m_comp_params.m_etc1s_quality_level = clamp(atoi(arg_v[arg_index + 1]), BASISU_QUALITY_MIN, BASISU_QUALITY_MAX); - arg_count++; + m_comp_params.m_perceptual = true; + m_comp_params.m_ktx2_and_basis_srgb_transfer_function = true; } else if (strcasecmp(pArg, "-output_file") == 0) { @@ -793,10 +1098,6 @@ class command_line_params m_comp_params.m_debug = true; enable_debug_printf(true); } - else if (strcasecmp(pArg, "-validate_etc1s") == 0) - { - m_comp_params.m_validate_etc1s = true; - } else if (strcasecmp(pArg, "-validate_output") == 0) { m_comp_params.m_validate_output_data = true; @@ -805,39 +1106,12 @@ class command_line_params m_comp_params.m_debug_images = true; else if (strcasecmp(pArg, "-stats") == 0) m_comp_params.m_compute_stats = true; - else if (strcasecmp(pArg, "-gen_global_codebooks") == 0) - { - // TODO - } - else if (strcasecmp(pArg, "-use_global_codebooks") == 0) - { - REMAINING_ARGS_CHECK(1); - m_etc1s_use_global_codebooks_file = arg_v[arg_index + 1]; - arg_count++; - } - else if (strcasecmp(pArg, "-comp_level") == 0) - { - REMAINING_ARGS_CHECK(1); - m_comp_params.m_compression_level = atoi(arg_v[arg_index + 1]); - arg_count++; - } - else if (strcasecmp(pArg, "-max_endpoints") == 0) - { - REMAINING_ARGS_CHECK(1); - m_comp_params.m_etc1s_max_endpoint_clusters = clamp(atoi(arg_v[arg_index + 1]), 1, BASISU_MAX_ENDPOINT_CLUSTERS); - arg_count++; - } - else if (strcasecmp(pArg, "-max_selectors") == 0) - { - REMAINING_ARGS_CHECK(1); - m_comp_params.m_etc1s_max_selector_clusters = clamp(atoi(arg_v[arg_index + 1]), 1, BASISU_MAX_SELECTOR_CLUSTERS); - arg_count++; - } else if (strcasecmp(pArg, "-y_flip") == 0) m_comp_params.m_y_flip = true; else if (strcasecmp(pArg, "-normal_map") == 0) { m_comp_params.m_perceptual = false; + m_comp_params.m_ktx2_and_basis_srgb_transfer_function = false; m_comp_params.m_mip_srgb = false; m_comp_params.m_no_selector_rdo = true; m_comp_params.m_no_endpoint_rdo = true; @@ -894,7 +1168,7 @@ class command_line_params else if (strcasecmp(pArg, "-max_threads") == 0) { REMAINING_ARGS_CHECK(1); - m_max_threads = atoi(arg_v[arg_index + 1]); + m_max_threads = maximum(1, atoi(arg_v[arg_index + 1])); arg_count++; } else if (strcasecmp(pArg, "-mipmap") == 0) @@ -910,19 +1184,9 @@ class command_line_params else if (strcasecmp(pArg, "-format_only") == 0) { REMAINING_ARGS_CHECK(1); - m_format_only = atoi(arg_v[arg_index + 1]); + m_unpack_format_only = atoi(arg_v[arg_index + 1]); arg_count++; } - else if (strcasecmp(pArg, "-etc1_only") == 0) - { - m_etc1_only = true; - m_format_only = (int)basist::transcoder_texture_format::cTFETC1_RGB; - } - else if (strcasecmp(pArg, "-disable_hierarchical_endpoint_codebooks") == 0) - m_comp_params.m_disable_hierarchical_endpoint_codebooks = true; - else if (check_for_hdr_options(arg_v, pArg, arg_index, num_remaining_args, arg_count)) - { - } else if (strcasecmp(pArg, "-opencl") == 0) { m_comp_params.m_use_opencl = true; @@ -940,7 +1204,7 @@ class command_line_params { REMAINING_ARGS_CHECK(1); m_comp_params.m_mip_filter = arg_v[arg_index + 1]; - // TODO: Check filter + // TODO: Check filter arg_count++; } else if (strcasecmp(pArg, "-mip_renorm") == 0) @@ -961,22 +1225,6 @@ class command_line_params m_comp_params.m_mip_srgb = true; else if (strcasecmp(pArg, "-mip_linear") == 0) m_comp_params.m_mip_srgb = false; - else if (strcasecmp(pArg, "-no_selector_rdo") == 0) - m_comp_params.m_no_selector_rdo = true; - else if (strcasecmp(pArg, "-selector_rdo_thresh") == 0) - { - REMAINING_ARGS_CHECK(1); - m_comp_params.m_selector_rdo_thresh = (float)atof(arg_v[arg_index + 1]); - arg_count++; - } - else if (strcasecmp(pArg, "-no_endpoint_rdo") == 0) - m_comp_params.m_no_endpoint_rdo = true; - else if (strcasecmp(pArg, "-endpoint_rdo_thresh") == 0) - { - REMAINING_ARGS_CHECK(1); - m_comp_params.m_endpoint_rdo_thresh = (float)atof(arg_v[arg_index + 1]); - arg_count++; - } else if (strcasecmp(pArg, "-userdata0") == 0) { REMAINING_ARGS_CHECK(1); @@ -1068,20 +1316,28 @@ class command_line_params arg_index += arg_count; assert(arg_index <= arg_c); } - - if (m_comp_params.m_etc1s_quality_level != -1) + + if (m_comp_params.m_quality_level != -1) // old-style -q X option { m_comp_params.m_etc1s_max_endpoint_clusters = 0; m_comp_params.m_etc1s_max_selector_clusters = 0; + + // -q also controls XUASTC LDR weight grid DCT quality level + m_comp_params.m_xuastc_ldr_use_dct = true; + + // Automatically enable lossy XUASTC supercompression if DCT is enabled. + if (!m_comp_params.m_xuastc_ldr_use_lossy_supercompression.was_changed()) + m_comp_params.m_xuastc_ldr_use_lossy_supercompression = true; } else if ((!m_comp_params.m_etc1s_max_endpoint_clusters) || (!m_comp_params.m_etc1s_max_selector_clusters)) { m_comp_params.m_etc1s_max_endpoint_clusters = 0; m_comp_params.m_etc1s_max_selector_clusters = 0; - m_comp_params.m_etc1s_quality_level = 128; + m_comp_params.m_quality_level = 128; } - + + // Ensure mip_srgb is set to match the perceptual flag if the user didn't explicitly set it. if (!m_comp_params.m_mip_srgb.was_changed()) { // They didn't specify what colorspace to do mipmap filtering in, so choose sRGB if they've specified that the texture is sRGB. @@ -1091,6 +1347,30 @@ class command_line_params m_comp_params.m_mip_srgb = false; } + // Handle new-style unified effort and quality levels across all codecs. + // We have so many codecs now that it's necessary to unify the primary quality/effort controls otherwise it's too confusing. + // If they've specified either -effort or -quality, assume they want the new unified API. + // If they haven't specified either, they get the old parameters/options. + if ((m_effort_level != -1) || (m_quality_level != -1)) + { + if (m_used_old_style_codec_config_param) + { + fmt_printf("WARNING: Mixing old and new-style (-effort and/or -quality) codec configuration parameters.\nNew-style parameters may overwrite your old-style codec configuration settings.\nPrefer using -effort X and -quality X."); + } + + const bool lossy_supercompression_changed = m_comp_params.m_xuastc_ldr_use_lossy_supercompression.was_changed(); + const bool lossy_supercompression_value = m_comp_params.m_xuastc_ldr_use_lossy_supercompression; + + // Set the new-style effort/quality level, but importantly don't override any settings already changed if they haven't explictly specified -effort or -quality. + m_comp_params.set_format_mode_and_quality_effort(m_comp_params.get_format_mode(), m_quality_level, m_effort_level, false); + + // Allow the user to override the lossy supercompression setting, independent of the quality/effort levels. + if (lossy_supercompression_changed) + { + m_comp_params.m_xuastc_ldr_use_lossy_supercompression = lossy_supercompression_value; + } + } + return true; } @@ -1121,28 +1401,28 @@ class command_line_params new_input_alpha_filenames.push_back(m_input_alpha_filenames[i]); } new_input_alpha_filenames.swap(m_input_alpha_filenames); - + return true; } basis_compressor_params m_comp_params; - + tool_mode m_mode; - + bool m_ktx2_mode; bool m_ktx2_zstandard; int m_ktx2_zstandard_level; uint32_t m_ktx2_animdata_duration; uint32_t m_ktx2_animdata_timescale; uint32_t m_ktx2_animdata_loopcount; - + basisu::vector m_input_filenames; basisu::vector m_input_alpha_filenames; std::string m_output_filename; std::string m_output_path; - int m_format_only; + int m_unpack_format_only; std::string m_multifile_printf; uint32_t m_multifile_first; @@ -1153,9 +1433,9 @@ class command_line_params std::string m_etc1s_use_global_codebooks_file; std::string m_test_file_dir; - + uint32_t m_max_threads; - + bool m_individual; bool m_no_ktx; bool m_ktx_only; @@ -1166,19 +1446,29 @@ class command_line_params bool m_compare_plot; bool m_parallel_compression; bool m_tonemap_dither_flag; + bool m_xuastc_ldr_disable_bc7_transcoding; + bool m_no_etc1s_transcoding_chroma_filtering; + bool m_higher_quality_transcoding; + bool m_force_deblocking; + bool m_disable_deblocking; + bool m_stronger_deblocking; + + int m_effort_level; + int m_quality_level; + bool m_used_old_style_codec_config_param; // true if the user has specified low-level or old-style codec configuration parameters }; static bool expand_multifile(command_line_params &opts) { if (!opts.m_multifile_printf.size()) return true; - + if (!opts.m_multifile_num) { error_printf("-multifile_printf specified, but not -multifile_num\n"); return false; } - + std::string fmt(opts.m_multifile_printf); // Workaround for MSVC debugger issues. Questionable to leave in here. size_t x = fmt.find_first_of('!'); @@ -1190,15 +1480,15 @@ static bool expand_multifile(command_line_params &opts) error_printf("Must include C-style printf() format character '%%' in -multifile_printf string\n"); return false; } - + for (uint32_t i = opts.m_multifile_first; i < opts.m_multifile_first + opts.m_multifile_num; i++) { char buf[1024]; -#ifdef _WIN32 +#ifdef _WIN32 sprintf_s(buf, sizeof(buf), fmt.c_str(), i); #else snprintf(buf, sizeof(buf), fmt.c_str(), i); -#endif +#endif if (buf[0]) opts.m_input_filenames.push_back(buf); @@ -1209,8 +1499,8 @@ static bool expand_multifile(command_line_params &opts) struct basis_data { - basis_data() : - m_transcoder() + basis_data() : + m_transcoder() { } uint8_vec m_file_data; @@ -1227,7 +1517,7 @@ static basis_data *load_basis_file(const char *pInput_filename, bool force_etc1s delete p; return nullptr; } - printf("Input file \"%s\"\n", pInput_filename); + printf("\nInput file \"%s\"\n", pInput_filename); if (!basis_data.size()) { error_printf("File is empty!\n"); @@ -1258,6 +1548,24 @@ static basis_data *load_basis_file(const char *pInput_filename, bool force_etc1s return p; } +static uint32_t get_transcode_flags_from_options(const command_line_params& opts) +{ + uint32_t transcode_flags = opts.m_higher_quality_transcoding ? basist::cDecodeFlagsHighQuality : 0; + + if (opts.m_disable_deblocking) + transcode_flags |= basist::cDecodeFlagsNoDeblockFiltering; + else if (opts.m_force_deblocking) + transcode_flags |= basist::cDecodeFlagsForceDeblockFiltering; + if (opts.m_stronger_deblocking) + transcode_flags |= basist::cDecodeFlagsStrongerDeblockFiltering; + if (opts.m_no_etc1s_transcoding_chroma_filtering) + transcode_flags |= basist::cDecodeFlagsNoETC1SChromaFiltering; + if (opts.m_xuastc_ldr_disable_bc7_transcoding) + transcode_flags |= basist::cDecodeFlagXUASTCLDRDisableFastBC7Transcoding; + + return transcode_flags; +} + static bool compress_mode(command_line_params &opts) { uint32_t num_threads = 1; @@ -1265,17 +1573,18 @@ static bool compress_mode(command_line_params &opts) if (opts.m_comp_params.m_multithreading) { // We use std::thread::hardware_concurrency() as a hint to determine the default # of threads to put into a pool. - num_threads = std::thread::hardware_concurrency(); + num_threads = get_num_hardware_threads(); if (num_threads < 1) num_threads = 1; if (num_threads > opts.m_max_threads) num_threads = opts.m_max_threads; } + // num_threads is the total thread pool size, *including* the calling thread. So 1=no extra threads. job_pool compressor_jpool(opts.m_parallel_compression ? 1 : num_threads); if (!opts.m_parallel_compression) opts.m_comp_params.m_pJob_pool = &compressor_jpool; - + if (!expand_multifile(opts)) { error_printf("-multifile expansion failed!\n"); @@ -1287,7 +1596,7 @@ static bool compress_mode(command_line_params &opts) error_printf("No input files to process!\n"); return false; } - + basis_data* pGlobal_codebook_data = nullptr; if (opts.m_etc1s_use_global_codebooks_file.size()) { @@ -1297,7 +1606,7 @@ static bool compress_mode(command_line_params &opts) printf("Loaded global codebooks from .basis file \"%s\"\n", opts.m_etc1s_use_global_codebooks_file.c_str()); } - + basis_compressor_params ¶ms = opts.m_comp_params; if (opts.m_ktx2_mode) @@ -1307,9 +1616,7 @@ static bool compress_mode(command_line_params &opts) params.m_ktx2_uastc_supercompression = basist::KTX2_SS_ZSTANDARD; else params.m_ktx2_uastc_supercompression = basist::KTX2_SS_NONE; - - params.m_ktx2_srgb_transfer_func = opts.m_comp_params.m_perceptual; - + if (params.m_tex_type == basist::basis_texture_type::cBASISTexTypeVideoFrames) { // Create KTXanimData key value entry @@ -1319,7 +1626,7 @@ static bool compress_mode(command_line_params &opts) const char* pAD = "KTXanimData"; kv.m_key.resize(strlen(pAD) + 1); strcpy((char*)kv.m_key.data(), pAD); - + basist::ktx2_animdata ad; ad.m_duration = opts.m_ktx2_animdata_duration; ad.m_timescale = opts.m_ktx2_animdata_timescale; @@ -1330,15 +1637,18 @@ static bool compress_mode(command_line_params &opts) params.m_ktx2_key_values.push_back(kv); } - + // TODO- expose this to command line. params.m_ktx2_zstd_supercompression_level = opts.m_ktx2_zstandard_level; } params.m_read_source_images = true; params.m_write_output_basis_or_ktx2_files = true; - params.m_pGlobal_codebooks = pGlobal_codebook_data ? &pGlobal_codebook_data->m_transcoder.get_lowlevel_etc1s_decoder() : nullptr; - + params.m_pGlobal_codebooks = pGlobal_codebook_data ? &pGlobal_codebook_data->m_transcoder.get_lowlevel_etc1s_decoder() : nullptr; + + // Get the transcode/decode flags used when validating the output by calling the transcoder from the options. + params.m_transcode_flags = get_transcode_flags_from_options(opts); + FILE *pCSV_file = nullptr; if (opts.m_csv_file.size()) { @@ -1354,7 +1664,7 @@ static bool compress_mode(command_line_params &opts) } printf("Processing %u total file(s)\n", (uint32_t)opts.m_input_filenames.size()); - + interval_timer all_tm; all_tm.start(); @@ -1403,7 +1713,7 @@ static bool compress_mode(command_line_params &opts) params.m_source_filenames = opts.m_input_filenames; params.m_source_alpha_filenames = opts.m_input_alpha_filenames; } - + if (opts.m_output_filename.size()) params.m_out_filename = opts.m_output_filename; else @@ -1459,9 +1769,11 @@ static bool compress_mode(command_line_params &opts) if (params.m_status_output) { - printf("Compression succeeded to file \"%s\" size %zu bytes in %3.3f secs\n", params.m_out_filename.c_str(), - opts.m_ktx2_mode ? c.get_output_ktx2_file().size() : c.get_output_basis_file().size(), - tm.get_elapsed_secs()); + fmt_printf("Compression succeeded to file \"{}\" size {} bytes in {3.3} secs, {3.3} bits/texel\n", + params.m_out_filename.c_str(), + opts.m_ktx2_mode ? (uint64_t)c.get_output_ktx2_file().size() : (uint64_t)c.get_output_basis_file().size(), + tm.get_elapsed_secs(), + opts.m_ktx2_mode ? c.get_ktx2_bits_per_texel() : c.get_basis_bits_per_texel()); } } else @@ -1479,6 +1791,15 @@ static bool compress_mode(command_line_params &opts) switch (ec) { + case basis_compressor::cECFailedInvalidParameters: + { + error_printf("Invalid compressor parameters (internal error)\n"); + + if (opts.m_individual) + exit_flag = false; + + break; + } case basis_compressor::cECFailedReadingSourceImages: { error_printf("Compressor failed reading a source image!\n"); @@ -1559,7 +1880,7 @@ static bool compress_mode(command_line_params &opts) fprintf(pCSV_file, "\"%s\", %u, %u, %u, %u, %u, %f, %f, %f, %f, %f, %u, %u, %f, %f, %f, %f, %f, %f, %f\n", params.m_out_filename.c_str(), - c.get_basis_file_size(), + (uint32_t)c.get_basis_file_size(), (uint32_t)c.get_stats().size(), c.get_stats()[0].m_width, c.get_stats()[0].m_height, (uint32_t)c.get_any_source_image_has_alpha(), c.get_basis_bits_per_texel(), @@ -1567,7 +1888,7 @@ static bool compress_mode(command_line_params &opts) c.get_stats()[0].m_basis_rgba_avg_psnr, c.get_stats()[0].m_basis_luma_709_psnr, c.get_stats()[0].m_best_etc1s_luma_709_psnr, - params.m_etc1s_quality_level, (int)params.m_compression_level, tm.get_elapsed_secs(), + params.m_quality_level, (int)params.m_etc1s_compression_level, tm.get_elapsed_secs(), rgb_avg_psnr_min, rgb_avg_psnr_avg, a_avg_psnr_min, a_avg_psnr_avg, luma_709_psnr_min, luma_709_psnr_avg); @@ -1591,7 +1912,7 @@ static bool compress_mode(command_line_params &opts) comp_params_vec, results); BASISU_NOTE_UNUSED(any_failed); - + for (uint32_t i = 0; i < comp_params_vec.size(); i++) { if (results[i].m_error_code != basis_compressor::cECSuccess) @@ -1600,8 +1921,8 @@ static bool compress_mode(command_line_params &opts) total_failures++; - error_printf("File %u (first source image: \"%s\", output file: \"%s\") failed with error code %i!\n", i, - comp_params_vec[i].m_source_filenames[0].c_str(), + error_printf("File %u (first source image: \"%s\", output file: \"%s\") failed with error code %i!\n", i, + comp_params_vec[i].m_source_filenames[0].c_str(), comp_params_vec[i].m_out_filename.c_str(), (int)results[i].m_error_code); } @@ -1610,11 +1931,11 @@ static bool compress_mode(command_line_params &opts) total_successes++; } } - + } // if (opts.m_parallel_compression) printf("Total successes: %u failures: %u\n", total_successes, total_failures); - + all_tm.stop(); if (total_files > 1) @@ -1625,7 +1946,7 @@ static bool compress_mode(command_line_params &opts) fclose(pCSV_file); pCSV_file = nullptr; } - delete pGlobal_codebook_data; + delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr; return result; @@ -1641,9 +1962,8 @@ static bool unpack_and_validate_ktx2_file( uint32_t& total_unpack_warnings, uint32_t& total_pvrtc_nonpow2_warnings) { - // TODO - (void)pCSV_file; - (void)file_index; + BASISU_NOTE_UNUSED(pCSV_file); + BASISU_NOTE_UNUSED(file_index); const bool validate_flag = (opts.m_mode == cValidate); @@ -1666,7 +1986,7 @@ static bool unpack_and_validate_ktx2_file( error_printf("ktx2_transcoder::start_transcoding() failed! File either uses an unsupported feature, is invalid, was corrupted, or this is a bug.\n"); return false; } - + printf("Resolution: %ux%u\n", dec.get_width(), dec.get_height()); fmt_printf("Block size: {}x{}\n", dec.get_block_width(), dec.get_block_height()); printf("Mipmap Levels: %u\n", dec.get_levels()); @@ -1676,10 +1996,13 @@ static bool unpack_and_validate_ktx2_file( if (dec.is_hdr()) fmt_printf("LDR to HDR upconversion nit multiplier: {}\n", dec.get_ldr_hdr_upconversion_nit_multiplier()); - + const bool is_etc1s = (dec.get_basis_tex_format() == basist::basis_tex_format::cETC1S); - + bool is_hdr = false; + //bool is_xuastc_ldr = false, is_astc_ldr = false; + + std::string fmt_str_temp; const char* pFmt_str = nullptr; switch (dec.get_basis_tex_format()) @@ -1689,7 +2012,7 @@ static bool unpack_and_validate_ktx2_file( pFmt_str = "ETC1S"; break; } - case basist::basis_tex_format::cUASTC4x4: + case basist::basis_tex_format::cUASTC_LDR_4x4: { pFmt_str = "UASTC_LDR_4x4"; break; @@ -1706,10 +2029,56 @@ static bool unpack_and_validate_ktx2_file( pFmt_str = "ASTC_HDR_6x6"; break; } - case basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE: + case basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE: { is_hdr = true; - pFmt_str = "ASTC_HDR_6x6_INTERMEDIATE"; + pFmt_str = "UASTC_HDR_6x6_INTERMEDIATE"; + break; + } + case basist::basis_tex_format::cXUASTC_LDR_4x4: + case basist::basis_tex_format::cXUASTC_LDR_5x4: + case basist::basis_tex_format::cXUASTC_LDR_5x5: + case basist::basis_tex_format::cXUASTC_LDR_6x5: + case basist::basis_tex_format::cXUASTC_LDR_6x6: + case basist::basis_tex_format::cXUASTC_LDR_8x5: + case basist::basis_tex_format::cXUASTC_LDR_8x6: + case basist::basis_tex_format::cXUASTC_LDR_10x5: + case basist::basis_tex_format::cXUASTC_LDR_10x6: + case basist::basis_tex_format::cXUASTC_LDR_8x8: + case basist::basis_tex_format::cXUASTC_LDR_10x8: + case basist::basis_tex_format::cXUASTC_LDR_10x10: + case basist::basis_tex_format::cXUASTC_LDR_12x10: + case basist::basis_tex_format::cXUASTC_LDR_12x12: + { + //is_xuastc_ldr = true; + + uint32_t block_width = 0, block_height = 0; + basist::get_basis_tex_format_block_size(dec.get_basis_tex_format(), block_width, block_height); + fmt_str_temp = fmt_string("XUASTC_LDR_{}x{}", block_width, block_height); + pFmt_str = fmt_str_temp.c_str(); + break; + } + case basist::basis_tex_format::cASTC_LDR_4x4: + case basist::basis_tex_format::cASTC_LDR_5x4: + case basist::basis_tex_format::cASTC_LDR_5x5: + case basist::basis_tex_format::cASTC_LDR_6x5: + case basist::basis_tex_format::cASTC_LDR_6x6: + case basist::basis_tex_format::cASTC_LDR_8x5: + case basist::basis_tex_format::cASTC_LDR_8x6: + case basist::basis_tex_format::cASTC_LDR_10x5: + case basist::basis_tex_format::cASTC_LDR_10x6: + case basist::basis_tex_format::cASTC_LDR_8x8: + case basist::basis_tex_format::cASTC_LDR_10x8: + case basist::basis_tex_format::cASTC_LDR_10x10: + case basist::basis_tex_format::cASTC_LDR_12x10: + case basist::basis_tex_format::cASTC_LDR_12x12: + { + //is_astc_ldr = true; + + uint32_t block_width = 0, block_height = 0; + basist::get_basis_tex_format_block_size(dec.get_basis_tex_format(), block_width, block_height); + fmt_str_temp = fmt_string("ASTC_LDR_{}x{}", block_width, block_height); + pFmt_str = fmt_str_temp.c_str(); break; } default: @@ -1720,7 +2089,7 @@ static bool unpack_and_validate_ktx2_file( } printf("Supercompression Format: %s\n", pFmt_str); - + printf("Supercompression Scheme: "); switch (dec.get_header().m_supercompression_scheme) { @@ -1733,9 +2102,9 @@ static bool unpack_and_validate_ktx2_file( } printf("Has Alpha: %u\n", (uint32_t)dec.get_has_alpha()); - + printf("\nKTX2 header vk_format: 0x%X (decimal %u)\n", (uint32_t)dec.get_header().m_vk_format, (uint32_t)dec.get_header().m_vk_format); - + printf("\nData Format Descriptor (DFD):\n"); printf("DFD length in bytes: %zu\n", dec.get_dfd().size()); printf("DFD color model: %u\n", dec.get_dfd_color_model()); @@ -1751,8 +2120,13 @@ static bool unpack_and_validate_ktx2_file( printf("DFD chan1: %s\n", basist::ktx2_get_etc1s_df_channel_id_str(dec.get_dfd_channel_id1())); } else + { printf("DFD chan0: %s\n", basist::ktx2_get_uastc_df_channel_id_str(dec.get_dfd_channel_id0())); + } + // For proper ASTC decoding we must know which ASTC decode profile to apply (sRGB or linear). + const bool actual_ktx2_srgb_transfer_func = (dec.get_dfd_transfer_func() == basist::KTX2_KHR_DF_TRANSFER_SRGB); + printf("DFD hex values:\n"); for (uint32_t i = 0; i < dec.get_dfd().size(); i++) { @@ -1764,6 +2138,25 @@ static bool unpack_and_validate_ktx2_file( } printf("\n"); + // the sRGB transfer function to use while unpacking astc content (ideally we want this to always match what we used during astc encoding) + bool srgb_transfer_func_astc_unpacking = actual_ktx2_srgb_transfer_func; + + // the sRGB transfer function to use when writing out files (we want to indicate to the caller if the data is sRGB or linear) + bool srgb_transfer_func_astc_writing = actual_ktx2_srgb_transfer_func; + + const bool is_uastc_ldr_4x4 = (dec.get_basis_tex_format() == basist::basis_tex_format::cUASTC_LDR_4x4); + if ((is_etc1s) || (is_uastc_ldr_4x4)) + { + // The ETC1S and UASTC LDR 4x4 transcoders supply ASTC LDR 4x4 data assuming the decoder will NOT be using the sRGB read decode profile, which is likely the most common case (in geospatial rendering scenarios). + // Note XUASTC/UASTC LDR 4x4-12x12 supports both linear and sRGB decode profiles throughout the entire pipeline (encoding/transcoding/decoding to raw pixels). + srgb_transfer_func_astc_unpacking = false; + + // This matches the behavior of our original tools. It ensures astcenc uses linear by default when reading our transcoded .KTX files. + srgb_transfer_func_astc_writing = false; + + if (actual_ktx2_srgb_transfer_func) + printf("Note: ETC1S/UASTC LDR 4x4 will always be decoded by this tool using the ASTC linear decode profile, regardless of the KTX2/.basis DFD transfer function field.\n"); + } printf("Total key values: %zu\n", dec.get_key_values().size()); for (uint32_t i = 0; i < dec.get_key_values().size(); i++) @@ -1772,13 +2165,13 @@ static bool unpack_and_validate_ktx2_file( if (dec.get_key_values()[i].m_value.size() > 256) continue; - + bool is_ascii = true; for (uint32_t j = 0; j < dec.get_key_values()[i].m_value.size(); j++) { uint8_t c = dec.get_key_values()[i].m_value[j]; - if (!( - ((c >= ' ') && (c < 0x80)) || + if (!( + ((c >= ' ') && (c < 0x80)) || ((j == dec.get_key_values()[i].m_value.size() - 1) && (!c)) )) { @@ -1851,10 +2244,10 @@ static bool unpack_and_validate_ktx2_file( error_printf("Failed retrieving image level information (%u %u %u)!\n", layer_index, level_index, face_index); return false; } - + fmt_printf("--- Level Index: {}, Layer Index: {}, Face Index: {}\n", level_info.m_level_index, level_info.m_layer_index, level_info.m_face_index); - + fmt_printf("Orig width/height: {}x{}\n", level_info.m_orig_width, level_info.m_orig_height); fmt_printf("Width/height: {}x{}\n", level_info.m_width, level_info.m_height); fmt_printf("Block width/height: {}x{}\n", level_info.m_block_width, level_info.m_block_height); @@ -1878,12 +2271,14 @@ static bool unpack_and_validate_ktx2_file( int first_format = 0; int last_format = (int)basist::transcoder_texture_format::cTFTotalTextureFormats; - if (opts.m_format_only > -1) + if (opts.m_unpack_format_only > -1) { - first_format = opts.m_format_only; + first_format = opts.m_unpack_format_only; last_format = first_format + 1; } + uint32_t transcode_flags = get_transcode_flags_from_options(opts); + for (int format_iter = first_format; format_iter < last_format; format_iter++) { basist::transcoder_texture_format tex_fmt = static_cast(format_iter); @@ -1934,7 +2329,7 @@ static bool unpack_and_validate_ktx2_file( if ((transcoder_tex_fmt == basist::transcoder_texture_format::cTFPVRTC1_4_RGB) || (transcoder_tex_fmt == basist::transcoder_texture_format::cTFPVRTC1_4_RGBA)) { - if (!is_pow2(level_info.m_width) || !is_pow2(level_info.m_height)) + if (!is_pow2(level_info.m_orig_width) || !is_pow2(level_info.m_orig_height)) { total_pvrtc_nonpow2_warnings++; @@ -1952,13 +2347,11 @@ static bool unpack_and_validate_ktx2_file( // Fill the buffer with psuedo-random bytes, to help more visibly detect cases where the transcoder fails to write to part of the output. fill_buffer_with_random_bytes(gi.get_ptr(), gi.get_size_in_bytes()); - - const uint32_t decode_flags = basist::cDecodeFlagsHighQuality; - + interval_timer tm; tm.start(); - if (!dec.transcode_image_level(level_index, layer_index, face_index, gi.get_ptr(), gi.get_total_blocks(), transcoder_tex_fmt, decode_flags)) + if (!dec.transcode_image_level(level_index, layer_index, face_index, gi.get_ptr(), gi.get_total_blocks(), transcoder_tex_fmt, transcode_flags)) { error_printf("Failed transcoding image level (%u %u %u %u)!\n", layer_index, level_index, face_index, format_iter); return false; @@ -1966,7 +2359,7 @@ static bool unpack_and_validate_ktx2_file( double total_time = tm.get_elapsed_ms(); - printf("Transcode of layer %u level %u face %u res %ux%u format %s succeeded in %3.3f ms\n", layer_index, level_index, face_index, + printf("Transcode of layer %u level %u face %u res %ux%u format %s succeeded in %3.3f ms\n", layer_index, level_index, face_index, level_info.m_orig_width, level_info.m_orig_height, basist::basis_get_format_name(transcoder_tex_fmt), total_time); } @@ -1980,14 +2373,14 @@ static bool unpack_and_validate_ktx2_file( if (validate_flag) return true; - // Now write KTX/DDS files and unpack them to individual PNG's/EXR's + // Now write KTX/DDS/ASTC files and unpack them to individual PNG's/EXR's const bool is_cubemap = (dec.get_faces() > 1); const bool is_array = (total_layers > 1); const bool is_cubemap_array = is_cubemap && is_array; const bool is_mipmapped = dec.get_levels() > 1; BASISU_NOTE_UNUSED(is_cubemap_array); BASISU_NOTE_UNUSED(is_mipmapped); - + // The maximum Direct3D array size is 2048. const uint32_t MAX_DDS_TEXARRAY_SIZE = 2048; @@ -1995,7 +2388,7 @@ static bool unpack_and_validate_ktx2_file( { const basist::transcoder_texture_format transcoder_tex_fmt = static_cast(format_iter); const basisu::texture_format tex_fmt = basis_get_basisu_texture_format(transcoder_tex_fmt); - + if (basist::basis_transcoder_format_is_uncompressed(transcoder_tex_fmt)) continue; if (!basis_is_format_supported(transcoder_tex_fmt, dec.get_basis_tex_format())) @@ -2003,8 +2396,10 @@ static bool unpack_and_validate_ktx2_file( if (transcoder_tex_fmt == basist::transcoder_texture_format::cTFBC7_ALT) continue; - // TODO: Could write DDS texture arrays. + const bool is_fmt_astc = basis_is_transcoder_texture_format_astc(transcoder_tex_fmt); + // TODO: Could write DDS texture arrays. + // No KTX tool that we know of supports cubemap arrays, so write individual cubemap files for each layer. if ((!opts.m_no_ktx) && (is_cubemap)) { @@ -2015,22 +2410,24 @@ static bool unpack_and_validate_ktx2_file( for (uint32_t face_index = 0; face_index < 6; face_index++) cubemap.push_back(gpu_images[format_iter][face_index][layer_index]); + // Write KTX1 file { std::string ktx_filename(base_filename + string_format("_transcoded_cubemap_%s_layer_%u.ktx", basist::basis_get_format_name(transcoder_tex_fmt), layer_index)); - if (!write_compressed_texture_file(ktx_filename.c_str(), cubemap, true, true)) + if (!write_compressed_texture_file(ktx_filename.c_str(), cubemap, true, is_fmt_astc ? srgb_transfer_func_astc_writing : actual_ktx2_srgb_transfer_func)) { error_printf("Failed writing KTX file \"%s\"!\n", ktx_filename.c_str()); return false; } printf("Wrote .KTX cubemap file \"%s\"\n", ktx_filename.c_str()); } - + + // Write .DDS file if (does_dds_support_format(cubemap[0][0].get_format())) { std::string dds_filename(base_filename + string_format("_transcoded_cubemap_%s_layer_%u.dds", basist::basis_get_format_name(transcoder_tex_fmt), layer_index)); - if (!write_compressed_texture_file(dds_filename.c_str(), cubemap, true, true)) + if (!write_compressed_texture_file(dds_filename.c_str(), cubemap, true, actual_ktx2_srgb_transfer_func)) { error_printf("Failed writing DDS file \"%s\"!\n", dds_filename.c_str()); return false; @@ -2052,7 +2449,7 @@ static bool unpack_and_validate_ktx2_file( std::string dds_filename(base_filename + string_format("_transcoded_array_%s.dds", basist::basis_get_format_name(transcoder_tex_fmt))); - if (!write_compressed_texture_file(dds_filename.c_str(), tex_array, is_cubemap, true)) + if (!write_compressed_texture_file(dds_filename.c_str(), tex_array, is_cubemap, actual_ktx2_srgb_transfer_func)) { error_printf("Failed writing DDS file \"%s\"!\n", dds_filename.c_str()); return false; @@ -2061,7 +2458,7 @@ static bool unpack_and_validate_ktx2_file( } } - // Now unpack each layer and face individually and write KTX/DDS/PNG/EXR files for each + // Now unpack each layer and face individually and write KTX/DDS/ASTC/PNG/EXR/OUT files for each for (uint32_t layer_index = 0; layer_index < total_layers; layer_index++) { for (uint32_t face_index = 0; face_index < dec.get_faces(); face_index++) @@ -2090,7 +2487,7 @@ static bool unpack_and_validate_ktx2_file( else ktx_filename = base_filename + string_format("_transcoded_%s_layer_%04u.ktx", basist::basis_get_format_name(transcoder_tex_fmt), layer_index); - if (!write_compressed_texture_file(ktx_filename.c_str(), gi, true)) + if (!write_compressed_texture_file(ktx_filename.c_str(), gi, is_fmt_astc ? srgb_transfer_func_astc_writing : actual_ktx2_srgb_transfer_func)) { error_printf("Failed writing KTX file \"%s\"!\n", ktx_filename.c_str()); return false; @@ -2107,7 +2504,7 @@ static bool unpack_and_validate_ktx2_file( else dds_filename = base_filename + string_format("_transcoded_%s_layer_%04u.dds", basist::basis_get_format_name(transcoder_tex_fmt), layer_index); - if (!write_compressed_texture_file(dds_filename.c_str(), gi, true)) + if (!write_compressed_texture_file(dds_filename.c_str(), gi, actual_ktx2_srgb_transfer_func)) { error_printf("Failed writing DDS file \"%s\"!\n", dds_filename.c_str()); return false; @@ -2153,11 +2550,30 @@ static bool unpack_and_validate_ktx2_file( } printf("Wrote .EXR file \"%s\"\n", rgb_filename.c_str()); } + + // Save .astc + if ((!opts.m_ktx_only) && basist::basis_is_transcoder_texture_format_astc(transcoder_tex_fmt)) + { + std::string astc_filename; + if (gi.size() > 1) + astc_filename = base_filename + string_format("_unpacked_%s_level_%u_face_%u_layer_%04u.astc", basist::basis_get_format_name(transcoder_tex_fmt), level_index, face_index, layer_index); + else + astc_filename = base_filename + string_format("_unpacked_%s_face_%u_layer_%04u.astc", basist::basis_get_format_name(transcoder_tex_fmt), face_index, layer_index); + + const gpu_image& level_g = gi[level_index]; + + if (!write_astc_file(astc_filename.c_str(), level_g.get_ptr(), level_g.get_block_width(), level_g.get_block_height(), level_info.m_width, level_info.m_height)) + { + error_printf("Failed writing to .ASTC file \"%s\"\n", astc_filename.c_str()); + return false; + } + printf("Wrote .ASTC file \"%s\"\n", astc_filename.c_str()); + } } else { image u; - if (!gi[level_index].unpack(u)) + if (!gi[level_index].unpack(u, srgb_transfer_func_astc_unpacking)) { printf("Warning: Failed unpacking GPU texture data (%u %u %u %u). Unpacking as much as possible.\n", format_iter, layer_index, level_index, face_index); total_unpack_warnings++; @@ -2182,23 +2598,7 @@ static bool unpack_and_validate_ktx2_file( } printf("Wrote .PNG file \"%s\"\n", rgb_filename.c_str()); } - - // Save .OUT - if ((transcoder_tex_fmt == basist::transcoder_texture_format::cTFFXT1_RGB) && (opts.m_write_out)) - { - std::string out_filename; - if (gi.size() > 1) - out_filename = base_filename + string_format("_unpacked_rgb_%s_level_%u_face_%u_layer_%04u.out", basist::basis_get_format_name(transcoder_tex_fmt), level_index, face_index, layer_index); - else - out_filename = base_filename + string_format("_unpacked_rgb_%s_face_%u_layer_%04u.out", basist::basis_get_format_name(transcoder_tex_fmt), face_index, layer_index); - if (!write_3dfx_out_file(out_filename.c_str(), gi[level_index])) - { - error_printf("Failed writing to OUT file \"%s\"\n", out_filename.c_str()); - return false; - } - printf("Wrote .OUT file \"%s\"\n", out_filename.c_str()); - } - + // Save alpha if (basis_transcoder_format_has_alpha(transcoder_tex_fmt) && (!opts.m_ktx_only) && (write_png)) { @@ -2227,6 +2627,41 @@ static bool unpack_and_validate_ktx2_file( printf("Wrote .PNG file \"%s\"\n", rgba_filename.c_str()); } + // Save .astc + if ((!opts.m_ktx_only) && basist::basis_is_transcoder_texture_format_astc(transcoder_tex_fmt)) + { + std::string astc_filename; + if (gi.size() > 1) + astc_filename = base_filename + string_format("_unpacked_%s_level_%u_face_%u_layer_%04u.astc", basist::basis_get_format_name(transcoder_tex_fmt), level_index, face_index, layer_index); + else + astc_filename = base_filename + string_format("_unpacked_%s_face_%u_layer_%04u.astc", basist::basis_get_format_name(transcoder_tex_fmt), face_index, layer_index); + + const gpu_image& level_g = gi[level_index]; + + if (!write_astc_file(astc_filename.c_str(), level_g.get_ptr(), level_g.get_block_width(), level_g.get_block_height(), level_info.m_width, level_info.m_height)) + { + error_printf("Failed writing to .ASTC file \"%s\"\n", astc_filename.c_str()); + return false; + } + printf("Wrote .ASTC file \"%s\"\n", astc_filename.c_str()); + } + + // Save .OUT + if ((transcoder_tex_fmt == basist::transcoder_texture_format::cTFFXT1_RGB) && (opts.m_write_out)) + { + std::string out_filename; + if (gi.size() > 1) + out_filename = base_filename + string_format("_unpacked_rgb_%s_level_%u_face_%u_layer_%04u.out", basist::basis_get_format_name(transcoder_tex_fmt), level_index, face_index, layer_index); + else + out_filename = base_filename + string_format("_unpacked_rgb_%s_face_%u_layer_%04u.out", basist::basis_get_format_name(transcoder_tex_fmt), face_index, layer_index); + if (!write_3dfx_out_file(out_filename.c_str(), gi[level_index])) + { + error_printf("Failed writing to OUT file \"%s\"\n", out_filename.c_str()); + return false; + } + printf("Wrote .OUT file \"%s\"\n", out_filename.c_str()); + } + } // is_hdr } // level_index @@ -2237,7 +2672,7 @@ static bool unpack_and_validate_ktx2_file( } // format_iter - if ((opts.m_format_only == -1) && (!validate_flag)) + if ((opts.m_unpack_format_only == -1) && (!validate_flag)) { if (is_hdr) { @@ -2267,7 +2702,7 @@ static bool unpack_and_validate_ktx2_file( interval_timer tm; tm.start(); - if (!dec.transcode_image_level(level_index, layer_index, face_index, half_img.data(), total_pixels, transcoder_tex_fmt, 0)) + if (!dec.transcode_image_level(level_index, layer_index, face_index, half_img.data(), total_pixels, transcoder_tex_fmt, transcode_flags)) { fmt_error_printf("Failed transcoding image level ({} {} {})!\n", layer_index, level_index, face_index); return false; @@ -2275,8 +2710,8 @@ static bool unpack_and_validate_ktx2_file( double total_transcode_time = tm.get_elapsed_ms(); - fmt_printf("Transcode of level {} layer {} face {} res {}x{} format {} succeeded in {} ms\n", - level_index, layer_index, face_index, + fmt_printf("Transcode of level {} layer {} face {} res {}x{} format {} succeeded in {} ms\n", + level_index, layer_index, face_index, level_info.m_orig_width, level_info.m_orig_height, basist::basis_get_format_name(transcoder_tex_fmt), total_transcode_time); if ((!validate_flag) && (!opts.m_ktx_only)) @@ -2299,7 +2734,7 @@ static bool unpack_and_validate_ktx2_file( } } // face_index - } // layer_index + } // layer_index } // level_index // RGB HALF @@ -2328,7 +2763,7 @@ static bool unpack_and_validate_ktx2_file( interval_timer tm; tm.start(); - if (!dec.transcode_image_level(level_index, layer_index, face_index, half_img.data(), total_pixels, transcoder_tex_fmt, 0)) + if (!dec.transcode_image_level(level_index, layer_index, face_index, half_img.data(), total_pixels, transcoder_tex_fmt, transcode_flags)) { fmt_error_printf("Failed transcoding image level ({} {} {})!\n", layer_index, level_index, face_index); return false; @@ -2360,10 +2795,10 @@ static bool unpack_and_validate_ktx2_file( } } // face_index - } // layer_index + } // layer_index } // level_index - // RGB HALF + // RGB_9E5 for (uint32_t level_index = 0; level_index < dec.get_levels(); level_index++) { for (uint32_t layer_index = 0; layer_index < total_layers; layer_index++) @@ -2389,7 +2824,7 @@ static bool unpack_and_validate_ktx2_file( interval_timer tm; tm.start(); - if (!dec.transcode_image_level(level_index, layer_index, face_index, rgb9e5_img.data(), total_pixels, transcoder_tex_fmt, 0)) + if (!dec.transcode_image_level(level_index, layer_index, face_index, rgb9e5_img.data(), total_pixels, transcoder_tex_fmt, transcode_flags)) { fmt_error_printf("Failed transcoding image level ({} {} {})!\n", layer_index, level_index, face_index); return false; @@ -2420,136 +2855,420 @@ static bool unpack_and_validate_ktx2_file( } } // face_index - } // layer_index + } // layer_index } // level_index - + } else { - // TODO: Add LDR uncompressed formats - } - } + // RGBA 32bpp + for (uint32_t level_index = 0; level_index < dec.get_levels(); level_index++) + { + for (uint32_t layer_index = 0; layer_index < total_layers; layer_index++) + { + for (uint32_t face_index = 0; face_index < dec.get_faces(); face_index++) + { + const basist::transcoder_texture_format transcoder_tex_fmt = basist::transcoder_texture_format::cTFRGBA32; - return true; -} + basist::ktx2_image_level_info level_info; -static bool unpack_and_validate_basis_file( - uint32_t file_index, - const std::string &base_filename, - uint8_vec &basis_file_data, - command_line_params& opts, - FILE *pCSV_file, - basis_data* pGlobal_codebook_data, - uint32_t &total_unpack_warnings, - uint32_t &total_pvrtc_nonpow2_warnings) -{ - const bool validate_flag = (opts.m_mode == cValidate); + if (!dec.get_image_level_info(level_info, level_index, layer_index, face_index)) + { + fmt_error_printf("Failed retrieving image level information ({} {} {})!\n", layer_index, level_index, face_index); + return false; + } - basist::basisu_transcoder dec; + const uint32_t total_pixels = level_info.m_orig_width * level_info.m_orig_height; - if (pGlobal_codebook_data) - { - dec.set_global_codebooks(&pGlobal_codebook_data->m_transcoder.get_lowlevel_etc1s_decoder()); - } + image img(level_info.m_orig_width, level_info.m_orig_height); - if (!opts.m_fuzz_testing) - { - // Skip the full validation, which CRC16's the entire file. + fill_buffer_with_random_bytes(img.get_ptr(), img.get_total_pixels() * sizeof(color_rgba)); - // Validate the file - note this isn't necessary for transcoding - if (!dec.validate_file_checksums(&basis_file_data[0], (uint32_t)basis_file_data.size(), true)) - { - error_printf("File version is unsupported, or file failed one or more CRC checks!\n"); + interval_timer tm; + tm.start(); - return false; - } - } + if (!dec.transcode_image_level(level_index, layer_index, face_index, img.get_ptr(), total_pixels, transcoder_tex_fmt, transcode_flags)) + { + fmt_error_printf("Failed transcoding image level ({} {} {})!\n", layer_index, level_index, face_index); + return false; + } - printf("File version and CRC checks succeeded\n"); + double total_transcode_time = tm.get_elapsed_ms(); - basist::basisu_file_info fileinfo; - if (!dec.get_file_info(&basis_file_data[0], (uint32_t)basis_file_data.size(), fileinfo)) - { - error_printf("Failed retrieving Basis file information!\n"); - return false; - } + fmt_printf("Transcode of level {} layer {} face {} res {}x{} format {} succeeded in {} ms\n", + level_index, layer_index, face_index, + level_info.m_orig_width, level_info.m_orig_height, basist::basis_get_format_name(transcoder_tex_fmt), total_transcode_time); - assert(fileinfo.m_total_images == fileinfo.m_image_mipmap_levels.size()); - assert(fileinfo.m_total_images == dec.get_total_images(&basis_file_data[0], (uint32_t)basis_file_data.size())); + if ((!validate_flag) && (!opts.m_ktx_only)) + { + std::string rgba_filename(base_filename + fmt_string("_unpacked_rgba_{}_level_{}_face_{}_layer{04}.png", basist::basis_get_format_name(transcoder_tex_fmt), level_index, face_index, layer_index)); + if (!save_png(rgba_filename, img, cImageSaveIgnoreAlpha)) + { + error_printf("Failed writing to .PNG file \"%s\"\n", rgba_filename.c_str()); + return false; + } - printf("File info:\n"); - printf(" Version: %X\n", fileinfo.m_version); - printf(" Total header size: %u\n", fileinfo.m_total_header_size); - printf(" Total selectors: %u\n", fileinfo.m_total_selectors); - printf(" Selector codebook size: %u\n", fileinfo.m_selector_codebook_size); - printf(" Total endpoints: %u\n", fileinfo.m_total_endpoints); - printf(" Endpoint codebook size: %u\n", fileinfo.m_endpoint_codebook_size); - printf(" Tables size: %u\n", fileinfo.m_tables_size); - printf(" Slices size: %u\n", fileinfo.m_slices_size); - fmt_printf(" Block Dimensions: {}x{}\n", fileinfo.m_block_width, fileinfo.m_block_height); + std::string rgb_filename(base_filename + fmt_string("_unpacked_rgb_{}_level_{}_face_{}_layer{04}.png", basist::basis_get_format_name(transcoder_tex_fmt), level_index, face_index, layer_index)); + if (!save_png(rgb_filename, img, cImageSaveIgnoreAlpha)) + { + error_printf("Failed writing to .PNG file \"%s\"\n", rgb_filename.c_str()); + return false; + } + printf("Wrote .PNG file \"%s\"\n", rgb_filename.c_str()); - bool is_hdr = false; + std::string a_filename(base_filename + fmt_string("_unpacked_a_{}_{}_{}_{04}.png", basist::basis_get_format_name(transcoder_tex_fmt), level_index, face_index, layer_index)); + if (!save_png(a_filename, img, cImageSaveGrayscale, 3)) + { + error_printf("Failed writing to .PNG file \"%s\"\n", a_filename.c_str()); + return false; + } + printf("Wrote .PNG file \"%s\"\n", a_filename.c_str()); + } - const char* pFmt_str = nullptr; - switch (fileinfo.m_tex_format) - { - case basist::basis_tex_format::cETC1S: - { - pFmt_str = "ETC1S"; - break; - } - case basist::basis_tex_format::cUASTC4x4: - { - pFmt_str = "UASTC_LDR_4x4"; - break; - } - case basist::basis_tex_format::cUASTC_HDR_4x4: - { - is_hdr = true; - pFmt_str = "UASTC_HDR_4x4"; - break; - } - case basist::basis_tex_format::cASTC_HDR_6x6: - { - is_hdr = true; - pFmt_str = "ASTC_HDR_6x6"; - break; - } - case basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE: - { - is_hdr = true; - pFmt_str = "ASTC_HDR_6x6_INTERMEDIATE"; - break; - } - default: - { - assert(0); - return false; - } - } + } // face_index + } // layer_index + } // level_index - fmt_printf(" Texture format: {}\n", pFmt_str); + // RGB565 + for (uint32_t level_index = 0; level_index < dec.get_levels(); level_index++) + { + for (uint32_t layer_index = 0; layer_index < total_layers; layer_index++) + { + for (uint32_t face_index = 0; face_index < dec.get_faces(); face_index++) + { + const basist::transcoder_texture_format transcoder_tex_fmt = basist::transcoder_texture_format::cTFRGB565; - printf(" Texture type: %s\n", basist::basis_get_texture_type_name(fileinfo.m_tex_type)); - printf(" us per frame: %u (%f fps)\n", fileinfo.m_us_per_frame, fileinfo.m_us_per_frame ? (1.0f / ((float)fileinfo.m_us_per_frame / 1000000.0f)) : 0.0f); - printf(" Total slices: %u\n", (uint32_t)fileinfo.m_slice_info.size()); - printf(" Total images: %i\n", fileinfo.m_total_images); - printf(" Y Flipped: %u, Has alpha slices: %u\n", fileinfo.m_y_flipped, fileinfo.m_has_alpha_slices); - printf(" userdata0: 0x%X userdata1: 0x%X\n", fileinfo.m_userdata0, fileinfo.m_userdata1); - printf(" Per-image mipmap levels: "); - for (uint32_t i = 0; i < fileinfo.m_total_images; i++) - printf("%u ", fileinfo.m_image_mipmap_levels[i]); - printf("\n"); + basist::ktx2_image_level_info level_info; - uint32_t total_texels = 0; + if (!dec.get_image_level_info(level_info, level_index, layer_index, face_index)) + { + fmt_error_printf("Failed retrieving image level information ({} {} {})!\n", layer_index, level_index, face_index); + return false; + } - printf("\nImage info:\n"); - for (uint32_t i = 0; i < fileinfo.m_total_images; i++) - { - basist::basisu_image_info ii; - if (!dec.get_image_info(&basis_file_data[0], (uint32_t)basis_file_data.size(), ii, i)) - { - error_printf("get_image_info() failed!\n"); + const uint32_t total_pixels = level_info.m_orig_width * level_info.m_orig_height; + + basisu::vector packed_img(total_pixels); + + fill_buffer_with_random_bytes(packed_img.get_ptr(), packed_img.size_in_bytes()); + + interval_timer tm; + tm.start(); + + if (!dec.transcode_image_level(level_index, layer_index, face_index, packed_img.get_ptr(), total_pixels, transcoder_tex_fmt, transcode_flags)) + { + fmt_error_printf("Failed transcoding image level ({} {} {})!\n", layer_index, level_index, face_index); + return false; + } + + double total_transcode_time = tm.get_elapsed_ms(); + + image img(level_info.m_orig_width, level_info.m_orig_height); + + for (uint32_t y = 0; y < level_info.m_orig_height; y++) + { + for (uint32_t x = 0; x < level_info.m_orig_width; x++) + { + const uint16_t p = packed_img[x + y * level_info.m_orig_width]; + uint32_t r = p >> 11, g = (p >> 5) & 63, b = p & 31; + r = (r << 3) | (r >> 2); + g = (g << 2) | (g >> 4); + b = (b << 3) | (b >> 2); + img(x, y).set(r, g, b, 255); + } + } + + fmt_printf("Transcode of level {} layer {} face {} res {}x{} format {} succeeded in {} ms\n", + level_index, layer_index, face_index, + level_info.m_orig_width, level_info.m_orig_height, basist::basis_get_format_name(transcoder_tex_fmt), total_transcode_time); + + if ((!validate_flag) && (!opts.m_ktx_only)) + { + std::string rgb_filename(base_filename + fmt_string("_unpacked_rgb_{}_level_{}_face_{}_layer{04}.png", basist::basis_get_format_name(transcoder_tex_fmt), level_index, face_index, layer_index)); + if (!save_png(rgb_filename, img, cImageSaveIgnoreAlpha)) + { + error_printf("Failed writing to .PNG file \"%s\"\n", rgb_filename.c_str()); + return false; + } + printf("Wrote .PNG file \"%s\"\n", rgb_filename.c_str()); + + } + + } // face_index + } // layer_index + } // level_index + + // RGBA4444 + for (uint32_t level_index = 0; level_index < dec.get_levels(); level_index++) + { + for (uint32_t layer_index = 0; layer_index < total_layers; layer_index++) + { + for (uint32_t face_index = 0; face_index < dec.get_faces(); face_index++) + { + const basist::transcoder_texture_format transcoder_tex_fmt = basist::transcoder_texture_format::cTFRGBA4444; + + basist::ktx2_image_level_info level_info; + + if (!dec.get_image_level_info(level_info, level_index, layer_index, face_index)) + { + fmt_error_printf("Failed retrieving image level information ({} {} {})!\n", layer_index, level_index, face_index); + return false; + } + + const uint32_t total_pixels = level_info.m_orig_width * level_info.m_orig_height; + + basisu::vector packed_img(total_pixels); + + fill_buffer_with_random_bytes(packed_img.get_ptr(), packed_img.size_in_bytes()); + + interval_timer tm; + tm.start(); + + if (!dec.transcode_image_level(level_index, layer_index, face_index, packed_img.get_ptr(), total_pixels, transcoder_tex_fmt, transcode_flags)) + { + fmt_error_printf("Failed transcoding image level ({} {} {})!\n", layer_index, level_index, face_index); + return false; + } + + double total_transcode_time = tm.get_elapsed_ms(); + + image img(level_info.m_orig_width, level_info.m_orig_height); + + for (uint32_t y = 0; y < level_info.m_orig_height; y++) + { + for (uint32_t x = 0; x < level_info.m_orig_width; x++) + { + const uint16_t p = packed_img[x + y * level_info.m_orig_width]; + uint32_t r = p >> 12, g = (p >> 8) & 15, b = (p >> 4) & 15, a = p & 15; + r = (r << 4) | r; + g = (g << 4) | g; + b = (b << 4) | b; + a = (a << 4) | a; + img(x, y).set(r, g, b, a); + } + } + + fmt_printf("Transcode of level {} layer {} face {} res {}x{} format {} succeeded in {} ms\n", + level_index, layer_index, face_index, + level_info.m_orig_width, level_info.m_orig_height, basist::basis_get_format_name(transcoder_tex_fmt), total_transcode_time); + + if ((!validate_flag) && (!opts.m_ktx_only)) + { + std::string rgba_filename(base_filename + fmt_string("_unpacked_rgba_{}_level_{}_face_{}_layer{04}.png", basist::basis_get_format_name(transcoder_tex_fmt), level_index, face_index, layer_index)); + if (!save_png(rgba_filename, img)) + { + error_printf("Failed writing to .PNG file \"%s\"\n", rgba_filename.c_str()); + return false; + } + + std::string rgb_filename(base_filename + fmt_string("_unpacked_rgb_{}_level_{}_face_{}_layer{04}.png", basist::basis_get_format_name(transcoder_tex_fmt), level_index, face_index, layer_index)); + if (!save_png(rgb_filename, img, cImageSaveIgnoreAlpha)) + { + error_printf("Failed writing to .PNG file \"%s\"\n", rgb_filename.c_str()); + return false; + } + printf("Wrote .PNG file \"%s\"\n", rgb_filename.c_str()); + + std::string a_filename(base_filename + fmt_string("_unpacked_a_{}_{}_{}_{04}.png", basist::basis_get_format_name(transcoder_tex_fmt), level_index, face_index, layer_index)); + if (!save_png(a_filename, img, cImageSaveGrayscale, 3)) + { + error_printf("Failed writing to .PNG file \"%s\"\n", a_filename.c_str()); + return false; + } + printf("Wrote .PNG file \"%s\"\n", a_filename.c_str()); + } + + } // face_index + } // layer_index + } // level_index + } + } + + return true; +} + +static bool unpack_and_validate_basis_file( + uint32_t file_index, + const std::string &base_filename, + uint8_vec &basis_file_data, + command_line_params& opts, + FILE *pCSV_file, + basis_data* pGlobal_codebook_data, + uint32_t &total_unpack_warnings, + uint32_t &total_pvrtc_nonpow2_warnings) +{ + const bool validate_flag = (opts.m_mode == cValidate); + + basist::basisu_transcoder dec; + + if (pGlobal_codebook_data) + { + dec.set_global_codebooks(&pGlobal_codebook_data->m_transcoder.get_lowlevel_etc1s_decoder()); + } + + if (!opts.m_fuzz_testing) + { + // Skip the full validation, which CRC16's the entire file. + + // Validate the file - note this isn't necessary for transcoding + if (!dec.validate_file_checksums(&basis_file_data[0], (uint32_t)basis_file_data.size(), true)) + { + error_printf("File version is unsupported, or file failed one or more CRC checks!\n"); + + return false; + } + } + + printf("File version and CRC checks succeeded\n"); + + basist::basisu_file_info fileinfo; + if (!dec.get_file_info(&basis_file_data[0], (uint32_t)basis_file_data.size(), fileinfo)) + { + error_printf("Failed retrieving Basis file information!\n"); + return false; + } + + assert(fileinfo.m_total_images == fileinfo.m_image_mipmap_levels.size()); + assert(fileinfo.m_total_images == dec.get_total_images(&basis_file_data[0], (uint32_t)basis_file_data.size())); + + printf("File info:\n"); + printf(" Version: %X\n", fileinfo.m_version); + printf(" Total header size: %u\n", fileinfo.m_total_header_size); + printf(" Total selectors: %u\n", fileinfo.m_total_selectors); + printf(" Selector codebook size: %u\n", fileinfo.m_selector_codebook_size); + printf(" Total endpoints: %u\n", fileinfo.m_total_endpoints); + printf(" Endpoint codebook size: %u\n", fileinfo.m_endpoint_codebook_size); + printf(" Tables size: %u\n", fileinfo.m_tables_size); + printf(" Slices size: %u\n", fileinfo.m_slices_size); + fmt_printf(" Block Dimensions: {}x{}\n", fileinfo.m_block_width, fileinfo.m_block_height); + + bool is_hdr = false; + + std::string fmt_str_temp; + + const char* pFmt_str = nullptr; + switch (fileinfo.m_tex_format) + { + case basist::basis_tex_format::cETC1S: + { + pFmt_str = "ETC1S"; + break; + } + case basist::basis_tex_format::cUASTC_LDR_4x4: + { + pFmt_str = "UASTC_LDR_4x4"; + break; + } + case basist::basis_tex_format::cUASTC_HDR_4x4: + { + is_hdr = true; + pFmt_str = "UASTC_HDR_4x4"; + break; + } + case basist::basis_tex_format::cASTC_HDR_6x6: + { + is_hdr = true; + pFmt_str = "ASTC_HDR_6x6"; + break; + } + case basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE: + { + is_hdr = true; + pFmt_str = "UASTC_HDR_6x6_INTERMEDIATE"; + break; + } + case basist::basis_tex_format::cXUASTC_LDR_4x4: + case basist::basis_tex_format::cXUASTC_LDR_5x4: + case basist::basis_tex_format::cXUASTC_LDR_5x5: + case basist::basis_tex_format::cXUASTC_LDR_6x5: + case basist::basis_tex_format::cXUASTC_LDR_6x6: + case basist::basis_tex_format::cXUASTC_LDR_8x5: + case basist::basis_tex_format::cXUASTC_LDR_8x6: + case basist::basis_tex_format::cXUASTC_LDR_10x5: + case basist::basis_tex_format::cXUASTC_LDR_10x6: + case basist::basis_tex_format::cXUASTC_LDR_8x8: + case basist::basis_tex_format::cXUASTC_LDR_10x8: + case basist::basis_tex_format::cXUASTC_LDR_10x10: + case basist::basis_tex_format::cXUASTC_LDR_12x10: + case basist::basis_tex_format::cXUASTC_LDR_12x12: + { + uint32_t block_width = 0, block_height = 0; + basist::get_basis_tex_format_block_size(fileinfo.m_tex_format, block_width, block_height); + fmt_str_temp = fmt_string("XUASTC_LDR_{}x{}", block_width, block_height); + pFmt_str = fmt_str_temp.c_str(); + break; + } + case basist::basis_tex_format::cASTC_LDR_4x4: + case basist::basis_tex_format::cASTC_LDR_5x4: + case basist::basis_tex_format::cASTC_LDR_5x5: + case basist::basis_tex_format::cASTC_LDR_6x5: + case basist::basis_tex_format::cASTC_LDR_6x6: + case basist::basis_tex_format::cASTC_LDR_8x5: + case basist::basis_tex_format::cASTC_LDR_8x6: + case basist::basis_tex_format::cASTC_LDR_10x5: + case basist::basis_tex_format::cASTC_LDR_10x6: + case basist::basis_tex_format::cASTC_LDR_8x8: + case basist::basis_tex_format::cASTC_LDR_10x8: + case basist::basis_tex_format::cASTC_LDR_10x10: + case basist::basis_tex_format::cASTC_LDR_12x10: + case basist::basis_tex_format::cASTC_LDR_12x12: + { + uint32_t block_width = 0, block_height = 0; + basist::get_basis_tex_format_block_size(fileinfo.m_tex_format, block_width, block_height); + fmt_str_temp = fmt_string("ASTC_LDR_{}x{}", block_width, block_height); + pFmt_str = fmt_str_temp.c_str(); + break; + } + default: + { + assert(0); + return false; + } + } + + fmt_printf(" Texture format: {}\n", pFmt_str); + + printf(" Texture type: %s\n", basist::basis_get_texture_type_name(fileinfo.m_tex_type)); + printf(" us per frame: %u (%f fps)\n", fileinfo.m_us_per_frame, fileinfo.m_us_per_frame ? (1.0f / ((float)fileinfo.m_us_per_frame / 1000000.0f)) : 0.0f); + printf(" Total slices: %u\n", (uint32_t)fileinfo.m_slice_info.size()); + printf(" Total images: %i\n", fileinfo.m_total_images); + printf(" Y Flipped: %u, Has alpha slices: %u, sRGB: %u\n", fileinfo.m_y_flipped, fileinfo.m_has_alpha_slices, fileinfo.m_srgb); + printf(" userdata0: 0x%X userdata1: 0x%X\n", fileinfo.m_userdata0, fileinfo.m_userdata1); + printf(" Per-image mipmap levels: "); + for (uint32_t i = 0; i < fileinfo.m_total_images; i++) + printf("%u ", fileinfo.m_image_mipmap_levels[i]); + printf("\n"); + + // the sRGB transfer function to use while astc unpacking (we want this to ideally match what we used during astc encoding) + bool srgb_transfer_func_astc_unpacking = fileinfo.m_srgb; + + // the sRGB transfer function to use when writing out files (we want to indicate to the caller if the data is sRGB or linear) + bool srgb_transfer_func_astc_writing = fileinfo.m_srgb; + + const bool is_etc1s = (fileinfo.m_tex_format == basist::basis_tex_format::cETC1S); + const bool is_uastc_ldr_4x4 = (fileinfo.m_tex_format == basist::basis_tex_format::cUASTC_LDR_4x4); + if ((is_etc1s) || (is_uastc_ldr_4x4)) + { + // The ETC1S and UASTC LDR 4x4 transcoders supply ASTC LDR 4x4 data assuming the decoder will NOT be using the sRGB read decode profile, which is likely the most common case (in geospatial rendering scenarios). + // Note XUASTC/UASTC LDR 4x4-12x12 supports both linear and sRGB decode profiles throughout the entire pipeline (encoding/transcoding/decoding to raw pixels). + srgb_transfer_func_astc_unpacking = false; + + // This matches the behavior of our original tools. It ensures astcenc uses linear by default when reading our transcoded .KTX files. + srgb_transfer_func_astc_writing = false; + + if (fileinfo.m_srgb) + printf("Note: ETC1S/UASTC LDR 4x4 will always be decoded by this tool using the ASTC linear decode profile, regardless of the KTX2/.basis DFD transfer function field.\n"); + } + + uint32_t total_texels = 0; + + printf("\nImage info:\n"); + for (uint32_t i = 0; i < fileinfo.m_total_images; i++) + { + basist::basisu_image_info ii; + if (!dec.get_image_info(&basis_file_data[0], (uint32_t)basis_file_data.size(), ii, i)) + { + error_printf("get_image_info() failed!\n"); return false; } @@ -2623,16 +3342,16 @@ static bool unpack_and_validate_basis_file( printf("start_transcoding time: %3.3f ms\n", start_transcoding_time_ms); basisu::vector< gpu_image_vec > gpu_images[(int)basist::transcoder_texture_format::cTFTotalTextureFormats]; - + double total_format_transcoding_time_ms[(int)basist::transcoder_texture_format::cTFTotalTextureFormats]; clear_obj(total_format_transcoding_time_ms); int first_format = 0; int last_format = (int)basist::transcoder_texture_format::cTFTotalTextureFormats; - if (opts.m_format_only > -1) + if (opts.m_unpack_format_only > -1) { - first_format = opts.m_format_only; + first_format = opts.m_unpack_format_only; last_format = first_format + 1; } @@ -2675,6 +3394,8 @@ static bool unpack_and_validate_basis_file( gpu_images[(int)tex_fmt][image_index].resize(fileinfo.m_image_mipmap_levels[image_index]); } + uint32_t transcode_flags = get_transcode_flags_from_options(opts); + // Now transcode the file to all supported texture formats and save mipmapped KTX files for (int format_iter = first_format; format_iter < last_format; format_iter++) { @@ -2701,30 +3422,30 @@ static bool unpack_and_validate_basis_file( if ((transcoder_tex_fmt == basist::transcoder_texture_format::cTFPVRTC1_4_RGB) || (transcoder_tex_fmt == basist::transcoder_texture_format::cTFPVRTC1_4_RGBA)) { - if (!is_pow2(level_info.m_width) || !is_pow2(level_info.m_height)) + if (!is_pow2(level_info.m_orig_width) || !is_pow2(level_info.m_orig_height)) { total_pvrtc_nonpow2_warnings++; printf("Warning: Will not transcode image %u level %u res %ux%u to PVRTC1 (one or more dimension is not a power of 2)\n", image_index, level_index, level_info.m_width, level_info.m_height); - // Can't transcode this image level to PVRTC because it's not a pow2 (we're going to support transcoding non-pow2 to the next larger pow2 soon) + // Can't transcode this image level to PVRTC because it's not a pow2 (we're going to support transcoding non-pow2 to the next "larger" pow2 soon) continue; } } basisu::texture_format tex_fmt = basis_get_basisu_texture_format(transcoder_tex_fmt); + fmt_printf("Transcoding format: {}\n", (uint32_t)tex_fmt); + gpu_image& gi = gpu_images[(int)transcoder_tex_fmt][image_index][level_index]; gi.init(tex_fmt, level_info.m_orig_width, level_info.m_orig_height); // Fill the buffer with psuedo-random bytes, to help more visibly detect cases where the transcoder fails to write to part of the output. fill_buffer_with_random_bytes(gi.get_ptr(), gi.get_size_in_bytes()); - - uint32_t decode_flags = basist::cDecodeFlagsHighQuality; - + tm.start(); - if (!dec.transcode_image_level(&basis_file_data[0], (uint32_t)basis_file_data.size(), image_index, level_index, gi.get_ptr(), gi.get_total_blocks(), transcoder_tex_fmt, decode_flags)) + if (!dec.transcode_image_level(&basis_file_data[0], (uint32_t)basis_file_data.size(), image_index, level_index, gi.get_ptr(), gi.get_total_blocks(), transcoder_tex_fmt, transcode_flags)) { error_printf("Failed transcoding image level (%u %u %u)!\n", image_index, level_index, format_iter); return false; @@ -2742,9 +3463,9 @@ static bool unpack_and_validate_basis_file( } // image_info - // Upack UASTC files seperately, to validate we can transcode slices to UASTC and unpack them to pixels. - // This is a special path because UASTC is not yet a valid transcoder_texture_format, but a lower-level block_format. - if (fileinfo.m_tex_format == basist::basis_tex_format::cUASTC4x4) + // Upack UASTC LDR 4x4 files seperately, to validate we can transcode slices to UASTC LDR 4x4 and unpack them to pixels. + // This is a special path because UASTC LDR 4x4 is not yet a valid transcoder_texture_format, but a lower-level block_format. + if (fileinfo.m_tex_format == basist::basis_tex_format::cUASTC_LDR_4x4) { for (uint32_t image_index = 0; image_index < fileinfo.m_total_images; image_index++) { @@ -2763,12 +3484,12 @@ static bool unpack_and_validate_basis_file( // Fill the buffer with psuedo-random bytes, to help more visibly detect cases where the transcoder fails to write to part of the output. fill_buffer_with_random_bytes(gi.get_ptr(), gi.get_size_in_bytes()); - + tm.start(); if (!dec.transcode_slice( - &basis_file_data[0], (uint32_t)basis_file_data.size(), - level_info.m_first_slice_index, gi.get_ptr(), gi.get_total_blocks(), basist::block_format::cUASTC_4x4, gi.get_bytes_per_block())) + &basis_file_data[0], (uint32_t)basis_file_data.size(), + level_info.m_first_slice_index, gi.get_ptr(), gi.get_total_blocks(), basist::block_format::cUASTC_4x4, gi.get_bytes_per_block(), transcode_flags)) { error_printf("Failed transcoding image level (%u %u) to UASTC!\n", image_index, level_index); return false; @@ -2781,9 +3502,9 @@ static bool unpack_and_validate_basis_file( if ((!validate_flag) && (!opts.m_ktx_only)) { image u; - if (!gi.unpack(u)) + if (!gi.unpack(u, srgb_transfer_func_astc_unpacking)) { - error_printf("Warning: Failed unpacking GPU texture data (%u %u) to UASTC. \n", image_index, level_index); + error_printf("Warning: Failed unpacking GPU texture data (%u %u). \n", image_index, level_index); return false; } //u.crop(level_info.m_orig_width, level_info.m_orig_height); @@ -2835,6 +3556,8 @@ static bool unpack_and_validate_basis_file( if (transcoder_tex_fmt == basist::transcoder_texture_format::cTFBC7_ALT) continue; + const bool is_fmt_astc = basis_is_transcoder_texture_format_astc(transcoder_tex_fmt); + if ((!opts.m_no_ktx) && (fileinfo.m_tex_type == basist::cBASISTexTypeCubemapArray)) { // No KTX tool that we know of supports cubemap arrays, so write individual cubemap files. @@ -2844,9 +3567,10 @@ static bool unpack_and_validate_basis_file( for (uint32_t i = 0; i < 6; i++) cubemap.push_back(gpu_images[format_iter][image_index + i]); + // KTX1 { std::string ktx_filename(base_filename + string_format("_transcoded_cubemap_%s_%u.ktx", basist::basis_get_format_name(transcoder_tex_fmt), image_index / 6)); - if (!write_compressed_texture_file(ktx_filename.c_str(), cubemap, true, true)) + if (!write_compressed_texture_file(ktx_filename.c_str(), cubemap, true, is_fmt_astc ? srgb_transfer_func_astc_writing : fileinfo.m_srgb)) { error_printf("Failed writing KTX file \"%s\"!\n", ktx_filename.c_str()); return false; @@ -2854,10 +3578,11 @@ static bool unpack_and_validate_basis_file( printf("Wrote .KTX file \"%s\"\n", ktx_filename.c_str()); } + // DDS if (does_dds_support_format(cubemap[0][0].get_format())) { std::string dds_filename(base_filename + string_format("_transcoded_cubemap_%s_%u.dds", basist::basis_get_format_name(transcoder_tex_fmt), image_index / 6)); - if (!write_compressed_texture_file(dds_filename.c_str(), cubemap, true, true)) + if (!write_compressed_texture_file(dds_filename.c_str(), cubemap, true, fileinfo.m_srgb)) { error_printf("Failed writing DDS file \"%s\"!\n", dds_filename.c_str()); return false; @@ -2884,9 +3609,10 @@ static bool unpack_and_validate_basis_file( if ((!opts.m_no_ktx) && (fileinfo.m_tex_type != basist::cBASISTexTypeCubemapArray)) { + // KTX1 { std::string ktx_filename(base_filename + string_format("_transcoded_%s_%04u.ktx", basist::basis_get_format_name(transcoder_tex_fmt), image_index)); - if (!write_compressed_texture_file(ktx_filename.c_str(), gi, true)) + if (!write_compressed_texture_file(ktx_filename.c_str(), gi, is_fmt_astc ? srgb_transfer_func_astc_writing : fileinfo.m_srgb)) { error_printf("Failed writing KTX file \"%s\"!\n", ktx_filename.c_str()); return false; @@ -2894,10 +3620,11 @@ static bool unpack_and_validate_basis_file( printf("Wrote .KTX file \"%s\"\n", ktx_filename.c_str()); } + // DDS if (does_dds_support_format(gi[0].get_format())) { std::string dds_filename(base_filename + string_format("_transcoded_%s_%04u.dds", basist::basis_get_format_name(transcoder_tex_fmt), image_index)); - if (!write_compressed_texture_file(dds_filename.c_str(), gi, true)) + if (!write_compressed_texture_file(dds_filename.c_str(), gi, fileinfo.m_srgb)) { error_printf("Failed writing DDS file \"%s\"!\n", dds_filename.c_str()); return false; @@ -2946,7 +3673,7 @@ static bool unpack_and_validate_basis_file( else { image u; - if (!gi[level_index].unpack(u)) + if (!gi[level_index].unpack(u, srgb_transfer_func_astc_unpacking)) { printf("Warning: Failed unpacking GPU texture data (%u %u %u). Unpacking as much as possible.\n", format_iter, image_index, level_index); total_unpack_warnings++; @@ -3012,7 +3739,7 @@ static bool unpack_and_validate_basis_file( } printf("Wrote .PNG file \"%s\"\n", rgba_filename.c_str()); } - + } // is_hdr } // level_index @@ -3026,7 +3753,7 @@ static bool unpack_and_validate_basis_file( uint32_t max_mipmap_levels = 0; //if (!opts.m_etc1_only) - if ((opts.m_format_only == -1) && (!validate_flag)) + if ((opts.m_unpack_format_only == -1) && (!validate_flag)) { if (is_hdr) { @@ -3052,8 +3779,8 @@ static bool unpack_and_validate_basis_file( tm.start(); - if (!dec.transcode_image_level(&basis_file_data[0], (uint32_t)basis_file_data.size(), image_index, level_index, - half_img.get_ptr(), total_pixels, transcoder_tex_fmt, 0, level_info.m_orig_width, nullptr, level_info.m_orig_height)) + if (!dec.transcode_image_level(&basis_file_data[0], (uint32_t)basis_file_data.size(), image_index, level_index, + half_img.get_ptr(), total_pixels, transcoder_tex_fmt, transcode_flags, level_info.m_orig_width, nullptr, level_info.m_orig_height)) { error_printf("Failed transcoding image level (%u %u %u)!\n", image_index, level_index, transcoder_tex_fmt); return false; @@ -3110,7 +3837,7 @@ static bool unpack_and_validate_basis_file( tm.start(); if (!dec.transcode_image_level(&basis_file_data[0], (uint32_t)basis_file_data.size(), image_index, level_index, - half_img.get_ptr(), total_pixels, transcoder_tex_fmt, 0, level_info.m_orig_width, nullptr, level_info.m_orig_height)) + half_img.get_ptr(), total_pixels, transcoder_tex_fmt, transcode_flags, level_info.m_orig_width, nullptr, level_info.m_orig_height)) { error_printf("Failed transcoding image level (%u %u %u)!\n", image_index, level_index, transcoder_tex_fmt); return false; @@ -3167,7 +3894,7 @@ static bool unpack_and_validate_basis_file( tm.start(); if (!dec.transcode_image_level(&basis_file_data[0], (uint32_t)basis_file_data.size(), image_index, level_index, - rgb9e5_img.get_ptr(), total_pixels, transcoder_tex_fmt, 0, level_info.m_orig_width, nullptr, level_info.m_orig_height)) + rgb9e5_img.get_ptr(), total_pixels, transcoder_tex_fmt, transcode_flags, level_info.m_orig_width, nullptr, level_info.m_orig_height)) { error_printf("Failed transcoding image level (%u %u %u)!\n", image_index, level_index, transcoder_tex_fmt); return false; @@ -3199,8 +3926,6 @@ static bool unpack_and_validate_basis_file( } // level_index } // image_index - - } else { @@ -3225,7 +3950,7 @@ static bool unpack_and_validate_basis_file( tm.start(); - if (!dec.transcode_image_level(&basis_file_data[0], (uint32_t)basis_file_data.size(), image_index, level_index, &img(0, 0).r, img.get_total_pixels(), transcoder_tex_fmt, 0, img.get_pitch(), nullptr, img.get_height())) + if (!dec.transcode_image_level(&basis_file_data[0], (uint32_t)basis_file_data.size(), image_index, level_index, &img(0, 0).r, img.get_total_pixels(), transcoder_tex_fmt, transcode_flags, img.get_pitch(), nullptr, img.get_height())) { error_printf("Failed transcoding image level (%u %u %u)!\n", image_index, level_index, transcoder_tex_fmt); return false; @@ -3280,7 +4005,7 @@ static bool unpack_and_validate_basis_file( tm.start(); - if (!dec.transcode_image_level(&basis_file_data[0], (uint32_t)basis_file_data.size(), image_index, level_index, &packed_img[0], (uint32_t)packed_img.size(), transcoder_tex_fmt, 0, level_info.m_orig_width, nullptr, level_info.m_orig_height)) + if (!dec.transcode_image_level(&basis_file_data[0], (uint32_t)basis_file_data.size(), image_index, level_index, &packed_img[0], (uint32_t)packed_img.size(), transcoder_tex_fmt, transcode_flags, level_info.m_orig_width, nullptr, level_info.m_orig_height)) { error_printf("Failed transcoding image level (%u %u %u)!\n", image_index, level_index, transcoder_tex_fmt); return false; @@ -3343,7 +4068,7 @@ static bool unpack_and_validate_basis_file( tm.start(); - if (!dec.transcode_image_level(&basis_file_data[0], (uint32_t)basis_file_data.size(), image_index, level_index, &packed_img[0], (uint32_t)packed_img.size(), transcoder_tex_fmt, 0, level_info.m_orig_width, nullptr, level_info.m_orig_height)) + if (!dec.transcode_image_level(&basis_file_data[0], (uint32_t)basis_file_data.size(), image_index, level_index, &packed_img[0], (uint32_t)packed_img.size(), transcoder_tex_fmt, transcode_flags, level_info.m_orig_width, nullptr, level_info.m_orig_height)) { error_printf("Failed transcoding image level (%u %u %u)!\n", image_index, level_index, transcoder_tex_fmt); return false; @@ -3391,10 +4116,10 @@ static bool unpack_and_validate_basis_file( } // level_index } // image_index - + } // is_hdr - } // if ((opts.m_format_only == -1) && (!validate_flag)) + } // if ((opts.m_unpack_format_only == -1) && (!validate_flag)) if (pCSV_file) { @@ -3432,7 +4157,7 @@ static bool unpack_and_validate_mode(command_line_params &opts) tm.start(); //const bool validate_flag = (opts.m_mode == cValidate); - + basis_data* pGlobal_codebook_data = nullptr; if (opts.m_etc1s_use_global_codebooks_file.size()) { @@ -3499,14 +4224,14 @@ static bool unpack_and_validate_mode(command_line_params &opts) delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr; return false; } - + bool is_ktx2 = false; if (file_data.size() >= sizeof(basist::g_ktx2_file_identifier)) { is_ktx2 = (memcmp(file_data.data(), basist::g_ktx2_file_identifier, sizeof(basist::g_ktx2_file_identifier)) == 0); } - printf("Input file \"%s\", KTX2: %u\n", pInput_filename, is_ktx2); + printf("\nInput file \"%s\", KTX2: %u\n", pInput_filename, is_ktx2); bool status; if (is_ktx2) @@ -3536,10 +4261,10 @@ static bool unpack_and_validate_mode(command_line_params &opts) if (!status) { - if (pCSV_file) + if (pCSV_file) fclose(pCSV_file); - delete pGlobal_codebook_data; + delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr; return false; @@ -3562,7 +4287,7 @@ static bool unpack_and_validate_mode(command_line_params &opts) fclose(pCSV_file); pCSV_file = nullptr; } - delete pGlobal_codebook_data; + delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr; return true; @@ -3608,7 +4333,7 @@ static bool hdr_compare_mode(command_line_params& opts) printf("Comparison image res: %ux%u\n", a.get_width(), a.get_height()); image_metrics im; - + im.calc_half(a, b, 0, 1, true); im.print("R "); @@ -3700,7 +4425,7 @@ static bool compare_mode(command_line_params &opts) im.calc(a, b, 0, 0, true, true); im.print("Y 601 " ); - + if (opts.m_compare_ssim) { vec4F s_rgb(compute_ssim(a, b, false, false)); @@ -3743,7 +4468,7 @@ static bool compare_mode(command_line_params &opts) save_png("delta_img_rgb.png", delta_img, cImageSaveIgnoreAlpha); printf("Wrote delta_img_rgb.png\n"); - + save_png("delta_img_a.png", delta_img, cImageSaveGrayscale, 3); printf("Wrote delta_img_a.png\n"); @@ -3886,7 +4611,7 @@ static bool compare_mode(command_line_params &opts) while ((int)strlen(tics2) < x) strcat(tics2, " "); - sprintf(buf, "0"); + snprintf(buf, sizeof(buf), "0"); strcat(tics, buf); } else if (((x & 7) == 0) || (x == X_SIZE)) @@ -3898,18 +4623,18 @@ static bool compare_mode(command_line_params &opts) strcat(tics2, " "); int v = (x - (int)X_SIZE / 2); - sprintf(buf, "%i", v / 10); + snprintf(buf, sizeof(buf), "%i", v / 10); strcat(tics, buf); if (v < 0) { if (-v < 10) - sprintf(buf, "%i", v % 10); + snprintf(buf, sizeof(buf), "%i", v % 10); else - sprintf(buf, " %i", -v % 10); + snprintf(buf, sizeof(buf), " %i", -v % 10); } else - sprintf(buf, "%i", v % 10); + snprintf(buf, sizeof(buf), "%i", v % 10); strcat(tics2, buf); } else @@ -3925,7 +4650,7 @@ static bool compare_mode(command_line_params &opts) } } // display_plot - + return true; } @@ -3964,7 +4689,7 @@ static bool split_image_mode(command_line_params& opts) } printf("Wrote file %s\n", buf); } - + return true; } @@ -4009,7 +4734,7 @@ static bool combine_images_mode(command_line_params& opts) const char* pOutput_filename = "combined.png"; if (opts.m_output_filename.size()) pOutput_filename = opts.m_output_filename.c_str(); - + if (!save_png(pOutput_filename, combined_img)) { fprintf(stderr, "Failed writing file %s\n", pOutput_filename); @@ -4051,7 +4776,7 @@ static bool tonemap_image_mode(command_line_params& opts) string_combine_path(output_filename, opts.m_output_path.c_str(), output_filename.c_str()); const char* pBasename = output_filename.c_str(); - + image srgb_img(width, height); image lin_img(width, height); @@ -4064,7 +4789,7 @@ static bool tonemap_image_mode(command_line_params& opts) p[0] = clamp(p[0], 0.0f, 1.0f); p[1] = clamp(p[1], 0.0f, 1.0f); p[2] = clamp(p[2], 0.0f, 1.0f); - + { int rc = (int)std::round(linear_to_srgb(p[0]) * 255.0f); int gc = (int)std::round(linear_to_srgb(p[1]) * 255.0f); @@ -4116,7 +4841,7 @@ static bool tonemap_image_mode(command_line_params& opts) for (int e = -6; e <= 6; e++) { const float scale = powf(2.0f, (float)e); - + tonemap_image_reinhard(tonemapped_img, hdr_img, scale, opts.m_tonemap_dither_flag); std::string filename(string_format("%s_reinhard_tonemapped_scale_%f.png", pBasename, scale)); @@ -4177,16 +4902,16 @@ static bool compsize_mode(command_line_params& opts) return true; } -const struct test_file +const struct etc1s_uastc_4x4_ldr_test_file { const char* m_pFilename; uint32_t m_etc1s_size; float m_etc1s_psnr; float m_uastc_psnr; - + uint32_t m_etc1s_128_size; float m_etc1s_128_psnr; -} g_test_files[] = +} g_etc1s_uastc_4x4_ldr_test_files[] = { { "black_1x1.png", 189, 100.0f, 100.0f, 189, 100.0f }, { "kodim01.png", 30993, 27.40f, 44.14f, 58354, 30.356064f }, @@ -4217,7 +4942,6 @@ const struct test_file { "wikipedia.png", 38961, 24.10f, 30.47f, 69558, 27.630802f }, { "alpha0.png", 766, 100.0f, 56.16f, 747, 100.000000f } }; -const uint32_t TOTAL_TEST_FILES = sizeof(g_test_files) / sizeof(g_test_files[0]); static bool test_mode_ldr(command_line_params& opts) { @@ -4236,14 +4960,16 @@ static bool test_mode_ldr(command_line_params& opts) #endif const float ETC1S_FILESIZE_THRESHOLD = .045f; - for (uint32_t i = 0; i < TOTAL_TEST_FILES; i++) + for (uint32_t i = 0; i < std::size(g_etc1s_uastc_4x4_ldr_test_files); i++) { + const auto& test_file = g_etc1s_uastc_4x4_ldr_test_files[i]; + std::string filename(opts.m_test_file_dir); if (filename.size()) { filename.push_back('/'); } - filename += std::string(g_test_files[i].m_pFilename); + filename += std::string(test_file.m_pFilename); basisu::vector source_images(1); @@ -4254,7 +4980,7 @@ static bool test_mode_ldr(command_line_params& opts) return false; } - printf("Loaded file \"%s\", dimemsions %ux%u has alpha: %u\n", filename.c_str(), source_image.get_width(), source_image.get_height(), source_image.has_alpha()); + printf("Loaded file \"%s\", dimensions %ux%u has alpha: %u\n", filename.c_str(), source_image.get_width(), source_image.get_height(), source_image.has_alpha()); image_stats stats; @@ -4264,7 +4990,7 @@ static bool test_mode_ldr(command_line_params& opts) // Test ETC1S flags_and_quality = (opts.m_comp_params.m_multithreading ? cFlagThreaded : 0) | cFlagPrintStats | cFlagPrintStatus; - + { printf("**** Testing ETC1S non-OpenCL level 1\n"); @@ -4279,16 +5005,16 @@ static bool test_mode_ldr(command_line_params& opts) printf("ETC1S level 1 Size: %u, PSNR: %f\n", (uint32_t)data_size, stats.m_basis_rgba_avg_psnr); - float file_size_ratio = fabs((data_size / (float)g_test_files[i].m_etc1s_size) - 1.0f); + float file_size_ratio = fabs((data_size / (float)test_file.m_etc1s_size) - 1.0f); if (file_size_ratio > ETC1S_FILESIZE_THRESHOLD) { - error_printf("Expected ETC1S file size was %u, but got %u instead!\n", g_test_files[i].m_etc1s_size, (uint32_t)data_size); + error_printf("Expected ETC1S file size was %u, but got %u instead!\n", test_file.m_etc1s_size, (uint32_t)data_size); total_mismatches++; } - if (fabs(stats.m_basis_rgba_avg_psnr - g_test_files[i].m_etc1s_psnr) > ETC1S_PSNR_THRESHOLD) + if (fabs(stats.m_basis_rgba_avg_psnr - test_file.m_etc1s_psnr) > ETC1S_PSNR_THRESHOLD) { - error_printf("Expected ETC1S RGBA Avg PSNR was %f, but got %f instead!\n", g_test_files[i].m_etc1s_psnr, stats.m_basis_rgba_avg_psnr); + error_printf("Expected ETC1S RGBA Avg PSNR was %f, but got %f instead!\n", test_file.m_etc1s_psnr, stats.m_basis_rgba_avg_psnr); total_mismatches++; } } @@ -4309,16 +5035,16 @@ static bool test_mode_ldr(command_line_params& opts) printf("ETC1S level 128 Size: %u, PSNR: %f\n", (uint32_t)data_size, stats.m_basis_rgba_avg_psnr); - float file_size_ratio = fabs((data_size / (float)g_test_files[i].m_etc1s_128_size) - 1.0f); + float file_size_ratio = fabs((data_size / (float)test_file.m_etc1s_128_size) - 1.0f); if (file_size_ratio > ETC1S_FILESIZE_THRESHOLD) { - error_printf("Expected ETC1S file size was %u, but got %u instead!\n", g_test_files[i].m_etc1s_128_size, (uint32_t)data_size); + error_printf("Expected ETC1S file size was %u, but got %u instead!\n", test_file.m_etc1s_128_size, (uint32_t)data_size); total_mismatches++; } - if (fabs(stats.m_basis_rgba_avg_psnr - g_test_files[i].m_etc1s_128_psnr) > ETC1S_PSNR_THRESHOLD) + if (fabs(stats.m_basis_rgba_avg_psnr - test_file.m_etc1s_128_psnr) > ETC1S_PSNR_THRESHOLD) { - error_printf("Expected ETC1S RGBA Avg PSNR was %f, but got %f instead!\n", g_test_files[i].m_etc1s_128_psnr, stats.m_basis_rgba_avg_psnr); + error_printf("Expected ETC1S RGBA Avg PSNR was %f, but got %f instead!\n", test_file.m_etc1s_128_psnr, stats.m_basis_rgba_avg_psnr); total_mismatches++; } } @@ -4340,36 +5066,36 @@ static bool test_mode_ldr(command_line_params& opts) printf("ETC1S+OpenCL Size: %u, PSNR: %f\n", (uint32_t)data_size, stats.m_basis_rgba_avg_psnr); - float file_size_ratio = fabs((data_size / (float)g_test_files[i].m_etc1s_size) - 1.0f); + float file_size_ratio = fabs((data_size / (float)test_file.m_etc1s_size) - 1.0f); if (file_size_ratio > .04f) { - error_printf("Expected ETC1S+OpenCL file size was %u, but got %u instead!\n", g_test_files[i].m_etc1s_size, (uint32_t)data_size); + error_printf("Expected ETC1S+OpenCL file size was %u, but got %u instead!\n", test_file.m_etc1s_size, (uint32_t)data_size); total_mismatches++; } - if (g_test_files[i].m_etc1s_psnr == 100.0f) + if (test_file.m_etc1s_psnr == 100.0f) { // TODO if (stats.m_basis_rgba_avg_psnr < 69.0f) { - error_printf("Expected ETC1S+OpenCL RGBA Avg PSNR was %f, but got %f instead!\n", g_test_files[i].m_etc1s_psnr, stats.m_basis_rgba_avg_psnr); + error_printf("Expected ETC1S+OpenCL RGBA Avg PSNR was %f, but got %f instead!\n", test_file.m_etc1s_psnr, stats.m_basis_rgba_avg_psnr); total_mismatches++; } } - else if (fabs(stats.m_basis_rgba_avg_psnr - g_test_files[i].m_etc1s_psnr) > .2f) + else if (fabs(stats.m_basis_rgba_avg_psnr - test_file.m_etc1s_psnr) > .2f) { - error_printf("Expected ETC1S+OpenCL RGBA Avg PSNR was %f, but got %f instead!\n", g_test_files[i].m_etc1s_psnr, stats.m_basis_rgba_avg_psnr); + error_printf("Expected ETC1S+OpenCL RGBA Avg PSNR was %f, but got %f instead!\n", test_file.m_etc1s_psnr, stats.m_basis_rgba_avg_psnr); total_mismatches++; } } // Test UASTC { - printf("**** Testing UASTC\n"); + printf("**** Testing UASTC LDR 4x4\n"); flags_and_quality = (opts.m_comp_params.m_multithreading ? cFlagThreaded : 0) | cFlagPrintStats | cFlagPrintStatus; - void* pData = basis_compress(basist::basis_tex_format::cUASTC4x4, source_images, flags_and_quality, uastc_rdo_quality, &data_size, &stats); + void* pData = basis_compress(basist::basis_tex_format::cUASTC_LDR_4x4, source_images, flags_and_quality, uastc_rdo_quality, &data_size, &stats); if (!pData) { error_printf("basis_compress() failed!\n"); @@ -4379,9 +5105,9 @@ static bool test_mode_ldr(command_line_params& opts) printf("UASTC Size: %u, PSNR: %f\n", (uint32_t)data_size, stats.m_basis_rgba_avg_psnr); - if (fabs(stats.m_basis_rgba_avg_psnr - g_test_files[i].m_uastc_psnr) > UASTC_PSNR_THRESHOLD) + if (fabs(stats.m_basis_rgba_avg_psnr - test_file.m_uastc_psnr) > UASTC_PSNR_THRESHOLD) { - error_printf("Expected UASTC RGBA Avg PSNR was %f, but got %f instead!\n", g_test_files[i].m_etc1s_psnr, stats.m_basis_rgba_avg_psnr); + error_printf("Expected UASTC RGBA Avg PSNR was %f, but got %f instead!\n", test_file.m_etc1s_psnr, stats.m_basis_rgba_avg_psnr); total_mismatches++; } } @@ -4475,7 +5201,7 @@ static bool test_mode_hdr(command_line_params& opts, basist::basis_tex_format te fmt_printf("test_mode_hdr: Testing basis_tex_format {}, lambda {}\n", (uint32_t)tex_fmt, lambda); uint32_t total_mismatches = 0; - + #ifdef USE_TIGHTER_TEST_TOLERANCES // The PSNR's above were created with a MSVC compiled executable, x64. Hopefully this is not too low a threshold. const float PSNR_THRESHOLD = .125f; @@ -4483,7 +5209,7 @@ static bool test_mode_hdr(command_line_params& opts, basist::basis_tex_format te // Minor differences in how floating point code is optimized can result in slightly different generated files. const float PSNR_THRESHOLD = .3f; #endif - + double highest_delta = 0.0f; // TODO: This doesn't test all 6x6 levels, but that's fine for now. @@ -4508,8 +5234,8 @@ static bool test_mode_hdr(command_line_params& opts, basist::basis_tex_format te return false; } - printf("Loaded file \"%s\", dimemsions %ux%u\n", filename.c_str(), source_image.get_width(), source_image.get_height()); - + printf("Loaded file \"%s\", dimensions %ux%u\n", filename.c_str(), source_image.get_width(), source_image.get_height()); + for (uint32_t uastc_hdr_level = 0; uastc_hdr_level <= MAX_ASTC_HDR_4x4_TEST_LEVEL; uastc_hdr_level++) { image_stats stats; @@ -4523,7 +5249,7 @@ static bool test_mode_hdr(command_line_params& opts, basist::basis_tex_format te flags_and_quality |= uastc_hdr_level; void* pData = basis_compress(tex_fmt, - source_imagesf, flags_and_quality, lambda, + source_imagesf, flags_and_quality, lambda, &data_size, &stats); if (!pData) { @@ -4534,7 +5260,7 @@ static bool test_mode_hdr(command_line_params& opts, basist::basis_tex_format te double delta1, delta2; - printf("ASTC PSNR: %f (expected %f, delta %f), BC6H PSNR: %f (expected %f, delta %f)\n", + printf("ASTC PSNR: %f (expected %f, delta %f), BC6H PSNR: %f (expected %f, delta %f)\n", stats.m_basis_rgb_avg_log2_psnr, pTest_files[i].m_level_psnr_astc[uastc_hdr_level], delta1 = fabs(stats.m_basis_rgb_avg_log2_psnr - pTest_files[i].m_level_psnr_astc[uastc_hdr_level]), stats.m_basis_rgb_avg_bc6h_log2_psnr, pTest_files[i].m_level_psnr_bc6h[uastc_hdr_level], delta2 = fabs(stats.m_basis_rgb_avg_bc6h_log2_psnr - pTest_files[i].m_level_psnr_bc6h[uastc_hdr_level])); @@ -4605,14 +5331,197 @@ static bool test_mode_hdr(command_line_params& opts, basist::basis_tex_format te return result; } -static bool clbench_mode(command_line_params& opts) +const uint32_t XUASTC_LDR_TEST_FILE_NUM_RUNS = 3; + +struct xuastc_ldr_test_file { - BASISU_NOTE_UNUSED(opts); + const char* m_pFilename; - bool opencl_failed = false; - bool use_cl = basis_benchmark_etc1s_opencl(&opencl_failed); - if (use_cl) - printf("OpenCL ETC1S encoding is faster on this machine\n"); + struct test_run + { + float m_dct_q; + uint32_t m_comp_size; + float m_rgba_psnr; + }; + + test_run m_test_runs[XUASTC_LDR_TEST_FILE_NUM_RUNS]; +}; + +xuastc_ldr_test_file g_xuastc_ldr_test_files_6x6[] = +{ + { "black_1x1.png", { { 100.000000f, 111, 100.000000f }, { 75.000000f, 112, 100.000000f }, { 35.000000f, 112, 100.000000f } } }, + { "kodim01.png", { { 100.000000f, 141064, 37.188324f }, { 75.000000f, 115385, 32.893822f }, { 35.000000f, 80001, 30.057878f } } }, + { "kodim02.png", { { 100.000000f, 135146, 40.280567f }, { 75.000000f, 82435, 36.618645f }, { 35.000000f, 57365, 34.556519f } } }, + { "kodim03.png", { { 100.000000f, 133654, 42.754337f }, { 75.000000f, 72161, 38.706654f }, { 35.000000f, 51462, 36.026749f } } }, + { "kodim04.png", { { 100.000000f, 138877, 40.671108f }, { 75.000000f, 84194, 36.773575f }, { 35.000000f, 61363, 34.570110f } } }, + { "kodim05.png", { { 100.000000f, 146600, 35.842682f }, { 75.000000f, 124004, 33.176735f }, { 35.000000f, 94508, 30.148199f } } }, + { "kodim06.png", { { 100.000000f, 134928, 38.721409f }, { 75.000000f, 94356, 34.459309f }, { 35.000000f, 65904, 31.435408f } } }, + { "kodim07.png", { { 100.000000f, 136807, 41.048141f }, { 75.000000f, 85150, 38.172615f }, { 35.000000f, 64387, 35.527702f } } }, + { "kodim08.png", { { 100.000000f, 145326, 35.896526f }, { 75.000000f, 119654, 33.047630f }, { 35.000000f, 92376, 29.980146f } } }, + { "kodim09.png", { { 100.000000f, 135074, 42.271267f }, { 75.000000f, 66568, 38.262554f }, { 35.000000f, 47686, 35.810940f } } }, + { "kodim10.png", { { 100.000000f, 137184, 41.879585f }, { 75.000000f, 73560, 37.980556f }, { 35.000000f, 54453, 35.449261f } } }, + { "kodim11.png", { { 100.000000f, 138275, 38.718960f }, { 75.000000f, 91902, 35.112244f }, { 35.000000f, 66243, 32.391891f } } }, + { "kodim12.png", { { 100.000000f, 132918, 42.822681f }, { 75.000000f, 71330, 38.155998f }, { 35.000000f, 49345, 35.743179f } } }, + { "kodim13.png", { { 100.000000f, 141033, 33.948277f }, { 75.000000f, 123631, 30.678318f }, { 35.000000f, 88403, 27.592640f } } }, + { "kodim14.png", { { 100.000000f, 141117, 36.902863f }, { 75.000000f, 108060, 33.896935f }, { 35.000000f, 77104, 31.451799f } } }, + { "kodim15.png", { { 100.000000f, 135981, 40.416115f }, { 75.000000f, 76564, 36.855175f }, { 35.000000f, 55002, 34.548985f } } }, + { "kodim16.png", { { 100.000000f, 134349, 42.286755f }, { 75.000000f, 80713, 36.828140f }, { 35.000000f, 55894, 33.982174f } } }, + { "kodim17.png", { { 100.000000f, 138778, 40.653671f }, { 75.000000f, 81391, 37.024017f }, { 35.000000f, 59293, 34.429058f } } }, + { "kodim18.png", { { 100.000000f, 142690, 36.400116f }, { 75.000000f, 104323, 33.398468f }, { 35.000000f, 74051, 30.714231f } } }, + { "kodim19.png", { { 100.000000f, 138584, 39.704021f }, { 75.000000f, 87574, 35.544052f }, { 35.000000f, 63776, 33.032051f } } }, + { "kodim20.png", { { 100.000000f, 121663, 41.099850f }, { 75.000000f, 64552, 37.174721f }, { 35.000000f, 44838, 34.739983f } } }, + { "kodim21.png", { { 100.000000f, 138337, 38.284393f }, { 75.000000f, 85878, 34.727512f }, { 35.000000f, 60879, 32.004494f } } }, + { "kodim22.png", { { 100.000000f, 142142, 38.583397f }, { 75.000000f, 93914, 35.047283f }, { 35.000000f, 65592, 32.702984f } } }, + { "kodim23.png", { { 100.000000f, 140280, 42.489117f }, { 75.000000f, 74579, 39.385365f }, { 35.000000f, 57354, 37.228970f } } }, + { "kodim24.png", { { 100.000000f, 138443, 36.158039f }, { 75.000000f, 101415, 33.512146f }, { 35.000000f, 75311, 30.575174f } } }, + { "white_1x1.png", { { 100.000000f, 111, 100.000000f }, { 75.000000f, 112, 100.000000f }, { 35.000000f, 112, 100.000000f } } }, + { "wikipedia.png", { { 100.000000f, 189589, 32.205330f }, { 75.000000f, 168732, 31.926851f }, { 35.000000f, 160971, 30.209082f } } }, + //{ "alpha0.png", { { 100.000000f, 1389, 49.883366f }, { 75.000000f, 1385, 49.125038f }, { 35.000000f, 1479, 42.865246f } } } // alpha0.png is minor nightmare for testing XUASTC LDR because it's very sensitive to tiny FP differences +}; + +static bool test_mode_xuastc_ldr(command_line_params& opts) +{ + uint32_t total_mismatches = 0; + + // Minor differences in how floating point code is optimized can result in slightly different generated files. + + // XUASTC LDR's IDCT is currently float - at low q's and high (>48) dB's tiny differences during decompression are noticeable + const float XUASTC_PSNR_THRESHOLD = 1.0f; + const float XUASTC_FILESIZE_THRESHOLD = .045f; + + struct run_stats + { + size_t m_comp_size; + image_stats m_stats; + }; + + basisu::vector2D< run_stats > run_image_stats((uint32_t)std::size(g_xuastc_ldr_test_files_6x6), XUASTC_LDR_TEST_FILE_NUM_RUNS); + + for (uint32_t i = 0; i < std::size(g_xuastc_ldr_test_files_6x6); i++) + { + const auto& test_file = g_xuastc_ldr_test_files_6x6[i]; + + std::string filename(opts.m_test_file_dir); + if (filename.size()) + { + filename.push_back('/'); + } + filename += std::string(test_file.m_pFilename); + + basisu::vector source_images(1); + + image& source_image = source_images[0]; + if (!load_png(filename.c_str(), source_image)) + { + error_printf("Failed loading test image \"%s\"\n", filename.c_str()); + return false; + } + + printf("Loaded file \"%s\", dimensions %ux%u has alpha: %u\n", filename.c_str(), source_image.get_width(), source_image.get_height(), source_image.has_alpha()); + + image_stats stats; + + uint32_t flags_and_quality; + + // Test XUASTC LDR + flags_and_quality = (opts.m_comp_params.m_multithreading ? cFlagThreaded : 0) | cFlagPrintStats | cFlagPrintStatus | cFlagSRGB; + + for (uint32_t run_index = 0; run_index < XUASTC_LDR_TEST_FILE_NUM_RUNS; run_index++) + { + const auto& test_run = test_file.m_test_runs[run_index]; + + float uastc_rdo_quality = 0.0f; + size_t data_size = 0; + + const uint32_t effort_level = 8; + + flags_and_quality &= ~0xFF; + flags_and_quality |= effort_level; + + if (test_run.m_dct_q < 100.0f) + { + uastc_rdo_quality = test_run.m_dct_q; + } + + basist::basis_tex_format tex_fmt = basist::basis_tex_format::cXUASTC_LDR_6x6; + + fmt_printf("**** Testing XUASTC LDR, DCT q {}, effort {}\n", test_run.m_dct_q, effort_level); + + void* pData = basis_compress(tex_fmt, source_images, flags_and_quality, uastc_rdo_quality, &data_size, &stats); + if (!pData) + { + error_printf("basis_compress() failed!\n"); + return false; + } + basis_free_data(pData); + + fmt_printf("XUASTC Size: {} (expected {}), RGBA PSNR: {3.3} dB (expected {3.3} dB)\n", + (uint32_t)data_size, test_run.m_comp_size, + stats.m_basis_rgba_avg_psnr, test_run.m_rgba_psnr); + + float file_size_ratio = fabs((data_size / (float)test_run.m_comp_size) - 1.0f); + + if (file_size_ratio > XUASTC_FILESIZE_THRESHOLD) + { + fmt_error_printf("Mismatch: Expected XUASTC LDR file size was {}, but got {} instead!\n", test_run.m_comp_size, (uint32_t)data_size); + total_mismatches++; + } + + if (fabs(stats.m_basis_rgba_avg_psnr - test_run.m_rgba_psnr) > XUASTC_PSNR_THRESHOLD) + { + fmt_error_printf("Mismatch: Expected XUASTC LDR RGBA Avg PSNR was {}, but got {} instead!\n", test_run.m_rgba_psnr, stats.m_basis_rgba_avg_psnr); + total_mismatches++; + } + + run_image_stats(i, run_index).m_comp_size = data_size; + run_image_stats(i, run_index).m_stats = stats; + } + } + +#if 0 + for (uint32_t i = 0; i < std::size(g_xuastc_ldr_test_files_6x6); i++) + { + fmt_printf("{{ \"{}\", {{", g_xuastc_ldr_test_files_6x6[i].m_pFilename); + + for (uint32_t j = 0; j < XUASTC_LDR_TEST_FILE_NUM_RUNS; j++) + { + fmt_printf(" {{ {}f, {}, {}f }", + g_xuastc_ldr_test_files_6x6[i].m_test_runs[j].m_dct_q, + run_image_stats(i, j).m_comp_size, + run_image_stats(i, j).m_stats.m_basis_rgba_avg_psnr); + + if (j != (XUASTC_LDR_TEST_FILE_NUM_RUNS - 1)) + fmt_printf(", "); + } + + fmt_printf(" } },\n"); + } +#endif + + printf("Total XUASTC LDR mismatches: %u\n", total_mismatches); + + bool result = true; + if (total_mismatches) + { + error_printf("XUASTC LDR test FAILED\n"); + result = false; + } + else + { + printf("XUASTC LDR test succeeded\n"); + } + + return result; +} + +static bool clbench_mode(command_line_params& opts) +{ + BASISU_NOTE_UNUSED(opts); + + bool opencl_failed = false; + bool use_cl = basis_benchmark_etc1s_opencl(&opencl_failed); + if (use_cl) + printf("OpenCL ETC1S encoding is faster on this machine\n"); else { if (opencl_failed) @@ -4639,17 +5548,513 @@ static void force_san_failure() } #endif // FORCE_SAN_FAILURE -static int main_internal(int argc, const char **argv) +static bool peek_astc_file(const char* pFilename) { - printf("Basis Universal LDR/HDR GPU Texture Compression and Transcoding System v" BASISU_TOOL_VERSION + fmt_printf("\nExamining .astc file: \"{}\"\n", pFilename); + + vector2D blocks; + uint32_t block_width, block_height, image_width, image_height; + if (!read_astc_file(pFilename, blocks, block_width, block_height, image_width, image_height)) + { + fmt_error_printf("Failed reading .astc file!\n"); + return false; + } + + const uint32_t total_block_pixels = block_width * block_height; + + fmt_printf("Block dimensions in pixels: {}x{}, {} total pixels\n", block_width, block_height, total_block_pixels); + fmt_printf("Image dimensions in pixels: {}x{}\n", image_width, image_height); + + fmt_printf("Extra cols/rows to pad image to ASTC block dimensions: {}x{}\n", + blocks.get_width() * block_width - image_width, + blocks.get_height() * block_height - image_height); + + image dec_image_srgb(image_width, image_height); + image dec_image_linear(image_width, image_height); + imagef dec_image_float(image_width, image_height); + + uint32_t cem_hist[16] = { }; + uint32_t cem_dp_hist[16] = { }; + uint32_t cem_used_bc_hist[16] = { }; + uint32_t total_dp = 0; + + uint32_t total_solid_blocks_ldr = 0; + uint32_t total_solid_blocks_hdr = 0; + uint32_t total_normal_blocks = 0; + + uint32_t part_hist[4] = { }; + uint32_t used_endpoint_levels_hist[astc_helpers::LAST_VALID_ENDPOINT_ISE_RANGE - astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE + 1] = { }; + uint32_t used_weight_levels_hist[astc_helpers::LAST_VALID_WEIGHT_ISE_RANGE - astc_helpers::FIRST_VALID_WEIGHT_ISE_RANGE + 1] = { }; + + uint32_t total_unequal_cem_blocks = 0; + uint32_t total_unequal_cem_blocks_2subsets = 0; + uint32_t total_unequal_cem_blocks_3subsets = 0; + uint32_t total_unequal_cem_blocks_4subsets = 0; + + uint32_t highest_part_seed = 0; + + int min_weight_grid_width = INT_MAX, min_weight_grid_height = INT_MAX; + int max_weight_grid_width = 0, max_weight_grid_height = 0; + + uint32_t total_ldr_blocks = 0, total_hdr_blocks = 0; + + basisu::hash_map weight_grid_histogram; + + struct log_astc_block_config_cmp_t + { + bool operator()(const astc_helpers::log_astc_block& a, + const astc_helpers::log_astc_block& b) const + { + // This only compares the ASTC configuration for equality, NOT the contents. + if (a.m_error_flag != b.m_error_flag) + return false; + if (a.m_error_flag) + return true; + + if (a.m_grid_width != b.m_grid_width) + return false; + if (a.m_grid_height != b.m_grid_height) + return false; + + if (a.m_solid_color_flag_ldr != b.m_solid_color_flag_ldr) + return false; + if (a.m_solid_color_flag_hdr != b.m_solid_color_flag_hdr) + return false; + + if (a.m_solid_color_flag_ldr || a.m_solid_color_flag_hdr) + return true; + + if (a.m_dual_plane != b.m_dual_plane) + return false; + if (a.m_color_component_selector != b.m_color_component_selector) + return false; + + if (a.m_num_partitions != b.m_num_partitions) + return false; + + if (a.m_endpoint_ise_range != b.m_endpoint_ise_range) + return false; + if (a.m_weight_ise_range != b.m_weight_ise_range) + return false; + + for (uint32_t i = 0; i < a.m_num_partitions; i++) + if (a.m_color_endpoint_modes[i] != b.m_color_endpoint_modes[i]) + return false; + + return true; + } + }; + + basisu::hash_map, log_astc_block_config_cmp_t > unique_config_histogram; + + for (uint32_t by = 0; by < blocks.get_height(); by++) + { + for (uint32_t bx = 0; bx < blocks.get_width(); bx++) + { + astc_helpers::log_astc_block log_blk; + + if (!astc_helpers::unpack_block(&blocks(bx, by), log_blk, block_width, block_height)) + { + fmt_error_printf("astc_helpers::unpack_block() failed on block {}x{}\n", bx, by); + return false; + } + + if (log_blk.m_error_flag) + { + fmt_error_printf("astc_helpers::unpack_block() returned an error flag on block {}x{}\n", bx, by); + return false; + } + + { + astc_helpers::log_astc_block scrubbed_log_blk; + memset(&scrubbed_log_blk, 0, sizeof(scrubbed_log_blk)); + + // just record the config, not the contents, so only the config hashes + scrubbed_log_blk.m_solid_color_flag_ldr = log_blk.m_solid_color_flag_ldr; + scrubbed_log_blk.m_solid_color_flag_hdr = log_blk.m_solid_color_flag_hdr; + scrubbed_log_blk.m_dual_plane = log_blk.m_dual_plane; + scrubbed_log_blk.m_color_component_selector = log_blk.m_color_component_selector; + scrubbed_log_blk.m_grid_width = log_blk.m_grid_width; + scrubbed_log_blk.m_grid_height = log_blk.m_grid_height; + scrubbed_log_blk.m_num_partitions = log_blk.m_num_partitions; + scrubbed_log_blk.m_color_endpoint_modes[0] = log_blk.m_color_endpoint_modes[0]; + scrubbed_log_blk.m_color_endpoint_modes[1] = log_blk.m_color_endpoint_modes[1]; + scrubbed_log_blk.m_color_endpoint_modes[2] = log_blk.m_color_endpoint_modes[2]; + scrubbed_log_blk.m_color_endpoint_modes[3] = log_blk.m_color_endpoint_modes[3]; + scrubbed_log_blk.m_weight_ise_range = log_blk.m_weight_ise_range; + scrubbed_log_blk.m_endpoint_ise_range = log_blk.m_endpoint_ise_range; + + auto ins_res(unique_config_histogram.insert(scrubbed_log_blk, 0)); + (ins_res.first)->second = (ins_res.first)->second + 1; + } + + bool is_hdr = log_blk.m_solid_color_flag_hdr; + + if (log_blk.m_solid_color_flag_ldr) + { + total_solid_blocks_ldr++; + total_ldr_blocks++; + } + else if (log_blk.m_solid_color_flag_hdr) + { + total_solid_blocks_hdr++; + total_hdr_blocks++; + } + else + { + total_normal_blocks++; + + min_weight_grid_width = minimum(min_weight_grid_width, log_blk.m_grid_width); + min_weight_grid_height = minimum(min_weight_grid_height, log_blk.m_grid_height); + + max_weight_grid_width = maximum(max_weight_grid_width, log_blk.m_grid_width); + max_weight_grid_height = maximum(max_weight_grid_height, log_blk.m_grid_height); + + { + uint32_t weight_grid_hash_key = log_blk.m_grid_width | (log_blk.m_grid_height << 8); + auto ins_res(weight_grid_histogram.insert(weight_grid_hash_key, 0)); + (ins_res.first)->second = (ins_res.first)->second + 1; + } + + if (log_blk.m_dual_plane) + total_dp++; + + part_hist[log_blk.m_num_partitions - 1]++; + + // For debugging seed packing bugs + highest_part_seed = basisu::maximum(highest_part_seed, log_blk.m_partition_id); + + uint32_t cur_endpoint_ofs = 0; + bool has_unequal_cems = false; + + for (uint32_t p = 0; p < log_blk.m_num_partitions; p++) + { + if (astc_helpers::is_cem_hdr(log_blk.m_color_endpoint_modes[p])) + is_hdr = true; + + cem_hist[log_blk.m_color_endpoint_modes[p]]++; + + if (log_blk.m_dual_plane) + cem_dp_hist[log_blk.m_color_endpoint_modes[p]]++; + + if ((p) && (log_blk.m_color_endpoint_modes[p] != log_blk.m_color_endpoint_modes[0])) + { + has_unequal_cems = true; + } + + if (astc_helpers::is_cem_ldr(log_blk.m_color_endpoint_modes[p])) + { + bool uses_bc = astc_helpers::used_blue_contraction(log_blk.m_color_endpoint_modes[p], log_blk.m_endpoints + cur_endpoint_ofs, log_blk.m_endpoint_ise_range); + + cem_used_bc_hist[log_blk.m_color_endpoint_modes[p]] += uses_bc; + } + + cur_endpoint_ofs += astc_helpers::get_num_cem_values(log_blk.m_color_endpoint_modes[p]); + } + + if (log_blk.m_num_partitions >= 2) + { + total_unequal_cem_blocks += has_unequal_cems; + + if (log_blk.m_num_partitions == 2) + total_unequal_cem_blocks_2subsets += has_unequal_cems; + else if (log_blk.m_num_partitions == 3) + total_unequal_cem_blocks_3subsets += has_unequal_cems; + else if (log_blk.m_num_partitions == 4) + total_unequal_cem_blocks_4subsets += has_unequal_cems; + } + + used_weight_levels_hist[open_range_check(log_blk.m_weight_ise_range - astc_helpers::FIRST_VALID_WEIGHT_ISE_RANGE, std::size(used_weight_levels_hist))]++; + used_endpoint_levels_hist[open_range_check(log_blk.m_endpoint_ise_range - astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE, std::size(used_endpoint_levels_hist))]++; + } + + if (is_hdr) + { + total_hdr_blocks++; + } + else + { + total_ldr_blocks++; + + color_rgba block_pixels[astc_helpers::MAX_BLOCK_PIXELS]; + + // sRGB8 decode profile unpack + bool status = astc_helpers::decode_block(log_blk, block_pixels, block_width, block_height, astc_helpers::cDecodeModeSRGB8); + if (!status) + { + fmt_error_printf("astc_helpers::decode_block() failed on block {}x{}\n", bx, by); + return false; + } + + dec_image_srgb.set_block_clipped(block_pixels, bx * block_width, by * block_height, block_width, block_height); + + // linear8 decode profile unpack + status = astc_helpers::decode_block(log_blk, block_pixels, block_width, block_height, astc_helpers::cDecodeModeLDR8); + if (!status) + { + fmt_error_printf("astc_helpers::decode_block() failed on block {}x{}\n", bx, by); + return false; + } + + dec_image_linear.set_block_clipped(block_pixels, bx * block_width, by * block_height, block_width, block_height); + } + + // half float unpack + { + basist::half_float block_pixels_half[astc_helpers::MAX_BLOCK_PIXELS][4]; + + bool status = astc_helpers::decode_block(log_blk, block_pixels_half, block_width, block_height, astc_helpers::cDecodeModeHDR16); + if (!status) + { + fmt_error_printf("astc_helpers::decode_block() failed on block {}x{}\n", bx, by); + return false; + } + + vec4F block_pixels_float[astc_helpers::MAX_BLOCK_PIXELS]; + for (uint32_t i = 0; i < total_block_pixels; i++) + for (uint32_t j = 0; j < 4; j++) + block_pixels_float[i][j] = basist::half_to_float(block_pixels_half[i][j]); + + dec_image_float.set_block_clipped(block_pixels_float, bx * block_width, by * block_height, block_width, block_height); + } + + } // bx + + } //by + + fmt_printf("Total LDR blocks: {}, total HDR blocks: {}\n", total_ldr_blocks, total_hdr_blocks); + + save_png("astc_decoded_srgb8_ldr.png", dec_image_srgb); + fmt_printf("Wrote astc_decoded_srgb8_ldr.png\n"); + + save_png("astc_decoded_linear8_ldr.png", dec_image_linear); + fmt_printf("Wrote astc_decoded_linear8_ldr.png\n"); + + write_exr("astc_decoded_half.exr", dec_image_float, 4, 0); + fmt_printf("Wrote astc_decoded_half.exr\n"); + + fmt_printf("\nASTC file statistics:\n"); + + const uint32_t total_blocks = (uint32_t)blocks.size(); + + fmt_printf("Total blocks: {}, total void extent LDR: {}, total void extent HDR: {}, total normal: {}\n", total_blocks, total_solid_blocks_ldr, total_solid_blocks_hdr, total_normal_blocks); + fmt_printf("Total dual plane: {} {3.2}%\n", total_dp, total_dp * 100.0f / (float)total_blocks); + + fmt_printf("Min weight grid dimensions: {}x{}\n", min_weight_grid_width, min_weight_grid_height); + fmt_printf("Max weight grid width: {}, height: {}\n", max_weight_grid_width, max_weight_grid_height); + + fmt_printf("\nPartition usage histogram:\n"); + for (uint32_t i = 0; i < 4; i++) + fmt_printf("{}: {} {3.2}%\n", i + 1, part_hist[i], (float)part_hist[i] * 100.0f / (float)total_blocks); + + fmt_printf("\nCEM usage histogram:\n"); + for (uint32_t i = 0; i < 15; i++) + { + fmt_printf("{}: {} {3.2}%, total BC: {} {3.2}%, total DP: {} {3.2}%\n", i, + cem_hist[i], (float)cem_hist[i] * 100.0f / (float)total_blocks, + cem_used_bc_hist[i], (float)cem_used_bc_hist[i] * 100.0f / (float)total_blocks, + cem_dp_hist[i], (float)cem_dp_hist[i] * 100.0f / (float)total_blocks); + } + + fmt_printf("\nUsed endpoint ISE levels:\n"); + for (uint32_t i = 0; i < std::size(used_endpoint_levels_hist); i++) + fmt_printf("{} levels: {}\n", astc_helpers::get_ise_levels(astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE + i), used_endpoint_levels_hist[i]); + + fmt_printf("\nUsed weight ISE levels:\n"); + for (uint32_t i = 0; i < std::size(used_weight_levels_hist); i++) + fmt_printf("{} levels: {}\n", astc_helpers::get_ise_levels(astc_helpers::FIRST_VALID_WEIGHT_ISE_RANGE + i), used_weight_levels_hist[i]); + + fmt_printf("\nTotal 2+ subset blocks using unequal CEM's: {} {3.2}%\n", total_unequal_cem_blocks, (float)total_unequal_cem_blocks * 100.0f / (float)total_blocks); + fmt_printf("Total 2 subset blocks using unequal CEM's: {} {3.2}%\n", total_unequal_cem_blocks_2subsets, (float)total_unequal_cem_blocks_2subsets * 100.0f / (float)total_blocks); + fmt_printf("Total 3 subset blocks using unequal CEM's: {} {3.2}%\n", total_unequal_cem_blocks_3subsets, (float)total_unequal_cem_blocks_3subsets * 100.0f / (float)total_blocks); + fmt_printf("Total 4 subset blocks using unequal CEM's: {} {3.2}%\n", total_unequal_cem_blocks_4subsets, (float)total_unequal_cem_blocks_4subsets * 100.0f / (float)total_blocks); + + fmt_printf("\nHighest part ID seed: {}, 0x{0x}\n", highest_part_seed, highest_part_seed); + + fmt_printf("\nWeight grid usage histogram:\n"); + + uint64_vec v; + for (auto it = weight_grid_histogram.begin(); it != weight_grid_histogram.end(); ++it) + v.push_back(((uint64_t)it->first << 32) | it->second); + + v.sort(); + + for (uint32_t i = 0; i < v.size(); i++) + fmt_printf(" {}x{}: total blocks {}\n", (v[i] >> 32) & 0xFF, (v[i] >> 40) & 0xFF, v[i] & UINT32_MAX); + + fmt_printf("\nTotal unique ASTC configurations: {}\n", unique_config_histogram.size_u32()); + + uint32_t config_idx = 0; + for (auto it = unique_config_histogram.begin(); it != unique_config_histogram.end(); ++it) + { + const auto& l = it->first; + const uint32_t total = it->second; + + fmt_printf(" {}. Used {} {3.2}% times: Solid LDR: {} HDR: {}, Grid: {}x{}, Dual Plane: {}, CCS: {}, NumParts: {}, CEMS: {} {} {} {}, WeightISERange: {}, EndpointISERange: {}\n", + config_idx, total, float(total) * 100.0f / total_blocks, + l.m_solid_color_flag_ldr, l.m_solid_color_flag_hdr, + l.m_grid_width, l.m_grid_height, + l.m_dual_plane, l.m_color_component_selector, + l.m_num_partitions, l.m_color_endpoint_modes[0], l.m_color_endpoint_modes[1], l.m_color_endpoint_modes[2], l.m_color_endpoint_modes[3], + l.m_weight_ise_range, l.m_endpoint_ise_range); + + config_idx++; + } + + fmt_printf("Success\n"); + + return true; +} + +bool xuastc_ldr_decoder_fuzz_test() +{ + basisu::rand rnd; + rnd.seed(1); + + const uint32_t N = 16; + + interval_timer itm; + double total_time_a = 0, total_time_b = 0; + + for (uint32_t blk_size_index = 0; blk_size_index < astc_helpers::NUM_ASTC_BLOCK_SIZES; blk_size_index++) + { + const uint32_t bw = astc_helpers::g_astc_block_sizes[blk_size_index][0]; + const uint32_t bh = astc_helpers::g_astc_block_sizes[blk_size_index][1]; + + fmt_printf("Testing block size {}x{}\n", bw, bh); + + const auto& trial_modes = basist::astc_ldr_t::g_encoder_trial_modes[blk_size_index]; + + if (!trial_modes.size()) + { + assert(0); + return false; + } + + for (uint32_t j = 0; j < trial_modes.size(); j++) + { + const auto& tm = trial_modes[j]; + + astc_helpers::log_astc_block log_blk; + log_blk.clear(); + + const bool test_solid = rnd.irand(0, 63) == 0; + + log_blk.m_grid_width = (uint8_t)tm.m_grid_width; + log_blk.m_grid_height = (uint8_t)tm.m_grid_height; + + log_blk.m_weight_ise_range = (uint8_t)tm.m_weight_ise_range; + log_blk.m_endpoint_ise_range = (uint8_t)tm.m_endpoint_ise_range; + + log_blk.m_dual_plane = tm.m_ccs_index != -1; + if (tm.m_ccs_index != -1) + log_blk.m_color_component_selector = (uint8_t)tm.m_ccs_index; + + log_blk.m_num_partitions = (uint8_t)tm.m_num_parts; + for (uint32_t s = 0; s < tm.m_num_parts; s++) + log_blk.m_color_endpoint_modes[s] = (uint8_t)tm.m_cem; + + for (uint32_t k = 0; k < N; k++) + { + if (log_blk.m_num_partitions > 1) + log_blk.m_partition_id = (uint16_t)rnd.irand(0, 1023); + + const uint32_t num_cem_endpoint_vals = astc_helpers::get_num_cem_values(tm.m_cem); + const uint32_t total_cem_endpoint_vals = num_cem_endpoint_vals * log_blk.m_num_partitions; + + for (uint32_t i = 0; i < total_cem_endpoint_vals; i++) + log_blk.m_endpoints[i] = (uint8_t)rnd.irand(0, astc_helpers::get_ise_levels(log_blk.m_endpoint_ise_range) - 1); + + const uint32_t num_weight_vals = (log_blk.m_dual_plane ? 2 : 1) * log_blk.m_grid_width * log_blk.m_grid_height; + for (uint32_t i = 0; i < num_weight_vals; i++) + log_blk.m_weights[i] = (uint8_t)rnd.irand(0, astc_helpers::get_ise_levels(log_blk.m_weight_ise_range) - 1); + + if (test_solid) + { + log_blk.clear(); + log_blk.m_solid_color_flag_ldr = true; + + uint32_t r = rnd.byte(); + uint32_t g = rnd.byte(); + uint32_t b = rnd.byte(); + uint32_t a = rnd.byte(); + + log_blk.m_solid_color[0] = (uint16_t)((r << 8) | r); + log_blk.m_solid_color[1] = (uint16_t)((g << 8) | g); + log_blk.m_solid_color[2] = (uint16_t)((b << 8) | b); + log_blk.m_solid_color[3] = (uint16_t)((a << 8) | a); + } + + const bool srgb = rnd.bit(); + + basist::color32 blk_a[astc_helpers::MAX_BLOCK_PIXELS]; + clear_obj(blk_a); + + itm.start(); + + bool status_a = astc_helpers::decode_block(log_blk, blk_a, bw, bh, srgb ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8); + if (!status_a) + { + error_printf("astc_helpers::decode_block() failed\n"); + return false; + } + + total_time_a += itm.get_elapsed_secs(); + + basist::color32 blk_b[astc_helpers::MAX_BLOCK_PIXELS]; + clear_obj(blk_b); + + itm.start(); + + bool status_b = astc_helpers::decode_block_xuastc_ldr(log_blk, blk_b, bw, bh, srgb ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8); + if (!status_b) + { + error_printf("astc_helpers::decode_block() failed\n"); + return false; + } + + total_time_b += itm.get_elapsed_secs(); + + for (uint32_t i = 0; i < bw * bh; i++) + { + if ((blk_a[i].r != blk_b[i].r) || (blk_a[i].g != blk_b[i].g) || (blk_a[i].b != blk_b[i].b) || (blk_a[i].a != blk_b[i].a)) + { + error_printf("decode block mismatch\n"); + return false; + } + } + + } // k + + } // j + + } // blk_size_index + + printf("ASTC block decoder vs. XUASTC LDR block decoding fuzz test succeeded\n"); + fmt_printf("Total time A: {}, B: {}\n", total_time_a, total_time_b); + + return true; +} + +static int main_internal(int argc, const char** argv) +{ + printf("Basis Universal LDR/HDR GPU Texture Supercompression System v" BASISU_TOOL_VERSION + #if defined(_ARM64EC_) || defined(_ARM64_) - " (ARM64)" + " (ARM64)" #elif defined(_M_IX86) - " (x86)" + " (x86)" #elif defined(_M_X64) || defined(_M_AMD64) - " (x64)" + " (x64)" +#elif defined(__wasi__) + " (WASI" + #if BASISU_WASI_THREADS + " Threaded" + #endif + ")" #endif - "\nCopyright (C) 2019-2025 Binomial LLC, All rights reserved\n"); + + "\nCopyright (C) 2019-2026 Binomial LLC, All rights reserved\n"); #ifdef FORCE_SAN_FAILURE force_san_failure(); @@ -4657,37 +6062,69 @@ static int main_internal(int argc, const char **argv) //interval_timer tm; //tm.start(); - + // See if OpenCL support has been disabled. We don't want to parse the command line until the lib is initialized bool use_opencl = false; bool opencl_force_serialization = false; + bool astc_peek_flag = false; + bool astc_fuzz_flag = false; for (int i = 1; i < argc; i++) { if ((strcmp(argv[i], "-opencl") == 0) || (strcmp(argv[i], "-clbench") == 0)) use_opencl = true; + if (strcmp(argv[i], "-opencl_serialize") == 0) opencl_force_serialization = true; + + if ((strcmp(argv[i], "-peek_astc") == 0) || (strcmp(argv[i], "-peek") == 0)) + astc_peek_flag = true; + + if (strcmp(argv[i], "-dev_astc_fuzz") == 0) + astc_fuzz_flag = true; } #if !BASISU_SUPPORT_OPENCL if (use_opencl) { - fprintf(stderr, "WARNING: -opencl specified, but OpenCL support was not enabled at compile time! With cmake, use -D BASISU_OPENCL=1. Falling back to CPU compression.\n"); + fprintf(stderr, "WARNING: -opencl specified, but OpenCL support was not defined or enabled at compile time! With cmake, use -D BASISU_OPENCL=1. Falling back to CPU compression.\n"); } #endif basisu_encoder_init(use_opencl, opencl_force_serialization); + + if (astc_fuzz_flag) + { + bool status = xuastc_ldr_decoder_fuzz_test(); + return status ? EXIT_SUCCESS : EXIT_FAILURE; + } - //printf("Encoder and transcoder libraries initialized in %3.3f ms\n", tm.get_elapsed_ms()); + if (astc_peek_flag) + { + if (argc != 3) + { + fmt_error_printf("Requires filename argument of .astc file\n"); + return EXIT_FAILURE; + } + + bool status = peek_astc_file(argv[2]); + return status ? EXIT_SUCCESS : EXIT_FAILURE; + } + //printf("Encoder and transcoder libraries initialized in %3.3f ms\n", tm.get_elapsed_ms()); + if (argc == 1) { print_usage(); return EXIT_FAILURE; } - + command_line_params opts; + +#if defined(__wasi__) && !BASISU_WASI_THREADS + opts.m_comp_params.m_multithreading = false; +#endif + if (!opts.parse(argc, argv)) { //print_usage(); @@ -4699,7 +6136,7 @@ static int main_internal(int argc, const char **argv) #else printf("No SSE, Multithreading: %u, Zstandard support: %u, OpenCL: %u\n", (uint32_t)opts.m_comp_params.m_multithreading, basist::basisu_transcoder_supports_ktx2_zstd(), opencl_is_available()); #endif - + if (!opts.process_listing_files()) return EXIT_FAILURE; @@ -4748,6 +6185,9 @@ static int main_internal(int argc, const char **argv) case cTestLDR: status = test_mode_ldr(opts); break; + case cTestXUASTCLDR: + status = test_mode_xuastc_ldr(opts); + break; case cTestHDR_4x4: status = test_mode_hdr(opts, basist::basis_tex_format::cUASTC_HDR_4x4, std::size(g_hdr_4x4_test_files), g_hdr_4x4_test_files, 0.0f); break; @@ -4755,11 +6195,11 @@ static int main_internal(int argc, const char **argv) status = test_mode_hdr(opts, basist::basis_tex_format::cASTC_HDR_6x6, std::size(g_hdr_6x6_test_files), g_hdr_6x6_test_files, 0.0f); break; case cTestHDR_6x6i: - status = test_mode_hdr(opts, basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE, std::size(g_hdr_6x6i_test_files), g_hdr_6x6i_test_files, 0.0f); - + status = test_mode_hdr(opts, basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE, std::size(g_hdr_6x6i_test_files), g_hdr_6x6i_test_files, 0.0f); + if (status) { - status = test_mode_hdr(opts, basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE, std::size(g_hdr_6x6i_l_test_files), g_hdr_6x6i_l_test_files, 500.0f); + status = test_mode_hdr(opts, basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE, std::size(g_hdr_6x6i_l_test_files), g_hdr_6x6i_l_test_files, 500.0f); } break; @@ -4786,7 +6226,7 @@ static int main_internal(int argc, const char **argv) //----------------------------------------------------------------------------------- #if CLEAR_WIN32_CONSOLE -void clear_console() +void clear_console() { //if (!IsDebuggerPresent()) // return; @@ -4810,6 +6250,39 @@ void clear_console() //----------------------------------------------------------------------------------- +// Attempt to detect AddressSanitizer (ASan) across compilers - only used for debug +// output purposes. +#ifndef DETECT_ASAN_H +#define DETECT_ASAN_H + +// Start with ASAN disabled +#undef USING_ASAN + +#if defined(__wasi__) + #define USING_ASAN 0 +#else + // --- Clang / Apple Clang: use __has_feature --- + #if defined(__has_feature) + # if __has_feature(address_sanitizer) + # define USING_ASAN 1 + # endif + #endif + + // --- GCC: __SANITIZE_ADDRESS__ --- + #if defined(__SANITIZE_ADDRESS__) + # define USING_ASAN 1 + #endif +#endif // #if defined(__wasi__) + +// If still undefined, ensure USING_ASAN is cleanly defined to 0 +#ifndef USING_ASAN +# define USING_ASAN 0 +#endif + +#endif // DETECT_ASAN_H + +//----------------------------------------------------------------------------------- + int main(int argc, const char** argv) { #ifdef _WIN32 @@ -4818,15 +6291,19 @@ int main(int argc, const char** argv) #if CLEAR_WIN32_CONSOLE clear_console(); + fmt_printf("{}\n", argv[0]); #endif #if defined(DEBUG) || defined(_DEBUG) - printf("DEBUG build\n"); + printf("DEBUG or _DEBUG defined\n"); +#endif +#if !defined(NDEBUG) + printf("NDEBUG is NOT defined\n"); #endif -#ifdef __SANITIZE_ADDRESS__ +#if USING_ASAN printf("Address sanitizer enabled\n"); #endif - + int status = EXIT_FAILURE; #if BASISU_CATCH_EXCEPTIONS @@ -4848,3 +6325,4 @@ int main(int argc, const char** argv) return status; } + diff --git a/external/basis_universal/basisu_tool_help.h b/external/basis_universal/basisu_tool_help.h new file mode 100644 index 0000000000..506bf01fd9 --- /dev/null +++ b/external/basis_universal/basisu_tool_help.h @@ -0,0 +1,711 @@ +// Don't edit directly - see python script in cmd_help directory. +"See project and wiki at: https://github.com/BinomialLLC/basis_universal\n" +"\n" +"The default processing mode is compression of one or more .PNG/.TGA/.JPG/.QOI/\n" +".DDS/.EXR/.HDR files to a LDR or HDR .KTX2 file. Alternate modes:\n" +"\n" +" -unpack: Use transcoder to unpack a .basis/.KTX2 file to one or more .KTX, \n" +" .DDS, .PNG, .ASTC, etc. files.\n" +"\n" +" -info: Display high-level information about a .basis/.KTX2 file\n" +" \n" +" -validate: Validate and display information about a .basis/.KTX2 file\n" +" \n" +" -compare: Compare two LDR PNG/BMP/TGA/JPG/QOI images specified with -file,\n" +" output PSNR and SSIM statistics and RGB/A delta images\n" +"\n" +" -compare_hdr: Compare two HDR .EXR/.HDR images specified with -file, output\n" +" PSNR statistics and RGB delta images\n" +"\n" +" -tonemap: Tonemap an HDR or EXR image to PNG at multiple exposures, use -file\n" +" to specify filename\n" +"\n" +" -peek_astc: Read an .astc file and calculate statistics (for testing/dev)\n" +"\n" +" -version or --version: Print version and exit\n" +"\n" +"--- Intro:\n" +"\n" +"This tool compresses LDR/SDR and HDR images and textures to a Basis Universal\n" +"supercompressed GPU texture, which can be written to supercompressed .basis or\n" +"standard .KTX2 files. It supports a number of SDR and HDR codecs, each with\n" +"different quality, transcoding performance, and bitrate tradeoffs. A SDR/HDR\n" +"mipmap generator is also included. This tool can also examine and\n" +"unpack .KTX2/.basis files to .PNG, .KTX (v1), .ASTC, or .DDS files using its\n" +"single source file transcoder library.\n" +"\n" +"The list of supported texture/supercompressed texture codecs:\n" +" -LDR: ETC1S (SDR default), RDO UASTC LDR 4x4, ASTC or XUASTC LDR 4x4-12x12\n" +" -HDR: UASTC HDR 4x4 (HDR default), RDO ASTC HDR 6x6, UASTC HDR 6x6\n" +" \n" +"RDO=Rate-Distortion Optimization. Two key parameters (quality and effort) have\n" +"been unified across all the codecs:\n" +"\n" +"- The -quality X parameter, where X ranges from [0, 100], controls the\n" +"compression quality vs. bitrate (output file size) tradeoff for those codecs\n" +"supporting supercompression or RDO (Rate-distortion optimization). 100=max\n" +"quality, and lower levels produce smaller files with more distortion.\n" +"\n" +"- The -effort X parameter, where X ranges from [0, 10], controls the\n" +"compression speed (and max CPU usage) vs. max achievable quality tradeoff.\n" +"Low efforts result in more distortion/artifacts, but faster compression. Lower\n" +"efforts result in less utilization of the underlying GPU block format's\n" +"capabilities.\n" +"\n" +"Key Definitions: \n" +"\"Quality\" controls the explicit tradeoff between output distortion and output\n" +"file bitrate (in bits per pixel or target). At max quality (100) each\n" +"compressor will output the lowest distortion it's capable of at its currently\n" +"configured effort level.\n" +"\n" +"\"Effort\" controls how intensely a compressor uses the CPU to focus on each\n" +"block format's encoding capabilities. Low effort levels only target a set of\n" +"core or basic capabilities (specific to each output target format), while\n" +"higher effort levels allow each compressor to explore more of each target's\n" +"features (at the cost of higher CPU time). Lower effort levels result in more\n" +"brittle compression (higher distortion on tough image/texture features).\n" +"Effort=0 fastest compression, effort=10=extremely slow.\n" +"\n" +"Unless an explicit mode is specified, if one or more files have the .basis\n" +"or .KTX2 extension this tool defaults to unpack mode.\n" +"\n" +"By default, the compressor assumes the input is in the sRGB colorspace (like\n" +"typical photos/albedo textures). If the input is NOT sRGB (like a normal map),\n" +"be sure to specify -linear for less artifacts. Depending on the content type,\n" +"some experimentation may be needed.\n" +"\n" +"The TinyEXR library is used to read .EXR images. Crucially, this small library\n" +"does not support all .EXR compression methods. For unsupported images, you can\n" +"use a tool like ImageMagick to convert them to uncompressed .EXR.\n" +"\n" +"For .DDS source files: Mipmapped or plain 2D textures (but not cubemaps) are\n" +"supported. Only uncompressed 32-bit RGBA/BGRA, half float RGBA, or float\n" +"RGBA .DDS files are supported. In -tex_array mode, if a .DDS file is specified,\n" +"all source files must be in .DDS format.\n" +"\n" +"Filenames prefixed with a @ symbol are read as filename listing files. Listing\n" +"text files specify which actual filenames to process (one filename per line).\n" +"\n" +"--- High-Level Texture Mode (Codec) Selection:\n" +"\n" +" 1. -etc1s: Encode to supercompressed ETC1S LDR (the default for SDR/LDR\n" +" inputs). Roughly .8-2.5 bpp. Supports temporal texture supercompression\n" +" (texture video) with skip blocks (Conditional Replenishment), with global\n" +" codebooks shared across all frames.\n" +"\n" +" 2. -uastc/-uastc_ldr: UASTC LDR 4x4. Encode to UASTC LDR 4x4, a custom high\n" +" quality virtual texture format designed for fast transcoding to numerous\n" +" GPU texture formats. Roughly 5-8 bpp. Supports RDO encoding using -lambda X\n" +" option - see options below. In this mode the multi-target compressor\n" +" optimizes for a balance of transcoded ASTC 4x4 LDR and BC7 quality.\n" +"\n" +" 3. -hdr/-hdr_4x4: UASTC HDR 4x4. Encode input as UASTC HDR 4x4 (the default if\n" +" any input file has the .EXR or .HDR extension, or if any .DDS file is HDR).\n" +" Output is standard, but constrained, ASTC HDR 4x4. Roughly 5-8 bpp. In this\n" +" mode the dual-target compressor optimizes for a balance of transcoded ASTC\n" +" 4x4 HDR and BC6H quality.\n" +"\n" +" 4. -hdr_6x6: ASTC HDR 6x6. Encode input as RDO or highest quality standard\n" +" ASTC HDR 6x6. Use -quality (preferred) or -lambda X (low-level, try\n" +" 100-20000 or higher) option to enable RDO ASTC HDR 6x6, where x controls\n" +" the quality vs. size tradeoff. Roughly 1.2-3.2 bpp.\n" +"\n" +" 5. -hdr_6x6i: UASTC HDR 6x6. Encode input as supercompressed UASTC HDR 6x6\n" +" intermediate. Use -quality (preferred) or -lambda X (low-level, try\n" +" 100-20000 or higher) option to enable RDO UASTC HDR 6x6, where x controls\n" +" the quality vs. size tradeoff. Roughly 1-3.2 bpp.\n" +"\n" +" 6. XUASTC LDR 4x4-12x12: -ldr_4x4i, -ldr_5x4i, -ldr_5x5i, -ldr_6x5i,\n" +" -ldr_6x6i, -ldr_8x5i, -ldr_8x6i, -ldr_10x5i, -ldr_10x6i, -ldr_8x8i,\n" +" -ldr_10x8i, -ldr_10x10i, -ldr_12x10i, -ldr_12x12i:\n" +" Compress to supercompressed XUASTC LDR/SDR using the specific\n" +" ASTC block size. See additional ASTC/XUASTC LDR specific options\n" +" (-effort, -quality, -xy, -ts, -tl, etc.) below. Roughly .3-5.7 bpp\n" +"\n" +" 7. ASTC LDR 4x4-12x12: -ldr_4x4, -ldr_5x4, -ldr_5x5, -ldr_6x5, -ldr_6x6,\n" +" -ldr_8x5, -ldr_8x6, -ldr_10x5, -ldr_10x6, -ldr_8x8, -ldr_10x8,\n" +" -ldr_10x10, -ldr_12x10, -ldr_12x12:\n" +" Compress to standard or ZStd supercompressed ASTC LDR/SDR using\n" +" the specific ASTC block size. See additional ASTC LDR specific\n" +" options (-effort, -quality, -xy, -ts, -tl, etc.) below. .89-8 bpp before\n" +" ZStd compression.\n" +"\n" +"--- Tool Options:\n" +"\n" +" -ktx2: Write .KTX2 files (the default). By default, UASTC LDR/HDR 4x4 and ASTC\n" +" 6x6 files will be compressed using Zstandard unless -ktx2_no_zstandard is\n" +" specified.\n" +"\n" +" -basis: Write .basis files instead of .KTX2 files.\n" +"\n" +" -file filename.png/tga/jpg/qoi/exr/hdr: Input image filename, multiple images\n" +" are OK, use -file X for each input filename (prefixing input filenames\n" +" with -file is optional)\n" +"\n" +" -alpha_file filename.png/tga/jpg/qoi: Input alpha image filename, multiple\n" +" images are OK, use -file X for each input filename (must be paired\n" +" with -file), images converted to REC709 grayscale and used as input alpha\n" +"\n" +" -quiet or -no_status_output: Disable compressor's status output to stdout\n" +"\n" +" -output_file filename: Output .basis/.KTX2 filename\n" +"\n" +" -output_path: Output .basis/.KTX2 files to specified directory.\n" +"\n" +" -debug or -verbose: Enable codec debug print to stdout (slightly slower).\n" +"\n" +" -debug_images: Enable codec debug images (much slower).\n" +"\n" +" -stats: Compute and display image quality metrics (slightly to much slower).\n" +"\n" +" -individual: Process input images individually and output\n" +" multiple .basis/.KTX2 files (not as a texture array - this is now the default\n" +" as of v1.16)\n" +"\n" +" -parallel: Compress multiple textures simultaneously (one per thread), instead\n" +" of one at a time. Compatible with OpenCL mode. This is much faster, but in\n" +" OpenCL mode the driver is pushed harder, and the CLI output will be jumbled.\n" +"\n" +" -linear: Use linear colorspace metrics (instead of the default sRGB or scaled\n" +" RGB for HDR), write linear transfer function setting to KTX2/basis file, and\n" +" by default linear (not sRGB) mipmap filtering (unless overridden). Same\n" +" as -tl.\n" +"\n" +" -srgb: Use sRGB colorspace metrics, write sRGB transfer function setting to\n" +" KTX2/basis file, and by default use sRGB mipmap filtering (unless\n" +" overridden). Same as -ts.\n" +"\n" +" -tex_type <2d, 2darray, 3d, video, cubemap>: Set Basis file header's texture\n" +" type field. Cubemap arrays require multiples of 6 images, in X+, X-, Y+, Y-,\n" +" Z+, Z- order, each image must be the same resolutions. 2d=arbitrary 2D\n" +" images, 2darray=2D array, 3D=volume texture slices, video=video frames,\n" +" cubemap=array of faces. For 2darray/3d/cubemaps/video, each source image's\n" +" dimensions and # of mipmap levels must be the same. For video, the .basis\n" +" file will be written with the first frame being an I-Frame, and subsequent\n" +" frames being P-Frames (using conditional replenishment). Playback must always\n" +" occur in order from first to last image.\n" +"\n" +" -cubemap: same as -tex_type cubemap\n" +"\n" +" -tex_array: Process input images as a single texture array and write a\n" +" single .basis/.KTX2 file (the former default before v1.16)\n" +"\n" +" -fuzz_testing: Use with -validate: Disables CRC16 validation of file contents\n" +" before transcoding\n" +"\n" +" -multifile_printf: printf() format string to use to compose multiple filenames\n" +"\n" +" -multifile_first: The index of the first file to process, default is 0 (must\n" +" specify -multifile_printf and -multifile_num)\n" +"\n" +" -multifile_num: The total number of files to process.\n" +"\n" +" -opencl: Enable OpenCL usage (currently only accelerates ETC1S encoding)\n" +"\n" +" -opencl_serialize: Serialize all calls to the OpenCL driver (to work around\n" +" buggy drivers, only useful with -parallel)\n" +"\n" +"--- ETC1S specific options (-etc1s - the LDR/SDR default):\n" +"\n" +" -quality X and -effort X: Set quality (1-100) and effort (0-10) levels\n" +"\n" +" -q X: Low-level ETC1S quality level, 1-255, default is 128, lower=better\n" +" compression/lower quality/faster, higher=less compression/higher\n" +" quality/slower, default is 128. For even higher quality,\n" +" use -max_endpoints/-max_selectors. (-quality is preferred.)\n" +"\n" +" -comp_level X: Low-level ETC1S speed vs. quality tradeoff. Range is 0-6,\n" +" default is 1. Higher values=MUCH slower, but slightly higher quality. Higher\n" +" levels intended for videos. (-effort is preferred.)\n" +"\n" +" -max_endpoints X: ETC1S: Manually set the max number of color endpoint\n" +" clusters from 1-16128, use instead of -q\n" +"\n" +" -max_selectors X: ETC1S: Manually set the max number of color selector\n" +" clusters from 1-16128, use instead of -q\n" +"\n" +"--- UASTC LDR/HDR 4x4 specific options (-uastc or -uastc_ldr):\n" +"\n" +" -quality X and -effort X: Set quality (1-100) and effort (0-10) levels\n" +"\n" +" -uastc, -uastc_ldr or -uastc_ldr_4x4: Enable UASTC LDR 4x4 texture mode,\n" +" instead of the default ETC1S mode. Significantly higher texture quality, but\n" +" much larger (~8bpp) files. (Note that UASTC LDR 4x4 .basis files must be\n" +" losslessly compressed by the user.)\n" +"\n" +" -uastc_level: Set low-level UASTC LDR/HDR 4x4 encoding effort level. LDR Range\n" +" is [0,4], default is 2, higher=slower but higher quality. 0=fastest/lowest\n" +" quality, 3=slowest practical option, 4=impractically slow/highest achievable\n" +" quality. UASTC HDR 4x4 range is [0,4]: higher=slower, but higher quality. HDR\n" +" 4x4 default level=1.\n" +"\n" +" -uastc_rdo_l X: Enable UASTC LDR 4x4 RDO post-processing and set the low-level\n" +" UASTC LDR 4x4 RDO quality scalar (lambda) to X. Lower values=higher\n" +" quality/larger LZ compressed files, higher values=lower quality/smaller LZ\n" +" compressed files. Good range to try is [.25-10]. Note: Previous versons used\n" +" the -uastc_rdo_q option, which was removed because the RDO algorithm was\n" +" changed.\n" +"\n" +" -uastc_rdo_d X: Set UASTC LDR 4x4 RDO dictionary size in bytes. Default is\n" +" 4096, max is 65536. Lower values=faster, but less compression.\n" +"\n" +" -uastc_rdo_b X: Set UASTC LDR 4x4 RDO max smooth block error scale. Range is\n" +" [1,300]. Default is 10.0, 1.0=disabled. Larger values suppress more artifacts\n" +" (and allocate more bits) on smooth blocks.\n" +"\n" +" -uastc_rdo_s X: Set UASTC LDR 4x4 RDO max smooth block standard deviation.\n" +" Range is [.01,65536]. Default is 18.0. Larger values expand the range of\n" +" blocks considered smooth.\n" +"\n" +" -uastc_rdo_f: Don't favor simpler UASTC LDR 4x4 modes in RDO mode.\n" +"\n" +" -uastc_rdo_m: Disable RDO multithreading (slightly higher compression,\n" +" deterministic).\n" +"\n" +"--- UASTC HDR 4x4 specific options (-hdr or -hdr_4x4 - the HDR default):\n" +"\n" +" -hdr, -hdr_4x4, or -uastc_hdr_4x4: Enable UASTC HDR 4x4 mode\n" +"\n" +" -quality X and -effort X: Set quality (1-100) and effort (0-10) levels\n" +"\n" +" -uastc_level X: Sets the low-level UASTC HDR 4x4 compressor's effort level.\n" +" Valid range is [0,4]: higher=slower but higher quality. HDR\n" +" default=1. Level 0=fastest/lowest quality, 3=highest practical\n" +" setting, 4=exhaustive\n" +"\n" +" -hdr_uber_mode: Allow the UASTC HDR 4x4 encoder to try varying the CEM 11\n" +" selectors more for slightly higher quality (slower). This may negatively\n" +" impact BC6H quality, however.\n" +"\n" +" -hdr_ultra_quant: UASTC HDR 4x4: Try to find better quantized CEM 7/11\n" +" endpoint values (slower).\n" +"\n" +" -hdr_favor_astc: UASTC HDR 4x4: By default the dual-target UASTC HDR 4x4\n" +" encoder tries to strike a balance or even slightly favor BC6H quality. If\n" +" this option is specified, ASTC HDR 4x4 quality is favored instead.\n" +"\n" +"--- ASTC/UASTC HDR 6x6 specific options (-hdr_6x6 or -hdr_6x6i):\n" +"\n" +"Internally both modes use the same compressor which can generate either\n" +"standard ASTC HDR 6x6 (with optional RDO) or UASTC HDR 6x6 (supercompressed\n" +"with a custom format).\n" +"\n" +" -hdr_6x6 or -astc_hdr_6x6: Enable RDO ASTC HDR 6x6 mode\n" +"\n" +" -hdr_6x6i or -uastc_hdr_6x6: Enable UASTC HDR 6x6 mode\n" +"\n" +" -quality X and -effort X: Set quality (1-100) and effort (0-10) levels\n" +"\n" +" -lambda X: Low-level option to enable rate distortion optimization (RDO) and\n" +" directly control the HDR 6x6 compressor's lambda setting. The\n" +" higher this value, the lower the quality, but the smaller the file\n" +" size. Try 100-20000, or higher values on some images. Upconverted\n" +" SDR images can generally tolerate much higher lambda settings vs.\n" +" true HDR images.\n" +"\n" +" -hdr_6x6_level X: Low-level option to set the codec to 6x6 HDR mode (same\n" +" as -hdr_6x6) and controls encoder performance vs. max quality\n" +" tradeoff. X may range from [0,12]. Default level is 2. Higher\n" +" values result in better quality but slower encoding. Values above\n" +" 10 are extremely slow.\n" +"\n" +" -hdr_6x6i_level X: Low-level option to set the codec to 6x6 HDR intermediate\n" +" mode (same as -hdr_6x6i) and controls encoder performance vs. max\n" +" quality tradeoff. X may range from [0,12]. Default level is 2.\n" +"\n" +" -rec_2020: The input image's gamut is Rec. 2020 vs. the default Rec. 709 - for\n" +" accurate colorspace error calculations. This value will also be\n" +" written to the KTX2 file's header in the DFD.\n" +"\n" +" -hdr_6x6_jnd\n" +" X, -hdr_6x6_extra_pats, -hdr_6x6_brute_force_pats,\n" +" -hdr_6x6_comp_levels X Y, or -hdr_6x6i_comp_levels X Y: Low-level\n" +" control over the encoder's configuration.\n" +"\n" +"--- SDR/LDR->HDR upconversion options (only used when encoding to HDR formats\n" +"from an LDR/SDR source image):\n" +"\n" +" -hdr_ldr_no_srgb_to_linear: If specified, LDR images will NOT be converted to\n" +" normalized linear light (via a sRGB->Linear conversion) during SDR->HDR\n" +" upconversion before compressing as HDR.\n" +"\n" +" -hdr_ldr_upconversion_nit_multiplier X: Specify how many nits (candelas per\n" +" sq. meter) LDR/SDR images are converted to after converting to linear\n" +" light. Default is 100 nits. Note: Previous builds used 1 nit. Common\n" +" values are 80-100 nits.\n" +"\n" +"--- ASTC LDR/XUASTC LDR specific options (-ldr_4x4 or -ldr_4x4i, up to 12x12):\n" +"\n" +"Internally both modes (ASTC 4x4-12x12 and XUASTC 4x4-12x12) use the same\n" +"core compressor but with different outputs. All 14 standard ASTC block \n" +"sizes are supported (see the list below).\n" +"\n" +" -ldr_4x4-12x12 or -astc_ldr_4x4-12x12: Enable ASTC LDR 4x4-12x12 mode\n" +"\n" +" -ldr_4x4i-12x12 or -xuastc_ldr_4x4-12x12: Enable XUASTC LDR 4x4-12x12 mode\n" +"\n" +" -quality X: Enables lossy weight grid DCT and sets DCT quality level [1,100]\n" +" (defaults to no DCT). Higher=better quality, but higher bitrate. Good values\n" +" to try are 30-90. Default is no weight grid DCT.\n" +"\n" +" -effort X: Set encoder effort level [0,10]: Encoding speed tradeoff, \n" +" higher=slower but potentially higher overall quality. Default=3, 10=Insane.\n" +" \n" +" -xuastc_arith, -xuastc_hybrid, -xuastc_zstd: Set transcoding speed vs.\n" +" compression ratio tradeoff by selecting the output profile/syntax. Default\n" +" is -xuastc_zstd (fastest, lowest ratio). ZStd is fastest/lowest ratio, arith\n" +" is slowest/highest ratio (3-15% better vs. ZStd).\n" +"\n" +" -xy: Enables lossy supercompression using windowed/bounded RDO for extra\n" +" compression (default is lossless supercompression of the XUASTC texture data\n" +" unless DCT is enabled)\n" +"\n" +" -xyd: Disables lossy supercompression (default, but automatically enabled\n" +" if -quality less than 100 is specified )\n" +"\n" +" -xs: Force disable 2-3 subset usage in all effort levels (lower quality but\n" +" faster compression and faster transcoding to BC7 at certain block sizes)\n" +"\n" +" -xp: Force disable RGB dual plane usage in all effort levels (lower quality\n" +" but faster compression and faster transcoding to BC7 at certain block sizes)\n" +"\n" +" -ts: Use LDR sRGB ASTC decoding profile - the default. This parameter should\n" +" match how the developer will decode or sample the ASTC texture data. Inverse\n" +" of -tl. Same as -srgb.\n" +"\n" +" -tl: Use LDR Linear ASTC decoding profile. Inverse of -ts. Same as -linear.\n" +"\n" +" -weights X Y Z W: Set unsigned integer channel error weights. Defaults are\n" +" 1,1,1,1. Useful to favor certain channels during compression.\n" +"\n" +" -ls_min_psnr X, -ls_min_alpha_psnr X, -ls_thresh_psnr X, -ls_thresh_alpha_psnr\n" +" X, -ls_thresh_edge_psnr X, -ls_thresh_edge_alpha_psnr X: Windowed/bounded RDO\n" +" settings (Lossy supercompression must be enabled, or -xy.)\n" +"\n" +" -xuastc_blurring: Experimental - enable blurred block candidates (MUCH slower\n" +" compression, higher quality).\n" +"\n" +"These modes support all the standard ASTC block sizes. The larger the block\n" +"size, the lower the bitrate on disk and in memory, but the more noticeable the\n" +"artifacts. Some block sizes (4x4, 6x6, and 8x6) have specially optimized direct\n" +"transcoding paths to BC7. For faster direct BC7 transcoding at these block\n" +"sizes, disable RGB dual plane (-xp) and subset usage (-xs).\n" +"\n" +" Block Size Base/Memory Size\n" +" 1. 4x4 8.00 bpp\n" +" 2. 5x4 6.40 bpp\n" +" 3. 5x5 5.12 bpp\n" +" 4. 6x5 4.27 bpp\n" +" 5. 6x6 3.56 bpp\n" +" 6. 8x5 3.20 bpp\n" +" 7. 8x6 2.67 bpp\n" +" 8. 10x5 2.56 bpp\n" +" 9. 10x6 2.13 bpp\n" +" 10. 8x8 2.00 bpp\n" +" 11. 10x8 1.60 bpp\n" +" 12. 10x10 1.28 bpp\n" +" 13. 12x10 1.07 bpp\n" +" 14. 12x12 0.89 bpp\n" +"\n" +"--- More options:\n" +"\n" +" -test: Run an automated LDR ETC1S/UASTC LDR 4x4 encoding and transcoding test.\n" +" Returns EXIT_FAILURE on any failures.\n" +"\n" +" -test_hdr_4x4/-test_hdr_6x6/-test_hdr_6x6i: Run automated UASTC HDR encoding\n" +" and transcoding tests. Returns EXIT_FAILURE on any failures.\n" +"\n" +" -test_xuastc: Run an automated XUASTC LDR encoding and transcoding test.\n" +" Returns EXIT_FAILURE on any failures.\n" +"\n" +" -test_dir: Optional directory of test files. Defaults to \"../test_files\".\n" +"\n" +" -y_flip: Flip input images vertically before compression\n" +"\n" +" -normal_map: Tunes codec parameters for better quality on normal maps (linear\n" +" colorspace metrics, linear mipmap filtering, no selector RDO, no sRGB)\n" +"\n" +" -no_alpha: Always output non-alpha basis files, even if one or more inputs has\n" +" alpha\n" +"\n" +" -force_alpha: Always output alpha basis files, even if no inputs has alpha\n" +"\n" +" -separate_rg_to_color_alpha: Separate input R and G channels to RGB and A (for\n" +" tangent space XY normal maps)\n" +"\n" +" -swizzle rgba: Specify swizzle for the 4 input color channels using r, g, b\n" +" and a (the -separate_rg_to_color_alpha flag is equivalent to rrrg)\n" +"\n" +" -renorm: Renormalize each input image before any further\n" +" processing/compression\n" +"\n" +" -no_multithreading: Disable multithreading\n" +"\n" +" -max_threads X: Use at most X threads total when multithreading is enabled\n" +" (this includes the main thread)\n" +"\n" +" -wasi_threads: Set number of threads to use in WASI threading builds\n" +" (default=8, only used in WASI threading builds)\n" +"\n" +" -no_ktx: Disable KTX writing when unpacking (faster, less output files)\n" +"\n" +" -ktx_only: Only write KTX files when unpacking (faster, less output files)\n" +"\n" +" -write_out: Write 3dfx OUT files when unpacking FXT1 textures\n" +"\n" +" -format_only: Only unpack the specified format, by its numeric code.\n" +"\n" +" -etc1_only: Only unpack to ETC1, skipping the other texture formats\n" +" during -unpack\n" +"\n" +" -disable_hierarchical_endpoint_codebooks: Disable hierarchical endpoint\n" +" codebook usage, slower but higher quality on some compression levels\n" +"\n" +" -compare_ssim: Compute and display SSIM of image comparison (slow)\n" +"\n" +" -compare_plot: Display histogram plots in -compare mode\n" +"\n" +" -bench: UASTC benchmark mode, for development only\n" +"\n" +" -resample X Y: Resample all input textures to XxY pixels using a box filter\n" +"\n" +" -resample_factor X: Resample all input textures by scale factor X using a box\n" +" filter\n" +"\n" +" -no_sse: Forbid all SSE instruction set usage\n" +"\n" +" -validate_etc1s: Validate internal ETC1S compressor's data structures during\n" +" compression (slower, intended for development).\n" +"\n" +" -ktx2_animdata_duration X: Set KTX2animData duration field to integer value X\n" +" (only valid/useful for -tex_type video, default is 1)\n" +"\n" +" -ktx2_animdata_timescale X: Set KTX2animData timescale field to integer value\n" +" X (only valid/useful for -tex_type video, default is 15)\n" +"\n" +" -ktx2_animdata_loopcount X: Set KTX2animData loopcount field to integer value\n" +" X (only valid/useful for -tex_type video, default is 0)\n" +"\n" +" -framerate X: Set framerate in .basis header to X/frames sec.\n" +"\n" +" -ktx2_no_zstandard: Don't compress UASTC texture data using Zstandard -- store\n" +" it uncompressed instead.\n" +"\n" +" -ktx2_zstandard_level X: Set ZStandard compression level to X (see Zstandard\n" +" documentation, default level is 6)\n" +"\n" +" -tonemap_dither: Dither tonemapper's 8-bit/component output by adding a small\n" +" amount of white noise, only used with -tonemap mode\n" +"\n" +"--- Mipmap Generator Options:\n" +"\n" +"By default, SDR textures will be converted from sRGB to linear light before\n" +"mipmap filtering, then back to sRGB (for the RGB color channels) unless -linear\n" +"is specified. You can override this behavior with -mip_srgb/-mip_linear.\n" +"\n" +" -mipmap: Generate mipmaps for each source image\n" +"\n" +" -mip_srgb: Convert image to linear before filtering, then back to sRGB.\n" +" (This is set automatically by default, unless you override it.)\n" +"\n" +" -mip_linear: Keep image in linear light during mipmap filtering (i.e. do not\n" +" convert to/from sRGB for filtering purposes). (This is set automatically by\n" +" default, unless you override it.)\n" +"\n" +" -mip_scale X: Set mipmap filter kernel's scale, lower=sharper, higher=more\n" +" blurry, default is 1.0 (quite conservative).\n" +"\n" +" -mip_filter X: Set mipmap filter kernel, default is kaiser. Supported filters:\n" +" box, tent, bell, b-spline, mitchell, blackman, lanczos3, lanczos4, lanczos6,\n" +" lanczos12, kaiser, gaussian, catmullrom, quadratic_interp, quadratic_approx,\n" +" quadratic_mix\n" +"\n" +" -mip_renorm: Renormalize normal map to unit length vectors after filtering\n" +"\n" +" -mip_clamp: Use clamp addressing on borders, instead of wrapping\n" +"\n" +" -mip_fast: Use faster mipmap generation (resample from previous mip, not\n" +" always first/largest mip level). The default.\n" +"\n" +" -mip_slow: Always resample each mipmap level starting from the largest mipmap.\n" +" Higher quality, but slower. Opposite of -mip_fast. \n" +"\n" +" -mip_smallest X: Set smallest pixel dimension for generated mipmaps, default\n" +" is 1 pixel \n" +"\n" +"--- Transcoding Options (used while unpacking, validating after compression):\n" +"\n" +"These settings control the \"decode flags\" used while transcoding:\n" +"\n" +" -higher_quality_transcoding: Enable higher quality, but slower, transcoding\n" +"\n" +" -no_deblocking: Always disable adaptive deblocking filter on all block sizes\n" +" (XUASTC/ASTC LDR 4x4-12x12 only). By default only block sizes >8x6 are\n" +" deblocked while transcoding. (No deblocking ever occurs when transcoding to\n" +" ASTC: only when re-encoding ASTC to another format, to lower artifacts.)\n" +"\n" +" -force_deblocking: Always use adaptive deblocking filter, even for block sizes\n" +" <= 8x6 (XUASTC/ASTC LDR 4x4-12x12 only)\n" +"\n" +" -stronger_deblocking: Use stronger adaptive deblocking filtering (XUASTC/ASTC\n" +" LDR 4x4-12x12 only)\n" +"\n" +" -no_etc1s_chroma_filtering: Disable adaptive ETC1S transcode chroma filter,\n" +" for faster transcoding to BC7.\n" +"\n" +" -fast_xuastc_ldr_bc7_transcoding: Use much faster, but lower quality, XUASTC\n" +" LDR 4x4/6x6/8x6 direct BC7 transcoders (the default)\n" +"\n" +" -no_fast_xuastc_ldr_bc7_transcoding: Disable much faster, but slightly lower\n" +" quality, XUASTC LDR 4x4/6x6/8x6 direct BC7 transcoders\n" +"\n" +"--- Low-Level ETC1S backend endpoint/selector RDO codec options:\n" +"\n" +" -no_selector_rdo: Disable backend's selector rate distortion optimizations\n" +" (slightly faster, less noisy output, but lower quality per output bit)\n" +"\n" +" -selector_rdo_thresh X: Set selector RDO quality threshold, default is 1.25,\n" +" lower is higher quality but less quality per output bit (try 1.0-3.0)\n" +"\n" +" -no_endpoint_rdo: Disable backend's endpoint rate distortion optimizations\n" +" (slightly faster, less noisy output, but lower quality per output bit)\n" +"\n" +" -endpoint_rdo_thresh X: Set endpoint RDO quality threshold, default is 1.5,\n" +" lower is higher quality but less quality per output bit (try 1.0-3.0)\n" +"\n" +"--- Set various low-level fields in the Basis file header:\n" +"\n" +" -userdata0 X: Set 32-bit userdata0 field in Basis file header to X (X is a\n" +" signed 32-bit int)\n" +"\n" +" -userdata1 X: Set 32-bit userdata1 field in Basis file header to X (X is a\n" +" signed 32-bit int)\n" +"\n" +"--- Example LDR ETC1S/UASTC LDR 4x4 command lines:\n" +"\n" +" - basisu x.png : Compress sRGB image x.png to x.ktx2 using default settings\n" +" (multiple filenames OK, use -tex_array if you want a tex array vs. multiple\n" +" output files)\n" +"\n" +" - basisu -basis x.qoi : Compress sRGB image x.qoi to x.basis (supports 24-bit\n" +" or 32-bit .QOI files)\n" +"\n" +" - basisu x.ktx2 : Unpack x.basis to PNG/KTX files (multiple filenames OK)\n" +"\n" +" - basisu x.basis : Unpack x.basis to PNG/KTX files (multiple filenames OK)\n" +"\n" +" - basisu -uastc x.png -uastc_rdo_l 2.0 -ktx2 -stats : Compress to a\n" +" UASTC .KTX2 file with RDO (rate distortion optimization) to reduce .KTX2\n" +" compressed file size\n" +"\n" +" - basisu -file x.png -mipmap -y_flip : Compress a mipmapped x.ktx2 file from\n" +" an sRGB image named x.png, Y flip each source image\n" +"\n" +" - basisu -validate -file x.basis : Validate x.basis (check header, check file\n" +" CRC's, attempt to transcode all slices)\n" +"\n" +" - basisu -unpack -file x.basis : Validates, transcodes and unpacks x.basis to\n" +" mipmapped .KTX and RGB/A .PNG files (transcodes to all supported GPU texture\n" +" formats)\n" +"\n" +" - basisu -q 255 -file x.png -mipmap -debug -stats : Compress sRGB x.png to\n" +" x.ktx2 at quality level 255 with compressor debug output/statistics\n" +"\n" +" - basisu -linear -max_endpoints 16128 -max_selectors 16128 -file x.png :\n" +" Compress non-sRGB x.png to x.ktx2 using the largest supported manually\n" +" specified codebook sizes\n" +"\n" +" - basisu -basis -comp_level 2 -max_selectors 8192 -max_endpoints\n" +" 8192 -tex_type video -framerate 20 -multifile_printf\n" +" \"x%02u.png\" -multifile_first 1 -multifile_num 20 : Compress a 20 sRGB source\n" +" image video sequence (x01.png, x02.png, x03.png, etc.) to x01.basis\n" +"\n" +"--- Example UASTC HDR 4x4 command lines:\n" +"\n" +" - basisu x.exr : Compress a HDR .EXR (or .HDR) image to a UASTC HDR 4x4 .KTX2\n" +" file. LDR/SDR images will be upconverted to linear light HDR before\n" +" compression. See HDR upconversion options, above.\n" +"\n" +" - basisu -hdr_4x4 x.exr : Compress a HDR .EXR image to a UASTC HDR 4x4 .KTX2\n" +" file.\n" +"\n" +" - basisu x.hdr -uastc_level 0 : Compress a HDR .hdr image to a UASTC HDR\n" +" 4x4 .KTX2 file, fastest encoding but lowest quality\n" +"\n" +" - basisu -hdr x.png : Compress a LDR .PNG image to UASTC HDR 4x4 (image is\n" +" converted from sRGB to linear light first, use -hdr_ldr_no_srgb_to_linear to\n" +" disable)\n" +"\n" +" - basisu x.hdr -uastc_level 3 : Compress a HDR .hdr image to UASTC HDR 4x4 at\n" +" higher quality (-uastc_level 4 is highest quality, but very slow encoding)\n" +"\n" +" - basisu x.hdr -uastc_level 3 -mipmap -basis -stats -debug -debug_images :\n" +" Compress a HDR .hdr image to UASTC HDR 4x4, .basis output file, at higher\n" +" quality, generate mipmaps, output statistics and debug information, and write\n" +" tone mapped debug images\n" +"\n" +" - basisu x.hdr -stats -hdr_favor_astc -hdr_uber_mode -uastc_level 4 : Highest\n" +" achievable ASTC HDR 4x4 quality (very slow encoding, BC6H quality is traded\n" +" off)\n" +"\n" +"--- Example RDO ASTC/UASTC HDR 6x6 command lines:\n" +"\n" +" - basisu -hdr_6x6 x.exr : Compress a HDR .EXR (or .HDR) image to a UASTC HDR\n" +" 6x6 .KTX2 file. LDR/SDR images will be upconverted to linear light HDR before\n" +" compression. See HDR upconversion options, above.\n" +"\n" +" - basisu -lambda 1000 -hdr_6x6 x.exr : Compress a HDR .EXR (or .HDR) image to\n" +" a UASTC HDR 6x6 .KTX2 file with rate-distortion optimization (RDO), at lambda\n" +" level 1000.\n" +"\n" +" - basisu -hdr_6x6i x.exr : Compress a HDR .EXR image to a compressed\n" +" intermediate format UASTC HDR 6x6 .KTX2 file.\n" +"\n" +" - basisu -lambda 1000 -hdr_6x6i x.exr : Compress a HDR .EXR image to a\n" +" compressed intermediate format UASTC HDR 6x6 .KTX2 file with rate-distortion\n" +" optimization (RDO), at lambda level 1000.\n" +"\n" +"--- Example ASTC/XUASTC LDR 4x4-12x12 command lines:\n" +"\n" +" - basisu -ldr_6x6i -q 75 -xuastc_arith test.png : Compress test.png to XUASTC\n" +" LDR 6x6 using weight grid DCT with setting 75 and the arith profile for\n" +" higher compression.\n" +"\n" +" - basisu -ldr_4x4 test.png : Compress test.png to ASTC LDR 4x4\n" +"\n" +" - basisu -mipmap -ldr_10x5i test.png : Compress test.png to XUASTC LDR 10x5,\n" +" using lossless ZStd supercompression, with mipmaps\n" +"\n" +"--- ETC1S Texture Video Notes: Use -comp_level 2 or higher for better codebook\n" +"generation, specify very large codebooks using -max_endpoints\n" +"and -max_selectors, and reduce the default endpoint RDO threshold\n" +"(-endpoint_rdo_thresh) to around 1.25. Videos may have mipmaps and alpha\n" +"channels. Videos must always be played back by the transcoder in first to last\n" +"image order. Video files currently use I-Frames on the first image, and\n" +"P-Frames using conditional replenishment on subsequent frames.\n" +"\n" +"--- Low-level ETC1S compression (Effort) Level (-comp_level X) Details \n" +"\n" +"This setting controls the ETC1S speed vs. quality tradeoff. (Use -q to control\n" +"the quality vs. compressed size tradeoff.):\n" +"\n" +" - Level 0: Fastest, but has marginal quality and can be brittle on complex\n" +"images. Avg. Y dB: 35.45\n" +"\n" +" - Level 1: Hierarchical codebook searching, faster ETC1S encoding. 36.87 dB,\n" +"~1.4x slower vs. level 0. (This is the default setting.)\n" +"\n" +" - Level 2: Use this or higher for video. Hierarchical codebook searching.\n" +"36.87 dB, ~1.4x slower vs. level 0. (This is the v1.12's default setting.)\n" +"\n" +" - Level 3: Full codebook searching. 37.13 dB, ~1.8x slower vs. level 0.\n" +"(Equivalent to the initial release's default settings.)\n" +"\n" +" - Level 4: Hierarchical codebook searching, codebook k-means iterations. 37.15\n" +"dB, ~4x slower vs. level 0\n" +"\n" +" - Level 5: Full codebook searching, codebook k-means iterations. 37.41 dB,\n" +"~5.5x slower vs. level 0.\n" +"\n" +" - Level 6: Full codebook searching, twice as many codebook k-means iterations,\n" +"best ETC1 endpoint opt. 37.43 dB, ~12x slower vs. level 0" diff --git a/external/basis_universal/bin/basisu_mt.wasm b/external/basis_universal/bin/basisu_mt.wasm new file mode 100644 index 0000000000..d3816fc10e Binary files /dev/null and b/external/basis_universal/bin/basisu_mt.wasm differ diff --git a/external/basis_universal/bin/basisu_st.wasm b/external/basis_universal/bin/basisu_st.wasm new file mode 100644 index 0000000000..fb51493882 Binary files /dev/null and b/external/basis_universal/bin/basisu_st.wasm differ diff --git a/external/basis_universal/bin/clean.bat b/external/basis_universal/bin/clean.bat index d31658fc0a..6952615b57 100644 --- a/external/basis_universal/bin/clean.bat +++ b/external/basis_universal/bin/clean.bat @@ -1,7 +1,13 @@ +@echo off del *.exr del *.png -del *.ktx -del *.ktx2 del *.dds del *.astc -del *.basis +del *.tga + +for %%F in (*.ktx) do ( + if /I "%%~xF"==".ktx" ( + echo Deleting "%%F" + del "%%F" + ) +) diff --git a/external/basis_universal/bin/clean.sh b/external/basis_universal/bin/clean.sh new file mode 100644 index 0000000000..e8b0b96b72 --- /dev/null +++ b/external/basis_universal/bin/clean.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +rm -f *.exr +rm -f *.png +rm -f *.dds +rm -f *.astc +rm -f *.ktx +rm -f *.tga + diff --git a/external/basis_universal/bin/example_capi_mt.wasm b/external/basis_universal/bin/example_capi_mt.wasm new file mode 100644 index 0000000000..218ba59bab Binary files /dev/null and b/external/basis_universal/bin/example_capi_mt.wasm differ diff --git a/external/basis_universal/bin/example_capi_st.wasm b/external/basis_universal/bin/example_capi_st.wasm new file mode 100644 index 0000000000..5dc3f13758 Binary files /dev/null and b/external/basis_universal/bin/example_capi_st.wasm differ diff --git a/external/basis_universal/bin/example_mt.wasm b/external/basis_universal/bin/example_mt.wasm new file mode 100644 index 0000000000..8d816c9463 Binary files /dev/null and b/external/basis_universal/bin/example_mt.wasm differ diff --git a/external/basis_universal/bin/example_st.wasm b/external/basis_universal/bin/example_st.wasm new file mode 100644 index 0000000000..b99a16b4ca Binary files /dev/null and b/external/basis_universal/bin/example_st.wasm differ diff --git a/external/basis_universal/bin/example_transcoding_mt.wasm b/external/basis_universal/bin/example_transcoding_mt.wasm new file mode 100644 index 0000000000..6ba4802684 Binary files /dev/null and b/external/basis_universal/bin/example_transcoding_mt.wasm differ diff --git a/external/basis_universal/bin/example_transcoding_st.wasm b/external/basis_universal/bin/example_transcoding_st.wasm new file mode 100644 index 0000000000..4824892875 Binary files /dev/null and b/external/basis_universal/bin/example_transcoding_st.wasm differ diff --git a/external/basis_universal/bin/runw.bat b/external/basis_universal/bin/runw.bat new file mode 100644 index 0000000000..016441957f --- /dev/null +++ b/external/basis_universal/bin/runw.bat @@ -0,0 +1,6 @@ +@ECHO OFF +REM Example: "runw.bat test_images/xmen.png" +REM Example: "runw.bat /bik/bik1.png" + +REM wasmtime --dir=. --dir=.. --dir=..\test_files --dir=d:/dev/test_images::/test_images --dir=d:/dev/test_images/bik::/bik basisu_st.wasm %* +wasmtime --dir=. --dir=.. --dir=..\test_files basisu_st.wasm %* diff --git a/external/basis_universal/bin/runw.sh b/external/basis_universal/bin/runw.sh new file mode 100644 index 0000000000..6b7f3952fd --- /dev/null +++ b/external/basis_universal/bin/runw.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +wasmtime run --dir=. --dir=../test_files ./basisu_st.wasm "$@" diff --git a/external/basis_universal/bin/runwt.bat b/external/basis_universal/bin/runwt.bat new file mode 100644 index 0000000000..d397419072 --- /dev/null +++ b/external/basis_universal/bin/runwt.bat @@ -0,0 +1,7 @@ +@ECHO OFF +REM Example: "runw.bat test_images/xmen.png" +REM Example: "runw.bat /bik/bik1.png" + +REM wasmtime --wasm threads=yes --wasi threads=yes --dir=. --dir=.. --dir=..\test_files::/test_files --dir=d:/dev/test_images::/test_images --dir=d:/dev/test_images/bik::/bik basisu_mt.wasm %* + +wasmtime --wasm threads=yes --wasi threads=yes --dir=. --dir=.. --dir=..\test_files::/test_files basisu_mt.wasm %* diff --git a/external/basis_universal/bin/runwt.sh b/external/basis_universal/bin/runwt.sh new file mode 100644 index 0000000000..3009600e42 --- /dev/null +++ b/external/basis_universal/bin/runwt.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +wasmtime run --dir=. --dir=../test_files --wasm threads=yes --wasi threads=yes ./basisu_mt.wasm "$@" diff --git a/external/basis_universal/bin/test_wasm.bat b/external/basis_universal/bin/test_wasm.bat new file mode 100644 index 0000000000..0091bbc5af --- /dev/null +++ b/external/basis_universal/bin/test_wasm.bat @@ -0,0 +1,2 @@ +wasmtime --dir=. --dir=.. --dir=..\test_files basisu.wasm -test + diff --git a/external/basis_universal/build_python_win.bat b/external/basis_universal/build_python_win.bat new file mode 100644 index 0000000000..6805cce7d2 --- /dev/null +++ b/external/basis_universal/build_python_win.bat @@ -0,0 +1,57 @@ +@echo off + +echo =========================================== +echo Building Python extensions (Windows) +echo =========================================== + +REM Set the Python executable path (edit if needed) +set PY_EXE=C:\Users\richg\AppData\Local\Programs\Python\Python312\python.exe + +REM Ensure Python exists +if not exist "%PY_EXE%" ( + echo ERROR: Python 3.12 executable not found: + echo %PY_EXE% + echo Please install Python 3.12 or update PY_EXE in this script. + exit /b 1 +) + +REM Create build directory if missing +if not exist build_python_win ( + echo Creating build_python_win directory... + mkdir build_python_win +) + +cd build_python_win + +echo Running CMake configure... +cmake -G "Visual Studio 17 2022" -A x64 ^ + -DBUILD_PYTHON=ON ^ + -DBUILD_WASM=OFF ^ + -DPYTHON_EXECUTABLE="%PY_EXE%" ^ + .. + +IF ERRORLEVEL 1 ( + echo. + echo *** CMake configure FAILED *** + exit /b 1 +) + +echo. +echo CMake configure OK. +echo Starting build... + +cmake --build . --config Release + +IF ERRORLEVEL 1 ( + echo. + echo *** Build FAILED *** + exit /b 1 +) + +echo. +echo =========================================== +echo Build SUCCESSFUL! +echo Output in: python/basisu_py/ +echo =========================================== + +exit /b 0 diff --git a/external/basis_universal/cmd_help/cmd_help.txt b/external/basis_universal/cmd_help/cmd_help.txt new file mode 100644 index 0000000000..231147db58 --- /dev/null +++ b/external/basis_universal/cmd_help/cmd_help.txt @@ -0,0 +1,713 @@ +Basis Universal LDR/HDR GPU Texture Supercompression System v1.65.0 (x64) +Copyright (C) 2019-2026 Binomial LLC, All rights reserved + +Usage: basisu filename [filename ...] +See project and wiki at: https://github.com/BinomialLLC/basis_universal + +The default processing mode is compression of one or more .PNG/.TGA/.JPG/.QOI/ +.DDS/.EXR/.HDR files to a LDR or HDR .KTX2 file. Alternate modes: + + -unpack: Use transcoder to unpack a .basis/.KTX2 file to one or more .KTX, + .DDS, .PNG, .ASTC, etc. files. + + -info: Display high-level information about a .basis/.KTX2 file + + -validate: Validate and display information about a .basis/.KTX2 file + + -compare: Compare two LDR PNG/BMP/TGA/JPG/QOI images specified with -file, + output PSNR and SSIM statistics and RGB/A delta images + + -compare_hdr: Compare two HDR .EXR/.HDR images specified with -file, output + PSNR statistics and RGB delta images + + -tonemap: Tonemap an HDR or EXR image to PNG at multiple exposures, use -file + to specify filename + + -peek_astc: Read an .astc file and calculate statistics (for testing/dev) + + -version or --version: Print version and exit + +--- Intro: + +This tool compresses LDR/SDR and HDR images and textures to a Basis Universal +supercompressed GPU texture, which can be written to supercompressed .basis or +standard .KTX2 files. It supports a number of SDR and HDR codecs, each with +different quality, transcoding performance, and bitrate tradeoffs. A SDR/HDR +mipmap generator is also included. This tool can also examine and +unpack .KTX2/.basis files to .PNG, .KTX (v1), .ASTC, or .DDS files using its +single source file transcoder library. + +The list of supported texture/supercompressed texture codecs: + -LDR: ETC1S (SDR default), RDO UASTC LDR 4x4, ASTC or XUASTC LDR 4x4-12x12 + -HDR: UASTC HDR 4x4 (HDR default), RDO ASTC HDR 6x6, UASTC HDR 6x6 + +RDO=Rate-Distortion Optimization. Two key parameters (quality and effort) have +been unified across all the codecs: + +- The -quality X parameter, where X ranges from [0, 100], controls the +compression quality vs. bitrate (output file size) tradeoff for those codecs +supporting supercompression or RDO (Rate-distortion optimization). 100=max +quality, and lower levels produce smaller files with more distortion. + +- The -effort X parameter, where X ranges from [0, 10], controls the +compression speed (and max CPU usage) vs. max achievable quality tradeoff. +Low efforts result in more distortion/artifacts, but faster compression. Lower +efforts result in less utilization of the underlying GPU block format's +capabilities. + +Key Definitions: +"Quality" controls the explicit tradeoff between output distortion and output +file bitrate (in bits per pixel or target). At max quality (100) each +compressor will output the lowest distortion it's capable of at its currently +configured effort level. + +"Effort" controls how intensely a compressor uses the CPU to focus on each +block format's encoding capabilities. Low effort levels only target a set of +core or basic capabilities (specific to each output target format), while +higher effort levels allow each compressor to explore more of each target's +features (at the cost of higher CPU time). Lower effort levels result in more +brittle compression (higher distortion on tough image/texture features). +Effort=0 fastest compression, effort=10=extremely slow. + +Unless an explicit mode is specified, if one or more files have the .basis +or .KTX2 extension this tool defaults to unpack mode. + +By default, the compressor assumes the input is in the sRGB colorspace (like +typical photos/albedo textures). If the input is NOT sRGB (like a normal map), +be sure to specify -linear for less artifacts. Depending on the content type, +some experimentation may be needed. + +The TinyEXR library is used to read .EXR images. Crucially, this small library +does not support all .EXR compression methods. For unsupported images, you can +use a tool like ImageMagick to convert them to uncompressed .EXR. + +For .DDS source files: Mipmapped or plain 2D textures (but not cubemaps) are +supported. Only uncompressed 32-bit RGBA/BGRA, half float RGBA, or float +RGBA .DDS files are supported. In -tex_array mode, if a .DDS file is specified, +all source files must be in .DDS format. + +Filenames prefixed with a @ symbol are read as filename listing files. Listing +text files specify which actual filenames to process (one filename per line). + +--- High-Level Texture Mode (Codec) Selection: + + 1. -etc1s: Encode to supercompressed ETC1S LDR (the default for SDR/LDR + inputs). Roughly .8-2.5 bpp. Supports temporal texture supercompression + (texture video) with skip blocks (Conditional Replenishment), with global + codebooks shared across all frames. + + 2. -uastc/-uastc_ldr: UASTC LDR 4x4. Encode to UASTC LDR 4x4, a custom high + quality virtual texture format designed for fast transcoding to numerous + GPU texture formats. Roughly 5-8 bpp. Supports RDO encoding using -lambda X + option - see options below. In this mode the multi-target compressor + optimizes for a balance of transcoded ASTC 4x4 LDR and BC7 quality. + + 3. -hdr/-hdr_4x4: UASTC HDR 4x4. Encode input as UASTC HDR 4x4 (the default if + any input file has the .EXR or .HDR extension, or if any .DDS file is HDR). + Output is standard, but constrained, ASTC HDR 4x4. Roughly 5-8 bpp. In this + mode the dual-target compressor optimizes for a balance of transcoded ASTC + 4x4 HDR and BC6H quality. + + 4. -hdr_6x6: ASTC HDR 6x6. Encode input as RDO or highest quality standard + ASTC HDR 6x6. Use -quality (preferred) or -lambda X (low-level, try + 100-20000 or higher) option to enable RDO ASTC HDR 6x6, where x controls + the quality vs. size tradeoff. Roughly 1.2-3.2 bpp. + + 5. -hdr_6x6i: UASTC HDR 6x6. Encode input as supercompressed UASTC HDR 6x6 + intermediate. Use -quality (preferred) or -lambda X (low-level, try + 100-20000 or higher) option to enable RDO UASTC HDR 6x6, where x controls + the quality vs. size tradeoff. Roughly 1-3.2 bpp. + + 6. XUASTC LDR 4x4-12x12: -ldr_4x4i, -ldr_5x4i, -ldr_5x5i, -ldr_6x5i, + -ldr_6x6i, -ldr_8x5i, -ldr_8x6i, -ldr_10x5i, -ldr_10x6i, -ldr_8x8i, + -ldr_10x8i, -ldr_10x10i, -ldr_12x10i, -ldr_12x12i: + Compress to supercompressed XUASTC LDR/SDR using the specific + ASTC block size. See additional ASTC/XUASTC LDR specific options + (-effort, -quality, -xy, -ts, -tl, etc.) below. Roughly .3-5.7 bpp + + 7. ASTC LDR 4x4-12x12: -ldr_4x4, -ldr_5x4, -ldr_5x5, -ldr_6x5, -ldr_6x6, + -ldr_8x5, -ldr_8x6, -ldr_10x5, -ldr_10x6, -ldr_8x8, -ldr_10x8, + -ldr_10x10, -ldr_12x10, -ldr_12x12: + Compress to standard or ZStd supercompressed ASTC LDR/SDR using + the specific ASTC block size. See additional ASTC LDR specific + options (-effort, -quality, -xy, -ts, -tl, etc.) below. .89-8 bpp before + ZStd compression. + +--- Tool Options: + + -ktx2: Write .KTX2 files (the default). By default, UASTC LDR/HDR 4x4 and ASTC + 6x6 files will be compressed using Zstandard unless -ktx2_no_zstandard is + specified. + + -basis: Write .basis files instead of .KTX2 files. + + -file filename.png/tga/jpg/qoi/exr/hdr: Input image filename, multiple images + are OK, use -file X for each input filename (prefixing input filenames + with -file is optional) + + -alpha_file filename.png/tga/jpg/qoi: Input alpha image filename, multiple + images are OK, use -file X for each input filename (must be paired + with -file), images converted to REC709 grayscale and used as input alpha + + -output_file filename: Output .basis/.KTX2 filename + + -output_path: Output .basis/.KTX2 files to specified directory. + + -debug or -verbose: Enable codec debug print to stdout (slightly slower). + + -debug_images: Enable codec debug images (much slower). + + -stats: Compute and display image quality metrics (slightly to much slower). + + -individual: Process input images individually and output + multiple .basis/.KTX2 files (not as a texture array - this is now the default + as of v1.16) + + -parallel: Compress multiple textures simultaneously (one per thread), instead + of one at a time. Compatible with OpenCL mode. This is much faster, but in + OpenCL mode the driver is pushed harder, and the CLI output will be jumbled. + + -linear: Use linear colorspace metrics (instead of the default sRGB or scaled + RGB for HDR), write linear transfer function setting to KTX2/basis file, and + by default linear (not sRGB) mipmap filtering (unless overridden). Same + as -tl. + + -srgb: Use sRGB colorspace metrics, write sRGB transfer function setting to + KTX2/basis file, and by default use sRGB mipmap filtering (unless + overridden). Same as -ts. + + -tex_type <2d, 2darray, 3d, video, cubemap>: Set Basis file header's texture + type field. Cubemap arrays require multiples of 6 images, in X+, X-, Y+, Y-, + Z+, Z- order, each image must be the same resolutions. 2d=arbitrary 2D + images, 2darray=2D array, 3D=volume texture slices, video=video frames, + cubemap=array of faces. For 2darray/3d/cubemaps/video, each source image's + dimensions and # of mipmap levels must be the same. For video, the .basis + file will be written with the first frame being an I-Frame, and subsequent + frames being P-Frames (using conditional replenishment). Playback must always + occur in order from first to last image. + + -cubemap: same as -tex_type cubemap + + -tex_array: Process input images as a single texture array and write a + single .basis/.KTX2 file (the former default before v1.16) + + -fuzz_testing: Use with -validate: Disables CRC16 validation of file contents + before transcoding + + -multifile_printf: printf() format string to use to compose multiple filenames + + -multifile_first: The index of the first file to process, default is 0 (must + specify -multifile_printf and -multifile_num) + + -multifile_num: The total number of files to process. + + -opencl: Enable OpenCL usage (currently only accelerates ETC1S encoding) + + -opencl_serialize: Serialize all calls to the OpenCL driver (to work around + buggy drivers, only useful with -parallel) + +--- ETC1S specific options (-etc1s - the LDR/SDR default): + + -quality X and -effort X: Set quality (1-100) and effort (0-10) levels + + -q X: Low-level ETC1S quality level, 1-255, default is 128, lower=better + compression/lower quality/faster, higher=less compression/higher + quality/slower, default is 128. For even higher quality, + use -max_endpoints/-max_selectors. (-quality is preferred.) + + -comp_level X: Low-level ETC1S speed vs. quality tradeoff. Range is 0-6, + default is 1. Higher values=MUCH slower, but slightly higher quality. Higher + levels intended for videos. (-effort is preferred.) + + -max_endpoints X: ETC1S: Manually set the max number of color endpoint + clusters from 1-16128, use instead of -q + + -max_selectors X: ETC1S: Manually set the max number of color selector + clusters from 1-16128, use instead of -q + +--- UASTC LDR/HDR 4x4 specific options (-uastc or -uastc_ldr): + + -quality X and -effort X: Set quality (1-100) and effort (0-10) levels + + -uastc, -uastc_ldr or -uastc_ldr_4x4: Enable UASTC LDR 4x4 texture mode, + instead of the default ETC1S mode. Significantly higher texture quality, but + much larger (~8bpp) files. (Note that UASTC LDR 4x4 .basis files must be + losslessly compressed by the user.) + + -uastc_level: Set low-level UASTC LDR/HDR 4x4 encoding effort level. LDR Range + is [0,4], default is 2, higher=slower but higher quality. 0=fastest/lowest + quality, 3=slowest practical option, 4=impractically slow/highest achievable + quality. UASTC HDR 4x4 range is [0,4]: higher=slower, but higher quality. HDR + 4x4 default level=1. + + -uastc_rdo_l X: Enable UASTC LDR 4x4 RDO post-processing and set the low-level + UASTC LDR 4x4 RDO quality scalar (lambda) to X. Lower values=higher + quality/larger LZ compressed files, higher values=lower quality/smaller LZ + compressed files. Good range to try is [.25-10]. Note: Previous versons used + the -uastc_rdo_q option, which was removed because the RDO algorithm was + changed. + + -uastc_rdo_d X: Set UASTC LDR 4x4 RDO dictionary size in bytes. Default is + 4096, max is 65536. Lower values=faster, but less compression. + + -uastc_rdo_b X: Set UASTC LDR 4x4 RDO max smooth block error scale. Range is + [1,300]. Default is 10.0, 1.0=disabled. Larger values suppress more artifacts + (and allocate more bits) on smooth blocks. + + -uastc_rdo_s X: Set UASTC LDR 4x4 RDO max smooth block standard deviation. + Range is [.01,65536]. Default is 18.0. Larger values expand the range of + blocks considered smooth. + + -uastc_rdo_f: Don't favor simpler UASTC LDR 4x4 modes in RDO mode. + + -uastc_rdo_m: Disable RDO multithreading (slightly higher compression, + deterministic). + +--- UASTC HDR 4x4 specific options (-hdr or -hdr_4x4 - the HDR default): + + -hdr, -hdr_4x4, or -uastc_hdr_4x4: Enable UASTC HDR 4x4 mode + + -quality X and -effort X: Set quality (1-100) and effort (0-10) levels + + -uastc_level X: Sets the low-level UASTC HDR 4x4 compressor's effort level. + Valid range is [0,4]: higher=slower but higher quality. HDR + default=1. Level 0=fastest/lowest quality, 3=highest practical + setting, 4=exhaustive + + -hdr_uber_mode: Allow the UASTC HDR 4x4 encoder to try varying the CEM 11 + selectors more for slightly higher quality (slower). This may negatively + impact BC6H quality, however. + + -hdr_ultra_quant: UASTC HDR 4x4: Try to find better quantized CEM 7/11 + endpoint values (slower). + + -hdr_favor_astc: UASTC HDR 4x4: By default the dual-target UASTC HDR 4x4 + encoder tries to strike a balance or even slightly favor BC6H quality. If + this option is specified, ASTC HDR 4x4 quality is favored instead. + +--- ASTC/UASTC HDR 6x6 specific options (-hdr_6x6 or -hdr_6x6i): + +Internally both modes use the same compressor which can generate either +standard ASTC HDR 6x6 (with optional RDO) or UASTC HDR 6x6 (supercompressed +with a custom format). + + -hdr_6x6 or -astc_hdr_6x6: Enable RDO ASTC HDR 6x6 mode + + -hdr_6x6i or -uastc_hdr_6x6: Enable UASTC HDR 6x6 mode + + -quality X and -effort X: Set quality (1-100) and effort (0-10) levels + + -lambda X: Low-level option to enable rate distortion optimization (RDO) and + directly control the HDR 6x6 compressor's lambda setting. The + higher this value, the lower the quality, but the smaller the file + size. Try 100-20000, or higher values on some images. Upconverted + SDR images can generally tolerate much higher lambda settings vs. + true HDR images. + + -hdr_6x6_level X: Low-level option to set the codec to 6x6 HDR mode (same + as -hdr_6x6) and controls encoder performance vs. max quality + tradeoff. X may range from [0,12]. Default level is 2. Higher + values result in better quality but slower encoding. Values above + 10 are extremely slow. + + -hdr_6x6i_level X: Low-level option to set the codec to 6x6 HDR intermediate + mode (same as -hdr_6x6i) and controls encoder performance vs. max + quality tradeoff. X may range from [0,12]. Default level is 2. + + -rec_2020: The input image's gamut is Rec. 2020 vs. the default Rec. 709 - for + accurate colorspace error calculations. This value will also be + written to the KTX2 file's header in the DFD. + + -hdr_6x6_jnd + X, -hdr_6x6_extra_pats, -hdr_6x6_brute_force_pats, + -hdr_6x6_comp_levels X Y, or -hdr_6x6i_comp_levels X Y: Low-level + control over the encoder's configuration. + +--- SDR/LDR->HDR upconversion options (only used when encoding to HDR formats +from an LDR/SDR source image): + + -hdr_ldr_no_srgb_to_linear: If specified, LDR images will NOT be converted to + normalized linear light (via a sRGB->Linear conversion) during SDR->HDR + upconversion before compressing as HDR. + + -hdr_ldr_upconversion_nit_multiplier X: Specify how many nits (candelas per + sq. meter) LDR/SDR images are converted to after converting to linear + light. Default is 100 nits. Note: Previous builds used 1 nit. Common + values are 80-100 nits. + +--- ASTC LDR/XUASTC LDR specific options (-ldr_4x4 or -ldr_4x4i, up to 12x12): + +Internally both modes (ASTC 4x4-12x12 and XUASTC 4x4-12x12) use the same +core compressor but with different outputs. All 14 standard ASTC block +sizes are supported (see the list below). + + -ldr_4x4-12x12 or -astc_ldr_4x4-12x12: Enable ASTC LDR 4x4-12x12 mode + + -ldr_4x4i-12x12 or -xuastc_ldr_4x4-12x12: Enable XUASTC LDR 4x4-12x12 mode + + -quality X: Enables lossy weight grid DCT and sets DCT quality level [1,100] + (defaults to no DCT). Higher=better quality, but higher bitrate. Good values + to try are 30-90. Default is no weight grid DCT. + + -effort X: Set encoder effort level [1,10]: Encoding speed tradeoff, + higher=slower but potentially higher overall quality. Default=3, 10=Insane. + + -xuastc_arith, -xuastc_hybrid, -xuastc_zstd: Set transcoding speed vs. + compression ratio tradeoff by selecting the output profile/syntax. Default + is -xuastc_zstd (fastest, lowest ratio). ZStd is fastest/lowest ratio, arith + is slowest/highest ratio (3-15% better vs. ZStd). + + -xy: Enables lossy supercompression using windowed/bounded RDO for extra + compression (default is lossless supercompression of the XUASTC texture data + unless DCT is enabled) + + -xyd: Disables lossy supercompression (default, but automatically enabled + if -quality less than 100 is specified ) + + -xs: Force disable 2-3 subset usage in all effort levels (lower quality but + faster compression and faster transcoding to BC7 at certain block sizes) + + -xp: Force disable RGB dual plane usage in all effort levels (lower quality + but faster compression and faster transcoding to BC7 at certain block sizes) + + -ts: Use LDR sRGB ASTC decoding profile - the default. This parameter should + match how the developer will decode or sample the ASTC texture data. Inverse + of -tl. Same as -srgb. + + -tl: Use LDR Linear ASTC decoding profile. Inverse of -ts. Same as -linear. + + -weights X Y Z W: Set unsigned integer channel error weights. Defaults are + 1,1,1,1. Useful to favor certain channels during compression. + + -ls_min_psnr X, -ls_min_alpha_psnr X, -ls_thresh_psnr X, -ls_thresh_alpha_psnr + X, -ls_thresh_edge_psnr X, -ls_thresh_edge_alpha_psnr X: Windowed/bounded RDO + settings (Lossy supercompression must be enabled, or -xy.) + + -xuastc_blurring: Experimental - enable blurred block candidates (MUCH slower + compression, higher quality). + +These modes support all the standard ASTC block sizes. The larger the block +size, the lower the bitrate on disk and in memory, but the more noticeable the +artifacts. Some block sizes (4x4, 6x6, and 8x6) have specially optimized direct +transcoding paths to BC7. For faster direct BC7 transcoding at these block +sizes, disable RGB dual plane (-xp) and subset usage (-xs). + + Block Size Base/Memory Size + 1. 4x4 8.00 bpp + 2. 5x4 6.40 bpp + 3. 5x5 5.12 bpp + 4. 6x5 4.27 bpp + 5. 6x6 3.56 bpp + 6. 8x5 3.20 bpp + 7. 8x6 2.67 bpp + 8. 10x5 2.56 bpp + 9. 10x6 2.13 bpp + 10. 8x8 2.00 bpp + 11. 10x8 1.60 bpp + 12. 10x10 1.28 bpp + 13. 12x10 1.07 bpp + 14. 12x12 0.89 bpp + +--- More options: + + -test: Run an automated LDR ETC1S/UASTC LDR 4x4 encoding and transcoding test. + Returns EXIT_FAILURE on any failures. + + -test_hdr_4x4/-test_hdr_6x6/-test_hdr_6x6i: Run automated UASTC HDR encoding + and transcoding tests. Returns EXIT_FAILURE on any failures. + + -test_xuastc: Run an automated XUASTC LDR encoding and transcoding test. + Returns EXIT_FAILURE on any failures. + + -test_dir: Optional directory of test files. Defaults to "../test_files". + + -y_flip: Flip input images vertically before compression + + -normal_map: Tunes codec parameters for better quality on normal maps (linear + colorspace metrics, linear mipmap filtering, no selector RDO, no sRGB) + + -no_alpha: Always output non-alpha basis files, even if one or more inputs has + alpha + + -force_alpha: Always output alpha basis files, even if no inputs has alpha + + -separate_rg_to_color_alpha: Separate input R and G channels to RGB and A (for + tangent space XY normal maps) + + -swizzle rgba: Specify swizzle for the 4 input color channels using r, g, b + and a (the -separate_rg_to_color_alpha flag is equivalent to rrrg) + + -renorm: Renormalize each input image before any further + processing/compression + + -no_multithreading: Disable multithreading + + -max_threads X: Use at most X threads total when multithreading is enabled + (this includes the main thread) + + -wasi_threads: Set number of threads to use in WASI threading builds + (default=8, only used in WASI threading builds) + + -no_ktx: Disable KTX writing when unpacking (faster, less output files) + + -ktx_only: Only write KTX files when unpacking (faster, less output files) + + -write_out: Write 3dfx OUT files when unpacking FXT1 textures + + -format_only: Only unpack the specified format, by its numeric code. + + -etc1_only: Only unpack to ETC1, skipping the other texture formats + during -unpack + + -disable_hierarchical_endpoint_codebooks: Disable hierarchical endpoint + codebook usage, slower but higher quality on some compression levels + + -compare_ssim: Compute and display SSIM of image comparison (slow) + + -compare_plot: Display histogram plots in -compare mode + + -bench: UASTC benchmark mode, for development only + + -resample X Y: Resample all input textures to XxY pixels using a box filter + + -resample_factor X: Resample all input textures by scale factor X using a box + filter + + -no_sse: Forbid all SSE instruction set usage + + -validate_etc1s: Validate internal ETC1S compressor's data structures during + compression (slower, intended for development). + + -ktx2_animdata_duration X: Set KTX2animData duration field to integer value X + (only valid/useful for -tex_type video, default is 1) + + -ktx2_animdata_timescale X: Set KTX2animData timescale field to integer value + X (only valid/useful for -tex_type video, default is 15) + + -ktx2_animdata_loopcount X: Set KTX2animData loopcount field to integer value + X (only valid/useful for -tex_type video, default is 0) + + -framerate X: Set framerate in .basis header to X/frames sec. + + -ktx2_no_zstandard: Don't compress UASTC texture data using Zstandard -- store + it uncompressed instead. + + -ktx2_zstandard_level X: Set ZStandard compression level to X (see Zstandard + documentation, default level is 6) + + -tonemap_dither: Dither tonemapper's 8-bit/component output by adding a small + amount of white noise, only used with -tonemap mode + +--- Mipmap Generator Options: + +By default, SDR textures will be converted from sRGB to linear light before +mipmap filtering, then back to sRGB (for the RGB color channels) unless -linear +is specified. You can override this behavior with -mip_srgb/-mip_linear. + + -mipmap: Generate mipmaps for each source image + + -mip_srgb: Convert image to linear before filtering, then back to sRGB. + (This is set automatically by default, unless you override it.) + + -mip_linear: Keep image in linear light during mipmap filtering (i.e. do not + convert to/from sRGB for filtering purposes). (This is set automatically by + default, unless you override it.) + + -mip_scale X: Set mipmap filter kernel's scale, lower=sharper, higher=more + blurry, default is 1.0 (quite conservative). + + -mip_filter X: Set mipmap filter kernel, default is kaiser. Supported filters: + box, tent, bell, b-spline, mitchell, blackman, lanczos3, lanczos4, lanczos6, + lanczos12, kaiser, gaussian, catmullrom, quadratic_interp, quadratic_approx, + quadratic_mix + + -mip_renorm: Renormalize normal map to unit length vectors after filtering + + -mip_clamp: Use clamp addressing on borders, instead of wrapping + + -mip_fast: Use faster mipmap generation (resample from previous mip, not + always first/largest mip level). The default. + + -mip_slow: Always resample each mipmap level starting from the largest mipmap. + Higher quality, but slower. Opposite of -mip_fast. + + -mip_smallest X: Set smallest pixel dimension for generated mipmaps, default + is 1 pixel + +--- Transcoding Options (used while unpacking, validating after compression): + +These settings control the "decode flags" used while transcoding: + + -higher_quality_transcoding: Enable higher quality, but slower, transcoding + + -no_deblocking: Always disable adaptive deblocking filter on all block sizes + (XUASTC/ASTC LDR 4x4-12x12 only). By default only block sizes >8x6 are + deblocked while transcoding. (No deblocking ever occurs when transcoding to + ASTC: only when re-encoding ASTC to another format, to lower artifacts.) + + -force_deblocking: Always use adaptive deblocking filter, even for block sizes + <= 8x6 (XUASTC/ASTC LDR 4x4-12x12 only) + + -stronger_deblocking: Use stronger adaptive deblocking filtering (XUASTC/ASTC + LDR 4x4-12x12 only) + + -no_etc1s_chroma_filtering: Disable adaptive ETC1S transcode chroma filter, + for faster transcoding to BC7. + + -fast_xuastc_ldr_bc7_transcoding: Use much faster, but lower quality, XUASTC + LDR 4x4/6x6/8x6 direct BC7 transcoders (the default) + + -no_fast_xuastc_ldr_bc7_transcoding: Disable much faster, but slightly lower + quality, XUASTC LDR 4x4/6x6/8x6 direct BC7 transcoders + +--- Low-Level ETC1S backend endpoint/selector RDO codec options: + + -no_selector_rdo: Disable backend's selector rate distortion optimizations + (slightly faster, less noisy output, but lower quality per output bit) + + -selector_rdo_thresh X: Set selector RDO quality threshold, default is 1.25, + lower is higher quality but less quality per output bit (try 1.0-3.0) + + -no_endpoint_rdo: Disable backend's endpoint rate distortion optimizations + (slightly faster, less noisy output, but lower quality per output bit) + + -endpoint_rdo_thresh X: Set endpoint RDO quality threshold, default is 1.5, + lower is higher quality but less quality per output bit (try 1.0-3.0) + +--- Set various low-level fields in the Basis file header: + + -userdata0 X: Set 32-bit userdata0 field in Basis file header to X (X is a + signed 32-bit int) + + -userdata1 X: Set 32-bit userdata1 field in Basis file header to X (X is a + signed 32-bit int) + +--- Example LDR ETC1S/UASTC LDR 4x4 command lines: + + - basisu x.png : Compress sRGB image x.png to x.ktx2 using default settings + (multiple filenames OK, use -tex_array if you want a tex array vs. multiple + output files) + + - basisu -basis x.qoi : Compress sRGB image x.qoi to x.basis (supports 24-bit + or 32-bit .QOI files) + + - basisu x.ktx2 : Unpack x.basis to PNG/KTX files (multiple filenames OK) + + - basisu x.basis : Unpack x.basis to PNG/KTX files (multiple filenames OK) + + - basisu -uastc x.png -uastc_rdo_l 2.0 -ktx2 -stats : Compress to a + UASTC .KTX2 file with RDO (rate distortion optimization) to reduce .KTX2 + compressed file size + + - basisu -file x.png -mipmap -y_flip : Compress a mipmapped x.ktx2 file from + an sRGB image named x.png, Y flip each source image + + - basisu -validate -file x.basis : Validate x.basis (check header, check file + CRC's, attempt to transcode all slices) + + - basisu -unpack -file x.basis : Validates, transcodes and unpacks x.basis to + mipmapped .KTX and RGB/A .PNG files (transcodes to all supported GPU texture + formats) + + - basisu -q 255 -file x.png -mipmap -debug -stats : Compress sRGB x.png to + x.ktx2 at quality level 255 with compressor debug output/statistics + + - basisu -linear -max_endpoints 16128 -max_selectors 16128 -file x.png : + Compress non-sRGB x.png to x.ktx2 using the largest supported manually + specified codebook sizes + + - basisu -basis -comp_level 2 -max_selectors 8192 -max_endpoints + 8192 -tex_type video -framerate 20 -multifile_printf + "x%02u.png" -multifile_first 1 -multifile_num 20 : Compress a 20 sRGB source + image video sequence (x01.png, x02.png, x03.png, etc.) to x01.basis + +--- Example UASTC HDR 4x4 command lines: + + - basisu x.exr : Compress a HDR .EXR (or .HDR) image to a UASTC HDR 4x4 .KTX2 + file. LDR/SDR images will be upconverted to linear light HDR before + compression. See HDR upconversion options, above. + + - basisu -hdr_4x4 x.exr : Compress a HDR .EXR image to a UASTC HDR 4x4 .KTX2 + file. + + - basisu x.hdr -uastc_level 0 : Compress a HDR .hdr image to a UASTC HDR + 4x4 .KTX2 file, fastest encoding but lowest quality + + - basisu -hdr x.png : Compress a LDR .PNG image to UASTC HDR 4x4 (image is + converted from sRGB to linear light first, use -hdr_ldr_no_srgb_to_linear to + disable) + + - basisu x.hdr -uastc_level 3 : Compress a HDR .hdr image to UASTC HDR 4x4 at + higher quality (-uastc_level 4 is highest quality, but very slow encoding) + + - basisu x.hdr -uastc_level 3 -mipmap -basis -stats -debug -debug_images : + Compress a HDR .hdr image to UASTC HDR 4x4, .basis output file, at higher + quality, generate mipmaps, output statistics and debug information, and write + tone mapped debug images + + - basisu x.hdr -stats -hdr_favor_astc -hdr_uber_mode -uastc_level 4 : Highest + achievable ASTC HDR 4x4 quality (very slow encoding, BC6H quality is traded + off) + +--- Example RDO ASTC/UASTC HDR 6x6 command lines: + + - basisu -hdr_6x6 x.exr : Compress a HDR .EXR (or .HDR) image to a UASTC HDR + 6x6 .KTX2 file. LDR/SDR images will be upconverted to linear light HDR before + compression. See HDR upconversion options, above. + + - basisu -lambda 1000 -hdr_6x6 x.exr : Compress a HDR .EXR (or .HDR) image to + a UASTC HDR 6x6 .KTX2 file with rate-distortion optimization (RDO), at lambda + level 1000. + + - basisu -hdr_6x6i x.exr : Compress a HDR .EXR image to a compressed + intermediate format UASTC HDR 6x6 .KTX2 file. + + - basisu -lambda 1000 -hdr_6x6i x.exr : Compress a HDR .EXR image to a + compressed intermediate format UASTC HDR 6x6 .KTX2 file with rate-distortion + optimization (RDO), at lambda level 1000. + +--- Example ASTC/XUASTC LDR 4x4-12x12 command lines: + + - basisu -ldr_6x6i -q 75 -xuastc_arith test.png : Compress test.png to XUASTC + LDR 6x6 using weight grid DCT with setting 75 and the arith profile for + higher compression. + + - basisu -ldr_4x4 test.png : Compress test.png to ASTC LDR 4x4 + + - basisu -mipmap -ldr_10x5i test.png : Compress test.png to XUASTC LDR 10x5, + using lossless ZStd supercompression, with mipmaps + +--- ETC1S Texture Video Notes: Use -comp_level 2 or higher for better codebook +generation, specify very large codebooks using -max_endpoints +and -max_selectors, and reduce the default endpoint RDO threshold +(-endpoint_rdo_thresh) to around 1.25. Videos may have mipmaps and alpha +channels. Videos must always be played back by the transcoder in first to last +image order. Video files currently use I-Frames on the first image, and +P-Frames using conditional replenishment on subsequent frames. + +--- Low-level ETC1S compression (Effort) Level (-comp_level X) Details + +This setting controls the ETC1S speed vs. quality tradeoff. (Use -q to control +the quality vs. compressed size tradeoff.): + + - Level 0: Fastest, but has marginal quality and can be brittle on complex +images. Avg. Y dB: 35.45 + + - Level 1: Hierarchical codebook searching, faster ETC1S encoding. 36.87 dB, +~1.4x slower vs. level 0. (This is the default setting.) + + - Level 2: Use this or higher for video. Hierarchical codebook searching. +36.87 dB, ~1.4x slower vs. level 0. (This is the v1.12's default setting.) + + - Level 3: Full codebook searching. 37.13 dB, ~1.8x slower vs. level 0. +(Equivalent to the initial release's default settings.) + + - Level 4: Hierarchical codebook searching, codebook k-means iterations. 37.15 +dB, ~4x slower vs. level 0 + + - Level 5: Full codebook searching, codebook k-means iterations. 37.41 dB, +~5.5x slower vs. level 0. + + - Level 6: Full codebook searching, twice as many codebook k-means iterations, +best ETC1 endpoint opt. 37.43 dB, ~12x slower vs. level 0 + diff --git a/external/basis_universal/cmd_help/process.py b/external/basis_universal/cmd_help/process.py new file mode 100644 index 0000000000..57b933a096 --- /dev/null +++ b/external/basis_universal/cmd_help/process.py @@ -0,0 +1,12 @@ +# convert_to_c_string.py +def to_c_string(path): + with open(path, "r", encoding="utf-8") as f: + text = f.read() + # escape backslashes and quotes + text = text.replace("\\", "\\\\").replace("\"", "\\\"") + # replace newlines with \n + text = text.replace("\n", "\\n\"\n\"") + return "\"" + text + "\"" + +if __name__ == "__main__": + print(to_c_string("cmd_help.txt")) diff --git a/external/basis_universal/contrib/single_file_transcoder/basisu_transcoder-in.cpp b/external/basis_universal/contrib/single_file_transcoder/basisu_transcoder-in.cpp index b073a37879..63d64b6395 100644 --- a/external/basis_universal/contrib/single_file_transcoder/basisu_transcoder-in.cpp +++ b/external/basis_universal/contrib/single_file_transcoder/basisu_transcoder-in.cpp @@ -29,9 +29,11 @@ #define BASISD_SUPPORT_FXT1 0 /* - * KTX2 support disabled. + * KTX2 support enabled. */ -#define BASISD_SUPPORT_KTX2 0 +#define BASISD_SUPPORT_KTX2 1 + +#define BASISU_ASTC_HELPERS_IMPLEMENTATION #include "basisu_transcoder.cpp" diff --git a/external/basis_universal/contrib/single_file_transcoder/combine.py b/external/basis_universal/contrib/single_file_transcoder/combine.py index 829d433116..8eac17e232 100755 --- a/external/basis_universal/contrib/single_file_transcoder/combine.py +++ b/external/basis_universal/contrib/single_file_transcoder/combine.py @@ -160,7 +160,7 @@ def add_file(file: Path, file_name: str = None) -> None: if (not file_name): file_name = file.name error_line(f'Processing: {file_name}') - with file.open('r', errors='replace') as opened: + with file.open('r', encoding='utf-8-sig', errors='replace') as opened: for line in opened: line = line.rstrip('\n') match_include = include_regex.match(line); diff --git a/external/basis_universal/contrib/single_file_transcoder/create_transcoder.sh b/external/basis_universal/contrib/single_file_transcoder/create_transcoder.sh index 0c7a435bfe..39616d0044 100755 --- a/external/basis_universal/contrib/single_file_transcoder/create_transcoder.sh +++ b/external/basis_universal/contrib/single_file_transcoder/create_transcoder.sh @@ -22,7 +22,7 @@ which cc > /dev/null if [ $? -ne 0 ]; then echo "(Skipping compile test)" else - cc -std=c++11 -Wall -Wextra -Wno-unused-value -Os -g0 -fno-exceptions -fno-rtti -fno-strict-aliasing -o $OUT_FILE examples/simple.cpp -lstdc++ -lm + cc -std=c++17 -Wall -Wextra -Wno-unused-value -Os -g0 -fno-exceptions -fno-rtti -fno-strict-aliasing -o $OUT_FILE examples/simple.cpp ../../zstd/zstd.c -lstdc++ -lm # Did compilation work? if [ $? -ne 0 ]; then echo "Compiling simple.cpp: FAILED" diff --git a/external/basis_universal/encoder/3rdparty/android_astc_decomp.cpp b/external/basis_universal/encoder/3rdparty/android_astc_decomp.cpp index f850d03d0c..75e8d82103 100644 --- a/external/basis_universal/encoder/3rdparty/android_astc_decomp.cpp +++ b/external/basis_universal/encoder/3rdparty/android_astc_decomp.cpp @@ -836,11 +836,13 @@ void decodeISETritBlock (ISEDecodedResult* dst, int numValues, BitAccessStream& m[4] = data.getNext(numBits); deUint32 T7 = data.getNext(1); +#ifndef __clang__ #ifndef __EMSCRIPTEN__ #ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wimplicit-fallthrough=" #endif +#endif #endif switch (numValues) { @@ -853,10 +855,12 @@ void decodeISETritBlock (ISEDecodedResult* dst, int numValues, BitAccessStream& default: DE_ASSERT(false); } +#ifndef __clang__ #ifndef __EMSCRIPTEN__ #ifdef __GNUC__ #pragma GCC diagnostic pop #endif +#endif #endif const deUint32 T = (T7 << 7) | (T56 << 5) | (T4 << 4) | (T23 << 2) | (T01 << 0); @@ -902,11 +906,13 @@ void decodeISEQuintBlock (ISEDecodedResult* dst, int numValues, BitAccessStream& m[2] = data.getNext(numBits); deUint32 Q56 = data.getNext(2); +#ifndef __clang__ #ifndef __EMSCRIPTEN__ #ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wimplicit-fallthrough=" #endif +#endif #endif switch (numValues) { @@ -917,10 +923,13 @@ void decodeISEQuintBlock (ISEDecodedResult* dst, int numValues, BitAccessStream& default: DE_ASSERT(false); } + +#ifndef __clang__ #ifndef __EMSCRIPTEN__ #ifdef __GNUC__ #pragma GCC diagnostic pop #endif +#endif #endif const deUint32 Q = (Q56 << 5) | (Q34 << 3) | (Q012 << 0); diff --git a/external/basis_universal/encoder/3rdparty/tinydds.h b/external/basis_universal/encoder/3rdparty/tinydds.h index 41e1d6f131..b1dda65231 100644 --- a/external/basis_universal/encoder/3rdparty/tinydds.h +++ b/external/basis_universal/encoder/3rdparty/tinydds.h @@ -828,7 +828,7 @@ typedef struct TinyDDS_Context { #define TINYDDS_MAKE_RIFFCODE(a, b, c, d) (a | (b << 8) | (c << 16) | (d << 24)) -static uint32_t TinyDDS_fileIdentifier = TINYDDS_MAKE_RIFFCODE('D', 'D', 'S', ' '); +//static uint32_t TinyDDS_fileIdentifier = TINYDDS_MAKE_RIFFCODE('D', 'D', 'S', ' '); static void TinyDDS_NullErrorFunc(void *user, char const *msg) { BASISU_NOTE_UNUSED(user); BASISU_NOTE_UNUSED(msg); } diff --git a/external/basis_universal/encoder/basisu_astc_hdr_6x6_enc.cpp b/external/basis_universal/encoder/basisu_astc_hdr_6x6_enc.cpp index fd1efe520e..b41d69b575 100644 --- a/external/basis_universal/encoder/basisu_astc_hdr_6x6_enc.cpp +++ b/external/basis_universal/encoder/basisu_astc_hdr_6x6_enc.cpp @@ -1,4 +1,16 @@ // File: basisu_astc_hdr_6x6_enc.cpp +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. #include "basisu_astc_hdr_6x6_enc.h" #include "basisu_enc.h" #include "basisu_astc_hdr_common.h" @@ -22,13 +34,13 @@ using namespace basist::astc_6x6_hdr; namespace astc_6x6_hdr { -static void atomic_max(std::atomic& atomic_var, uint32_t new_value) +static void atomic_max(std::atomic& atomic_var, uint32_t new_value) { uint32_t current = atomic_var.load(std::memory_order_relaxed); for ( ; ; ) { uint32_t new_max = std::max(current, new_value); - if (atomic_var.compare_exchange_weak(current, new_max, std::memory_order_relaxed, std::memory_order_relaxed)) + if (atomic_var.compare_exchange_weak(current, new_max, std::memory_order_relaxed, std::memory_order_relaxed)) break; } } @@ -164,7 +176,7 @@ static float inversePQ(float E) // Highest error is for values less than SMALLEST_PQ_VAL_IN. // // Approximation is round trip lossless for 10-12 bits at [0,10000] nits: -// for x [0,1024] (SCALE=1023) or for x [0,4095] (SCALE=4096): +// for x [0,1024] (SCALE=1023) or for x [0,4095] (SCALE=4096): // round(forwardPQTab(inversePQ(x / SCALE)) * SCALE) == x // // bfloat16 has enough precision to handle 8-bit sRGB to linear conversions: @@ -176,7 +188,7 @@ const int PQ_APPROX_EXP_RANGE = (PQ_APPROX_MAX_EXP - PQ_APPROX_MIN_EXP + 1); const float SMALLEST_PQ_VAL_IN = 0.000015258829080f; const float SMALLEST_PQ_VAL = 0.000551903737f; // forwardPQ(SMALLEST_PQ_VAL_IN) -const float LARGEST_PQ_VAL = 1.251312f; +const float LARGEST_PQ_VAL = 1.251312f; float g_pq_approx_tabs[PQ_APPROX_EXP_RANGE][128]; @@ -248,7 +260,7 @@ static inline float forwardPQTab(float v) } // 100 nits = ~.5 i -// This converts absolute linear RGB light in either REC 709 or REC2020/BT2100 color gamut to ICtCp, a coding space where Ct is scaled by 2. +// This converts absolute linear RGB light in either REC 709 or REC2020/BT2100 color gamut to ICtCp, a coding space where Ct is scaled by 2. // To convert to perceptual ITP for error/distance calculations, multiply the result Ct by .5 (or set itp_flag to true). // Assumes REC 709 input, or REC 2020/BT.2100 RGB input if rec2020_bt2100_color_gamut is true. // @@ -268,7 +280,7 @@ static inline float forwardPQTab(float v) static void linear_rgb_to_ictcp(const vec3F& rgb_in, vec3F& ictcp, bool itp_flag = false, bool rec2020_bt2100_color_gamut = false) { vec3F rgb_2100(rgb_in); - + float l, m, s; if (!rec2020_bt2100_color_gamut) { @@ -613,7 +625,7 @@ struct partition_pattern_vec operator size_t() const { - return basisu::hash_hsieh(m_parts, sizeof(m_parts)); + return basist::hash_hsieh(m_parts, sizeof(m_parts)); } }; @@ -662,7 +674,7 @@ class vp_tree m_nodes[0].m_outer_node = -1; uint_vec inner_list, outer_list; - + inner_list.reserve(n / 2); outer_list.reserve(n / 2); @@ -711,8 +723,8 @@ class vp_tree enum { MaxSupportedSize = 256 + 1 }; public: - result_queue() : - m_cur_size(0) + result_queue() : + m_cur_size(0) { } @@ -765,14 +777,14 @@ class vp_tree bool pop() { - if (m_cur_size == 0) + if (m_cur_size == 0) return false; m_elements[1] = m_elements[m_cur_size--]; down_heap(1); return true; } - + float get_highest_dist() const { if (!m_cur_size) @@ -780,7 +792,7 @@ class vp_tree return top().m_dist; } - + private: result_array_type m_elements; size_t m_cur_size; @@ -814,7 +826,7 @@ class vp_tree } } }; - + void find_nearest(uint32_t num_subsets, const partition_pattern_vec& desired_pat, result_queue& results, uint32_t max_results) { assert((num_subsets >= 2) && (num_subsets <= 3)); @@ -879,7 +891,7 @@ class vp_tree if (m_nodes[node_index].m_outer_node >= 0) { - if ( (results.get_size() < max_results) || + if ( (results.get_size() < max_results) || ((m_nodes[node_index].m_dist - best_dist_to_vantage) <= results.get_highest_dist()) ) { @@ -895,7 +907,7 @@ class vp_tree if (m_nodes[node_index].m_inner_node >= 0) { - if ( (results.get_size() < max_results) || + if ( (results.get_size() < max_results) || ((best_dist_to_vantage - m_nodes[node_index].m_dist) <= results.get_highest_dist()) ) { @@ -904,13 +916,13 @@ class vp_tree } } } - + void find_nearest_at_node_non_recursive(int init_node_index, uint32_t num_desired_pats, const partition_pattern_vec* pDesired_pats, result_queue& results, uint32_t max_results) { uint_vec node_stack; node_stack.reserve(16); node_stack.push_back(init_node_index); - + do { const uint32_t node_index = node_stack.back(); @@ -985,7 +997,7 @@ class vp_tree m_nodes.resize(m_nodes.size() + 1); const uint32_t new_node_index = m_nodes.size_u32() - 1; - + m_nodes[new_node_index].m_vantage_point = pUnique_pats[root_idx.first]; m_nodes[new_node_index].m_point_index = root_idx.first; m_nodes[new_node_index].m_dist = root_idx.second; @@ -1039,17 +1051,17 @@ class vp_tree basisu::vector< std::pair > dists; dists.reserve(n); - + float_vec float_dists; float_dists.reserve(n); - + for (uint32_t pat_indices_iter = 0; pat_indices_iter < n; pat_indices_iter++) { const uint32_t split_pat_index = pat_indices[pat_indices_iter]; assert(split_pat_index < num_unique_pats); const partition_pattern_vec& trial_vantage = pUnique_pats[split_pat_index]; - + dists.resize(0); float_dists.resize(0); @@ -1060,7 +1072,7 @@ class vp_tree if (pat_index == split_pat_index) continue; - + float dist = trial_vantage.get_distance(pUnique_pats[pat_index]); dists.emplace_back(std::pair(dist, pat_index)); @@ -1080,13 +1092,13 @@ class vp_tree split_dist = (split_dist + dists[(num_dists / 2) - 1].first) * .5f; uint32_t total_inner = 0, total_outer = 0; - + for (uint32_t j = 0; j < n; j++) { const uint32_t pat_index = pat_indices[j]; if (pat_index == split_pat_index) continue; - + float dist = trial_vantage.get_distance(pUnique_pats[pat_index]); if (dist <= split_dist) @@ -1096,7 +1108,7 @@ class vp_tree } float split_metric = (float)minimum(total_inner, total_outer) / (float)maximum(total_inner, total_outer); - + if ( (split_metric > best_split_metric) || ((split_metric == best_split_metric) && (s.m_var > best_split_var)) ) { @@ -1115,7 +1127,7 @@ struct partition { uint64_t m_p; - inline partition() : + inline partition() : m_p(0) { } @@ -1145,7 +1157,7 @@ struct partition inline operator size_t() const { - return hash_hsieh((const uint8_t *)&m_p, sizeof(m_p)); + return basist::hash_hsieh((const uint8_t *)&m_p, sizeof(m_p)); } }; @@ -1172,19 +1184,19 @@ static void init_partitions2_6x6() { uint64_t p_bits = 0; uint64_t p_bits_inv = 0; - + for (uint32_t y = 0; y < 6; y++) { for (uint32_t x = 0; x < 6; x++) { uint64_t p = astc_helpers::compute_texel_partition(i, x, y, 0, 2, false); assert(p < 2); - + p_bits |= (p << (x + y * 6)); p_bits_inv |= ((1 - p) << (x + y * 6)); } } - + if (!p_bits) continue; if (p_bits == ((1ULL << 36) - 1)) @@ -1206,13 +1218,13 @@ static void init_partitions2_6x6() BASISU_NOTE_UNUSED(res); } } - + uint32_t num_unique_partitions2 = 0; - + for (const auto& r : phash) { assert(r.second < 1024); - + const uint32_t unique_index = num_unique_partitions2; assert(unique_index < NUM_UNIQUE_PARTITIONS2); @@ -1221,7 +1233,7 @@ static void init_partitions2_6x6() pat_vec[i] = (uint8_t)((r.first >> i) & 1); g_partitions2[unique_index] = pat_vec; - + assert(g_part2_unique_index_to_seed[unique_index] == r.second); g_part2_seed_to_unique_index[r.second] = unique_index; @@ -1260,7 +1272,7 @@ static bool estimate_partition2_6x6( int* pBest_parts, uint32_t num_best_parts) { const uint32_t BLOCK_W = 6, BLOCK_H = 6, BLOCK_T = BLOCK_W * BLOCK_H; - + vec3F training_vecs[BLOCK_T], mean(0.0f); for (uint32_t i = 0; i < BLOCK_T; i++) @@ -1321,10 +1333,10 @@ static bool estimate_partition2_6x6( desired_part.m_parts[i] = proj < 0.0f; } #endif - + //interval_timer tm; //tm.start(); - + #if BRUTE_FORCE_PART_SEARCH uint32_t part_similarity[NUM_UNIQUE_PARTITIONS2]; @@ -1432,7 +1444,7 @@ static bool encode_block_2_subsets( part_pixel_index[part_index][l] = (uint8_t)(x + y * BLOCK_W); part_total_pixels[part_index] = l + 1; - } // x + } // x } // y uint8_t blk_endpoints[2][basist::NUM_MODE11_ENDPOINTS]; @@ -1529,7 +1541,7 @@ static bool encode_block_2_subsets( grid_w, grid_h, // dest/to dimension (grid size) desired_weights, // these are dequantized weights, NOT ISE symbols, [by][bx] downsampled_weights); // [wy][wx] - + best_log_blk.m_partition_id = (uint16_t)p_seed; memcpy(best_log_blk.m_endpoints, blk_endpoints[0], num_endpoint_vals); memcpy(best_log_blk.m_endpoints + num_endpoint_vals, blk_endpoints[1], num_endpoint_vals); @@ -1589,7 +1601,7 @@ static void init_partitions3_6x6() partition3_hash_map part3_hash; part3_hash.reserve(512); - + for (uint32_t seed_index = 0; seed_index < 1024; seed_index++) { partition_pattern_vec p3; @@ -1645,6 +1657,7 @@ static bool estimate_partition3_6x6( float brightest_inten = 0.0f, darkest_inten = BIG_FLOAT_VAL; vec3F cluster_centroids[NUM_SUBSETS]; + clear_obj(cluster_centroids); for (uint32_t i = 0; i < BLOCK_T; i++) { @@ -1692,23 +1705,23 @@ static bool estimate_partition3_6x6( if ((cluster_centroids[0] == cluster_centroids[2]) || (cluster_centroids[1] == cluster_centroids[2])) return false; - + uint32_t cluster_pixels[NUM_SUBSETS][BLOCK_T]; uint32_t num_cluster_pixels[NUM_SUBSETS]; vec3F new_cluster_means[NUM_SUBSETS]; const uint32_t NUM_ITERS = 4; - + for (uint32_t s = 0; s < NUM_ITERS; s++) { memset(num_cluster_pixels, 0, sizeof(num_cluster_pixels)); - memset(new_cluster_means, 0, sizeof(new_cluster_means)); + memset((void *)new_cluster_means, 0, sizeof(new_cluster_means)); for (uint32_t i = 0; i < BLOCK_T; i++) { - float d[NUM_SUBSETS] = { - training_vecs[i].squared_distance(cluster_centroids[0]), - training_vecs[i].squared_distance(cluster_centroids[1]), + float d[NUM_SUBSETS] = { + training_vecs[i].squared_distance(cluster_centroids[0]), + training_vecs[i].squared_distance(cluster_centroids[1]), training_vecs[i].squared_distance(cluster_centroids[2]) }; float min_d = d[0]; @@ -1735,7 +1748,7 @@ static bool estimate_partition3_6x6( cluster_centroids[j] = new_cluster_means[j] / (float)num_cluster_pixels[j]; } } // s - + partition_pattern_vec desired_part; for (uint32_t p = 0; p < NUM_SUBSETS; p++) { @@ -1770,7 +1783,7 @@ static bool estimate_partition3_6x6( } // part_index; std::sort(part_similarity, part_similarity + NUM_UNIQUE_PARTITIONS3); - + for (uint32_t i = 0; i < num_best_parts; i++) pBest_parts[i] = part_similarity[i] & 0xFFFF; #else @@ -1798,15 +1811,15 @@ static bool encode_block_3_subsets( astc_hdr_codec_base_options& coptions, bool uber_mode_flag, const int* pEst_patterns, int num_est_patterns, - uint32_t comp_level, + uint32_t comp_level, opt_mode_t mode11_opt_mode) { BASISU_NOTE_UNUSED(uber_mode_flag); const uint32_t BLOCK_W = 6, BLOCK_H = 6, NUM_SUBSETS = 3; const uint32_t num_endpoint_vals = astc_helpers::get_num_cem_values(cem); - + res.m_valid = false; - + double best_e = BIG_FLOAT_VAL; astc_helpers::log_astc_block best_log_blk; @@ -1848,7 +1861,7 @@ static bool encode_block_3_subsets( part_pixel_index[part_index][l] = (uint8_t)(x + y * BLOCK_W); part_total_pixels[part_index] = l + 1; - } // x + } // x } // y uint8_t blk_endpoints[NUM_SUBSETS][basist::NUM_MODE11_ENDPOINTS]; @@ -1886,7 +1899,7 @@ static bool encode_block_3_subsets( blk_endpoints[part_iter], blk_weights[part_iter], coptions, - false, best_log_blk.m_endpoint_ise_range, uber_mode_flag, false, + false, best_log_blk.m_endpoint_ise_range, uber_mode_flag, false, FIRST_MODE11_SUBMODE_INDEX, MAX_MODE11_SUBMODE_INDEX, false, mode11_opt_mode); } @@ -1915,7 +1928,7 @@ static bool encode_block_3_subsets( for (uint32_t p = 0; p < NUM_SUBSETS; p++) memcpy(best_log_blk.m_endpoints + num_endpoint_vals * p, blk_endpoints[p], num_endpoint_vals); - + memcpy(best_log_blk.m_weights, ise_weights, BLOCK_W * BLOCK_H); } } @@ -1948,7 +1961,7 @@ static bool encode_block_3_subsets( astc_helpers::log_astc_block trial_blk(best_log_blk); trial_blk.m_partition_id = (uint16_t)g_part3_unique_index_to_seed[unique_part_index]; - + for (uint32_t p = 0; p < NUM_SUBSETS; p++) memcpy(trial_blk.m_endpoints + num_endpoint_vals * p, blk_endpoints[p], num_endpoint_vals); @@ -2018,7 +2031,7 @@ static uint32_t encode_values(bitwise_coder &coder, uint32_t total_values, const uint32_t total_tq_values = 0, tq_accum = 0, tq_mul = 1; assert((total_values) && (total_values <= MAX_VALS)); - + const uint32_t ep_bits = astc_helpers::g_ise_range_table[endpoint_range][0]; const uint32_t ep_trits = astc_helpers::g_ise_range_table[endpoint_range][1]; const uint32_t ep_quints = astc_helpers::g_ise_range_table[endpoint_range][2]; @@ -2061,7 +2074,7 @@ static uint32_t encode_values(bitwise_coder &coder, uint32_t total_values, const } uint32_t total_bits_output = 0; - + for (uint32_t i = 0; i < total_tq_values; i++) { const uint32_t num_bits = ep_trits ? 8 : 7; @@ -2132,7 +2145,7 @@ static void code_block(bitwise_coder& coder, { const int unique_partition_index = g_part2_seed_to_unique_index[log_blk.m_partition_id]; assert(unique_partition_index != -1); - + coder.put_truncated_binary(unique_partition_index, NUM_UNIQUE_PARTITIONS2); } else if (log_blk.m_num_partitions == 3) @@ -2142,7 +2155,7 @@ static void code_block(bitwise_coder& coder, coder.put_truncated_binary(unique_partition_index, NUM_UNIQUE_PARTITIONS3); } - + encode_values(coder, num_endpoint_vals * log_blk.m_num_partitions, log_blk.m_endpoints, log_blk.m_endpoint_ise_range); } @@ -2176,7 +2189,7 @@ struct smooth_map_params // 3x3 region m_max_smooth_std_dev = 100.0f; m_smooth_max_mse_scale = 13000.0f; - + // 7x7 region m_max_med_smooth_std_dev = 9.0f; m_med_smooth_max_mse_scale = 15000.0f; @@ -2244,7 +2257,7 @@ static void filter_block(uint32_t grid_x, uint32_t grid_y, const vec3F* pSrc_blo for (uint32_t c = 0; c < 3; c++) { const basist::half_float h = basist::float_to_half(temp_block[y][x][c]); - + pDst_block_half3[x + y * 6][c] = h; pDst_block_q16[x + y * 6][c] = (float)half_to_qlog16(h); } @@ -2265,9 +2278,9 @@ static void filter_block(uint32_t grid_x, uint32_t grid_y, const vec3F* pSrc_blo for (uint32_t i = 0; i < pCol_lists[y].n; i++) p += temp_block[pCol_lists[y].p[i].pixel][x] * pCol_lists[y].p[i].weight; - + p.clamp(0.0f, basist::ASTC_HDR_MAX_VAL); - + for (uint32_t c = 0; c < 3; c++) { const basist::half_float h = basist::float_to_half(p[c]); @@ -2277,7 +2290,7 @@ static void filter_block(uint32_t grid_x, uint32_t grid_y, const vec3F* pSrc_blo } pDst_block_q16[x + y * 6][3] = 0.0f; - + } // x } // y } @@ -2368,7 +2381,7 @@ static void filter_block(uint32_t grid_x, uint32_t grid_y, const vec3F* pSrc_blo for (uint32_t i = 0; i < pRow_lists[x].n; i++) p += vec3F(pSrc_block[y * 6 + pRow_lists[x].p[i].pixel]) * pRow_lists[x].p[i].weight; - + temp_block[y][x] = p; } // x } // y @@ -2404,7 +2417,7 @@ static float diff_blocks(const vec4F* pA, const vec4F* pB) float diff = 0.0f; for (uint32_t i = 0; i < BLOCK_T; i++) diff += square(pA[i][0] - pB[i][0]) + square(pA[i][1] - pB[i][1]) + square(pA[i][2] - pB[i][2]); - + return diff * (1.0f / (float)BLOCK_T); } @@ -2446,13 +2459,14 @@ static void create_smooth_maps2( const uint32_t height = orig_img.get_height(); //const uint32_t total_pixels = orig_img.get_total_pixels(); const uint32_t num_comps = 3; - + if (params.m_no_mse_scaling) { smooth_block_mse_scales.set_all(1.0f); return; } - + + // TODO: - move up before the no mse scaling check (harmless as that is only a debug aid) smooth_block_mse_scales.resize(width, height); image smooth_vis, med_smooth_vis, ultra_smooth_vis; @@ -2552,7 +2566,7 @@ static void create_smooth_maps2( float yl = clampf(max_std_dev / params.m_max_ultra_smooth_std_dev, 0.0f, 1.0f); yl = powf(yl, 2.0f); - + smooth_block_mse_scales(x, y) = lerp(params.m_ultra_smooth_max_mse_scale, smooth_block_mse_scales(x, y), yl); if (params.m_debug_images) @@ -2595,12 +2609,12 @@ static float compute_pixel_mse_itp(const vec3F& orig_pixel_itp, const vec3F& com float delta_i = orig_pixel_itp[0] - comp_pixel_itp[0]; float delta_t = orig_pixel_itp[1] - comp_pixel_itp[1]; float delta_p = orig_pixel_itp[2] - comp_pixel_itp[2]; - + float err = (delta_i * delta_i) + (delta_t * delta_t) + (delta_p * delta_p); if (delta_itp_dark_adjustment) { - // We have to process a large range of inputs, including extremely dark inputs. + // We have to process a large range of inputs, including extremely dark inputs. // Artifically amplify MSE on very dark pixels - otherwise they'll be overly compressed at higher lambdas. // This is to better handle very dark signals which could be explictly overexposed. float s = bu_math::smoothstep(0.0f, REALLY_DARK_I_THRESHOLD, orig_pixel_itp[0]); @@ -2688,7 +2702,7 @@ static float compute_pixel_delta_itp(const vec3F& a, const vec3F& b, const vec3F float err = 720.0f * sqrtf((delta_i * delta_i) + (delta_t * delta_t) + (delta_p * delta_p)); float s = bu_math::smoothstep(0.0f, REALLY_DARK_I_THRESHOLD, orig[0]); - + if (delta_itp_dark_adjustment) { // This is to better handle very dark signals which could be explictly overexposed. @@ -2702,22 +2716,22 @@ static float compute_pixel_delta_itp(const vec3F& a, const vec3F& b, const vec3F struct candidate_encoding { encoding_type m_encoding_type; - + basist::half_float m_solid_color[3]; uint32_t m_run_len; vec3F m_comp_pixels[MAX_BLOCK_H][MAX_BLOCK_W]; // [y][x] vec3F m_comp_pixels_itp[MAX_BLOCK_H][MAX_BLOCK_W]; // [y][x] - + endpoint_mode m_endpoint_mode; block_mode m_block_mode; bitwise_coder m_coder; - + // The block to code, which may not be valid ASTC. This may have to be transcoded (by requantizing the weights/endpoints) before it's valid ASTC. // Note the endpoints may be coded endpoints OR transcoded endpoints, depending on the encoding type. - astc_helpers::log_astc_block m_coded_log_blk; + astc_helpers::log_astc_block m_coded_log_blk; // The block the decoder outputs. astc_helpers::log_astc_block m_decomp_log_blk; @@ -2725,9 +2739,9 @@ struct candidate_encoding int m_reuse_delta_index; // m_t can get VERY large - double m_t, m_d; + double m_t, m_d; float m_bits; - + candidate_encoding() { clear(); @@ -2758,7 +2772,7 @@ struct candidate_encoding m_coded_log_blk = rhs.m_coded_log_blk; m_decomp_log_blk = rhs.m_decomp_log_blk; m_reuse_delta_index = rhs.m_reuse_delta_index; - + return *this; } @@ -2790,19 +2804,19 @@ struct candidate_encoding m_run_len = 0; clear_obj(m_comp_pixels); - + m_endpoint_mode = endpoint_mode::cInvalid; m_block_mode = block_mode::cInvalid; m_coder.restart(); - + m_coded_log_blk.clear(); m_decomp_log_blk.clear(); m_t = 0; m_d = 0; m_bits = 0; - + m_reuse_delta_index = 0; } }; @@ -2826,7 +2840,7 @@ bool decode_astc_block(uint32_t block_w, uint32_t block_h, astc_helpers::log_ast basist::half_to_float(decoded_pixels_half4[x + y * block_w][0]), basist::half_to_float(decoded_pixels_half4[x + y * block_w][1]), basist::half_to_float(decoded_pixels_half4[x + y * block_w][2])); - } // x + } // x } //y return true; @@ -2857,12 +2871,12 @@ static bool decode_file(const uint8_vec& comp_data, vector2D MAX_ASTC_HDR_6X6_DIM) || (height > MAX_ASTC_HDR_6X6_DIM)) return false; @@ -2879,11 +2893,11 @@ static bool decode_file(const uint8_vec& comp_data, vector2D num_blocks_remaining) return false; - + uint32_t prev_bx = cur_bx, prev_by = cur_by; if (cur_bx) @@ -2983,7 +2997,7 @@ static bool decode_file(const uint8_vec& comp_data, vector2D { const uint32_t width = src_img.get_width(); const uint32_t height = src_img.get_height(); - + if (pPacked_bc6h_img) pPacked_bc6h_img->resize(width, height); @@ -3369,7 +3383,7 @@ static bool pack_bc6h_image(const imagef &src_img, vector2D const uint32_t num_blocks_y = src_img.get_block_height(4); bc6h_blocks.resize(num_blocks_x, num_blocks_y); - + for (uint32_t by = 0; by < num_blocks_y; by++) { for (uint32_t bx = 0; bx < num_blocks_x; bx++) @@ -3415,7 +3429,7 @@ static bool pack_bc6h_image(const imagef &src_img, vector2D fmt_error_printf("unpack_bc6h() failed\n"); return false; } - + for (uint32_t y = 0; y < 4; y++) { for (uint32_t x = 0; x < 4; x++) @@ -3457,7 +3471,7 @@ static void estimate_partitions_mode7_and_11( uint32_t num_pats_to_examine, const uint32_t* pUnique_pat_indices_to_examine, // indices of pats to examine const vec3F *pHalf_pixels_as_floats, // block's half pixel values casted to floats const astc_hdr_codec_base_options& coptions, // options - uint32_t num_desired_pats, + uint32_t num_desired_pats, int *pDesired_pat_indices_mode11, int *pDesired_pat_indices_mode7) // output indices { BASISU_NOTE_UNUSED(coptions); @@ -3480,7 +3494,7 @@ static void estimate_partitions_mode7_and_11( candidate_res mode7_candidates[MAX_CANDIDATES]; const vec3F grayscale_axis(0.5773502691f); - + for (uint32_t examine_iter = 0; examine_iter < num_pats_to_examine; examine_iter++) { const uint32_t unique_part_index = pUnique_pat_indices_to_examine[examine_iter]; @@ -3506,7 +3520,7 @@ static void estimate_partitions_mode7_and_11( } // x } // y - + for (uint32_t i = 0; i < num_parts; i++) { assert(part_total_texels[i]); @@ -3542,7 +3556,7 @@ static void estimate_partitions_mode7_and_11( for (uint32_t part_index = 0; part_index < num_parts; part_index++) total_variance[part_index] = part_cov[part_index][0] + part_cov[part_index][3] + part_cov[part_index][5]; - vec3F part_axis[MAX_PARTS]; + //vec3F part_axis[MAX_PARTS]; float mode11_eigenvalue_est[MAX_PARTS]; // For each partition, compute the variance along the principle axis float mode7_eigenvalue_est[MAX_PARTS]; // For each partition, compute the variance along the principle axis @@ -3551,7 +3565,7 @@ static void estimate_partitions_mode7_and_11( float* pCov = &part_cov[part_index][0]; float xr = .9f, xg = 1.0f, xb = .7f; - + const uint32_t NUM_POWER_ITERS = 4; for (uint32_t iter = 0; iter < NUM_POWER_ITERS; iter++) { @@ -3564,7 +3578,7 @@ static void estimate_partitions_mode7_and_11( if (m >= 1e-10f) { m = 1.0f / m; - + r *= m; g *= m; b *= m; @@ -3576,7 +3590,7 @@ static void estimate_partitions_mode7_and_11( } float len_sq = xr * xr + xg * xg + xb * xb; - + if (len_sq < 1e-10f) { xr = grayscale_axis[0]; @@ -3591,7 +3605,7 @@ static void estimate_partitions_mode7_and_11( xg *= len_sq; xb *= len_sq; } - + { // Transform the principle axis by the covariance matrix, which will scale the vector by its eigenvalue (the variance of the dataset projected onto the principle axis). float r = xr * pCov[0] + xg * pCov[1] + xb * pCov[2]; @@ -3602,13 +3616,13 @@ static void estimate_partitions_mode7_and_11( // The result is the variance along the principle axis. //float z1 = sqrtf(r * r + g * g + b * b); // this works with the principle axis //float z2 = r * xr + g * xg + b * xb; // compute length projected along xr,xg,xb - + mode11_eigenvalue_est[part_index] = r * xr + g * xg + b * xb; } { const float yrgb = grayscale_axis[0]; - + // Transform the grayscale axis by the covariance matrix, which will scale the vector by the eigenvalue (which is the variance of the dataset projected onto this vector). float r = yrgb * pCov[0] + yrgb * pCov[1] + yrgb * pCov[2]; float g = yrgb * pCov[1] + yrgb * pCov[3] + yrgb * pCov[4]; @@ -3618,7 +3632,7 @@ static void estimate_partitions_mode7_and_11( } } // part_index - + // Compute the total variance (squared error) of the other 2 axes by subtracting the total variance of all channels by the variance of the principle axis. // TODO: Could also compute the ratio of the principle axis's variance vs. the total variance. float mode11_total_sq_dist_to_line_alt = 0.0f; @@ -3731,7 +3745,7 @@ static void estimate_partitions_mode7( } vec3F part_axis(0.5773502691f); - + // TODO: This total distance can be computed rapidly. First compute the total variance of each channel (sum the diag entries of the covar matrix), // then compute the principle eigenvalue, and subtract. The result is the variance of the projection distances. float total_sq_dist_to_line = 0.0f; @@ -3797,7 +3811,7 @@ static float calc_deblocking_penalty_itp( const vec3F& q_pixel_itp = pass_src_img_itp(qx, qy); const vec3F &d_pixel_itp = candidate.m_comp_pixels_itp[qy - by * 6][qx - bx * 6]; // compressed block - + vec3F orig_delta_v(o_pixel_itp - q_pixel_itp); total_orig_mse += square(orig_delta_v[0]) + square(orig_delta_v[1]) + square(orig_delta_v[2]); @@ -3861,15 +3875,15 @@ static bool calc_strip_size( else { rows_per_strip = (num_blocks_y / total_strips) & ~1; - + if (rows_per_strip < 2) rows_per_strip = 2;// num_blocks_y; } - + assert((rows_per_strip == num_blocks_y) || ((rows_per_strip & 1) == 0)); total_strips = (num_blocks_y + rows_per_strip - 1) / rows_per_strip; - + if (global_cfg.m_debug_output) { fmt_printf("num_blocks_y: {}, total_threads : {}, Total strips : {}\n", num_blocks_y, total_threads, total_strips); @@ -3979,7 +3993,7 @@ struct uastc_hdr_6x6_debug_state std::mutex m_vis_image_mutex; std::atomic m_comp_level_hist[ASTC_HDR_6X6_MAX_COMP_LEVEL + 1]; - + std::atomic m_total_jnd_replacements; std::mutex m_stats_mutex; @@ -3995,7 +4009,7 @@ struct uastc_hdr_6x6_debug_state } } } - + void init(uint32_t width, uint32_t height) { m_stat_vis.resize(width, height); @@ -4009,7 +4023,7 @@ struct uastc_hdr_6x6_debug_state basisu::clear_obj(m_endpoint_mode_hist); basisu::clear_obj(m_block_mode_hist); basisu::clear_obj(m_block_mode_total_bits); - + for (uint32_t i = 0; i < (uint32_t)block_mode::cBMTotalModes; i++) { for (uint32_t j = 0; j < 3; j++) @@ -4036,7 +4050,7 @@ struct uastc_hdr_6x6_debug_state for (uint32_t i = 0; i < std::size(m_total_part2_stats); i++) m_total_part2_stats[i].store(0); - + for (uint32_t i = 0; i < std::size(m_dp_stats); i++) m_dp_stats[i].store(0); @@ -4160,9 +4174,9 @@ struct uastc_hdr_6x6_debug_state struct uastc_hdr_6x6_encode_state { astc_hdr_codec_base_options master_coptions; - + imagef src_img; - + imagef src_img_filtered1; imagef src_img_filtered2; @@ -4188,7 +4202,7 @@ static bool compress_strip_task( { BASISU_NOTE_UNUSED(num_blocks_y); BASISU_NOTE_UNUSED(total_strips); - + vec3F prev_comp_pixels[BLOCK_H][BLOCK_W]; // [y][x] basisu::clear_obj(prev_comp_pixels); @@ -4427,15 +4441,15 @@ static bool compress_strip_task( for (uint32_t i = 0; i < 3; i++) { #if 0 - // 9/5/2025, wrong metric, we're iterating channels pairs here, not individual channels. + // 9/5/2025, wrong metric, we're iterating channels pairs here, not individual channels. // On 3 active channel blocks this causes no difference. - if (half_comp_stats[i].m_range > 0.0f) + if (half_comp_stats[i].m_range > 0.0f) #else static const uint8_t s_chan_pairs[3][2] = { {0, 1}, {0, 2}, {1, 2} }; - + const uint32_t chanA = s_chan_pairs[i][0]; const uint32_t chanB = s_chan_pairs[i][1]; - + if ((half_comp_stats[chanA].m_range > 0.0f) && (half_comp_stats[chanB].m_range > 0.0f)) #endif { @@ -4670,7 +4684,7 @@ static bool compress_strip_task( } // Create the block the decoder would transcode into. - copy_weight_grid(dual_plane, grid_x, grid_y, transcode_weights, decomp_log_blk); + copy_weight_grid(dual_plane, grid_x, grid_y, transcode_weights, decomp_log_blk, false); } else if (prev_coded_log_blk.m_num_partitions == 2) { @@ -4697,7 +4711,7 @@ static bool compress_strip_task( part_half_pixels[part_index][l] = half_pixels[y][x]; part_total_pixels[part_index] = l + 1; - } // x + } // x } // y uint8_t blk_weights[2][BLOCK_W * BLOCK_H]; @@ -4748,7 +4762,7 @@ static bool compress_strip_task( basist::astc_6x6_hdr::requantize_astc_weights(num_grid_samples, coded_log_blk.m_weights, coded_log_blk.m_weight_ise_range, transcode_weights, decomp_log_blk.m_weight_ise_range); // Create the block the decoder would transcode into. - copy_weight_grid(dual_plane, grid_x, grid_y, transcode_weights, decomp_log_blk); + copy_weight_grid(dual_plane, grid_x, grid_y, transcode_weights, decomp_log_blk, false); } else if (prev_coded_log_blk.m_num_partitions == 3) { @@ -4775,7 +4789,7 @@ static bool compress_strip_task( part_half_pixels[part_index][l] = half_pixels[y][x]; part_total_pixels[part_index] = l + 1; - } // x + } // x } // y uint8_t blk_weights[3][BLOCK_W * BLOCK_H]; @@ -4818,7 +4832,7 @@ static bool compress_strip_task( basist::astc_6x6_hdr::requantize_astc_weights(num_grid_samples, coded_log_blk.m_weights, coded_log_blk.m_weight_ise_range, transcode_weights, decomp_log_blk.m_weight_ise_range); // Create the block the decoder would transcode into. - copy_weight_grid(dual_plane, grid_x, grid_y, transcode_weights, decomp_log_blk); + copy_weight_grid(dual_plane, grid_x, grid_y, transcode_weights, decomp_log_blk, false); } if (!validate_log_blk(decomp_log_blk)) @@ -5361,7 +5375,7 @@ static bool compress_strip_task( memcpy(decomp_blk.m_endpoints, transcode_endpoints, num_endpoint_vals); - copy_weight_grid(dual_plane, grid_x, grid_y, transcode_weights, decomp_blk); + copy_weight_grid(dual_plane, grid_x, grid_y, transcode_weights, decomp_blk, false); if (!validate_log_blk(decomp_blk)) { @@ -5593,7 +5607,7 @@ static bool compress_strip_task( memcpy(decomp_blk.m_endpoints, transcode_endpoints, num_endpoint_vals); - copy_weight_grid(dual_plane, grid_x, grid_y, transcode_weights, decomp_blk); + copy_weight_grid(dual_plane, grid_x, grid_y, transcode_weights, decomp_blk, false); if (!validate_log_blk(decomp_blk)) { @@ -5915,7 +5929,7 @@ static bool compress_strip_task( basist::astc_6x6_hdr::requantize_ise_endpoints(mode_desc.m_cem, mode_desc.m_endpoint_ise_range, coded_log_blk.m_endpoints, mode_desc.m_transcode_endpoint_ise_range, decomp_blk.m_endpoints); - copy_weight_grid(dual_plane, grid_x, grid_y, transcode_weights, decomp_blk); + copy_weight_grid(dual_plane, grid_x, grid_y, transcode_weights, decomp_blk, false); if (!validate_log_blk(decomp_blk)) { @@ -6117,13 +6131,13 @@ static bool compress_strip_task( mode_penalty *= (complex_block ? RUN_PENALTY * 2.0f : RUN_PENALTY); float candidate_bits = (float)candidate.m_coder.get_total_bits(); - + double candidate_d = (double)candidate_mse * mode_penalty; const float D_POWER = 2.0f; - + // this value can get VERY large after squaring on random (fuzzed) HDR inputs - double candidate_t = perceptual_scale * pow(candidate_d, D_POWER) + candidate_bits * (global_cfg.m_lambda * 1000.0f); + double candidate_t = perceptual_scale * pow(candidate_d, D_POWER) + candidate_bits * (global_cfg.m_lambda * 1000.0f); candidate.m_t = candidate_t; candidate.m_d = candidate_d; @@ -6140,7 +6154,7 @@ static bool compress_strip_task( if (best_candidate_index < 0) { assert(0); - + // Should never happen best_candidate_index = 0; } @@ -6160,7 +6174,7 @@ static bool compress_strip_task( debug_state.m_total_gaussian2_blocks.fetch_add(1, std::memory_order_relaxed); continue; } - + if (global_cfg.m_rdo_candidate_diversity_boost) { // candidate diversity boosting - consider candidates along/near the Pareto front @@ -6390,7 +6404,7 @@ static bool compress_strip_task( const uint32_t p = pat[x + y * 6]; debug_state.m_part_vis.set_clipped(bx * 6 + x, by * 6 + y, color_rgba(p ? 100 : 0, 128, p ? 100 : 0, 255)); } // x - } // y + } // y } else if (best_candidate.m_decomp_log_blk.m_num_partitions == 3) { @@ -6413,7 +6427,7 @@ static bool compress_strip_task( c.set(0, 100, 150, 255); debug_state.m_part_vis.set_clipped(bx * 6 + x, by * 6 + y, c); } // x - } // y + } // y } else if (best_candidate.m_decomp_log_blk.m_dual_plane) { @@ -6526,7 +6540,7 @@ void global_init() tm.start(); init_pq_tables(); - + init_partitions2_6x6(); init_partitions3_6x6(); @@ -6543,7 +6557,7 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa assert(g_initialized); if (!g_initialized) return false; - + assert(pJob_pool); if (orig_global_cfg.m_debug_output) @@ -6589,17 +6603,17 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa f = basist::ASTC_HDR_MAX_VAL; enc_state.src_img(x, y)[c] = f; - + } // c - + } // x } // y - + if (global_cfg.m_debug_images) { write_exr((global_cfg.m_debug_image_prefix + "orig.exr").c_str(), enc_state.src_img, 3, 0); } - + image src_img_compressed; tonemap_image_compressive2(src_img_compressed, enc_state.src_img); @@ -6624,10 +6638,10 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa enc_state.src_img_filtered1.resize(width, height); image_resample(enc_state.src_img, enc_state.src_img_filtered1, "gaussian", global_cfg.m_gaussian1_strength); //1.45f); - + enc_state.src_img_filtered2.resize(width, height); image_resample(enc_state.src_img, enc_state.src_img_filtered2, "gaussian", global_cfg.m_gaussian2_strength); //1.83f); - + if (global_cfg.m_debug_images) { write_exr((global_cfg.m_debug_image_prefix + "blurred1.exr").c_str(), enc_state.src_img_filtered1, 3, 0); @@ -6639,10 +6653,10 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa enc_state.src_img_itp.resize(width, height); convet_rgb_image_to_itp(enc_state.src_img, enc_state.src_img_itp, global_cfg); - + enc_state.src_img_filtered1_itp.resize(width, height); convet_rgb_image_to_itp(enc_state.src_img_filtered1, enc_state.src_img_filtered1_itp, global_cfg); - + enc_state.src_img_filtered2_itp.resize(width, height); convet_rgb_image_to_itp(enc_state.src_img_filtered2, enc_state.src_img_filtered2_itp, global_cfg); @@ -6655,20 +6669,20 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa fmt_error_printf("compress_photo: Failed computing strip sizes\n"); return false; } - + if (global_cfg.m_debug_output) fmt_printf("lambda: {}, comp_level: {}, highest_comp_level: {}, extra patterns: {}\n", global_cfg.m_lambda, global_cfg.m_master_comp_level, global_cfg.m_highest_comp_level, global_cfg.m_extra_patterns_flag); - + enc_state.coded_blocks.resize(num_blocks_x, num_blocks_y); - + bitwise_coder coded_bits; - coded_bits.put_bits(0xABCD, 16); + coded_bits.put_bits(UASTC_6x6_HDR_SIG1, 16); coded_bits.put_bits(width, 16); coded_bits.put_bits(height, 16); - + enc_state.packed_img.resize(width, height); - + enc_state.strip_bits.resize(total_strips); enc_state.final_astc_blocks.resize(num_blocks_x, num_blocks_y); @@ -6679,7 +6693,7 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa debug_state.init(width, height); else debug_state.init(0, 0); - + interval_timer tm; tm.start(); @@ -6689,7 +6703,7 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa for (uint32_t strip_index = 0; strip_index < total_strips; strip_index++) { const uint32_t strip_first_by = strip_index * rows_per_strip; - + uint32_t strip_last_by = minimum(strip_first_by + rows_per_strip - 1, num_blocks_y); if (strip_index == (total_strips - 1)) strip_last_by = num_blocks_y - 1; @@ -6715,7 +6729,7 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa if (any_failed_flag) break; - + } // strip_index pJob_pool->wait_for_all(); @@ -6725,7 +6739,7 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa fmt_error_printf("One or more strips failed during compression\n"); return false; } - + if (global_cfg.m_debug_output) fmt_printf("Encoding time: {} secs\n", tm.get_elapsed_secs()); @@ -6744,7 +6758,7 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa for (uint32_t i = 0; i < total_strips; i++) coded_bits.append(enc_state.strip_bits[i]); - + coded_bits.put_bits(0xA742, 16); coded_bits.flush(); @@ -6753,13 +6767,13 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa { write_exr((global_cfg.m_output_image_prefix + "comp.exr").c_str(), enc_state.packed_img, 3, 0); } - + if (global_cfg.m_debug_output) fmt_printf("\nTotal intermediate output bits/pixel: {3.4}\n", (float)coded_bits.get_total_bits() / (float)(width * height)); vector2D decoded_blocks1; vector2D decoded_blocks2; - + if (global_cfg.m_debug_output) fmt_printf("decode_file\n"); @@ -6856,7 +6870,7 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa fmt_error_printf("unpack_physical_astc_block() failed\n"); return false; } - + unpacked_astc_img.set_block_clipped(pixels, x * BLOCK_W, y * BLOCK_H, BLOCK_W, BLOCK_H); vec4F pixels_google[MAX_BLOCK_W * MAX_BLOCK_H]; @@ -6879,7 +6893,7 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa } } } - + if (global_cfg.m_debug_output) fmt_printf("\nUnpack succeeded\n"); @@ -6887,9 +6901,9 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa { vector2D bc6h_blocks; - + fast_bc6h_params enc_params; - + bool pack_status = pack_bc6h_image(unpacked_astc_img, bc6h_blocks, &unpacked_bc6h_img, enc_params); if (!pack_status) { @@ -6898,7 +6912,7 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa } unpacked_bc6h_img.crop(width, height); - + if (global_cfg.m_output_images) { write_exr((global_cfg.m_output_image_prefix + "unpacked_bc6h.exr").c_str(), unpacked_bc6h_img, 3, 0); @@ -6907,7 +6921,7 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa unpacked_astc_img.crop(width, height); unpacked_astc_google_img.crop(width, height); - + if (global_cfg.m_output_images) { write_exr((global_cfg.m_output_image_prefix + "unpacked_astc.exr").c_str(), unpacked_astc_img, 3, 0); @@ -6932,7 +6946,7 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa im.print_hp(); } } - + metrics.m_im_astc_log2.calc(enc_state.src_img, unpacked_astc_img, 0, 3, true, true); if (global_cfg.m_debug_output) @@ -6982,7 +6996,7 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa for (uint32_t i = 0; i < 3; i++) { im.calc(enc_state.src_img, unpacked_bc6h_img, i, 1, true, true); - + if (global_cfg.m_debug_output) { printf("%c: ", "RGBA"[i]); @@ -7004,14 +7018,14 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa if (global_cfg.m_image_stats) { image_metrics im; - + if (global_cfg.m_debug_output) printf("BC6H half float space error metrics (a piecewise linear approximation of log2 error):\n"); for (uint32_t i = 0; i < 3; i++) { im.calc_half(enc_state.src_img, unpacked_bc6h_img, i, 1, true); - + if (global_cfg.m_debug_output) { printf("%c: ", "RGBA"[i]); @@ -7020,7 +7034,7 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa } metrics.m_im_bc6h_half.calc_half(enc_state.src_img, unpacked_bc6h_img, 0, 3, true); - + if (global_cfg.m_debug_output) { printf("RGB: "); diff --git a/external/basis_universal/encoder/basisu_astc_hdr_6x6_enc.h b/external/basis_universal/encoder/basisu_astc_hdr_6x6_enc.h index fe89c5b703..bcc4bfd2bb 100644 --- a/external/basis_universal/encoder/basisu_astc_hdr_6x6_enc.h +++ b/external/basis_universal/encoder/basisu_astc_hdr_6x6_enc.h @@ -5,12 +5,13 @@ namespace astc_6x6_hdr { + const uint32_t ASTC_HDR_6X6_DEF_USER_COMP_LEVEL = 2; const uint32_t ASTC_HDR_6X6_MAX_USER_COMP_LEVEL = 12; - + const uint32_t ASTC_HDR_6X6_MAX_COMP_LEVEL = 4; - + const float LDR_BLACK_BIAS = 0.0f;// .49f; - + // Note: This struct is copied several times, so do not place any heavyweight objects in here. struct astc_hdr_6x6_global_config { @@ -18,9 +19,11 @@ namespace astc_6x6_hdr // This encoder computes colorspace error in the ICtCp (or more accurately the delta ITP, where CT is scaled by .5 vs. ICtCp to become T) colorspace, so getting this correct is important. // By default the encoder assumes the input is in absolute luminance (in nits or candela per square meter, cd/m^2), specified as positive-only linear light RGB, using the REC 709 colorspace gamut (but NOT the sRGB transfer function, i.e. linear light). // If the m_rec2020_bt2100_color_gamut flag is true, the input colorspace is treated as REC 2020/BT.2100 (which is wider than 709). - // For SDR/LDR->HDR upconversion, the REC 709 sRGB input should be converted to linear light (sRGB->linear) and the resulting normalized linear RGB values scaled by either 80 or 100 nits (the luminance of a typical SDR monitor). + // For SDR/LDR->HDR upconversion, the REC 709 sRGB input should be converted to linear light (sRGB->linear) and the resulting normalized linear RGB values scaled by either 80 or 100 nits (the luminance of a typical SDR monitor). // SDR upconversion to normalized [0,1] (i.e. non-absolute) luminances may work but is not supported because ITP errors will not be predicted correctly. - bool m_rec2020_bt2100_color_gamut = false; + // 11/3/2025: This flag is always copied straight into the output KTX2 DFD colorspace, even for non-HDR formats. + // TODO: Move this parameter to reflect this. + bool m_rec2020_bt2100_color_gamut = false; // levels 0-3 normal levels, 4=exhaustive uint32_t m_master_comp_level = 0; @@ -35,13 +38,13 @@ namespace astc_6x6_hdr float m_jnd_delta_itp_thresh = .75f; bool m_force_one_strip = false; - + bool m_gaussian1_fallback = true; // def to true, if this is disabled m_gaussian2_fallback should be disabled too float m_gaussian1_strength = 1.45f; bool m_gaussian2_fallback = true; // def to true, hopefully rarely kicks in float m_gaussian2_strength = 1.83f; - + // m_disable_delta_endpoint_usage may give a slight increase in RDO ASTC encoding efficiency. It's also faster. bool m_disable_delta_endpoint_usage = false; @@ -67,7 +70,7 @@ namespace astc_6x6_hdr bool m_disable_twothree_subsets = false; // def to false bool m_use_solid_blocks = true; // def to true bool m_use_runs = true; // def to true - bool m_block_stat_optimizations_flag = true; // def to true + bool m_block_stat_optimizations_flag = true; // def to true bool m_rdo_candidate_diversity_boost = true; // def to true float m_rdo_candidate_diversity_boost_bit_window_weight = 1.2f; @@ -96,7 +99,7 @@ namespace astc_6x6_hdr basisu::fmt_debug_printf("m_rdo_candidate_diversity_boost: {}, m_rdo_candidate_diversity_boost_bit_window_weight: {}\n", m_rdo_candidate_diversity_boost, m_rdo_candidate_diversity_boost_bit_window_weight); basisu::fmt_debug_printf("m_favor_higher_compression: {}, m_num_reuse_xy_deltas: {}\n", m_favor_higher_compression, m_num_reuse_xy_deltas); } - + astc_hdr_6x6_global_config() { } @@ -121,7 +124,7 @@ namespace astc_6x6_hdr basisu::image_metrics m_im_bc6h_log2; basisu::image_metrics m_im_bc6h_half; }; - + // The input image should be unpadded to 6x6 boundaries, i.e. the original unexpanded image. bool compress_photo(const basisu::imagef& orig_src_img, const astc_hdr_6x6_global_config& global_cfg, basisu::job_pool* pJob_pool, basisu::uint8_vec& intermediate_tex_data, basisu::uint8_vec& astc_tex_data, result_metrics& metrics); diff --git a/external/basis_universal/encoder/basisu_astc_hdr_common.cpp b/external/basis_universal/encoder/basisu_astc_hdr_common.cpp index 65be44c1ee..d4df682861 100644 --- a/external/basis_universal/encoder/basisu_astc_hdr_common.cpp +++ b/external/basis_universal/encoder/basisu_astc_hdr_common.cpp @@ -1,4 +1,16 @@ // File: basisu_astc_hdr_common.cpp +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. #include "basisu_enc.h" #include "basisu_gpu_texture.h" #include "../transcoder/basisu_astc_helpers.h" @@ -14,6 +26,7 @@ using namespace basist; namespace basisu { +// Beware: the first entry is the # of weight levels for that BISE range. const uint8_t g_ise_weight_lerps[MAX_SUPPORTED_ISE_WEIGHT_INDEX + 1][33] = { { 2, 0, 64 }, // 0, note ise range=0 is invalid for 4x4 block sizes (<24 weight bits in the block) @@ -117,7 +130,7 @@ static void compute_half_to_qlog_table(uint32_t bits, uint16_t* pTable, const ba float best_err = BIG_FLOAT_VAL; uint32_t best_qlog = 0; - + double prev_err = BIG_FLOAT_VAL; // For all possible qlog's @@ -141,13 +154,13 @@ static void compute_half_to_qlog_table(uint32_t bits, uint16_t* pTable, const ba } prev_err = err; - + // Find best if (err < best_err) { best_err = err; best_qlog = i; - + if (best_err == 0.0f) break; } @@ -171,7 +184,7 @@ static void init_qlog_tables() #if BASISU_MULTITHREADED_INIT job_pool jp(3); - + for (uint32_t bits = HALF_TO_QLOG_TABS_MIN_BITS; bits <= HALF_TO_QLOG_TABS_MAX_BITS; bits++) { jp.add_job( [bits, &qlog16_to_float]() { compute_half_to_qlog_table(bits, g_pHalf_to_qlog_tabs[bits - HALF_TO_QLOG_TABS_MIN_BITS], qlog16_to_float); }); @@ -328,7 +341,7 @@ static bool compute_least_squares_endpoints_rgb( uint32_t N, const uint8_t* pSelectors, const vec4F* pSelector_weights, vec3F* pXl, vec3F* pXh, const vec4F* pColors, const aabb3F& input_box) { - // Least squares using normal equations: http://www.cs.cornell.edu/~bindel/class/cs3220-s12/notes/lec10.pdf + // Least squares using normal equations: http://www.cs.cornell.edu/~bindel/class/cs3220-s12/notes/lec10.pdf // https://web.archive.org/web/20150319232457/http://www.cs.cornell.edu/~bindel/class/cs3220-s12/notes/lec10.pdf // I did this in matrix form first, expanded out all the ops, then optimized it a bit. float z00 = 0.0f, z01 = 0.0f, z10 = 0.0f, z11 = 0.0f; @@ -339,7 +352,7 @@ static bool compute_least_squares_endpoints_rgb( for (uint32_t i = 0; i < N; i++) { const uint32_t sel = pSelectors[i]; - + z00 += pSelector_weights[sel][0]; z10 += pSelector_weights[sel][1]; z11 += pSelector_weights[sel][2]; @@ -373,7 +386,7 @@ static bool compute_least_squares_endpoints_rgb( iz01 = -z01 * det; iz10 = -z10 * det; iz11 = z00 * det; - + (*pXl)[0] = (float)(iz00 * q00_r + iz01 * q10_r); (*pXh)[0] = (float)(iz10 * q00_r + iz11 * q10_r); @@ -392,7 +405,7 @@ static bool compute_least_squares_endpoints_rgb( l = input_box[0][c]; h = input_box[1][c]; } - + (*pXl)[c] = l; (*pXh)[c] = h; } @@ -429,17 +442,17 @@ static bool compute_least_squares_endpoints_rgb( } static bool compute_least_squares_endpoints_rgb_raw_weights( - uint32_t N, const uint8_t* pRaw_weights, + uint32_t N, const uint8_t* pRaw_weights, vec3F* pXl, vec3F* pXh, const vec4F* pColors, const aabb3F& input_box) { - // Least squares using normal equations: http://www.cs.cornell.edu/~bindel/class/cs3220-s12/notes/lec10.pdf + // Least squares using normal equations: http://www.cs.cornell.edu/~bindel/class/cs3220-s12/notes/lec10.pdf // https://web.archive.org/web/20150319232457/http://www.cs.cornell.edu/~bindel/class/cs3220-s12/notes/lec10.pdf // I did this in matrix form first, expanded out all the ops, then optimized it a bit. float z00 = 0.0f, z01 = 0.0f, z10 = 0.0f, z11 = 0.0f; float q00_r = 0.0f, q10_r = 0.0f, t_r = 0.0f; float q00_g = 0.0f, q10_g = 0.0f, t_g = 0.0f; float q00_b = 0.0f, q10_b = 0.0f, t_b = 0.0f; - + for (uint32_t i = 0; i < N; i++) { const float wt = (float)pRaw_weights[i] * (1.0f / 64.0f); @@ -541,13 +554,13 @@ static bool compute_least_squares_endpoints_2D( uint32_t N, const uint8_t* pSelectors, const vec4F* pSelector_weights, vec2F* pXl, vec2F* pXh, const vec2F* pColors, const aabb2F& input_box) { - // Least squares using normal equations: http://www.cs.cornell.edu/~bindel/class/cs3220-s12/notes/lec10.pdf + // Least squares using normal equations: http://www.cs.cornell.edu/~bindel/class/cs3220-s12/notes/lec10.pdf // https://web.archive.org/web/20150319232457/http://www.cs.cornell.edu/~bindel/class/cs3220-s12/notes/lec10.pdf // I did this in matrix form first, expanded out all the ops, then optimized it a bit. float z00 = 0.0f, z01 = 0.0f, z10 = 0.0f, z11 = 0.0f; float q00_r = 0.0f, q10_r = 0.0f, t_r = 0.0f; float q00_g = 0.0f, q10_g = 0.0f, t_g = 0.0f; - + for (uint32_t i = 0; i < N; i++) { const uint32_t sel = pSelectors[i]; @@ -599,7 +612,7 @@ static bool compute_least_squares_endpoints_2D( (*pXl)[c] = l; (*pXh)[c] = h; } - + pXl->clamp(0.0f, MAX_QLOG16_VAL); pXh->clamp(0.0f, MAX_QLOG16_VAL); @@ -610,7 +623,7 @@ static bool compute_least_squares_endpoints_1D( uint32_t N, const uint8_t* pSelectors, const vec4F* pSelector_weights, vec1F* pXl, vec1F* pXh, const vec1F* pColors, const aabb1F& input_box) { - // Least squares using normal equations: http://www.cs.cornell.edu/~bindel/class/cs3220-s12/notes/lec10.pdf + // Least squares using normal equations: http://www.cs.cornell.edu/~bindel/class/cs3220-s12/notes/lec10.pdf // https://web.archive.org/web/20150319232457/http://www.cs.cornell.edu/~bindel/class/cs3220-s12/notes/lec10.pdf // I did this in matrix form first, expanded out all the ops, then optimized it a bit. float z00 = 0.0f, z01 = 0.0f, z10 = 0.0f, z11 = 0.0f; @@ -668,10 +681,10 @@ static bool compute_least_squares_endpoints_1D( } static bool compute_weighted_least_squares_endpoints_rgb( - uint32_t N, + uint32_t N, const uint8_t* pSelectors, const vec4F* pSelector_weights, const float* pRaw_weights, /* ti */ const float* pEmphasis_weights /* wi */, - vec3F* pXl, vec3F* pXh, + vec3F* pXl, vec3F* pXh, const vec4F* pColors, /* pi */ const aabb3F& input_box) { @@ -702,7 +715,7 @@ static bool compute_weighted_least_squares_endpoints_rgb( const float pi_r = pColors[i][0], pi_g = pColors[i][1], pi_b = pColors[i][2]; weighted_mean_tw += wi * ti; - + weighted_mean_pw[0] += wi * pi_r; weighted_mean_pw[1] += wi * pi_g; weighted_mean_pw[2] += wi * pi_b; @@ -722,7 +735,7 @@ static bool compute_weighted_least_squares_endpoints_rgb( const float wi = pEmphasis_weights[i]; const float ti = pSelectors ? pSelector_weights[pSelectors[i]][3] : pRaw_weights[i]; const float pi_r = pColors[i][0], pi_g = pColors[i][1], pi_b = pColors[i][2]; - + spt[0] += wi * (pi_r - weighted_mean_pw[0]) * (ti - weighted_mean_tw); spt[1] += wi * (pi_g - weighted_mean_pw[1]) * (ti - weighted_mean_tw); spt[2] += wi * (pi_b - weighted_mean_pw[2]) * (ti - weighted_mean_tw); @@ -737,7 +750,7 @@ static bool compute_weighted_least_squares_endpoints_rgb( { float h = weighted_mean_pw[i] + (spt[i] / stt) * (1.0f - weighted_mean_tw); float l = weighted_mean_pw[i] - (spt[i] / stt) * weighted_mean_tw; - + (*pXh)[i] = h; (*pXl)[i] = l; } @@ -748,10 +761,10 @@ static bool compute_weighted_least_squares_endpoints_rgb( return true; } -static vec4F g_astc_ls_weights_ise[MAX_SUPPORTED_ISE_WEIGHT_INDEX + 1][MAX_SUPPORTED_WEIGHT_LEVELS]; +vec4F g_astc_ls_weights_ise[MAX_SUPPORTED_ISE_WEIGHT_INDEX + 1][MAX_SUPPORTED_WEIGHT_LEVELS]; -static uint8_t g_map_astc_to_linear_order[MAX_SUPPORTED_ISE_WEIGHT_INDEX + 1][MAX_SUPPORTED_WEIGHT_LEVELS]; // [ise_range][astc_index] -> linear index -static uint8_t g_map_linear_to_astc_order[MAX_SUPPORTED_ISE_WEIGHT_INDEX + 1][MAX_SUPPORTED_WEIGHT_LEVELS]; // [ise_range][linear_index] -> astc_index +uint8_t g_map_astc_to_linear_order[MAX_SUPPORTED_ISE_WEIGHT_INDEX + 1][MAX_SUPPORTED_WEIGHT_LEVELS]; // [ise_range][astc_index] -> linear index +uint8_t g_map_linear_to_astc_order[MAX_SUPPORTED_ISE_WEIGHT_INDEX + 1][MAX_SUPPORTED_WEIGHT_LEVELS]; // [ise_range][linear_index] -> astc_index static void encode_astc_hdr_init() { @@ -800,7 +813,7 @@ void astc_hdr_enc_init() astc_hdr_core_init(); - astc_helpers::init_tables(true); + astc_helpers::init_tables(); init_qlog_tables(); @@ -821,7 +834,7 @@ void interpolate_qlog12_colors( { for (uint32_t j = 0; j < 3; j++) { - assert(in_range(e[i][j], 0, 0xFFF)); + assert(is_in_range(e[i][j], 0, 0xFFF)); } } @@ -1128,7 +1141,7 @@ double eval_selectors( const vec3F low_color((float)pDecoded_half[lo_index * 3 + 0], (float)pDecoded_half[lo_index * 3 + 1], (float)pDecoded_half[lo_index * 3 + 2]); const vec3F high_color((float)pDecoded_half[hi_index * 3 + 0], (float)pDecoded_half[hi_index * 3 + 1], (float)pDecoded_half[hi_index * 3 + 2]); const vec3F mid_color((float)pDecoded_half[mid_index * 3 + 0], (float)pDecoded_half[mid_index * 3 + 1], (float)pDecoded_half[mid_index * 3 + 2]); - + const vec3F block_dir(high_color - low_color); for (uint32_t p = 0; p < num_pixels; p++) @@ -1140,11 +1153,11 @@ double eval_selectors( const int64_t desired_half_r_q = q2(desired_r, coptions.m_q_log_bias); const int64_t desired_half_g_q = q2(desired_g, coptions.m_q_log_bias); const int64_t desired_half_b_q = q2(desired_b, coptions.m_q_log_bias); - + // Determine which side of the middle plane the point is for a modest gain vec3F c((float)desired_r - mid_color[0], (float)desired_g - mid_color[1], (float)desired_b - mid_color[2]); float d = c.dot(block_dir); - + int i = 0, high_index = (num_weight_levels / 2) + 1; if (d >= 0.0f) { @@ -1240,11 +1253,11 @@ double eval_selectors_dual_plane( const uint32_t first_channel = (channel_index + 1) % 3; const uint32_t second_channel = (channel_index + 2) % 3; - + // First plane const double first_channel_weight = channel_weights[first_channel]; const double second_channel_weight = channel_weights[second_channel]; - + for (uint32_t p = 0; p < num_pixels; p++) { const half_float* pDesired_half = &pBlock_pixels_half[p * 3]; @@ -1284,7 +1297,7 @@ double eval_selectors_dual_plane( const half_float* pDesired_half = &pBlock_pixels_half[p * 3]; const double desired_half_a_q = q(pDesired_half[channel_index], coptions.m_q_log_bias); - + double lowest_e = BIG_FLOAT_VAL; // this is an approximation of MSLE @@ -1664,7 +1677,7 @@ bool pack_astc_mode11_submode(uint32_t submode, uint8_t* pEndpoints, int val_q[2 v4 |= (best_vd0 & 31); v5 |= (best_vd1 & 31); - assert(in_range(v0, 0, 255) && in_range(v1, 0, 255) && in_range(v2, 0, 255) && in_range(v3, 0, 255) && in_range(v4, 0, 255) && in_range(v5, 0, 255)); + assert(is_in_range(v0, 0, 255) && is_in_range(v1, 0, 255) && is_in_range(v2, 0, 255) && is_in_range(v3, 0, 255) && is_in_range(v4, 0, 255) && is_in_range(v5, 0, 255)); pEndpoints[0] = (uint8_t)v0; pEndpoints[1] = (uint8_t)v1; @@ -1711,7 +1724,7 @@ bool pack_astc_mode11_submode(uint32_t submode, uint8_t* pEndpoints, int val_q[2 bool pack_astc_mode11_submode(uint32_t submode, uint8_t* pEndpoints, const vec3F& low_q16, const vec3F& high_q16, int& max_clamp_mag, bool early_out_if_clamped, int max_clamp_mag_accept_thresh) { assert(submode <= 7); - + const uint32_t a_bits = 9 + (submode >> 1); const int max_a_val = (1 << a_bits) - 1; @@ -1803,7 +1816,7 @@ void pack_astc_mode11_direct(uint8_t* pEndpoints, vec3F l_q16, vec3F h_q16) // this quantizes R and G as 7 bits vs. 8, for grayscale. //l_q = g_half_to_qlog7[bounds_check((uint32_t)l_half, 0U, 32768U)] << 1; //h_q = g_half_to_qlog7[bounds_check((uint32_t)h_half, 0U, 32768U)] << 1; - + l_q = minimum(l_q, MAX_QLOG8); h_q = minimum(h_q, MAX_QLOG8); } @@ -1982,7 +1995,7 @@ bool pack_astc_mode7_submode(uint32_t submode, uint8_t* pEndpoints, const vec3F& x1 = get_bit(qlog[0], 8); // R8 x2 = get_bit(qlog[0], 7); // R7 x3 = get_bit(qlog[0], 10); // R10 - x4 = get_bit(qlog[0], 6); // R6 + x4 = get_bit(qlog[0], 6); // R6 x5 = get_bit(qlog[3], 6); // S6 x6 = get_bit(qlog[3], 5); // S5 break; @@ -1996,7 +2009,7 @@ bool pack_astc_mode7_submode(uint32_t submode, uint8_t* pEndpoints, const vec3F& x1 = get_bit(qlog[1], 5); // G5 x2 = get_bit(qlog[0], 7); // R7 x3 = get_bit(qlog[2], 5); // B5 - x4 = get_bit(qlog[0], 6); // R6 + x4 = get_bit(qlog[0], 6); // R6 x5 = get_bit(qlog[0], 10); // R10 x6 = get_bit(qlog[0], 9); // R9 break; @@ -2010,7 +2023,7 @@ bool pack_astc_mode7_submode(uint32_t submode, uint8_t* pEndpoints, const vec3F& x1 = get_bit(qlog[0], 8); // R8 x2 = get_bit(qlog[0], 7); // R7 x3 = get_bit(qlog[0], 6); // R6 - x4 = get_bit(qlog[3], 7); // S7 + x4 = get_bit(qlog[3], 7); // S7 x5 = get_bit(qlog[3], 6); // S6 x6 = get_bit(qlog[3], 5); // S5 break; @@ -2024,7 +2037,7 @@ bool pack_astc_mode7_submode(uint32_t submode, uint8_t* pEndpoints, const vec3F& x1 = get_bit(qlog[1], 5); // G5 x2 = get_bit(qlog[0], 7); // R7 x3 = get_bit(qlog[2], 5); // B5 - x4 = get_bit(qlog[0], 6); // R6 + x4 = get_bit(qlog[0], 6); // R6 x5 = get_bit(qlog[3], 6); // S6 x6 = get_bit(qlog[3], 5); // S5 break; @@ -2039,7 +2052,7 @@ bool pack_astc_mode7_submode(uint32_t submode, uint8_t* pEndpoints, const vec3F& x1 = get_bit(qlog[1], 5); // G5 x2 = get_bit(qlog[2], 6); // B6 x3 = get_bit(qlog[2], 5); // B5 - x4 = get_bit(qlog[0], 6); // R6 + x4 = get_bit(qlog[0], 6); // R6 x5 = get_bit(qlog[0], 7); // R7 x6 = get_bit(qlog[3], 5); // S5 break; @@ -2052,7 +2065,7 @@ bool pack_astc_mode7_submode(uint32_t submode, uint8_t* pEndpoints, const vec3F& x1 = get_bit(qlog[1], 5); // G5 x2 = get_bit(qlog[2], 6); // B6 x3 = get_bit(qlog[2], 5); // B5 - x4 = get_bit(qlog[0], 6); // R6 + x4 = get_bit(qlog[0], 6); // R6 x5 = get_bit(qlog[3], 6); // S6 x6 = get_bit(qlog[3], 5); // S5 break; @@ -2143,7 +2156,7 @@ bool pack_mode11(mode11_log_desc& desc, uint8_t* pEndpoints) pEndpoints[1] = (uint8_t)desc.m_b1; pEndpoints[3] = (uint8_t)desc.m_d0; pEndpoints[5] = (uint8_t)desc.m_d1 | 128; - + return true; } @@ -2161,9 +2174,9 @@ bool pack_mode11(mode11_log_desc& desc, uint8_t* pEndpoints) return false; const int va = desc.m_a, vb0 = desc.m_b0, vb1 = desc.m_b1, vc = desc.m_c, vd0 = desc.m_d0, vd1 = desc.m_d1; - + int v0 = 0, v1 = 0, v2 = 0, v3 = 0, v4 = 0, v5 = 0; - + int x0 = 0, x1 = 0, x2 = 0, x3 = 0, x4 = 0, x5 = 0; switch (desc.m_submode) { @@ -2222,7 +2235,7 @@ bool pack_mode11(mode11_log_desc& desc, uint8_t* pEndpoints) v4 |= (vd0 & 31); v5 |= (vd1 & 31); - assert(in_range(v0, 0, 255) && in_range(v1, 0, 255) && in_range(v2, 0, 255) && in_range(v3, 0, 255) && in_range(v4, 0, 255) && in_range(v5, 0, 255)); + assert(is_in_range(v0, 0, 255) && is_in_range(v1, 0, 255) && is_in_range(v2, 0, 255) && is_in_range(v3, 0, 255) && is_in_range(v4, 0, 255) && is_in_range(v5, 0, 255)); pEndpoints[0] = (uint8_t)v0; pEndpoints[1] = (uint8_t)v1; @@ -2236,7 +2249,7 @@ bool pack_mode11(mode11_log_desc& desc, uint8_t* pEndpoints) static inline int astc_hdr_sign_extend(int src, int num_src_bits) { - assert(basisu::in_range(num_src_bits, 2, 31)); + assert(basisu::is_in_range(num_src_bits, 2, 31)); const bool negative = (src & (1 << (num_src_bits - 1))) != 0; if (negative) @@ -2261,7 +2274,7 @@ void unpack_mode11(const uint8_t* pEndpoints, mode11_log_desc& desc) desc.m_b1 = pEndpoints[1]; desc.m_d0 = pEndpoints[3]; desc.m_d1 = pEndpoints[5] & 0x7F; - + return; } @@ -2384,7 +2397,7 @@ void decode_cem_7_config(const uint8_t* pEndpoints, int& submode_index, int &maj bool pack_mode11( const vec3F& low_color_q16, const vec3F& high_color_q16, - uint32_t ise_endpoint_range, uint8_t* pEndpoints, + uint32_t ise_endpoint_range, uint8_t* pEndpoints, const astc_hdr_codec_base_options& coptions, bool direct_only, int32_t first_submode, int32_t last_submode, bool ignore_clamping, uint32_t& submode_used) { @@ -2521,7 +2534,7 @@ bool pack_mode11( } // d } // c } // if (coptions.m_ultra_quant) - + submode_used = best_submode + 1; return (best_trial_dist != BIG_FLOAT_VAL); @@ -3039,7 +3052,7 @@ bool try_mode7( clear_obj(best_trial_endpoints); double best_trial_dist = BIG_FLOAT_VAL; int best_trial_submode = 0; - + for (int submode = first_submode; submode <= last_submode; submode++) { const int MAX_CLAMP_MAG_ACCEPT_THRESH = 16; @@ -3216,6 +3229,8 @@ double encode_astc_hdr_block_mode_11( float l = BIG_FLOAT_VAL, h = -BIG_FLOAT_VAL; vec3F low_color_q16, high_color_q16; + low_color_q16.clear(); + high_color_q16.clear(); for (uint32_t i = 0; i < num_pixels; i++) { @@ -3236,9 +3251,9 @@ double encode_astc_hdr_block_mode_11( high_color_q16 = pBlock_pixels_q16[i]; } } - + vec3F old_low_color_q16(low_color_q16), old_high_color_q16(high_color_q16); - + for (uint32_t i = 0; i < 3; i++) { low_color_q16[i] = lerp(old_low_color_q16[i], old_high_color_q16[i], 1.0f / 64.0f); @@ -3253,7 +3268,7 @@ double encode_astc_hdr_block_mode_11( clear_obj(trial_blk_weights); double trial_blk_error = BIG_FLOAT_VAL; - + bool did_improve = try_mode11(num_pixels, trial_blk_endpoints, trial_blk_weights, trial_blk_error, trial_best_submode, low_color_q16, high_color_q16, pBlock_pixels_half, num_weight_levels, ise_weight_range, coptions, direct_only, ise_endpoint_range, constrain_ise_weight_selectors, @@ -3330,7 +3345,7 @@ double encode_astc_hdr_block_mode_11( if (!compute_least_squares_endpoints_rgb(num_pixels, trial_blk_weights, &g_astc_ls_weights_ise[ise_weight_range][0], &l_q16, &h_q16, pBlock_pixels_q16, color_box_q16)) break; - + bool was_improved = try_mode11(num_pixels, blk_endpoints, blk_weights, cur_block_error, best_submode, l_q16, h_q16, pBlock_pixels_half, num_weight_levels, ise_weight_range, coptions, direct_only, ise_endpoint_range, constrain_ise_weight_selectors, @@ -3359,7 +3374,7 @@ double encode_astc_hdr_block_mode_11( float lw = LOW_EMPHASIS_WEIGHT, mw = MIDDLE_EMPHASIS_WEIGHT, hw = HIGH_EMPHASIS_WEIGHT; if (opt_mode == cWeightedLeastSquaresHeavy) lw = LOW_EMPHASIS_WEIGHT_HEAVY, mw = MIDDLE_EMPHASIS_WEIGHT_HEAVY, hw = HIGH_EMPHASIS_WEIGHT_HEAVY; - + for (uint32_t i = 0; i < num_pixels; i++) { vec3F k(vec3F(pBlock_pixels_q16[i]) - block_mean_color_q16); @@ -3368,7 +3383,7 @@ double encode_astc_hdr_block_mode_11( assert((kd >= l) && (kd <= h)); float v = (kd - l) / (h - l); - + if (v < mid) v = lerp(lw, mw, v / mid); else @@ -3617,6 +3632,8 @@ double encode_astc_hdr_block_downsampled_mode_11( float l = BIG_FLOAT_VAL, h = -BIG_FLOAT_VAL; vec3F low_color_q16, high_color_q16; + low_color_q16.clear(); + high_color_q16.clear(); for (uint32_t i = 0; i < num_pixels; i++) { @@ -3655,7 +3672,7 @@ double encode_astc_hdr_block_downsampled_mode_11( clear_obj(trial_blk_endpoints); clear_obj(trial_blk_weights); - + double trial_blk_error = BIG_FLOAT_VAL; bool could_pack = try_mode11(num_pixels, trial_blk_endpoints, trial_blk_weights, trial_blk_error, trial_best_submode, @@ -3696,7 +3713,7 @@ double encode_astc_hdr_block_downsampled_mode_11( } else if (pass) break; - + if ((opt_mode == cWeightedLeastSquares) || (opt_mode == cWeightedLeastSquaresHeavy)) { float emphasis_weights[MAX_ASTC_HDR_ENC_BLOCK_PIXELS]; @@ -3797,7 +3814,7 @@ double encode_astc_hdr_block_mode_11_dual_plane( assert((first_submode >= FIRST_MODE11_SUBMODE_INDEX) && (first_submode <= last_submode)); assert(last_submode <= MAX_MODE11_SUBMODE_INDEX); - + assert(num_pixels <= MAX_ASTC_HDR_ENC_BLOCK_PIXELS); best_submode = 0; @@ -3873,7 +3890,7 @@ double encode_astc_hdr_block_mode_11_dual_plane( double trial_blk_error = BIG_FLOAT_VAL; bool did_improve = try_mode11_dual_plane(channel_index, num_pixels, trial_blk_endpoints, trial_blk_weights0, trial_blk_weights1, trial_blk_error, trial_best_submode, - low_color_q16, high_color_q16, + low_color_q16, high_color_q16, pBlock_pixels_half, num_weight_levels, ise_weight_range, coptions, direct_only, ise_endpoint_range, constrain_ise_weight_selectors, first_submode, last_submode, ignore_clamping); @@ -3947,7 +3964,7 @@ double encode_astc_hdr_block_mode_11_dual_plane( memcpy(trial_blk_weights1, blk_weights1, num_pixels); } // pass - + return cur_block_error; } @@ -3962,7 +3979,7 @@ double encode_astc_hdr_block_mode_7( uint8_t* blk_endpoints, //[4] uint8_t* blk_weights, // [num_pixels] const astc_hdr_codec_base_options& coptions, - uint32_t ise_endpoint_range, + uint32_t ise_endpoint_range, int first_submode, int last_submode, const encode_astc_block_stats* pBlock_stats) { @@ -4008,7 +4025,7 @@ double encode_astc_hdr_block_mode_7( vec3F diff(high_color_q16 - low_color_q16); - // The mul here (* block_axis_q16[0]) is because the "S" or scale value is subtracted from the high color with a scale of 1.0, + // The mul here (* block_axis_q16[0]) is because the "S" or scale value is subtracted from the high color with a scale of 1.0, // i.e. it's equivalent to a vector of (1,1,1) multiplied by scale before the sub. We want to actually move along the grayscale axis, or (0.577350259, 0.577350259, 0.577350259). float s_q16 = diff.dot(block_axis_q16) * block_axis_q16[0]; @@ -4081,7 +4098,7 @@ double encode_astc_hdr_block_mode_7( vec3F alt_diff(alt_high_color_q16 - alt_low_color_q16); - // The mul here (* block_axis_q16[0]) is because the "S" or scale value is subtracted from the high color with a scale of 1.0, + // The mul here (* block_axis_q16[0]) is because the "S" or scale value is subtracted from the high color with a scale of 1.0, // i.e. it's equivalent to a vector of (1,1,1) multiplied by scale before the sub. We want to actually move along the grayscale axis, or (0.577350259, 0.577350259, 0.577350259). float alt_s_q16 = alt_diff.dot(block_axis_q16) * block_axis_q16[0]; @@ -4217,6 +4234,7 @@ void dequantize_astc_weights(uint32_t n, const uint8_t* pSrc_ise_vals, uint32_t } //-------------------------------------------------------------------------------------------------------------------------- +// Precomputed matrices via SLSQP (Sequential Least-Squares Quadratic Programming - scipy.optimize.minimize). Sharper results vs. other methods (like adjoint). // For each output (2x2) sample, the weight of each input (6x6) sample. static const float g_weight_downsample_6x6_to_2x2[4][36] = { @@ -4720,11 +4738,14 @@ static const float g_weight_downsample_6x6_to_6x6[36][36] = { //-------------------------------------------------------------------------------------------------------------------------- -const struct downsample_matrix_6x6 +struct downsample_matrix { uint32_t m_grid_width, m_grid_height; const float* m_p; -} g_downsample_matrices_6x6[] = { +}; + +downsample_matrix g_downsample_matrices_6x6[] = +{ { 2, 2, (const float*)g_weight_downsample_6x6_to_2x2 }, { 3, 2, (const float*)g_weight_downsample_6x6_to_3x2 }, { 4, 2, (const float*)g_weight_downsample_6x6_to_4x2 }, @@ -4753,11 +4774,892 @@ const struct downsample_matrix_6x6 }; //const uint32_t NUM_DOWNSAMPLE_MATRICES_6x6 = sizeof(g_downsample_matrices_6x6) / sizeof(g_downsample_matrices_6x6[0]); +//-------------------------------------------------------------------------------------------------------------------------- +// +// For each output (2x2) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_2x2[4][48] = { +{0.137431f, 0.119592f, 0.085575f, 0.056401f, 0.030751f, 0.000000f, 0.000000f, 0.000000f, 0.108851f, 0.086312f, 0.064884f, 0.039119f, 0.027653f, 0.000000f, 0.000000f, 0.000000f, 0.073703f, 0.067584f, 0.045034f, 0.032697f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024414f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.033828f, 0.058911f, 0.081870f, 0.120975f, 0.137384f, 0.000000f, 0.000000f, 0.000000f, 0.026912f, 0.038126f, 0.065247f, 0.083628f, 0.109730f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.037909f, 0.044325f, 0.065160f, 0.074043f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.021952f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024645f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.074133f, 0.065243f, 0.043065f, 0.035114f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.105931f, 0.087385f, 0.065848f, 0.035699f, 0.030068f, 0.000000f, 0.000000f, 0.000000f, 0.136321f, 0.121324f, 0.086171f, 0.057503f, 0.031553f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024251f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.037022f, 0.042379f, 0.063662f, 0.075871f, 0.000000f, 0.000000f, 0.000000f, 0.031315f, 0.037129f, 0.065785f, 0.084055f, 0.107841f, 0.000000f, 0.000000f, 0.000000f, 0.030537f, 0.057932f, 0.086040f, 0.120055f, 0.136127f}, +}; + +// For each output (3x2) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_3x2[6][48] = { +{0.212556f, 0.137038f, 0.067006f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.172663f, 0.105023f, 0.058944f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.113989f, 0.074111f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.037147f, 0.021524f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.077366f, 0.142656f, 0.145067f, 0.074900f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.048644f, 0.106713f, 0.104141f, 0.052434f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.048972f, 0.079367f, 0.079508f, 0.040229f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.064479f, 0.139823f, 0.212207f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.053987f, 0.104596f, 0.171728f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.026564f, 0.071759f, 0.119334f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.035524f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.037522f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.115689f, 0.072510f, 0.021389f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.170967f, 0.106096f, 0.061696f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.210888f, 0.137969f, 0.065274f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.045147f, 0.080905f, 0.078591f, 0.043486f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.045421f, 0.106778f, 0.106427f, 0.050794f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.079169f, 0.139959f, 0.144180f, 0.079143f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033940f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.021724f, 0.070791f, 0.117496f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.059938f, 0.109787f, 0.170583f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.064517f, 0.139526f, 0.211698f}, +}; + +// For each output (4x2) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_4x2[8][48] = { +{0.275657f, 0.133248f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.225305f, 0.089819f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.147466f, 0.079439f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.049065f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.071558f, 0.188360f, 0.141460f, 0.027429f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.068719f, 0.139588f, 0.107851f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024602f, 0.112032f, 0.076880f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.019401f, 0.000000f, 0.022120f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.025244f, 0.140416f, 0.189606f, 0.065541f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.021281f, 0.106671f, 0.142270f, 0.062848f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.068039f, 0.102306f, 0.026541f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.023517f, 0.025720f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.136533f, 0.275463f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.086827f, 0.223674f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.077361f, 0.153684f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.046457f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.048293f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.149189f, 0.077647f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.222753f, 0.093443f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.273639f, 0.135036f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.022695f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.027966f, 0.116923f, 0.074704f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.066610f, 0.140552f, 0.119791f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.070250f, 0.192769f, 0.140414f, 0.027327f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.026026f, 0.032280f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.073723f, 0.105102f, 0.027631f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.113307f, 0.139466f, 0.059915f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.027161f, 0.140907f, 0.189935f, 0.064546f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.045275f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.074412f, 0.151685f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.094074f, 0.223897f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.136604f, 0.274053f}, +}; + +// For each output (5x2) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_5x2[10][48] = { +{0.298257f, 0.099048f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.242705f, 0.083012f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.155959f, 0.035340f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.054463f, 0.031217f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.149629f, 0.250491f, 0.037003f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.113317f, 0.192720f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.093738f, 0.138010f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.025093f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.193314f, 0.196494f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.163178f, 0.158983f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.112334f, 0.115733f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.028572f, 0.031390f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.028975f, 0.256222f, 0.142262f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.191874f, 0.111703f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.137754f, 0.096234f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.034976f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.105369f, 0.297279f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.081692f, 0.239675f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.031939f, 0.162333f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.031404f, 0.050308f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.053972f, 0.028379f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.158432f, 0.035219f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.238959f, 0.089734f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.294641f, 0.100664f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.034176f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.090008f, 0.147020f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.103221f, 0.190008f, 0.024843f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.139784f, 0.245082f, 0.025860f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.032527f, 0.032618f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.117780f, 0.108323f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.155910f, 0.159880f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.197210f, 0.195753f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.042681f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.138684f, 0.099059f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.186926f, 0.105714f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.029545f, 0.254477f, 0.142915f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.029953f, 0.051219f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.029174f, 0.163463f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.087461f, 0.240531f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.103819f, 0.294380f}, +}; + +// For each output (6x2) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_6x2[12][48] = { +{0.362153f, 0.050427f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.296074f, 0.031598f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.192551f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.067197f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.240020f, 0.169624f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.196469f, 0.128913f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.131714f, 0.098049f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.035210f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.105361f, 0.301218f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.086270f, 0.220336f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.047552f, 0.171037f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.022966f, 0.045259f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.287211f, 0.111854f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.224383f, 0.097742f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.167408f, 0.037607f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.036827f, 0.036969f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.152162f, 0.235841f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.108280f, 0.202388f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.091687f, 0.151852f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.057789f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.051343f, 0.374208f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.304381f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.207583f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.062485f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.064793f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.193058f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.290484f, 0.038424f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.357650f, 0.055589f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.035640f, 0.019558f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.133571f, 0.100435f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.184400f, 0.125111f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.228117f, 0.173168f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.044711f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.043438f, 0.175074f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.089766f, 0.235789f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.108452f, 0.302770f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.037495f, 0.032008f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.168503f, 0.033572f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.226763f, 0.101709f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.292934f, 0.107016f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.019003f, 0.018791f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.100854f, 0.125828f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.107572f, 0.206978f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.169736f, 0.251237f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.060542f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024678f, 0.204824f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.301594f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.040204f, 0.368158f}, +}; + +// For each output (7x2) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_7x2[14][48] = { +{0.396534f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.324924f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.210380f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.068162f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.365804f, 0.047637f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.288211f, 0.031570f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.215416f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.051362f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.277573f, 0.121338f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.219048f, 0.084370f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.023178f, 0.000000f, 0.161469f, 0.031346f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.034866f, 0.046814f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.194115f, 0.218789f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.163854f, 0.137782f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020281f, 0.000000f, 0.127129f, 0.138049f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.089911f, 0.279003f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.100285f, 0.229490f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.026109f, 0.164969f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.036219f, 0.074014f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033369f, 0.385493f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.300028f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.222803f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.058307f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.395806f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.320906f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.218670f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.064618f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.064591f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.213009f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.324054f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.398346f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.052403f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.218943f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.280900f, 0.028228f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.364696f, 0.054830f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.040226f, 0.027986f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.172678f, 0.019447f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.228976f, 0.118935f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.278251f, 0.113500f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.017206f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.022203f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.022373f, 0.000000f, 0.138786f, 0.130317f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024343f, 0.000000f, 0.127713f, 0.134415f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.187440f, 0.195205f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033347f, 0.041046f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.029210f, 0.133093f, 0.000000f, 0.020285f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.102427f, 0.246296f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.104431f, 0.289864f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.027153f, 0.048478f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.032573f, 0.217822f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.278933f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.022617f, 0.372424f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.061793f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.219494f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.324119f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.394594f}, +}; + +// For each output (8x2) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_8x2[16][48] = { +{0.397679f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.325539f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.208885f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.067897f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.394986f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.323551f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.218305f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.063158f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.400685f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.325867f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.214372f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.059075f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.398573f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.319207f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.212413f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.069808f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.401571f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.323398f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.212771f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.062260f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.404990f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.322008f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.207631f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.065371f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.396891f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.320883f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.212780f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.069447f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.396345f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.321731f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.217640f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.064285f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.064801f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.212540f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.324204f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.398456f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.063907f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.221286f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.319039f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.395768f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.064375f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.221627f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.320522f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.393476f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.067161f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.214405f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.322795f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.395638f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.065100f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.209382f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.325769f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.399749f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.072177f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.207268f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.318619f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.401935f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.063557f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.217484f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.316546f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.402413f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.061762f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.218082f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.324604f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.395552f}, +}; + +// For each output (2x3) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_2x3[6][48] = { +{0.205910f, 0.181220f, 0.131230f, 0.084091f, 0.045598f, 0.000000f, 0.000000f, 0.000000f, 0.115248f, 0.106195f, 0.073083f, 0.057425f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.054674f, 0.092055f, 0.125587f, 0.176378f, 0.202284f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.055452f, 0.075306f, 0.102574f, 0.115689f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.044070f, 0.029520f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.136903f, 0.115512f, 0.084403f, 0.050846f, 0.035490f, 0.000000f, 0.000000f, 0.000000f, 0.143459f, 0.115683f, 0.085020f, 0.053056f, 0.036572f, 0.000000f, 0.000000f, 0.000000f, 0.043466f, 0.026000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.025190f, 0.040099f, 0.000000f, 0.000000f, 0.000000f, 0.037965f, 0.050927f, 0.083471f, 0.112563f, 0.137468f, 0.000000f, 0.000000f, 0.000000f, 0.033927f, 0.046348f, 0.085573f, 0.114643f, 0.134372f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024810f, 0.028641f, 0.044003f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.111326f, 0.107232f, 0.073233f, 0.050676f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.204047f, 0.179532f, 0.131819f, 0.088809f, 0.053325f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.023277f, 0.054224f, 0.067723f, 0.100097f, 0.113199f, 0.000000f, 0.000000f, 0.000000f, 0.047881f, 0.085543f, 0.130088f, 0.176198f, 0.201769f}, +}; + +// For each output (3x3) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_3x3[9][48] = { +{0.327238f, 0.215195f, 0.108640f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.184524f, 0.118385f, 0.046018f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.109423f, 0.206952f, 0.207632f, 0.108494f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.064973f, 0.120899f, 0.114663f, 0.066964f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.107663f, 0.213426f, 0.326644f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.045643f, 0.119988f, 0.186636f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.060005f, 0.030140f, 0.020392f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.193258f, 0.127396f, 0.061395f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.196600f, 0.132656f, 0.063337f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.060793f, 0.029915f, 0.024113f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.032682f, 0.042599f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.070428f, 0.145040f, 0.144782f, 0.074883f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.069308f, 0.145612f, 0.133265f, 0.071190f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.035901f, 0.034311f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.030350f, 0.056939f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.060846f, 0.125850f, 0.201518f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.063906f, 0.129434f, 0.203119f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.035006f, 0.026673f, 0.066360f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.184897f, 0.119434f, 0.045977f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.328093f, 0.217057f, 0.104542f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.064974f, 0.120280f, 0.118724f, 0.069494f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.111457f, 0.199814f, 0.204785f, 0.110472f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.038193f, 0.124885f, 0.182125f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.105011f, 0.218548f, 0.331237f}, +}; + +// For each output (4x3) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_4x3[12][48] = { +{0.424820f, 0.213734f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.237540f, 0.123907f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.101064f, 0.293828f, 0.214193f, 0.045263f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.051229f, 0.170008f, 0.124414f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.043452f, 0.216897f, 0.293802f, 0.110908f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.114842f, 0.173267f, 0.046832f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.204747f, 0.427412f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.126209f, 0.241633f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.087490f, 0.023647f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.277233f, 0.116842f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.282751f, 0.124394f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.087642f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024375f, 0.043221f, 0.025504f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.075199f, 0.165822f, 0.130107f, 0.031544f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.074010f, 0.171441f, 0.131257f, 0.016920f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.037357f, 0.043775f, 0.029468f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.034358f, 0.046676f, 0.025003f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.026567f, 0.127081f, 0.172282f, 0.077309f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.028046f, 0.132256f, 0.162992f, 0.075728f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033213f, 0.036679f, 0.021810f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.083610f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.116623f, 0.293550f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.118246f, 0.292686f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.095285f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.234002f, 0.132935f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.422801f, 0.210262f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.037740f, 0.173712f, 0.127636f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.107054f, 0.296425f, 0.213343f, 0.044090f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.122782f, 0.174732f, 0.044321f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.046279f, 0.214323f, 0.289278f, 0.108285f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.125079f, 0.236461f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.208583f, 0.429877f}, +}; + +// For each output (5x3) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_5x3[15][48] = { +{0.490219f, 0.168976f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.273361f, 0.067444f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.213329f, 0.380538f, 0.048722f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.138224f, 0.219188f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.309867f, 0.312289f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.189101f, 0.188743f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.037522f, 0.380550f, 0.216834f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.225818f, 0.139276f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.164462f, 0.488476f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.072635f, 0.274427f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.085550f, 0.041856f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.277218f, 0.100778f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.279523f, 0.102655f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.086943f, 0.025474f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.018474f, 0.000000f, 0.000000f, 0.023807f, 0.063654f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.142638f, 0.245307f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.145790f, 0.254064f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.047600f, 0.058666f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.047090f, 0.051660f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.197880f, 0.207261f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.205538f, 0.186457f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.052816f, 0.051298f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.018852f, 0.055366f, 0.033613f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.247747f, 0.138008f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.030549f, 0.240788f, 0.147930f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.066598f, 0.020549f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.031861f, 0.081013f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.095562f, 0.286515f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.091897f, 0.287997f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.038590f, 0.086564f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.268683f, 0.083034f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.485628f, 0.162655f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.121869f, 0.229484f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.218817f, 0.384593f, 0.045237f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.182342f, 0.183530f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.320205f, 0.313923f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.217960f, 0.138650f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.051048f, 0.375126f, 0.217217f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.064150f, 0.273673f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.169346f, 0.492831f}, +}; + +// For each output (6x3) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_6x3[18][48] = { +{0.567729f, 0.085252f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.316321f, 0.030698f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.359927f, 0.264711f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.204426f, 0.170936f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.160854f, 0.493683f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.055911f, 0.289551f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.471204f, 0.180222f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.281132f, 0.067442f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.244512f, 0.369052f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.158920f, 0.227515f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.066465f, 0.597036f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.336500f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.104579f, 0.023148f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.338908f, 0.039468f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.344319f, 0.042826f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.106751f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.059448f, 0.022978f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.245888f, 0.156583f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.251094f, 0.164427f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.073868f, 0.025715f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.047831f, 0.060057f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.116572f, 0.271105f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.108894f, 0.276085f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.039515f, 0.079942f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.080438f, 0.048264f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.267123f, 0.113138f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.263081f, 0.110654f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.077711f, 0.039591f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020193f, 0.059109f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.154371f, 0.249388f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.148917f, 0.263084f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.021121f, 0.083817f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024900f, 0.107003f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.375065f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.378856f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.114175f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.311342f, 0.043011f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.565421f, 0.080225f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.018768f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.192162f, 0.168731f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.354606f, 0.265733f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.069515f, 0.282839f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.159765f, 0.487881f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.278646f, 0.072312f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.480532f, 0.168510f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.157488f, 0.194745f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.261639f, 0.386129f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.043524f, 0.320675f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.055191f, 0.580610f}, +}; + +// For each output (7x3) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_7x3[21][48] = { +{0.641452f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.358548f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.571435f, 0.068076f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.330216f, 0.030272f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.442607f, 0.191771f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.243785f, 0.063036f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.018329f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.019157f, 0.000000f, 0.021315f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.273064f, 0.307420f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.195541f, 0.177034f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.022294f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024647f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.151030f, 0.456644f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.078617f, 0.291813f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.021896f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.060980f, 0.596856f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.342163f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.639429f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.360571f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.114797f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.378786f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.387691f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.118726f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.090755f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.356378f, 0.041502f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.359468f, 0.040845f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.091221f, 0.019830f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.078340f, 0.030772f, 0.000000f, 0.017555f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.267597f, 0.100863f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.271447f, 0.100798f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.064330f, 0.068296f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.044982f, 0.034940f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.021793f, 0.000000f, 0.194246f, 0.216278f, 0.000000f, 0.022234f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.203237f, 0.184740f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.019217f, 0.018086f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.023471f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.016776f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.047044f, 0.060726f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.086110f, 0.270497f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.100587f, 0.267194f, 0.000000f, 0.020092f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.050739f, 0.097011f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.023976f, 0.094747f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.036130f, 0.353791f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.032724f, 0.369552f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.089080f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.107420f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.386732f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.390932f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.114916f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.354042f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.645958f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.337170f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.589668f, 0.073162f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.281005f, 0.071771f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.450506f, 0.196718f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.021998f, 0.000000f, 0.000000f, 0.025261f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.032091f, 0.000000f, 0.182952f, 0.186377f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.270805f, 0.280517f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020667f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.064614f, 0.248064f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.182212f, 0.484444f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.046780f, 0.341462f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.041817f, 0.569940f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.355095f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.644905f}, +}; + +// For each output (8x3) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_8x3[24][48] = { +{0.642405f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.357595f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.643957f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.356043f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.642833f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.357167f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.637580f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.362420f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.642714f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.357286f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.637481f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.362519f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.646282f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.353718f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.640587f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.359413f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.113933f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.379885f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.389232f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.116950f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.104449f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.396859f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.400104f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.098588f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.102359f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.394242f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.401732f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.101667f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.096440f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.392155f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.400404f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.111000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.114593f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.389960f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.382704f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.112742f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.109021f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.396881f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.388517f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.105580f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.108474f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.389562f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.401518f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.100446f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.106886f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.387604f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.392295f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.113215f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.353573f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.646427f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.356921f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.643079f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.363744f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.636256f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.356177f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.643823f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.354225f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.645775f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.359749f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.640251f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.364443f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.635557f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.353912f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.646088f}, +}; + +// For each output (2x4) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_2x4[8][48] = { +{0.266475f, 0.237248f, 0.170961f, 0.108932f, 0.059980f, 0.000000f, 0.000000f, 0.000000f, 0.069153f, 0.052080f, 0.035172f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.071584f, 0.118291f, 0.158003f, 0.229344f, 0.262308f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.040608f, 0.047117f, 0.072745f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.133546f, 0.123736f, 0.085634f, 0.071146f, 0.020522f, 0.000000f, 0.000000f, 0.000000f, 0.181365f, 0.152470f, 0.109189f, 0.071277f, 0.051114f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.068769f, 0.083081f, 0.122611f, 0.135462f, 0.000000f, 0.000000f, 0.000000f, 0.052661f, 0.073804f, 0.122675f, 0.158233f, 0.182705f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.185771f, 0.157833f, 0.115265f, 0.071389f, 0.049909f, 0.000000f, 0.000000f, 0.000000f, 0.134315f, 0.122577f, 0.090159f, 0.072782f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.049580f, 0.068443f, 0.120275f, 0.155720f, 0.183091f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.072223f, 0.092680f, 0.123123f, 0.134866f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.061367f, 0.051211f, 0.034360f, 0.000000f, 0.028160f, 0.000000f, 0.000000f, 0.000000f, 0.255536f, 0.224675f, 0.167736f, 0.113503f, 0.063453f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033855f, 0.000000f, 0.030092f, 0.044250f, 0.067673f, 0.000000f, 0.000000f, 0.000000f, 0.059731f, 0.111955f, 0.169044f, 0.224131f, 0.259268f}, +}; + +// For each output (3x4) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_3x4[12][48] = { +{0.405143f, 0.264455f, 0.127900f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.105076f, 0.051679f, 0.045747f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.025952f, 0.148689f, 0.283429f, 0.283899f, 0.145415f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.061558f, 0.051058f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.124702f, 0.268998f, 0.405480f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.043101f, 0.052379f, 0.105340f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.214261f, 0.145181f, 0.047508f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.296952f, 0.196156f, 0.099941f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.084673f, 0.137735f, 0.144414f, 0.077484f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.086806f, 0.178074f, 0.179109f, 0.089543f, 0.022161f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.050723f, 0.149013f, 0.214357f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.101549f, 0.190388f, 0.293970f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.293440f, 0.200404f, 0.104808f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.212205f, 0.141684f, 0.047458f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.085757f, 0.179609f, 0.175648f, 0.084745f, 0.021210f, 0.000000f, 0.000000f, 0.000000f, 0.083231f, 0.140659f, 0.147264f, 0.081878f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.104715f, 0.195444f, 0.297105f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.052478f, 0.135662f, 0.214595f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.105858f, 0.047177f, 0.044681f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.407919f, 0.269431f, 0.124933f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.066066f, 0.061881f, 0.023069f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.149307f, 0.272481f, 0.277246f, 0.149950f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.036865f, 0.065377f, 0.096438f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.123758f, 0.269301f, 0.408262f}, +}; + +// For each output (4x4) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_4x4[16][48] = { +{0.550981f, 0.273527f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.143555f, 0.031938f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.122629f, 0.360487f, 0.261668f, 0.049773f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.061033f, 0.081604f, 0.062805f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.049839f, 0.269578f, 0.365997f, 0.133966f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.048352f, 0.083803f, 0.048464f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.267525f, 0.553972f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.034129f, 0.144375f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.277118f, 0.159322f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.390449f, 0.173111f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.047384f, 0.191890f, 0.131656f, 0.024565f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.109738f, 0.256529f, 0.192107f, 0.046132f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.031695f, 0.141682f, 0.193059f, 0.054775f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.036195f, 0.182374f, 0.246275f, 0.113945f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.160040f, 0.281798f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.166904f, 0.391257f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.392178f, 0.179451f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.279598f, 0.148773f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.107261f, 0.247609f, 0.198942f, 0.036907f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.054678f, 0.195067f, 0.134127f, 0.025410f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.017019f, 0.017319f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.032887f, 0.182133f, 0.239063f, 0.107658f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.026552f, 0.139058f, 0.187193f, 0.051118f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.169923f, 0.395389f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.148923f, 0.285765f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.142165f, 0.038534f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.547445f, 0.271856f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.044944f, 0.076529f, 0.068448f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.125039f, 0.368874f, 0.262015f, 0.054151f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.059929f, 0.083064f, 0.044633f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.053433f, 0.265593f, 0.362429f, 0.130919f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.045972f, 0.135681f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.264414f, 0.553933f}, +}; + +// For each output (5x4) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_5x4[20][48] = { +{0.596845f, 0.198746f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.148428f, 0.055981f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.278053f, 0.491329f, 0.050522f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.064229f, 0.115868f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.404918f, 0.399709f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.097883f, 0.097489f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.050295f, 0.498737f, 0.280436f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.117869f, 0.052664f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.200415f, 0.589668f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.063856f, 0.146061f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.306027f, 0.097934f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.428737f, 0.167302f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.155850f, 0.258285f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.187173f, 0.344891f, 0.035315f, 0.000000f, 0.018485f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.212411f, 0.213232f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.283532f, 0.290826f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.022380f, 0.255191f, 0.169763f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020378f, 0.342025f, 0.190264f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.089095f, 0.316913f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.159089f, 0.434903f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.436982f, 0.169707f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.310539f, 0.082773f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.187439f, 0.337224f, 0.031428f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.167442f, 0.252995f, 0.023472f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.298614f, 0.285810f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.206405f, 0.209172f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.019544f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033200f, 0.325724f, 0.185761f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.030366f, 0.251622f, 0.153784f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.161862f, 0.437691f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.086681f, 0.313765f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.149673f, 0.068654f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.589414f, 0.192260f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.038852f, 0.121054f, 0.025391f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.280331f, 0.492424f, 0.041948f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.095308f, 0.102698f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.407796f, 0.394198f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.106939f, 0.057645f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.058299f, 0.489157f, 0.287960f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.063501f, 0.142763f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.196593f, 0.597142f}, +}; + +// For each output (6x4) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_6x4[24][48] = { +{0.723801f, 0.094637f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.181562f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.476584f, 0.344817f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.116143f, 0.062457f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.194537f, 0.608409f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.061561f, 0.135493f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.579284f, 0.209203f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.135477f, 0.076035f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.308340f, 0.460085f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.052476f, 0.139411f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.019970f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.019719f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.082209f, 0.732181f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.185611f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.358932f, 0.060659f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.503915f, 0.076494f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.237301f, 0.199098f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.332364f, 0.231237f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.088364f, 0.322995f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.173711f, 0.414930f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.312366f, 0.093336f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.392413f, 0.164056f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.019281f, 0.018548f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.178453f, 0.229682f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.214423f, 0.359860f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.017582f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.071976f, 0.390475f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.537548f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.515147f, 0.078582f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.364623f, 0.041649f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.337054f, 0.220008f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.249141f, 0.193797f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.168802f, 0.423188f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.084285f, 0.323725f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.411061f, 0.182411f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.329651f, 0.076877f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.193953f, 0.352033f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.188543f, 0.265471f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.050266f, 0.555034f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.394700f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.179003f, 0.029987f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.700087f, 0.090924f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.019171f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.099147f, 0.059028f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.470203f, 0.352451f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.075527f, 0.135452f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.184084f, 0.604937f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.136189f, 0.084874f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.576900f, 0.202037f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.041868f, 0.099347f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.343377f, 0.515408f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.044581f, 0.169532f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.062013f, 0.723875f}, +}; + +// For each output (7x4) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_7x4[28][48] = { +{0.798509f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.201491f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.716711f, 0.085583f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.167498f, 0.030208f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.538182f, 0.218008f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.114187f, 0.070138f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020226f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020777f, 0.000000f, 0.000000f, 0.018482f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.367283f, 0.403492f, 0.000000f, 0.017972f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.071839f, 0.050645f, 0.000000f, 0.023445f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020007f, 0.000000f, 0.000000f, 0.000000f, 0.022030f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.023286f, 0.000000f, 0.000000f}, +{0.000000f, 0.026415f, 0.000000f, 0.000000f, 0.165810f, 0.526162f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.086343f, 0.166394f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.028875f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.068792f, 0.750632f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.180576f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.798640f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.201360f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.401325f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.563541f, 0.035134f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.393109f, 0.035360f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.514780f, 0.056751f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.286324f, 0.066048f, 0.000000f, 0.022966f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.397320f, 0.167136f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024391f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.018733f, 0.017081f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.228689f, 0.212401f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.027812f, 0.000000f, 0.230123f, 0.251307f, 0.000000f, 0.015952f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.018366f, 0.015349f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.089768f, 0.272262f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.165947f, 0.450195f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.021828f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.064329f, 0.394519f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.021491f, 0.519661f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.420154f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.579846f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.561993f, 0.042727f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.395280f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.507366f, 0.060806f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.388432f, 0.043397f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.017057f, 0.019075f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.399856f, 0.181694f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.283918f, 0.098400f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.018320f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.261768f, 0.263599f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.210680f, 0.218119f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.027513f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.019283f, 0.018776f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.156143f, 0.407378f, 0.000000f, 0.018410f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.081168f, 0.298842f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.043712f, 0.524648f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.025861f, 0.405779f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.027775f, 0.567781f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.404444f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.202734f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.797266f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.164849f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.736579f, 0.098573f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.028573f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.139627f, 0.082102f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.529383f, 0.220315f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020496f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.031087f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.029563f, 0.000000f, 0.069934f, 0.077745f, 0.000000f, 0.000000f, 0.000000f, 0.019031f, 0.000000f, 0.000000f, 0.369058f, 0.383087f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.072848f, 0.128566f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.206674f, 0.591912f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.028891f, 0.164765f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.054845f, 0.751498f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.186782f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.813218f}, +}; + +// For each output (8x4) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_8x4[32][48] = { +{0.800445f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.199555f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.801084f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.198916f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.802438f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.197562f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.800166f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.199834f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.808142f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.191858f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.801414f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.198586f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.798600f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.201400f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.800453f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.199547f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.415774f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.584226f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.409782f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.590218f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.407361f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.592639f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.411487f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.588513f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.416734f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.583266f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.409794f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.590206f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.409782f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.590218f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.419797f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.580203f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.588149f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.411851f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.591287f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.408713f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.587561f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.412439f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.589820f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.410180f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.585460f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.414540f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.590541f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.409459f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.587115f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.412885f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.584462f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.415538f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.200471f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.799529f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.195628f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.804372f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.195562f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.804438f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.194079f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.805921f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.205775f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.794225f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.197129f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.802871f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.193175f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.806825f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.185493f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.814507f}, +}; + +// For each output (2x5) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_2x5[10][48] = { +{0.314987f, 0.280141f, 0.203583f, 0.129696f, 0.071593f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.085378f, 0.141565f, 0.188187f, 0.272403f, 0.312467f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.255395f, 0.217105f, 0.170584f, 0.106646f, 0.072684f, 0.000000f, 0.000000f, 0.000000f, 0.072766f, 0.046537f, 0.029920f, 0.000000f, 0.028363f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.069530f, 0.105913f, 0.164044f, 0.215260f, 0.255339f, 0.000000f, 0.000000f, 0.000000f, 0.025591f, 0.000000f, 0.036814f, 0.050349f, 0.077160f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.152274f, 0.142699f, 0.102993f, 0.080565f, 0.018558f, 0.000000f, 0.000000f, 0.000000f, 0.157267f, 0.135460f, 0.099077f, 0.089287f, 0.021820f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.026396f, 0.087011f, 0.099835f, 0.143472f, 0.149274f, 0.000000f, 0.000000f, 0.000000f, 0.019143f, 0.078700f, 0.099557f, 0.143621f, 0.152993f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.071546f, 0.054560f, 0.034641f, 0.000000f, 0.026492f, 0.000000f, 0.000000f, 0.000000f, 0.253751f, 0.217970f, 0.167740f, 0.101477f, 0.071823f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.031122f, 0.000000f, 0.038539f, 0.044578f, 0.068079f, 0.000000f, 0.000000f, 0.000000f, 0.074011f, 0.104132f, 0.176778f, 0.213248f, 0.249513f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.309516f, 0.271823f, 0.202932f, 0.138334f, 0.077394f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.073235f, 0.136322f, 0.204986f, 0.270837f, 0.314620f}, +}; + +// For each output (3x5) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_3x5[15][48] = { +{0.506870f, 0.329427f, 0.163702f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.029175f, 0.167327f, 0.319880f, 0.321166f, 0.162451f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.158719f, 0.334975f, 0.506306f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.410647f, 0.270965f, 0.135943f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.101890f, 0.048392f, 0.032162f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.022675f, 0.131363f, 0.257700f, 0.263834f, 0.126043f, 0.021278f, 0.000000f, 0.000000f, 0.000000f, 0.022613f, 0.064121f, 0.066389f, 0.023985f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.131149f, 0.266568f, 0.407438f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.041342f, 0.046648f, 0.106854f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.259144f, 0.176197f, 0.070648f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.256402f, 0.170550f, 0.067060f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.085864f, 0.160352f, 0.153663f, 0.093488f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.093065f, 0.165400f, 0.162870f, 0.085298f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.069632f, 0.177258f, 0.252242f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.066495f, 0.178932f, 0.255440f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.109165f, 0.056989f, 0.043673f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.396795f, 0.263538f, 0.129840f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.022525f, 0.061369f, 0.062101f, 0.020335f, 0.000000f, 0.000000f, 0.000000f, 0.022912f, 0.129308f, 0.258462f, 0.259250f, 0.129291f, 0.034446f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.042198f, 0.051815f, 0.111374f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.136459f, 0.257176f, 0.400979f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.509094f, 0.334982f, 0.155925f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.175231f, 0.321060f, 0.327712f, 0.175997f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.154955f, 0.336566f, 0.508479f}, +}; + +// For each output (4x5) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_4x5[20][48] = { +{0.669318f, 0.330682f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.147967f, 0.437694f, 0.317636f, 0.064825f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.031879f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.060625f, 0.318845f, 0.433756f, 0.158597f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.028176f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.324316f, 0.675684f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.585012f, 0.264010f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.150977f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.134170f, 0.326735f, 0.247128f, 0.055953f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.060565f, 0.080612f, 0.050606f, 0.022675f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.021555f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.065736f, 0.255091f, 0.336456f, 0.141260f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020320f, 0.056879f, 0.083295f, 0.040963f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.247404f, 0.561749f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.037270f, 0.153576f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.313615f, 0.178768f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.317328f, 0.167805f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.022484f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.056200f, 0.226923f, 0.169203f, 0.032339f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.060880f, 0.227803f, 0.168145f, 0.036277f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.022230f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020809f, 0.161103f, 0.242215f, 0.080276f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.037660f, 0.170123f, 0.226083f, 0.061733f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.170517f, 0.314573f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.183677f, 0.312560f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.018674f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.150066f, 0.037627f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.563093f, 0.249214f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.017288f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.047237f, 0.083719f, 0.064159f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.141594f, 0.343865f, 0.254176f, 0.047961f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.060771f, 0.083714f, 0.056548f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.055519f, 0.260450f, 0.341460f, 0.141538f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033365f, 0.158801f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.243363f, 0.564471f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.027870f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.650693f, 0.321437f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.154390f, 0.455517f, 0.321763f, 0.068330f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.030540f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.067841f, 0.315774f, 0.431982f, 0.153863f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.029780f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.315631f, 0.654589f}, +}; + +// For each output (5x5) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_5x5[25][48] = { +{0.728974f, 0.241827f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.029199f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.326790f, 0.583809f, 0.061650f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.027751f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.474659f, 0.471971f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.027161f, 0.026208f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.064479f, 0.600103f, 0.335418f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.245795f, 0.727343f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.026862f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.577450f, 0.212083f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.146821f, 0.063646f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.278532f, 0.501669f, 0.039082f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.051617f, 0.129101f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.401558f, 0.402789f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.088129f, 0.087552f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.019972f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.039177f, 0.470310f, 0.275467f, 0.000000f, 0.000000f, 0.000000f, 0.020182f, 0.000000f, 0.000000f, 0.131064f, 0.041994f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.021806f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.201719f, 0.586252f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.071189f, 0.140839f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.390859f, 0.113288f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.395284f, 0.100569f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.180479f, 0.291419f, 0.034269f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.179460f, 0.288259f, 0.026114f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.232294f, 0.235881f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.249972f, 0.265992f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.015860f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020495f, 0.297441f, 0.200057f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.300629f, 0.181378f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.094856f, 0.384959f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.114338f, 0.382484f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.023363f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.142672f, 0.067752f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.579242f, 0.210334f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.050987f, 0.132705f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.278585f, 0.484125f, 0.053597f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.026554f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.092842f, 0.065201f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.385798f, 0.387342f, 0.000000f, 0.000000f, 0.021183f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.021080f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020712f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.044924f, 0.106062f, 0.061499f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.047893f, 0.466019f, 0.252890f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020637f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.058939f, 0.143896f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.202796f, 0.573732f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033403f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.730809f, 0.235788f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.032140f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.330176f, 0.584667f, 0.053018f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.026110f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.492274f, 0.481616f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.065854f, 0.592001f, 0.342145f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.037025f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.240768f, 0.722207f}, +}; + +// For each output (6x5) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_6x5[30][48] = { +{0.858351f, 0.111195f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.030454f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.561719f, 0.406108f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.032173f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.234049f, 0.720564f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.045387f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.699282f, 0.247085f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.053633f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.389024f, 0.574352f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.036624f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.092315f, 0.907685f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.700837f, 0.094616f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.181782f, 0.022766f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.478824f, 0.322377f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.106995f, 0.067586f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024218f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.020740f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.019187f, 0.000000f, 0.211821f, 0.554939f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.076920f, 0.116393f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.528826f, 0.215423f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.129030f, 0.084167f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.021007f, 0.021548f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.285851f, 0.511729f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.045516f, 0.156904f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.061737f, 0.729570f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.185199f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.023495f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.426048f, 0.065346f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.437353f, 0.050722f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020531f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.015946f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.269275f, 0.220699f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.271762f, 0.222318f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.107929f, 0.387609f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.097175f, 0.384787f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.022500f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.018661f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.393619f, 0.098786f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.415799f, 0.073135f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.219562f, 0.256847f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.228262f, 0.295329f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.020203f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.066094f, 0.437807f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.023625f, 0.426898f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.025372f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.179453f, 0.029939f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.702329f, 0.088278f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024531f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.109211f, 0.062119f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.483375f, 0.320765f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.017885f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.077080f, 0.134573f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.212908f, 0.535331f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.022223f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.119888f, 0.115275f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.556098f, 0.208739f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.022346f, 0.116179f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.324515f, 0.536960f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.039522f, 0.193447f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.040639f, 0.726391f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033823f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.857552f, 0.108625f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024057f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.029799f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.542169f, 0.403976f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.052699f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.223511f, 0.723790f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.052693f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.702269f, 0.245038f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.402547f, 0.597453f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.031996f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.086881f, 0.881123f}, +}; + +// For each output (7x5) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_7x5[35][48] = { +{0.964445f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.035555f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.853417f, 0.094561f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.052022f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.657134f, 0.277797f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020663f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.023601f, 0.000000f, 0.000000f, 0.020806f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.380325f, 0.419839f, 0.000000f, 0.023060f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.032462f, 0.000000f, 0.000000f, 0.025415f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.022865f, 0.000000f, 0.028258f, 0.000000f, 0.023082f, 0.020352f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024341f, 0.000000f, 0.000000f}, +{0.000000f, 0.031003f, 0.000000f, 0.000000f, 0.218422f, 0.657212f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024308f, 0.033400f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.035654f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.070868f, 0.871307f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.057825f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.964400f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.035600f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.771715f, 0.027473f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.200812f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.681017f, 0.087709f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.170219f, 0.037187f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.023867f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.019162f, 0.000000f, 0.019267f, 0.000000f, 0.521425f, 0.210553f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.107845f, 0.064833f, 0.000000f, 0.000000f, 0.000000f, 0.023456f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.016876f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.016582f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.374490f, 0.378533f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.037317f, 0.000000f, 0.070870f, 0.081690f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.019460f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020149f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.017492f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.198514f, 0.553647f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.069444f, 0.178395f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.077267f, 0.707241f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.191176f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024316f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.777498f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.197118f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.025384f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.457893f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.477045f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024793f, 0.020109f, 0.000000f, 0.020160f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.453272f, 0.036882f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.449988f, 0.037704f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.022154f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.390518f, 0.119870f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.380701f, 0.108911f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.016500f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.017868f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.216278f, 0.228953f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.240939f, 0.263209f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.016253f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.029917f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.096934f, 0.340899f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.088970f, 0.426562f, 0.000000f, 0.000000f, 0.016718f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.021872f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.073754f, 0.459232f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.422925f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.022217f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.019775f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.473981f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020534f, 0.461485f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024225f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.200471f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.772740f, 0.026789f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.025642f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.165170f, 0.033854f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.660678f, 0.089428f, 0.000000f, 0.000000f, 0.000000f, 0.025229f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.016453f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.117847f, 0.083344f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.528281f, 0.230342f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.023732f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.043833f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.077971f, 0.049154f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.382849f, 0.385195f, 0.000000f, 0.022790f, 0.000000f, 0.000000f, 0.020308f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.017900f}, +{0.000000f, 0.000000f, 0.018444f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.017477f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.086693f, 0.093631f, 0.000000f, 0.032653f, 0.000000f, 0.000000f, 0.019144f, 0.000000f, 0.199637f, 0.532319f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020247f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.035464f, 0.208022f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.065940f, 0.670327f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.209616f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.790384f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.036613f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.963387f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.046570f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.849248f, 0.104183f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020833f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.049999f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.649521f, 0.279647f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.030284f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.025099f, 0.000000f, 0.000000f, 0.017993f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.028953f, 0.000000f, 0.027848f, 0.031988f, 0.000000f, 0.000000f, 0.000000f, 0.022049f, 0.000000f, 0.000000f, 0.397216f, 0.418570f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.026723f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.038960f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.243424f, 0.690894f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.050705f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.071869f, 0.877426f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.036401f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.963599f}, +}; + +// For each output (8x5) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_8x5[40][48] = { +{0.966296f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033704f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.966306f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033694f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.966296f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033704f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.966298f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033702f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.966291f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033709f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.966291f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033709f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.966295f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033705f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.966296f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033704f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.793476f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.206524f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.803849f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.196151f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.803624f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.196376f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.797993f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.202007f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.776552f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.195983f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.027465f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.793721f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.206279f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.806466f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.193534f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.797656f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.202344f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.476380f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.496730f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.026890f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.490205f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.485068f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024727f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.498077f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.476651f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.025272f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.474340f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.480228f, 0.045432f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.478505f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.521495f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.478679f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.483579f, 0.000000f, 0.000000f, 0.037742f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.521456f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.478544f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.507379f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.492621f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.204896f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.795104f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.196765f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.803235f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.199650f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.800350f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.203568f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.796432f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.179104f, 0.025788f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.795108f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.198542f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.801458f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.212749f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.787251f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.210279f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.789721f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033704f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.966296f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033709f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.966291f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033700f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.966300f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033711f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.966289f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033705f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.966295f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033692f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.966308f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033717f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.966283f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033731f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.966269f}, +}; + +// For each output (2x6) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_2x6[12][48] = { +{0.316864f, 0.281020f, 0.203578f, 0.128737f, 0.069800f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.084099f, 0.140260f, 0.188810f, 0.272909f, 0.313922f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.309774f, 0.274434f, 0.201401f, 0.144203f, 0.070188f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.065514f, 0.142636f, 0.201399f, 0.276345f, 0.314107f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.317592f, 0.277500f, 0.192959f, 0.141457f, 0.070491f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.073241f, 0.142588f, 0.198561f, 0.278233f, 0.307377f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.320020f, 0.275328f, 0.193983f, 0.143663f, 0.067007f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.069519f, 0.132193f, 0.205168f, 0.279209f, 0.313912f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.314759f, 0.279613f, 0.202284f, 0.130432f, 0.072912f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.077965f, 0.136688f, 0.207007f, 0.271208f, 0.307132f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.311744f, 0.272206f, 0.202758f, 0.136022f, 0.077269f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.072611f, 0.134437f, 0.204577f, 0.271631f, 0.316744f}, +}; + +// For each output (3x6) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_3x6[18][48] = { +{0.509323f, 0.329513f, 0.161164f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.025207f, 0.165943f, 0.323432f, 0.324818f, 0.160600f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.157414f, 0.335022f, 0.507564f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.511584f, 0.329744f, 0.158672f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.031983f, 0.159222f, 0.310218f, 0.312506f, 0.158287f, 0.027785f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.156210f, 0.333357f, 0.510434f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.515123f, 0.331176f, 0.153701f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.026619f, 0.155693f, 0.312956f, 0.312469f, 0.159059f, 0.033204f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.156669f, 0.330733f, 0.512598f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.503816f, 0.332794f, 0.163390f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024597f, 0.154193f, 0.318347f, 0.305757f, 0.159499f, 0.037605f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.158978f, 0.332267f, 0.508755f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.512301f, 0.329905f, 0.157794f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.034639f, 0.152702f, 0.307204f, 0.309309f, 0.167621f, 0.028524f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.152238f, 0.331031f, 0.516731f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.511179f, 0.335760f, 0.153061f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.173463f, 0.322489f, 0.329811f, 0.174238f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.152159f, 0.337011f, 0.510830f}, +}; + +// For each output (4x6) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_4x6[24][48] = { +{0.671100f, 0.328900f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.148979f, 0.456693f, 0.330185f, 0.064143f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.058158f, 0.330805f, 0.451065f, 0.159972f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.322150f, 0.677850f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.677593f, 0.322407f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.167723f, 0.446276f, 0.319975f, 0.066025f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.073990f, 0.323047f, 0.441943f, 0.161020f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.326071f, 0.673929f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.679042f, 0.320958f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.152853f, 0.450375f, 0.323919f, 0.072853f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.061203f, 0.320863f, 0.451270f, 0.166664f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.319746f, 0.680254f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.676510f, 0.323490f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.162624f, 0.457726f, 0.332137f, 0.047514f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.063329f, 0.328068f, 0.444798f, 0.163805f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.320574f, 0.679426f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.678066f, 0.321934f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.166497f, 0.448536f, 0.320669f, 0.064298f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.065578f, 0.323791f, 0.452649f, 0.157982f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.322175f, 0.677825f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.671500f, 0.328500f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.150795f, 0.460955f, 0.323971f, 0.064280f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.066061f, 0.327767f, 0.449877f, 0.156295f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.322687f, 0.677313f}, +}; + +// For each output (5x6) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_5x6[30][48] = { +{0.754364f, 0.245636f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.335285f, 0.602164f, 0.062551f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.500479f, 0.499521f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.057582f, 0.607199f, 0.335218f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.249634f, 0.750366f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.757244f, 0.242756f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.346204f, 0.598435f, 0.055362f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.501490f, 0.498510f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.060219f, 0.591314f, 0.348467f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.244713f, 0.755287f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.752634f, 0.247366f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.342331f, 0.595920f, 0.061748f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.496285f, 0.503715f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.055875f, 0.601113f, 0.343013f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.245684f, 0.754316f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.754642f, 0.245358f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.341881f, 0.605457f, 0.052662f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.506471f, 0.493529f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.052276f, 0.594038f, 0.353686f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.243659f, 0.756341f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.752998f, 0.247002f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.343161f, 0.587149f, 0.069691f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.497737f, 0.502263f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.068745f, 0.600800f, 0.330455f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.249755f, 0.750245f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.760155f, 0.239845f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.341132f, 0.607027f, 0.051841f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.505602f, 0.494398f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.063784f, 0.594541f, 0.341675f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.246784f, 0.753216f}, +}; + +// For each output (6x6) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_6x6[36][48] = { +{0.891095f, 0.108905f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.581832f, 0.418168f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.242153f, 0.757847f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.741976f, 0.258024f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.403606f, 0.596394f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.087517f, 0.912483f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.889771f, 0.110229f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.562123f, 0.416930f, 0.000000f, 0.020947f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.239798f, 0.760202f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.745430f, 0.254570f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.386117f, 0.613883f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.079820f, 0.920180f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.881826f, 0.118174f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.573611f, 0.426389f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.253276f, 0.746724f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.743647f, 0.256353f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.401870f, 0.598130f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.084584f, 0.915416f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.886496f, 0.113504f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.579329f, 0.420671f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.247079f, 0.752921f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.738480f, 0.261520f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.387849f, 0.612151f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.084296f, 0.915704f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.887045f, 0.112955f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.566292f, 0.413182f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020526f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.245603f, 0.754397f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.743664f, 0.256336f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.400389f, 0.599611f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.085951f, 0.914049f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.893377f, 0.106623f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.023576f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.559870f, 0.416555f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.230693f, 0.769307f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.743815f, 0.256185f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.401590f, 0.598410f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.084902f, 0.915098f}, +}; + +// For each output (7x6) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_7x6[42][48] = { +{1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.898749f, 0.101251f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.666832f, 0.285944f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024418f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.022807f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.408751f, 0.452880f, 0.000000f, 0.022279f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020101f, 0.000000f, 0.026406f, 0.000000f, 0.021392f, 0.021638f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.026554f, 0.000000f, 0.000000f}, +{0.000000f, 0.030824f, 0.000000f, 0.000000f, 0.224222f, 0.683355f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.025094f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.036505f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.074156f, 0.925844f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.898226f, 0.101774f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.026159f, 0.000000f, 0.029283f, 0.000000f, 0.659538f, 0.261285f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.023736f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.465472f, 0.460934f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.047490f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.026105f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.265328f, 0.711299f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.023373f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.070803f, 0.929197f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.898251f, 0.101749f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.031089f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.687826f, 0.256342f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024743f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.032537f, 0.022458f, 0.000000f, 0.000000f, 0.021138f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.410489f, 0.430095f, 0.000000f, 0.017967f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.018781f, 0.000000f, 0.000000f, 0.026567f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.019968f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.246938f, 0.753062f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.075058f, 0.924942f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.901297f, 0.098703f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.030068f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.643429f, 0.251859f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020470f, 0.024955f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.029220f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.018980f, 0.000000f, 0.000000f, 0.026284f, 0.019861f, 0.000000f, 0.028010f, 0.000000f, 0.000000f, 0.000000f, 0.445381f, 0.461483f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.238827f, 0.737044f, 0.000000f, 0.024129f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.068567f, 0.931433f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.900397f, 0.099603f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.026865f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.019850f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.657029f, 0.271313f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024943f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033697f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020611f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.026549f, 0.000000f, 0.440563f, 0.453523f, 0.000000f, 0.025057f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.030966f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.255214f, 0.713821f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.075429f, 0.924571f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.889492f, 0.110508f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.021347f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.683322f, 0.295331f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.027000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.027842f, 0.000000f, 0.000000f, 0.031873f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.028367f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.023945f, 0.000000f, 0.000000f, 0.417773f, 0.443199f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.026004f, 0.000000f, 0.000000f, 0.024319f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.243621f, 0.706056f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.075547f, 0.924453f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f}, +}; + +// For each output (8x6) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_8x6[48][48] = { +{1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f}, +}; + +downsample_matrix g_downsample_matrices_8x6[] = +{ + { 2, 2, (const float*)g_weight_downsample_8x6_to_2x2 }, + { 3, 2, (const float*)g_weight_downsample_8x6_to_3x2 }, + { 4, 2, (const float*)g_weight_downsample_8x6_to_4x2 }, + { 5, 2, (const float*)g_weight_downsample_8x6_to_5x2 }, + { 6, 2, (const float*)g_weight_downsample_8x6_to_6x2 }, + { 7, 2, (const float*)g_weight_downsample_8x6_to_7x2 }, + { 8, 2, (const float*)g_weight_downsample_8x6_to_8x2 }, + { 2, 3, (const float*)g_weight_downsample_8x6_to_2x3 }, + { 3, 3, (const float*)g_weight_downsample_8x6_to_3x3 }, + { 4, 3, (const float*)g_weight_downsample_8x6_to_4x3 }, + { 5, 3, (const float*)g_weight_downsample_8x6_to_5x3 }, + { 6, 3, (const float*)g_weight_downsample_8x6_to_6x3 }, + { 7, 3, (const float*)g_weight_downsample_8x6_to_7x3 }, + { 8, 3, (const float*)g_weight_downsample_8x6_to_8x3 }, + { 2, 4, (const float*)g_weight_downsample_8x6_to_2x4 }, + { 3, 4, (const float*)g_weight_downsample_8x6_to_3x4 }, + { 4, 4, (const float*)g_weight_downsample_8x6_to_4x4 }, + { 5, 4, (const float*)g_weight_downsample_8x6_to_5x4 }, + { 6, 4, (const float*)g_weight_downsample_8x6_to_6x4 }, + { 7, 4, (const float*)g_weight_downsample_8x6_to_7x4 }, + { 8, 4, (const float*)g_weight_downsample_8x6_to_8x4 }, + { 2, 5, (const float*)g_weight_downsample_8x6_to_2x5 }, + { 3, 5, (const float*)g_weight_downsample_8x6_to_3x5 }, + { 4, 5, (const float*)g_weight_downsample_8x6_to_4x5 }, + { 5, 5, (const float*)g_weight_downsample_8x6_to_5x5 }, + { 6, 5, (const float*)g_weight_downsample_8x6_to_6x5 }, + { 7, 5, (const float*)g_weight_downsample_8x6_to_7x5 }, + { 8, 5, (const float*)g_weight_downsample_8x6_to_8x5 }, + { 2, 6, (const float*)g_weight_downsample_8x6_to_2x6 }, + { 3, 6, (const float*)g_weight_downsample_8x6_to_3x6 }, + { 4, 6, (const float*)g_weight_downsample_8x6_to_4x6 }, + { 5, 6, (const float*)g_weight_downsample_8x6_to_5x6 }, + { 6, 6, (const float*)g_weight_downsample_8x6_to_6x6 }, + { 7, 6, (const float*)g_weight_downsample_8x6_to_7x6 }, + { 8, 6, (const float*)g_weight_downsample_8x6_to_8x6 } +}; + //-------------------------------------------------------------------------------------------------------------------------- const float* get_6x6_downsample_matrix(uint32_t grid_width, uint32_t grid_height) { - // TODO: Use hash or map lookup. + // TODO: Use hash or map lookup, or calc the index directly for (const auto& m : g_downsample_matrices_6x6) if ((m.m_grid_width == grid_width) && (m.m_grid_height == grid_height)) return m.m_p; @@ -4766,6 +5668,121 @@ const float* get_6x6_downsample_matrix(uint32_t grid_width, uint32_t grid_height return nullptr; } +const float* get_8x6_downsample_matrix(uint32_t grid_width, uint32_t grid_height) +{ + // TODO: Use hash or map lookup, or calc the index directly + for (const auto& m : g_downsample_matrices_8x6) + if ((m.m_grid_width == grid_width) && (m.m_grid_height == grid_height)) + return m.m_p; + + assert(0); + return nullptr; +} + +//-------------------------------------------------------------------------------------------------------------------------- + +void compute_upsample_matrix(basisu::vector2D& upsample_matrix, uint32_t block_width, uint32_t block_height, uint32_t grid_width, uint32_t grid_height) +{ + assert((block_width >= 2) && (block_width <= astc_helpers::MAX_BLOCK_DIM)); + assert((block_height >= 2) && (block_height <= astc_helpers::MAX_BLOCK_DIM)); + assert((grid_width >= 2) && (grid_width <= block_width)); + assert((grid_height >= 2) && (grid_height <= block_height)); + + const uint32_t num_block_samples = block_width * block_height; + const uint32_t num_grid_samples = grid_width * grid_height; + + astc_helpers::weighted_sample samples[astc_helpers::MAX_BLOCK_DIM * astc_helpers::MAX_BLOCK_DIM]; + clear_obj(samples); + + astc_helpers::compute_upsample_weights(block_width, block_height, grid_width, grid_height, samples); + + // Compute upsample matrix: output num_block_samples (rows), input num_grid_samples (cols) + upsample_matrix.resize_rows_cols(num_block_samples, num_grid_samples); + + basisu::vector weights(num_grid_samples); + + // compute which source sample(s) contribute to it. + for (uint32_t d = 0; d < num_block_samples; d++) + { + const astc_helpers::weighted_sample& ws = samples[d]; + + weights.set_all(0.0f); + + for (uint32_t y = 0; y < 2; y++) + { + for (uint32_t x = 0; x < 2; x++) + { + float w = ws.m_weights[y][x] * (1.0f / 16.0f); + if (!w) + continue; + + assert((ws.m_src_x + x) < grid_width); + assert((ws.m_src_y + y) < grid_height); + + assert(weights[(ws.m_src_x + x) + (ws.m_src_y + y) * grid_width] == 0.0f); + weights[(ws.m_src_x + x) + (ws.m_src_y + y) * grid_width] = w; + } // x + } // y + + for (uint32_t i = 0; i < num_grid_samples; i++) + upsample_matrix.at_row_col(d, i) = weights[i]; + + } // d +} + +//-------------------------------------------------------------------------------------------------------------------------- +// compute At - used for gradient descent + +void compute_upsample_matrix_transposed(basisu::vector& unweighted_downsample_matrix, uint32_t block_width, uint32_t block_height, uint32_t grid_width, uint32_t grid_height) +{ + assert((block_width >= 2) && (block_width <= astc_helpers::MAX_BLOCK_DIM)); + assert((block_height >= 2) && (block_height <= astc_helpers::MAX_BLOCK_DIM)); + assert((grid_width >= 2) && (grid_width <= block_width)); + assert((grid_height >= 2) && (grid_height <= block_height)); + + const uint32_t num_block_samples = block_width * block_height; + const uint32_t num_grid_samples = grid_width * grid_height; + + // Compute upsample matrix: output num_block_samples (rows), input num_grid_samples (cols) + vector2D upsample_matrix; + compute_upsample_matrix(upsample_matrix, block_width, block_height, grid_width, grid_height); + + // downsample matrix At (without any scaling): num_grid_samples (rows), num_block_samples (cols) + unweighted_downsample_matrix.resize(num_grid_samples * num_block_samples); + unweighted_downsample_matrix.set_all(0.0f); + + for (uint32_t j = 0; j < num_grid_samples; ++j) + for (uint32_t i = 0; i < num_block_samples; ++i) + unweighted_downsample_matrix[j * num_block_samples + i] = upsample_matrix.at_row_col(i, j); +} + +//-------------------------------------------------------------------------------------------------------------------------- +// Computes downsample matrices - simpler alternative to SLSQP + +//-------------------------------------------------------------------------------------------------------------------------- +// pDst_vec[] - size must be >= num_grid_samples +// vector used for gradient descent + +void compute_diag_AtA_vector(uint32_t block_width, uint32_t block_height, uint32_t grid_width, uint32_t grid_height, const vector2D &upsample_matrix, float* pDst_vec) +{ + const uint32_t num_block_samples = block_width * block_height; + const uint32_t num_grid_samples = grid_width * grid_height; + + memset(pDst_vec, 0, sizeof(float) * num_grid_samples); + + for (uint32_t r = 0; r < num_block_samples; ++r) + { + for (uint32_t c = 0; c < num_grid_samples; ++c) + { + const float arc = upsample_matrix.at_row_col(r, c); + + pDst_vec[c] += arc * arc; + } + } +} + +//-------------------------------------------------------------------------------------------------------------------------- + void downsample_weight_grid( const float* pMatrix_weights, uint32_t bx, uint32_t by, // source/from dimension (block size) @@ -4803,7 +5820,7 @@ void downsample_ise_weights( assert((block_w <= MAX_ASTC_HDR_BLOCK_W) && (block_h <= MAX_ASTC_HDR_BLOCK_H)); assert((grid_w >= 2) && (grid_w <= MAX_ASTC_HDR_BLOCK_W)); assert((grid_h >= 2) && (grid_h <= MAX_ASTC_HDR_BLOCK_H)); - + assert(dequant_weight_ise_range >= astc_helpers::FIRST_VALID_WEIGHT_ISE_RANGE); assert(dequant_weight_ise_range <= astc_helpers::LAST_VALID_WEIGHT_ISE_RANGE); @@ -4902,7 +5919,7 @@ static bool refine_endpoints_mode11( { for (uint32_t i = 0; i < num_block_pixels; i++) def_pixel_block_ofs[i] = (uint8_t)i; - + pPixel_block_ofs = def_pixel_block_ofs; } @@ -4928,7 +5945,7 @@ static bool refine_endpoints_mode11( trial_blk_raw_weights[i] = upsampled_weights[pPixel_block_ofs[i]]; trial_blk_raw_weightsf[i] = (float)trial_blk_raw_weights[i] * (1.0f / 64.0f); } - + vec3F l_q16, h_q16; if (opt_mode == cOrdinaryLeastSquares) { @@ -4959,7 +5976,7 @@ static bool refine_endpoints_mode11( { float mid = (0.0f - l) / (h - l); mid = clamp(mid, .01f, .99f); - + float lw = LOW_EMPHASIS_WEIGHT, mw = MIDDLE_EMPHASIS_WEIGHT, hw = HIGH_EMPHASIS_WEIGHT; if (opt_mode == cWeightedLeastSquaresHeavy) lw = LOW_EMPHASIS_WEIGHT_HEAVY, mw = MIDDLE_EMPHASIS_WEIGHT_HEAVY, hw = HIGH_EMPHASIS_WEIGHT_HEAVY; @@ -5019,7 +6036,7 @@ static bool refine_endpoints_mode11( } uint8_t trial_endpoints[NUM_MODE11_ENDPOINTS]; - + uint32_t submode_used; bool pack_succeeded = pack_mode11(l_q16, h_q16, endpoint_ise_range, trial_endpoints, coptions, direct_only, first_submode, last_submode, false, submode_used); @@ -5046,7 +6063,7 @@ static bool refine_endpoints_mode11( const float R_WEIGHT = coptions.m_r_err_scale, G_WEIGHT = coptions.m_g_err_scale; double cur_error = 0, trial_error = 0; - + for (uint32_t p = 0; p < num_pixels; p++) { const half_float* pDesired_half = &pBlock_pixels_half[p][0]; @@ -5173,7 +6190,7 @@ static bool refine_endpoints_mode7( vec3F block_mean_color_q16(calc_mean(num_pixels, pBlock_pixels_q16)); vec3F new_high_color_q16(block_mean_color_q16); - + const float one_over_num_pixels = 1.0f / (float)num_pixels; for (uint32_t i = 0; i < num_pixels; i++) @@ -5185,7 +6202,7 @@ static bool refine_endpoints_mode7( new_high_color_q16[1] += k; new_high_color_q16[2] += k; } - + // Given a set of selectors and a high color, try to compute a better S. float t = 0.0f; @@ -5197,7 +6214,7 @@ static bool refine_endpoints_mode7( } t *= one_over_num_pixels; - + if (fabs(t) < .0000125f) return false; @@ -5223,7 +6240,7 @@ static bool refine_endpoints_mode7( if (!decode_mode7_to_qlog12(trial_endpoints, trial_e, nullptr, endpoint_ise_range)) return false; - + // -- for (uint32_t i = 0; i < 3; i++) @@ -5354,3 +6371,4 @@ bool refine_endpoints( } } // namespace basisu + diff --git a/external/basis_universal/encoder/basisu_astc_hdr_common.h b/external/basis_universal/encoder/basisu_astc_hdr_common.h index 18f424fe5a..e01999248a 100644 --- a/external/basis_universal/encoder/basisu_astc_hdr_common.h +++ b/external/basis_universal/encoder/basisu_astc_hdr_common.h @@ -12,13 +12,13 @@ namespace basisu const uint32_t MODE11_TOTAL_SUBMODES = 8; // plus an extra hidden submode, directly encoded, for direct, so really 9 (see tables 99/100 of the ASTC spec) const uint32_t MODE7_TOTAL_SUBMODES = 6; - + // [ise_range][0] = # levels // [ise_range][1...] = lerp value [0,64] // in ASTC order // Supported ISE weight ranges: 0 to 11, 12 total const uint32_t MIN_SUPPORTED_ISE_WEIGHT_INDEX = astc_helpers::BISE_2_LEVELS; // ISE 0=2 levels - const uint32_t MAX_SUPPORTED_ISE_WEIGHT_INDEX = astc_helpers::BISE_32_LEVELS; // ISE 11=16 levels + const uint32_t MAX_SUPPORTED_ISE_WEIGHT_INDEX = astc_helpers::BISE_32_LEVELS; // ISE 11=32 levels const uint32_t MIN_SUPPORTED_WEIGHT_LEVELS = 2; const uint32_t MAX_SUPPORTED_WEIGHT_LEVELS = 32; @@ -29,13 +29,17 @@ namespace basisu const float LDR_TO_HDR_NITS = 100.0f; + extern vec4F g_astc_ls_weights_ise[MAX_SUPPORTED_ISE_WEIGHT_INDEX + 1][MAX_SUPPORTED_WEIGHT_LEVELS]; + extern uint8_t g_map_astc_to_linear_order[MAX_SUPPORTED_ISE_WEIGHT_INDEX + 1][MAX_SUPPORTED_WEIGHT_LEVELS]; // [ise_range][astc_index] -> linear index + extern uint8_t g_map_linear_to_astc_order[MAX_SUPPORTED_ISE_WEIGHT_INDEX + 1][MAX_SUPPORTED_WEIGHT_LEVELS]; // [ise_range][linear_index] -> astc_index + struct astc_hdr_codec_base_options { float m_r_err_scale, m_g_err_scale; float m_q_log_bias; - + bool m_ultra_quant; - + // If true, the ASTC HDR compressor is allowed to more aggressively vary weight indices for slightly higher compression in non-fastest mode. This will hurt BC6H quality, however. bool m_allow_uber_mode; @@ -45,7 +49,7 @@ namespace basisu bool m_take_first_non_clamping_mode7_submode; bool m_disable_weight_plane_optimization; - + astc_hdr_codec_base_options() { init(); } void init(); @@ -173,7 +177,7 @@ namespace basisu basist::half_float* pDecoded_half, vec3F* pDecoded_float, uint32_t n, uint32_t ise_weight_range, uint32_t ise_endpoint_range); - + // Fast high precision piecewise linear approximation of log2(bias+x). // Half may be zero, positive or denormal. No NaN/Inf/negative. BASISU_FORCE_INLINE double q(basist::half_float x, float log_bias) @@ -183,7 +187,7 @@ namespace basisu fi.f = fast_half_to_float_pos_not_inf_or_nan(x); assert(fi.f >= 0.0f); - + fi.f += log_bias; return (double)fi.u; // approx log2f(fi.f), need to return double for the precision @@ -196,7 +200,7 @@ namespace basisu fi.f = fast_half_to_float_pos_not_inf_or_nan(x); assert(fi.f >= 0.0f); - + fi.f += log_bias; return fi.u; @@ -298,8 +302,8 @@ namespace basisu uint32_t ise_endpoint_range, bool uber_mode, bool constrain_ise_weight_selectors, - int32_t first_submode, int32_t last_submode, bool ignore_clamping, - opt_mode_t opt_mode, + int32_t first_submode, int32_t last_submode, bool ignore_clamping, + opt_mode_t opt_mode, const encode_astc_block_stats *pBlock_stats = nullptr); double encode_astc_hdr_block_downsampled_mode_11( @@ -325,7 +329,7 @@ namespace basisu uint32_t ise_endpoint_range, bool uber_mode, bool constrain_ise_weight_selectors, - int32_t first_submode, int32_t last_submode, + int32_t first_submode, int32_t last_submode, bool ignore_clamping); double encode_astc_hdr_block_mode_7( @@ -337,8 +341,8 @@ namespace basisu uint8_t* blk_endpoints, //[4] uint8_t* blk_weights, // [num_pixels] const astc_hdr_codec_base_options& coptions, - uint32_t ise_endpoint_range, - int first_submode = 0, int last_submode = MAX_MODE7_SUBMODE_INDEX, + uint32_t ise_endpoint_range, + int first_submode = 0, int last_submode = MAX_MODE7_SUBMODE_INDEX, const encode_astc_block_stats *pBlock_stats = nullptr); //-------------------------------------------------------------------------------------------------------------------------- @@ -373,17 +377,23 @@ namespace basisu bool pack_astc_mode11_submode(uint32_t submode, uint8_t* pEndpoints, int val_q[2][3], int& max_clamp_mag, bool early_out_if_clamped = false, int max_clamp_mag_accept_thresh = 0); bool pack_astc_mode11_submode(uint32_t submode, uint8_t* pEndpoints, const vec3F& low_q16, const vec3F& high_q16, int& max_clamp_mag, bool early_out_if_clamped = false, int max_clamp_mag_accept_thresh = 0); void pack_astc_mode11_direct(uint8_t* pEndpoints, vec3F l_q16, vec3F h_q16); - + bool pack_mode11(mode11_log_desc& desc, uint8_t* pEndpoints); void unpack_mode11(const uint8_t* pEndpoints, mode11_log_desc& desc); void decode_cem_11_config(const uint8_t* pEndpoints, int& submode_index, int& maj_index); void decode_cem_7_config(const uint8_t* pEndpoints, int& submode_index, int& maj_index); - + void dequantize_astc_weights(uint32_t n, const uint8_t* pSrc_ise_vals, uint32_t from_ise_range, uint8_t* pDst_raw_weights); const float* get_6x6_downsample_matrix(uint32_t grid_width, uint32_t grid_height); - + const float* get_8x6_downsample_matrix(uint32_t grid_width, uint32_t grid_height); + + void compute_upsample_matrix(basisu::vector2D& upsample_matrix, uint32_t block_width, uint32_t block_height, uint32_t grid_width, uint32_t grid_height); + void compute_upsample_matrix_transposed(basisu::vector& unweighted_downsample_matrix, uint32_t block_width, uint32_t block_height, uint32_t grid_width, uint32_t grid_height); + + void compute_diag_AtA_vector(uint32_t block_width, uint32_t block_height, uint32_t grid_width, uint32_t grid_height, const vector2D& upsample_matrix, float* pDst_vec); + void downsample_weight_grid( const float* pMatrix_weights, uint32_t bx, uint32_t by, // source/from dimension (block size) @@ -413,10 +423,11 @@ namespace basisu uint32_t num_pixels, const basist::half_float pBlock_pixels_half[][3], const vec4F pBlock_pixels_q16[], const uint8_t* pPixel_block_ofs, // maps this subset's pixels to block offsets astc_hdr_codec_base_options& coptions, opt_mode_t opt_mode); - + extern bool g_astc_hdr_enc_initialized; // This MUST be called before encoding any blocks. void astc_hdr_enc_init(); } // namespace basisu + diff --git a/external/basis_universal/encoder/basisu_astc_ldr_common.cpp b/external/basis_universal/encoder/basisu_astc_ldr_common.cpp new file mode 100644 index 0000000000..770966faf4 --- /dev/null +++ b/external/basis_universal/encoder/basisu_astc_ldr_common.cpp @@ -0,0 +1,5667 @@ +// File: basisu_astc_ldr_common.cpp +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "basisu_enc.h" +#include "../transcoder/basisu_astc_helpers.h" +#include "../transcoder/basisu_astc_hdr_core.h" +#include "basisu_astc_hdr_common.h" +#include "basisu_astc_ldr_common.h" + +#define BASISU_ASTC_LDR_DEBUG_MSGS (1) + +namespace basisu +{ + +namespace astc_ldr +{ + static bool g_initialized; + static vec4F g_astc_ls_raw_weights_ise[ASTC_LDR_MAX_RAW_WEIGHTS]; + + color_rgba blue_contract_enc(color_rgba orig, bool& did_clamp, int encoded_b) + { + color_rgba enc; + + int tr = orig.r * 2 - encoded_b; + int tg = orig.g * 2 - encoded_b; + if ((tr < 0) || (tr > 255) || (tg < 0) || (tg > 255)) + did_clamp = true; + + enc.r = (uint8_t)basisu::clamp(tr, 0, 255); + enc.g = (uint8_t)basisu::clamp(tg, 0, 255); + enc.b = (uint8_t)orig.b; + enc.a = orig.a; + return enc; + } + + color_rgba blue_contract_dec(int enc_r, int enc_g, int enc_b, int enc_a) + { + color_rgba dec; + dec.r = (uint8_t)((enc_r + enc_b) >> 1); + dec.g = (uint8_t)((enc_g + enc_b) >> 1); + dec.b = (uint8_t)enc_b; + dec.a = (uint8_t)enc_a; + return dec; + } + + void global_init() + { + if (g_initialized) + return; + + // Precomputed weight constants used during least fit determination. For each entry: w * w, (1.0f - w) * w, (1.0f - w) * (1.0f - w), w + for (uint32_t iw = 0; iw <= 64; iw++) + { + float w = (float)iw * (1.0f / 64.0f); + + g_astc_ls_raw_weights_ise[iw].set(w * w, (1.0f - w) * w, (1.0f - w) * (1.0f - w), w); + } + + g_initialized = true; + } + + static inline const vec4F* get_ls_weights_ise(uint32_t weight_ise_range) + { + assert((weight_ise_range <= astc_helpers::BISE_32_LEVELS) || (weight_ise_range == astc_helpers::BISE_64_LEVELS)); + + // astc_helpers::BISE_64_LEVELS indicates raw [0,64] weights (65 total), otherwise ISE weights (<= 32 levels total) + return (weight_ise_range == astc_helpers::BISE_64_LEVELS) ? g_astc_ls_raw_weights_ise : &g_astc_ls_weights_ise[weight_ise_range][0]; + } + + static bool compute_least_squares_endpoints_1D( + uint32_t N, const uint8_t* pSelectors, const vec4F* pSelector_weights, + float* pXl, float* pXh, const float* pVals, float bounds_min, float bounds_max) + { + float z00 = 0.0f, z01 = 0.0f, z10 = 0.0f, z11 = 0.0f; + float q00_r = 0.0f, q10_r = 0.0f, t_r = 0.0f; + + for (uint32_t i = 0; i < N; i++) + { + const uint32_t sel = pSelectors[i]; + + z00 += pSelector_weights[sel][0]; + z10 += pSelector_weights[sel][1]; + z11 += pSelector_weights[sel][2]; + + float w = pSelector_weights[sel][3]; + + q00_r += w * pVals[i]; + t_r += pVals[i]; + } + + q10_r = t_r - q00_r; + + z01 = z10; + + float det = z00 * z11 - z01 * z10; + if (fabs(det) < 1e-8f) + return false; + + det = 1.0f / det; + + float iz00, iz01, iz10, iz11; + iz00 = z11 * det; + iz01 = -z01 * det; + iz10 = -z10 * det; + iz11 = z00 * det; + + *pXh = (float)(iz00 * q00_r + iz01 * q10_r); *pXl = (float)(iz10 * q00_r + iz11 * q10_r); + + float l = saturate(*pXl), h = saturate(*pXh); + + if (bounds_min == bounds_max) + { + l = bounds_min; + h = bounds_max; + } + + *pXl = l; + *pXh = h; + + return true; + } + + static bool compute_least_squares_endpoints_2D( + uint32_t N, const uint8_t* pSelectors, const vec4F* pSelector_weights, + vec2F* pXl, vec2F* pXh, const vec2F* pColors, const vec2F& bounds_min, const vec2F& bounds_max) + { + float z00 = 0.0f, z01 = 0.0f, z10 = 0.0f, z11 = 0.0f; + float q00_r = 0.0f, q10_r = 0.0f, t_r = 0.0f; + float q00_g = 0.0f, q10_g = 0.0f, t_g = 0.0f; + + for (uint32_t i = 0; i < N; i++) + { + const uint32_t sel = pSelectors[i]; + + z00 += pSelector_weights[sel][0]; + z10 += pSelector_weights[sel][1]; + z11 += pSelector_weights[sel][2]; + + float w = pSelector_weights[sel][3]; + + q00_r += w * pColors[i][0]; + t_r += pColors[i][0]; + + q00_g += w * pColors[i][1]; + t_g += pColors[i][1]; + } + + q10_r = t_r - q00_r; + q10_g = t_g - q00_g; + + z01 = z10; + + float det = z00 * z11 - z01 * z10; + if (fabs(det) < 1e-8f) + return false; + + det = 1.0f / det; + + float iz00, iz01, iz10, iz11; + iz00 = z11 * det; + iz01 = -z01 * det; + iz10 = -z10 * det; + iz11 = z00 * det; + + (*pXh)[0] = (float)(iz00 * q00_r + iz01 * q10_r); (*pXl)[0] = (float)(iz10 * q00_r + iz11 * q10_r); + (*pXh)[1] = (float)(iz00 * q00_g + iz01 * q10_g); (*pXl)[1] = (float)(iz10 * q00_g + iz11 * q10_g); + + for (uint32_t c = 0; c < 2; c++) + { + float l = saturate((*pXl)[c]), h = saturate((*pXh)[c]); + + if (bounds_min[c] == bounds_max[c]) + { + l = bounds_min[c]; + h = bounds_max[c]; + } + + (*pXl)[c] = l; + (*pXh)[c] = h; + } + + return true; + } + + static bool compute_least_squares_endpoints_3D( + uint32_t N, const uint8_t* pSelectors, const vec4F* pSelector_weights, + vec4F* pXl, vec4F* pXh, const vec4F* pColors, const vec4F& bounds_min, const vec4F& bounds_max) + { + float z00 = 0.0f, z01 = 0.0f, z10 = 0.0f, z11 = 0.0f; + float q00_r = 0.0f, q10_r = 0.0f, t_r = 0.0f; + float q00_g = 0.0f, q10_g = 0.0f, t_g = 0.0f; + float q00_b = 0.0f, q10_b = 0.0f, t_b = 0.0f; + + for (uint32_t i = 0; i < N; i++) + { + const uint32_t sel = pSelectors[i]; + + z00 += pSelector_weights[sel][0]; + z10 += pSelector_weights[sel][1]; + z11 += pSelector_weights[sel][2]; + + float w = pSelector_weights[sel][3]; + + q00_r += w * pColors[i][0]; + t_r += pColors[i][0]; + + q00_g += w * pColors[i][1]; + t_g += pColors[i][1]; + + q00_b += w * pColors[i][2]; + t_b += pColors[i][2]; + } + + q10_r = t_r - q00_r; + q10_g = t_g - q00_g; + q10_b = t_b - q00_b; + + z01 = z10; + + float det = z00 * z11 - z01 * z10; + if (fabs(det) < 1e-8f) + return false; + + det = 1.0f / det; + + float iz00, iz01, iz10, iz11; + iz00 = z11 * det; + iz01 = -z01 * det; + iz10 = -z10 * det; + iz11 = z00 * det; + + (*pXh)[0] = (float)(iz00 * q00_r + iz01 * q10_r); (*pXl)[0] = (float)(iz10 * q00_r + iz11 * q10_r); + (*pXh)[1] = (float)(iz00 * q00_g + iz01 * q10_g); (*pXl)[1] = (float)(iz10 * q00_g + iz11 * q10_g); + (*pXh)[2] = (float)(iz00 * q00_b + iz01 * q10_b); (*pXl)[2] = (float)(iz10 * q00_b + iz11 * q10_b); + + (*pXh)[3] = 0; + (*pXl)[3] = 0; + + for (uint32_t c = 0; c < 3; c++) + { + float l = saturate((*pXl)[c]), h = saturate((*pXh)[c]); + + if (bounds_min[c] == bounds_max[c]) + { + l = bounds_min[c]; + h = bounds_max[c]; + } + + (*pXl)[c] = l; + (*pXh)[c] = h; + } + + return true; + } + + static bool compute_least_squares_endpoints_4D( + uint32_t N, const uint8_t* pSelectors, const vec4F* pSelector_weights, + vec4F* pXl, vec4F* pXh, const vec4F* pColors, const vec4F& bounds_min, const vec4F& bounds_max) + { + float z00 = 0.0f, z01 = 0.0f, z10 = 0.0f, z11 = 0.0f; + float q00_r = 0.0f, q10_r = 0.0f, t_r = 0.0f; + float q00_g = 0.0f, q10_g = 0.0f, t_g = 0.0f; + float q00_b = 0.0f, q10_b = 0.0f, t_b = 0.0f; + float q00_a = 0.0f, q10_a = 0.0f, t_a = 0.0f; + + for (uint32_t i = 0; i < N; i++) + { + const uint32_t sel = pSelectors[i]; + z00 += pSelector_weights[sel][0]; + z10 += pSelector_weights[sel][1]; + z11 += pSelector_weights[sel][2]; + + float w = pSelector_weights[sel][3]; + q00_r += w * pColors[i][0]; t_r += pColors[i][0]; + q00_g += w * pColors[i][1]; t_g += pColors[i][1]; + q00_b += w * pColors[i][2]; t_b += pColors[i][2]; + q00_a += w * pColors[i][3]; t_a += pColors[i][3]; + } + + q10_r = t_r - q00_r; + q10_g = t_g - q00_g; + q10_b = t_b - q00_b; + q10_a = t_a - q00_a; + + z01 = z10; + + float det = z00 * z11 - z01 * z10; + if (fabs(det) < 1e-8f) + return false; + + det = 1.0f / det; + + float iz00, iz01, iz10, iz11; + iz00 = z11 * det; + iz01 = -z01 * det; + iz10 = -z10 * det; + iz11 = z00 * det; + + (*pXh)[0] = (float)(iz00 * q00_r + iz01 * q10_r); (*pXl)[0] = (float)(iz10 * q00_r + iz11 * q10_r); + (*pXh)[1] = (float)(iz00 * q00_g + iz01 * q10_g); (*pXl)[1] = (float)(iz10 * q00_g + iz11 * q10_g); + (*pXh)[2] = (float)(iz00 * q00_b + iz01 * q10_b); (*pXl)[2] = (float)(iz10 * q00_b + iz11 * q10_b); + (*pXh)[3] = (float)(iz00 * q00_a + iz01 * q10_a); (*pXl)[3] = (float)(iz10 * q00_a + iz11 * q10_a); + + for (uint32_t c = 0; c < 4; c++) + { + float l = saturate((*pXl)[c]), h = saturate((*pXh)[c]); + + if (bounds_min[c] == bounds_max[c]) + { + l = bounds_min[c]; + h = bounds_max[c]; + } + + (*pXl)[c] = l; + (*pXh)[c] = h; + } + + return true; + } + +#if 0 + static void dequant_astc_weights(uint32_t n, const uint8_t* pSrc_ise_vals, uint32_t from_ise_range, uint8_t* pDst_raw_weights) + { + const auto& dequant_tab = astc_helpers::g_dequant_tables.get_weight_tab(from_ise_range).m_ISE_to_val; + + for (uint32_t i = 0; i < n; i++) + pDst_raw_weights[i] = dequant_tab[pSrc_ise_vals[i]]; + } +#endif + +#if 0 + static void dequant_astc_endpoints(uint32_t n, const uint8_t* pSrc_ise_vals, uint32_t from_ise_range, uint8_t* pDst_raw_weights) + { + const auto& dequant_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(from_ise_range).m_ISE_to_val; + + for (uint32_t i = 0; i < n; i++) + pDst_raw_weights[i] = dequant_tab[pSrc_ise_vals[i]]; + } +#endif + + int apply_delta_to_bise_weight_val(uint32_t weight_ise_range, int ise_val, int delta) + { + if (delta == 0) + return ise_val; + + uint32_t num_ise_levels = astc_helpers::get_ise_levels(weight_ise_range); + + const auto& ISE_to_rank = astc_helpers::g_dequant_tables.get_weight_tab(weight_ise_range).m_ISE_to_rank; + const auto& rank_to_ISE = astc_helpers::g_dequant_tables.get_weight_tab(weight_ise_range).m_rank_to_ISE; + + int cur_rank = ISE_to_rank[ise_val]; + int new_rank = basisu::clamp(cur_rank + delta, 0, (int)num_ise_levels - 1); + + return rank_to_ISE[new_rank]; + } + + // v must be [0,1] + // converts to nearest ISE index with proper precise rounding + static uint8_t precise_round_bise_endpoint_val(float v, uint32_t endpoint_ise_range) + { + assert((v >= 0) && (v <= 1.0f)); + + const auto& quant_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(endpoint_ise_range).m_val_to_ise; + const auto& dequant_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(endpoint_ise_range).m_ISE_to_val; + + v = saturate(v); + + const int iv = clamp((int)std::roundf(v * 255.0f), 0, 255); + + uint8_t ise_index = 0; + + float best_err = BIG_FLOAT_VAL; + for (int iscale_delta = -1; iscale_delta <= 1; iscale_delta++) + { + const int trial_ise_index = astc_helpers::apply_delta_to_bise_endpoint_val(endpoint_ise_range, quant_tab[iv], iscale_delta); + + const float dequant_val = dequant_tab[trial_ise_index] * (1.0f / 255.0f); + + const float dequant_err = fabs(dequant_val - v); + if (dequant_err < best_err) + { + best_err = dequant_err; + ise_index = (uint8_t)trial_ise_index; + } + } // iscale_delta + + return ise_index; + } + + // returns true if blue contraction was actually used + // note the encoded endpoints may be swapped + // TODO: Pass in vec4F l/h and let it more precisely quantize in here. + struct cem_encode_ldr_rgb_or_rgba_direct_result + { + bool m_is_blue_contracted; + bool m_endpoints_are_swapped; + bool m_any_degen; + }; + + static cem_encode_ldr_rgb_or_rgba_direct_result cem_encode_ldr_rgb_or_rgba_direct( + uint32_t cem_index, uint32_t endpoint_ise_range, const color_rgba& l, const color_rgba& h, uint8_t* pEndpoint_vals, + bool try_blue_contract) + { + assert((cem_index == astc_helpers::CEM_LDR_RGB_DIRECT) || (cem_index == astc_helpers::CEM_LDR_RGBA_DIRECT)); + + cem_encode_ldr_rgb_or_rgba_direct_result res; + + bool& endpoints_are_swapped = res.m_endpoints_are_swapped; + bool& any_degen = res.m_any_degen; + bool& is_blue_contracted = res.m_is_blue_contracted; + + assert((cem_index == astc_helpers::CEM_LDR_RGB_DIRECT) || (cem_index == astc_helpers::CEM_LDR_RGBA_DIRECT)); + + const bool has_alpha = (cem_index == astc_helpers::CEM_LDR_RGBA_DIRECT); + + const auto& quant_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(endpoint_ise_range).m_val_to_ise; + const auto& dequant_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(endpoint_ise_range).m_ISE_to_val; + + //const auto &ISE_to_rank = astc_helpers::g_dequant_tables.get_endpoint_tab(endpoint_ise_range).m_ISE_to_rank; + //const auto &rank_to_ISE = astc_helpers::g_dequant_tables.get_endpoint_tab(endpoint_ise_range).m_rank_to_ISE; + + color_rgba enc_l(l), enc_h(h); + endpoints_are_swapped = false; + + is_blue_contracted = false; + if (try_blue_contract) + { + int enc_v4 = quant_tab[enc_l.b], enc_v5 = quant_tab[enc_h.b]; + int dec_v4 = dequant_tab[enc_v4], dec_v5 = dequant_tab[enc_v5]; + + bool did_clamp = false; + enc_l = blue_contract_enc(h, did_clamp, dec_v5); // yes, they're swapped in the spec + enc_h = blue_contract_enc(l, did_clamp, dec_v4); + + if (!did_clamp) + { + is_blue_contracted = true; + endpoints_are_swapped = true; + } + else + { + enc_l = l; + enc_h = h; + } + } + + int enc_v0 = quant_tab[enc_l.r], enc_v2 = quant_tab[enc_l.g], enc_v4 = quant_tab[enc_l.b]; + int enc_v1 = quant_tab[enc_h.r], enc_v3 = quant_tab[enc_h.g], enc_v5 = quant_tab[enc_h.b]; + + int enc_v6 = 0, enc_v7 = 0; + if (has_alpha) + { + enc_v6 = quant_tab[enc_l.a]; + enc_v7 = quant_tab[enc_h.a]; + } + + any_degen = false; + if ((enc_v0 == enc_v1) && (l.r != h.r)) + any_degen = true; + if ((enc_v2 == enc_v3) && (l.g != h.g)) + any_degen = true; + if ((enc_v4 == enc_v5) && (l.b != h.b)) + any_degen = true; + if (has_alpha) + { + if ((enc_v6 == enc_v7) && (l.a != h.a)) + any_degen = true; + } + + int dec_v0 = dequant_tab[enc_v0], dec_v2 = dequant_tab[enc_v2], dec_v4 = dequant_tab[enc_v4]; + int dec_v1 = dequant_tab[enc_v1], dec_v3 = dequant_tab[enc_v3], dec_v5 = dequant_tab[enc_v5]; + + int s0 = dec_v0 + dec_v2 + dec_v4; + int s1 = dec_v1 + dec_v3 + dec_v5; + + bool should_swap = false; + + if ((s1 == s0) && (is_blue_contracted)) + { + // if sums are equal we can't use blue contraction at all, so undo it + enc_l = l; + enc_h = h; + + is_blue_contracted = false; + endpoints_are_swapped = false; + + enc_v0 = quant_tab[enc_l.r], enc_v2 = quant_tab[enc_l.g], enc_v4 = quant_tab[enc_l.b]; + enc_v1 = quant_tab[enc_h.r], enc_v3 = quant_tab[enc_h.g], enc_v5 = quant_tab[enc_h.b]; + + dec_v0 = dequant_tab[enc_v0], dec_v2 = dequant_tab[enc_v2], dec_v4 = dequant_tab[enc_v4]; + dec_v1 = dequant_tab[enc_v1], dec_v3 = dequant_tab[enc_v3], dec_v5 = dequant_tab[enc_v5]; + + if (has_alpha) + { + enc_v6 = quant_tab[enc_l.a]; + enc_v7 = quant_tab[enc_h.a]; + } + + s0 = dec_v0 + dec_v2 + dec_v4; + s1 = dec_v1 + dec_v3 + dec_v5; + } + + if (s1 >= s0) + { + if (is_blue_contracted) + should_swap = true; + } + else + { + if (!is_blue_contracted) + should_swap = true; + } + + if (should_swap) + { + endpoints_are_swapped = !endpoints_are_swapped; + + std::swap(enc_v0, enc_v1); + std::swap(enc_v2, enc_v3); + std::swap(enc_v4, enc_v5); + std::swap(enc_v6, enc_v7); + } + + pEndpoint_vals[0] = (uint8_t)enc_v0; + pEndpoint_vals[1] = (uint8_t)enc_v1; + + pEndpoint_vals[2] = (uint8_t)enc_v2; + pEndpoint_vals[3] = (uint8_t)enc_v3; + + pEndpoint_vals[4] = (uint8_t)enc_v4; + pEndpoint_vals[5] = (uint8_t)enc_v5; + + if (has_alpha) + { + pEndpoint_vals[6] = (uint8_t)enc_v6; + pEndpoint_vals[7] = (uint8_t)enc_v7; + } + + #ifdef _DEBUG + { + int check_s0 = dequant_tab[enc_v0] + dequant_tab[enc_v2] + dequant_tab[enc_v4]; + int check_s1 = dequant_tab[enc_v1] + dequant_tab[enc_v3] + dequant_tab[enc_v5]; + + if (check_s1 >= check_s0) + { + assert(!is_blue_contracted); + } + else + { + assert(is_blue_contracted); + } + } + #endif + + return res; + } + + // Cannot fail + // scale=1 cannot be packed + static void cem_encode_ldr_rgb_or_rgba_base_scale( + uint32_t cem_index, uint32_t endpoint_ise_range, float scale, float l_a, const vec4F& h, uint8_t* pEndpoint_vals) + { + assert((cem_index == astc_helpers::CEM_LDR_RGB_BASE_SCALE) || (cem_index == astc_helpers::CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A)); + assert((scale >= 0.0f) && (scale < 1.0f)); + + const bool has_alpha = (cem_index == astc_helpers::CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A); + + const auto& quant_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(endpoint_ise_range).m_val_to_ise; + const auto& dequant_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(endpoint_ise_range).m_ISE_to_val; + + const uint32_t total_vals_to_pack = has_alpha ? 6 : 4; + + float vals_to_pack[6] = { 0 }; + + vals_to_pack[0] = h[0]; + vals_to_pack[1] = h[1]; + vals_to_pack[2] = h[2]; + vals_to_pack[3] = clamp(scale * (256.0f / 255.0f), 0.0f, 1.0f); + + if (has_alpha) + { + vals_to_pack[4] = l_a; + vals_to_pack[5] = h[3]; + } + + for (uint32_t c = 0; c < total_vals_to_pack; c++) + { + const float v = vals_to_pack[c]; + const int iv = clamp((int)std::roundf(v * 255.0f), 0, 255); + + float best_err = BIG_FLOAT_VAL; + for (int iscale_delta = -1; iscale_delta <= 1; iscale_delta++) + { + const int trial_ise_index = astc_helpers::apply_delta_to_bise_endpoint_val(endpoint_ise_range, quant_tab[iv], iscale_delta); + + const float dequant_val = dequant_tab[trial_ise_index] * (1.0f / 255.0f); + + const float dequant_err = fabs(dequant_val - v); + if (dequant_err < best_err) + { + best_err = dequant_err; + pEndpoint_vals[c] = (uint8_t)trial_ise_index; + } + } // iscale_delta + + } // c + } + +#if 0 + static int clamp6(int val, bool& was_clamped) + { + if (val < -32) + { + val = -32; + was_clamped = true; + } + else if (val > 31) + { + val = 31; + was_clamped = true; + } + return val; + } +#endif + + // returns true if blue contraction was used + // note the encoded endpoints may be swapped + struct rgb_base_offset_res + { + bool m_failed_flag; + bool m_used_blue_contraction; + bool m_blue_contraction_clamped; + bool m_delta_clamped; + bool m_any_degen; + bool m_endpoints_swapped; + }; + + // May fail if the tiebreaking logic isn't strong enough. + static rgb_base_offset_res cem_encode_ldr_rgb_or_rgba_base_offset(uint32_t cem_index, uint32_t endpoint_ise_range, const color_rgba& orig_l, const color_rgba& orig_h, uint8_t* pEndpoint_vals, bool use_blue_contract) + { + assert((cem_index == astc_helpers::CEM_LDR_RGB_BASE_PLUS_OFFSET) || (cem_index == astc_helpers::CEM_LDR_RGBA_BASE_PLUS_OFFSET)); + + const bool has_alpha = (cem_index == astc_helpers::CEM_LDR_RGBA_BASE_PLUS_OFFSET); + + rgb_base_offset_res res; + res.m_failed_flag = false; + res.m_used_blue_contraction = false; + res.m_blue_contraction_clamped = false; + res.m_delta_clamped = false; + res.m_any_degen = false; + res.m_endpoints_swapped = false; + + bool blue_contraction_clamped = false; + + bool status = basist::astc_ldr_t::pack_base_offset( + cem_index, endpoint_ise_range, pEndpoint_vals, + convert_to_basist_color_rgba(orig_l), convert_to_basist_color_rgba(orig_h), + use_blue_contract, true, + blue_contraction_clamped, res.m_delta_clamped, res.m_endpoints_swapped); + + assert(status); + + if (!status) + { + res.m_failed_flag = true; + return res; + } + + // Verify the actual BC status by unpacking to be absolutely sure + res.m_used_blue_contraction = astc_helpers::used_blue_contraction(cem_index, pEndpoint_vals, endpoint_ise_range); + + color_rgba dec_l, dec_h; + astc_ldr::decode_endpoints(cem_index, pEndpoint_vals, endpoint_ise_range, dec_l, dec_h); + + const uint32_t num_comps = (has_alpha ? 4 : 3); + for (uint32_t c = 0; c < num_comps; c++) + { + if (orig_l[c] != orig_h[c]) + continue; + + // Desired L/H are not equal, but packed are equal=degenerate pack (loss of freedom). + if (dec_l[c] == dec_h[c]) + { + res.m_any_degen = true; + break; + } + } // c + + return res; + } + + // L or LA direct + static void encode_cem0_4(uint32_t cem_index, float lum_l, float lum_h, float a_l, float a_h, uint32_t endpoint_ise_range, uint8_t* pEndpoints) + { + assert((cem_index == astc_helpers::CEM_LDR_LUM_DIRECT) || (cem_index == astc_helpers::CEM_LDR_LUM_ALPHA_DIRECT)); + + const bool has_alpha = (cem_index == astc_helpers::CEM_LDR_LUM_ALPHA_DIRECT); + + pEndpoints[0] = precise_round_bise_endpoint_val(lum_l, endpoint_ise_range); + pEndpoints[1] = precise_round_bise_endpoint_val(lum_h, endpoint_ise_range); + + if (has_alpha) + { + pEndpoints[2] = precise_round_bise_endpoint_val(a_l, endpoint_ise_range); + pEndpoints[3] = precise_round_bise_endpoint_val(a_h, endpoint_ise_range); + } + } + + // Returned in ISE order + uint32_t get_colors(const color_rgba& l, const color_rgba& h, uint32_t weight_ise_index, color_rgba* pColors, bool decode_mode_srgb) + { + const uint32_t total_weights = astc_helpers::get_ise_levels(weight_ise_index); + + for (uint32_t i = 0; i < total_weights; i++) + { + uint32_t w = basisu::g_ise_weight_lerps[weight_ise_index][1 + i]; + + for (uint32_t c = 0; c < 4; c++) + { + int le = l[c], he = h[c]; + + // TODO: Investigate alpha handling here vs. latest spec. + // https://raw.githubusercontent.com/KhronosGroup/DataFormat/refs/heads/main/astc.txt + // The safest thing to do may be to assume non-sRGB in the encoder. I don't know yet. + // How should alpha be handled here for lowest divergence from actual ASTC decoding hardware? + if (decode_mode_srgb) + { + le = (le << 8) | 0x80; + he = (he << 8) | 0x80; + } + else + { + le = (le << 8) | le; + he = (he << 8) | he; + } + + uint32_t k = astc_helpers::weight_interpolate(le, he, w); + + // See https://registry.khronos.org/OpenGL/extensions/EXT/EXT_texture_compression_astc_decode_mode.txt + // All channels including alpha >>8. + pColors[i][c] = (uint8_t)(k >> 8); + } // c + } // i + + return total_weights; + } + + // Returns 65 colors (NOT just 64 - 0-64 weight levels, so 65). + uint32_t get_colors_raw_weights(const color_rgba& l, const color_rgba& h, color_rgba* pColors, bool decode_mode_srgb) + { + for (uint32_t w = 0; w <= 64; w++) + { + for (uint32_t c = 0; c < 4; c++) + { + int le = l[c], he = h[c]; + + // TODO: Investigate alpha handling here vs. latest spec. + // https://raw.githubusercontent.com/KhronosGroup/DataFormat/refs/heads/main/astc.txt + // The safest thing to do may be to assume non-sRGB in the encoder. I don't know yet. + // How should alpha be handled here for lowest divergence from actual ASTC decoding hardware? + if (decode_mode_srgb) + { + le = (le << 8) | 0x80; + he = (he << 8) | 0x80; + } + else + { + le = (le << 8) | le; + he = (he << 8) | he; + } + + uint32_t k = astc_helpers::weight_interpolate(le, he, w); + + // See https://registry.khronos.org/OpenGL/extensions/EXT/EXT_texture_compression_astc_decode_mode.txt + // All channels including alpha >>8. + pColors[w][c] = (uint8_t)(k >> 8); + + } // c + } // i + + return ASTC_LDR_MAX_RAW_WEIGHTS; + } + + // Assumes ise 20 (256 levels) + void decode_endpoints_ise20(uint32_t cem_index, const uint8_t* pEndpoint_vals, color_rgba& l, color_rgba& h) + { + assert(astc_helpers::is_cem_ldr(cem_index)); + + int ldr_endpoints[4][2]; + astc_helpers::decode_endpoint(cem_index, ldr_endpoints, pEndpoint_vals); + + for (uint32_t c = 0; c < 4; c++) + { + assert((ldr_endpoints[c][0] >= 0) && (ldr_endpoints[c][0] <= 255)); + assert((ldr_endpoints[c][1] >= 0) && (ldr_endpoints[c][1] <= 255)); + + l[c] = (uint8_t)ldr_endpoints[c][0]; + h[c] = (uint8_t)ldr_endpoints[c][1]; + } + } + + void decode_endpoints(uint32_t cem_index, const uint8_t* pEndpoint_vals, uint32_t endpoint_ise_index, color_rgba& l, color_rgba& h, float* pScale) + { + const uint32_t total_endpoint_vals = astc_helpers::get_num_cem_values(cem_index); + + const auto& endpoint_dequant_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(endpoint_ise_index).m_ISE_to_val; + + uint8_t dequantized_endpoints[astc_helpers::MAX_CEM_ENDPOINT_VALS]; + for (uint32_t i = 0; i < total_endpoint_vals; i++) + dequantized_endpoints[i] = endpoint_dequant_tab[pEndpoint_vals[i]]; + + decode_endpoints_ise20(cem_index, dequantized_endpoints, l, h); + + if ((pScale) && ((cem_index == astc_helpers::CEM_LDR_RGB_BASE_SCALE) || (cem_index == astc_helpers::CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A))) + { + *pScale = (float)dequantized_endpoints[3] * (1.0f / 256.0f); + } + } + + uint32_t get_colors(uint32_t cem_index, const uint8_t* pEndpoint_vals, uint32_t endpoint_ise_index, uint32_t weight_ise_index, color_rgba* pColors, bool decode_mode_srgb) + { + color_rgba l, h; + decode_endpoints(cem_index, pEndpoint_vals, endpoint_ise_index, l, h); + + return get_colors(l, h, weight_ise_index, pColors, decode_mode_srgb); + } + + // Decodes 65 colors + uint32_t get_colors_raw_weights(uint32_t cem_index, const uint8_t* pEndpoint_vals, uint32_t endpoint_ise_index, color_rgba* pColors, bool decode_mode_srgb) + { + color_rgba l, h; + decode_endpoints(cem_index, pEndpoint_vals, endpoint_ise_index, l, h); + + return get_colors_raw_weights(l, h, pColors, decode_mode_srgb); + } + +#if 0 + static vec4F calc_incremental_pca_4D(uint32_t num_pixels, const vec4F* pPixels, const vec4F& mean_f) + { + vec4F mean_axis(0.0f); + + for (uint32_t i = 0; i < num_pixels; i++) + { + vec4F orig_color(pPixels[i]); + + vec4F color(orig_color - mean_f); + + vec4F a(color * color[0]); + vec4F b(color * color[1]); + vec4F c(color * color[2]); + vec4F d(color * color[3]); + vec4F n(i ? mean_axis : color); + + n.normalize_in_place(); + + mean_axis[0] += a.dot(n); + mean_axis[1] += b.dot(n); + mean_axis[2] += c.dot(n); + mean_axis[3] += d.dot(n); + } + + if (mean_axis.norm() < 1e-5f) + mean_axis = vec4F(1.0f, 1.0f, 1.0f, 1.0f); + + mean_axis.normalize_in_place(); + + return mean_axis; + } +#endif + + // TODO: Try two-step Lanczos iteration/Rayleigh-Ritz approximation in a 2-dimensional Krylov subspace method vs. power method. + static vec4F calc_pca_4D(uint32_t num_pixels, const vec4F* pPixels, const vec4F& mean_f) + { + float m00 = 0, m01 = 0, m02 = 0, m03 = 0; + float m11 = 0, m12 = 0, m13 = 0; + float m22 = 0, m23 = 0; + float m33 = 0; + + for (size_t i = 0; i < num_pixels; ++i) + { + const vec4F v(pPixels[i] - mean_f); + + m00 += v[0] * v[0]; m01 += v[0] * v[1]; m02 += v[0] * v[2]; m03 += v[0] * v[3]; + m11 += v[1] * v[1]; m12 += v[1] * v[2]; m13 += v[1] * v[3]; + m22 += v[2] * v[2]; m23 += v[2] * v[3]; + m33 += v[3] * v[3]; + } + + // TODO: Seed from channel variances + vec4F v(.6f, .75f, .4f, .75f); + + const uint32_t NUM_POW_ITERS = 6; // must be even + for (uint32_t i = 0; i < NUM_POW_ITERS; ++i) + { + vec4F w( + m00 * v[0] + m01 * v[1] + m02 * v[2] + m03 * v[3], + m01 * v[0] + m11 * v[1] + m12 * v[2] + m13 * v[3], + m02 * v[0] + m12 * v[1] + m22 * v[2] + m23 * v[3], + m03 * v[0] + m13 * v[1] + m23 * v[2] + m33 * v[3] + ); + + if (i & 1) + w.normalize_in_place(); + v = w; + } + + if (v.norm() < 1e-5f) + v = vec4F(.5f, .5f, .5f, .5f); + + return v; + } + + static vec4F calc_pca_3D(uint32_t num_pixels, const vec4F* pPixels, const vec4F& mean_f) + { + float cov[6] = { 0, 0, 0, 0, 0, 0 }; + + for (uint32_t i = 0; i < num_pixels; i++) + { + const vec4F& v = pPixels[i]; + float r = v[0] - mean_f[0]; + float g = v[1] - mean_f[1]; + float b = v[2] - mean_f[2]; + cov[0] += r * r; cov[1] += r * g; cov[2] += r * b; cov[3] += g * g; cov[4] += g * b; cov[5] += b * b; + } + + float xr = .9f, xg = 1.0f, xb = .7f; + for (uint32_t iter = 0; iter < 3; iter++) + { + float r = xr * cov[0] + xg * cov[1] + xb * cov[2]; + float g = xr * cov[1] + xg * cov[3] + xb * cov[4]; + float b = xr * cov[2] + xg * cov[4] + xb * cov[5]; + + float m = maximumf(maximumf(fabsf(r), fabsf(g)), fabsf(b)); + if (m > 1e-10f) + { + m = 1.0f / m; + r *= m; g *= m; b *= m; + } + + xr = r; xg = g; xb = b; + } + + float nrm = xr * xr + xg * xg + xb * xb; + + vec4F axis(0.57735027f, 0.57735027f, 0.57735027f, 0.0f); + if (nrm > 1e-5f) + { + float inv_nrm = 1.0f / sqrtf(nrm); + xr *= inv_nrm; xg *= inv_nrm; xb *= inv_nrm; + axis.set(xr, xg, xb, 0); + } + + return axis; + } + + void pixel_stats_t::init(uint32_t num_pixels, const color_rgba* pPixels) + { + m_num_pixels = num_pixels; + m_has_alpha = false; + + m_min.set(255, 255, 255, 255); + m_max.set(0, 0, 0, 0); + + m_mean_f.clear(); + + for (uint32_t i = 0; i < m_num_pixels; i++) + { + const color_rgba& px = pPixels[i]; + + m_pixels[i] = px; + + m_pixels_f[i].set((float)px.r * (1.0f / 255.0f), (float)px.g * (1.0f / 255.0f), (float)px.b * (1.0f / 255.0f), (float)px.a * (1.0f / 255.0f)); + + m_mean_f += m_pixels_f[i]; + + m_min.r = basisu::minimum(m_min.r, px.r); + m_min.g = basisu::minimum(m_min.g, px.g); + m_min.b = basisu::minimum(m_min.b, px.b); + m_min.a = basisu::minimum(m_min.a, px.a); + + m_max.r = basisu::maximum(m_max.r, px.r); + m_max.g = basisu::maximum(m_max.g, px.g); + m_max.b = basisu::maximum(m_max.b, px.b); + m_max.a = basisu::maximum(m_max.a, px.a); + } + + m_mean_f *= (1.0f / (float)m_num_pixels); + m_mean_f.clamp(0.0f, 1.0f); + + m_min_f.set(m_min.r * (1.0f / 255.0f), m_min.g * (1.0f / 255.0f), m_min.b * (1.0f / 255.0f), m_min.a * (1.0f / 255.0f)); + m_max_f.set(m_max.r * (1.0f / 255.0f), m_max.g * (1.0f / 255.0f), m_max.b * (1.0f / 255.0f), m_max.a * (1.0f / 255.0f)); + + m_has_alpha = (m_min.a < 255); + + // Mean and zero relative RGB (3D) PCA axes + m_mean_rel_axis3 = calc_pca_3D(m_num_pixels, m_pixels_f, m_mean_f); + m_zero_rel_axis3 = calc_pca_3D(m_num_pixels, m_pixels_f, vec4F(0.0f)); + + // Mean and zero relative RGBA (4D) PCA axes + m_mean_rel_axis4 = calc_pca_4D(m_num_pixels, m_pixels_f, m_mean_f); + + for (uint32_t c = 0; c < 4u; c++) + m_rgba_stats[c].calc_simplified_with_range(m_num_pixels, &m_pixels_f[0][c], 4); + } + + static inline uint32_t square_of_diff(int a, int b) + { + assert((a >= 0) && (a <= 255)); + assert((b >= 0) && (b <= 255)); + + int d = a - b; + return (uint32_t)(d * d); + } + + uint64_t eval_solution( + const pixel_stats_t& pixel_stats, + uint32_t total_weights, const color_rgba* pWeight_colors, + uint8_t* pWeight_vals, uint32_t weight_ise_index, + const cem_encode_params& params) + { + BASISU_NOTE_UNUSED(weight_ise_index); + assert((total_weights <= 32) || (total_weights == 65)); + + uint64_t total_err = 0; + + if (params.m_pForced_weight_vals0) + { + for (uint32_t c = 0; c < pixel_stats.m_num_pixels; c++) + { + const color_rgba& px = pixel_stats.m_pixels[c]; + + const uint32_t w = params.m_pForced_weight_vals0[c]; + assert(w < total_weights); + + uint32_t err = + params.m_comp_weights[0] * square_of_diff(px.r, pWeight_colors[w].r) + + params.m_comp_weights[1] * square_of_diff(px.g, pWeight_colors[w].g) + + params.m_comp_weights[2] * square_of_diff(px.b, pWeight_colors[w].b) + + params.m_comp_weights[3] * square_of_diff(px.a, pWeight_colors[w].a); + + total_err += err; + + pWeight_vals[c] = (uint8_t)w; + } + } + else + { + for (uint32_t c = 0; c < pixel_stats.m_num_pixels; c++) + { + const color_rgba& px = pixel_stats.m_pixels[c]; + + uint32_t best_err = UINT32_MAX; + uint32_t best_sel = 0; + + for (uint32_t i = 0; i < total_weights; i++) + { + uint32_t err = + params.m_comp_weights[0] * square_of_diff(px.r, pWeight_colors[i].r) + + params.m_comp_weights[1] * square_of_diff(px.g, pWeight_colors[i].g) + + params.m_comp_weights[2] * square_of_diff(px.b, pWeight_colors[i].b) + + params.m_comp_weights[3] * square_of_diff(px.a, pWeight_colors[i].a); + + if (err < best_err) + { + best_err = err; + best_sel = i; + } + } + + total_err += best_err; + pWeight_vals[c] = (uint8_t)best_sel; + } + } // if (params.m_pForced_weight_vals0) + + return total_err; + } + + // Evaluates against raw weights [0,64], or to ISE quantized weights, depending on weight_ise_index. + uint64_t eval_solution( + const pixel_stats_t& pixel_stats, + uint32_t cem_index, + const uint8_t* pEndpoint_vals, uint32_t endpoint_ise_index, + uint8_t* pWeight_vals, uint32_t weight_ise_index, + const cem_encode_params& params) + { + assert((weight_ise_index <= astc_helpers::BISE_32_LEVELS) || (weight_ise_index == astc_helpers::BISE_64_LEVELS)); + + color_rgba weight_colors[ASTC_LDR_MAX_RAW_WEIGHTS]; + uint32_t num_weights; + + assert((weight_ise_index <= astc_helpers::BISE_32_LEVELS) || (weight_ise_index == astc_helpers::BISE_64_LEVELS)); + + // 64 levels isn't valid ASTC. It's used for raw weight mode. + if (weight_ise_index == astc_helpers::BISE_64_LEVELS) + num_weights = get_colors_raw_weights(cem_index, pEndpoint_vals, endpoint_ise_index, weight_colors, params.m_decode_mode_srgb); + else + num_weights = get_colors(cem_index, pEndpoint_vals, endpoint_ise_index, weight_ise_index, weight_colors, params.m_decode_mode_srgb); + + assert(num_weights <= std::size(weight_colors)); + + uint64_t trial_err = eval_solution( + pixel_stats, + num_weights, weight_colors, + pWeight_vals, weight_ise_index, + params); + + return trial_err; + } + + // Evaluates against raw weights [0,64], or to ISE quantized weights, depending on weight_ise_index. + uint64_t eval_solution_dp( + uint32_t ccs_index, + const pixel_stats_t& pixel_stats, + uint32_t total_weights, const color_rgba* pWeight_colors, + uint8_t* pWeight_vals0, uint8_t* pWeight_vals1, uint32_t weight_ise_index, + const cem_encode_params& params) + { + BASISU_NOTE_UNUSED(weight_ise_index); + + assert(ccs_index <= 3); + assert((total_weights <= 32) || (total_weights == 65)); + + uint64_t total_err = 0; + + if (params.m_pForced_weight_vals0) + { + for (uint32_t c = 0; c < pixel_stats.m_num_pixels; c++) + { + const color_rgba& px = pixel_stats.m_pixels[c]; + + const uint32_t w = params.m_pForced_weight_vals0[c]; + assert(w < total_weights); + + uint32_t err = 0; + for (uint32_t o = 0; o < 4; o++) + if (o != ccs_index) + err += params.m_comp_weights[o] * square_of_diff(px[o], pWeight_colors[w][o]); + + total_err += err; + + pWeight_vals0[c] = (uint8_t)w; + } + } + else + { + for (uint32_t c = 0; c < pixel_stats.m_num_pixels; c++) + { + const color_rgba& px = pixel_stats.m_pixels[c]; + + uint32_t best_err = UINT32_MAX; + uint32_t best_sel = 0; + + for (uint32_t i = 0; i < total_weights; i++) + { + uint32_t err = 0; + for (uint32_t o = 0; o < 4; o++) + if (o != ccs_index) + err += params.m_comp_weights[o] * square_of_diff(px[o], pWeight_colors[i][o]); + + if (err < best_err) + { + best_err = err; + best_sel = i; + } + } + + total_err += best_err; + pWeight_vals0[c] = (uint8_t)best_sel; + } + } + + if (params.m_pForced_weight_vals1) + { + for (uint32_t c = 0; c < pixel_stats.m_num_pixels; c++) + { + const color_rgba& px = pixel_stats.m_pixels[c]; + + const uint32_t w = params.m_pForced_weight_vals1[c]; + assert(w < total_weights); + + uint32_t err = square_of_diff(px[ccs_index], pWeight_colors[w][ccs_index]); + + total_err += err * params.m_comp_weights[ccs_index]; + pWeight_vals1[c] = (uint8_t)w; + } + } + else + { + for (uint32_t c = 0; c < pixel_stats.m_num_pixels; c++) + { + const color_rgba& px = pixel_stats.m_pixels[c]; + + uint32_t best_err = UINT32_MAX; + uint32_t best_sel = 0; + + for (uint32_t i = 0; i < total_weights; i++) + { + uint32_t err = square_of_diff(px[ccs_index], pWeight_colors[i][ccs_index]); + + if (err < best_err) + { + best_err = err; + best_sel = i; + } + } + + total_err += best_err * params.m_comp_weights[ccs_index]; + pWeight_vals1[c] = (uint8_t)best_sel; + } + } + + return total_err; + } + + // Evaluates against raw weights [0,64], or to ISE quantized weights, depending on weight_ise_index. + uint64_t eval_solution_dp( + const pixel_stats_t& pixel_stats, + uint32_t cem_index, uint32_t ccs_index, + const uint8_t* pEndpoint_vals, uint32_t endpoint_ise_index, + uint8_t* pWeight_vals0, uint8_t* pWeight_vals1, uint32_t weight_ise_index, + const cem_encode_params& params) + { + assert((weight_ise_index <= astc_helpers::BISE_32_LEVELS) || (weight_ise_index == astc_helpers::BISE_64_LEVELS)); + + color_rgba weight_colors[ASTC_LDR_MAX_RAW_WEIGHTS]; + uint32_t num_weights; + + // 64 levels isn't valid ASTC. It's used for raw weight mode. + if (weight_ise_index == astc_helpers::BISE_64_LEVELS) + num_weights = get_colors_raw_weights(cem_index, pEndpoint_vals, endpoint_ise_index, weight_colors, params.m_decode_mode_srgb); + else + num_weights = get_colors(cem_index, pEndpoint_vals, endpoint_ise_index, weight_ise_index, weight_colors, params.m_decode_mode_srgb); + + uint64_t trial_err = eval_solution_dp( + ccs_index, + pixel_stats, + num_weights, weight_colors, + pWeight_vals0, pWeight_vals1, weight_ise_index, + params); + + return trial_err; + } + + // Direct - refine ISE quantized endpoints from float endpoints + static void refine_cem8_or_12_endpoints(uint32_t cem_index, uint32_t endpoint_ise_range, uint8_t* pTrial_endpoint_vals, const vec4F& low_color_f, const vec4F& high_color_f, bool endpoints_are_swapped) + { + assert((cem_index == astc_helpers::CEM_LDR_RGB_DIRECT) || (cem_index == astc_helpers::CEM_LDR_RGBA_DIRECT)); + + if (endpoint_ise_range == astc_helpers::BISE_256_LEVELS) + return; + + const uint32_t total_comps = (cem_index == astc_helpers::CEM_LDR_RGBA_DIRECT) ? 4 : 3; + + assert((cem_index == astc_helpers::CEM_LDR_RGB_DIRECT) || (cem_index == astc_helpers::CEM_LDR_RGBA_DIRECT)); + assert((endpoint_ise_range >= astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE) && (endpoint_ise_range <= astc_helpers::LAST_VALID_ENDPOINT_ISE_RANGE)); + + const uint32_t total_endpoint_vals = astc_helpers::get_num_cem_values(cem_index); + const uint32_t num_endpoint_ise_levels = astc_helpers::get_ise_levels(endpoint_ise_range); + + const auto& endpoint_dequant_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(endpoint_ise_range).m_ISE_to_val; + + const auto& ISE_to_rank = astc_helpers::g_dequant_tables.get_endpoint_tab(endpoint_ise_range).m_ISE_to_rank; + const auto& rank_to_ISE = astc_helpers::g_dequant_tables.get_endpoint_tab(endpoint_ise_range).m_rank_to_ISE; + + const bool orig_used_blue_contraction = astc_helpers::cem8_or_12_used_blue_contraction(cem_index, pTrial_endpoint_vals, endpoint_ise_range); + + uint32_t first_comp = 0; + + uint8_t refined_endpoint_vals[astc_helpers::NUM_MODE12_ENDPOINTS]; + memcpy(refined_endpoint_vals, pTrial_endpoint_vals, total_endpoint_vals); + + if (orig_used_blue_contraction) + { + // TODO expensive: 2*3*9 = 54 tries + for (uint32_t e = 0; e < 2; e++) + { + float best_err = BIG_FLOAT_VAL; + uint8_t best_refined_endpoint_vals[3] = { 0, 0, 0 }; + + for (int b_delta = -1; b_delta <= 1; b_delta++) + { + for (int k = 0; k < 9; k++) + { + const int r_delta = (k % 3) - 1; + const int g_delta = (k / 3) - 1; + + const int comp_deltas[3] = { r_delta, g_delta, b_delta }; + + uint8_t trial_refined_endpoint_vals[3] = { 0, 0, 0 }; + + for (uint32_t c = 0; c < 3; c++) + { + const int enc_val = pTrial_endpoint_vals[c * 2 + e]; + + const int orig_rank = ISE_to_rank[enc_val]; + + const int v_delta = comp_deltas[c]; + const int new_rank = basisu::clamp(orig_rank + v_delta, 0, (int)num_endpoint_ise_levels - 1); + const int new_enc_ise_val = rank_to_ISE[new_rank]; + + trial_refined_endpoint_vals[c] = (uint8_t)new_enc_ise_val; + + } // c + + color_rgba trial_refined_endpoints_dequant(blue_contract_dec(endpoint_dequant_tab[trial_refined_endpoint_vals[0]], endpoint_dequant_tab[trial_refined_endpoint_vals[1]], endpoint_dequant_tab[trial_refined_endpoint_vals[2]], 255)); + + vec3F trial_refined_endpoints_dequant_f(0.0f); + for (uint32_t c = 0; c < 3; c++) + trial_refined_endpoints_dequant_f[c] = (float)trial_refined_endpoints_dequant[c] * (1.0f / 255.0f); + + vec3F desired_endpoint; + if (endpoints_are_swapped) + desired_endpoint = (e == 0) ? vec3F(high_color_f) : vec3F(low_color_f); + else + desired_endpoint = (e == 0) ? vec3F(low_color_f) : vec3F(high_color_f); + + float trial_err = desired_endpoint.squared_distance(trial_refined_endpoints_dequant_f); + if (trial_err < best_err) + { + best_err = trial_err; + memcpy(best_refined_endpoint_vals, trial_refined_endpoint_vals, 3); + } + + } // k + + } // b_delta + + for (uint32_t c = 0; c < 3; c++) + { + refined_endpoint_vals[c * 2 + e] = best_refined_endpoint_vals[c]; + } // c + + } // e + + // just refine A now (if it exists) + first_comp = 3; + } + + if (first_comp < total_comps) + { + for (uint32_t e = 0; e < 2; e++) + { + for (uint32_t c = first_comp; c < total_comps; c++) + { + const uint32_t idx = c * 2 + e; + const int enc_val = pTrial_endpoint_vals[idx]; + + const int orig_rank = ISE_to_rank[enc_val]; + + int best_rank = orig_rank; + float best_err = BIG_FLOAT_VAL; + for (int v_delta = -1; v_delta <= 1; v_delta++) + { + int new_rank = basisu::clamp(orig_rank + v_delta, 0, (int)num_endpoint_ise_levels - 1); + int new_enc_ise_val = rank_to_ISE[new_rank]; + + float dequant_val = (float)endpoint_dequant_tab[new_enc_ise_val] * (1.0f / 255.0f); + + float orig_val; + if (endpoints_are_swapped) + orig_val = (e == 0) ? high_color_f[c] : low_color_f[c]; + else + orig_val = (e == 0) ? low_color_f[c] : high_color_f[c]; + + float err = fabsf(dequant_val - orig_val); + if (err < best_err) + { + best_err = err; + best_rank = new_rank; + } + } + + refined_endpoint_vals[idx] = (uint8_t)rank_to_ISE[best_rank]; + + } // c + } // e + } + + bool refined_used_blue_contraction = astc_helpers::cem8_or_12_used_blue_contraction(cem_index, refined_endpoint_vals, endpoint_ise_range); + if (refined_used_blue_contraction == orig_used_blue_contraction) + { + memcpy(pTrial_endpoint_vals, refined_endpoint_vals, total_endpoint_vals); + } + } + + // Direct L/LA, single plane + static bool try_cem0_or_4(uint32_t cem_index, + const pixel_stats_t& pixel_stats, const cem_encode_params& enc_params, + uint32_t endpoint_ise_range, uint32_t weight_ise_range, + float lum_l, float lum_h, float a_l, float a_h, + uint8_t* pTrial_endpoint_vals, uint8_t* pTrial_weight_vals, uint64_t& trial_blk_error) + { + assert(g_initialized); + assert((cem_index == astc_helpers::CEM_LDR_LUM_DIRECT) || (cem_index == astc_helpers::CEM_LDR_LUM_ALPHA_DIRECT)); + + const bool cem_has_alpha = (cem_index == astc_helpers::CEM_LDR_LUM_ALPHA_DIRECT); + + const uint32_t num_endpoint_vals = astc_helpers::get_num_cem_values(cem_index); + + uint8_t trial_endpoint_vals[astc_helpers::NUM_MODE4_ENDPOINTS] = { 0 }; + uint8_t trial_weight_vals[ASTC_LDR_MAX_BLOCK_PIXELS] = { 0 }; + + encode_cem0_4(cem_index, lum_l, lum_h, a_l, a_h, endpoint_ise_range, trial_endpoint_vals); + + uint64_t trial_err = eval_solution( + pixel_stats, + cem_index, trial_endpoint_vals, endpoint_ise_range, + trial_weight_vals, weight_ise_range, + enc_params); + + bool improved_flag = false; + if (trial_err < trial_blk_error) + { + trial_blk_error = trial_err; + memcpy(pTrial_endpoint_vals, trial_endpoint_vals, astc_helpers::get_num_cem_values(cem_index)); + memcpy(pTrial_weight_vals, trial_weight_vals, pixel_stats.m_num_pixels); + improved_flag = true; + } + + bool any_degen = false; + if ((trial_endpoint_vals[0] == trial_endpoint_vals[1]) && (lum_l != lum_h)) + any_degen = true; + + if (cem_has_alpha) + { + if ((trial_endpoint_vals[2] == trial_endpoint_vals[3]) && (a_l != a_h)) + any_degen = true; + } + + if (any_degen) + { + const int l_delta = (lum_l < lum_h) ? -1 : 1; + const int a_delta = (a_l < a_h) ? -1 : 1; + + for (uint32_t t = 1; t <= 3; t++) + { + uint8_t fixed_endpoint_vals[astc_helpers::NUM_MODE4_ENDPOINTS]; + memcpy(fixed_endpoint_vals, trial_endpoint_vals, num_endpoint_vals); + + if (t & 1) + { + if ((trial_endpoint_vals[0] == trial_endpoint_vals[1]) && (lum_l != lum_h)) + fixed_endpoint_vals[0] = (uint8_t)astc_helpers::apply_delta_to_bise_endpoint_val(endpoint_ise_range, trial_endpoint_vals[0], l_delta); + + if (cem_has_alpha) + { + if ((trial_endpoint_vals[2] == trial_endpoint_vals[3]) && (a_l != a_h)) + fixed_endpoint_vals[2] = (uint8_t)astc_helpers::apply_delta_to_bise_endpoint_val(endpoint_ise_range, trial_endpoint_vals[2], a_delta); + } + } + + if (t & 2) + { + if ((trial_endpoint_vals[0] == trial_endpoint_vals[1]) && (lum_l != lum_h)) + fixed_endpoint_vals[1] = (uint8_t)astc_helpers::apply_delta_to_bise_endpoint_val(endpoint_ise_range, trial_endpoint_vals[1], -l_delta); + + if (cem_has_alpha) + { + if ((trial_endpoint_vals[2] == trial_endpoint_vals[3]) && (a_l != a_h)) + fixed_endpoint_vals[3] = (uint8_t)astc_helpers::apply_delta_to_bise_endpoint_val(endpoint_ise_range, trial_endpoint_vals[3], -a_delta); + } + } + + trial_err = eval_solution( + pixel_stats, + cem_index, fixed_endpoint_vals, endpoint_ise_range, + trial_weight_vals, weight_ise_range, + enc_params); + + if (trial_err < trial_blk_error) + { + trial_blk_error = trial_err; + memcpy(pTrial_endpoint_vals, fixed_endpoint_vals, astc_helpers::get_num_cem_values(cem_index)); + memcpy(pTrial_weight_vals, trial_weight_vals, pixel_stats.m_num_pixels); + improved_flag = true; + } + + } // t + } + + return improved_flag; + } + + static bool try_cem4_dp_a(uint32_t cem_index, + const pixel_stats_t& pixel_stats, const cem_encode_params& enc_params, + uint32_t endpoint_ise_range, uint32_t weight_ise_range, + float lum_l, float lum_h, float a_l, float a_h, + uint8_t* pTrial_endpoint_vals, uint8_t* pTrial_weight_vals0, uint8_t* pTrial_weight_vals1, uint64_t& trial_blk_error) + { + assert(g_initialized); + assert(cem_index == astc_helpers::CEM_LDR_LUM_ALPHA_DIRECT); + + const bool cem_has_alpha = (cem_index == astc_helpers::CEM_LDR_LUM_ALPHA_DIRECT); + + const uint32_t num_endpoint_vals = astc_helpers::get_num_cem_values(cem_index); + + uint8_t trial_endpoint_vals[astc_helpers::NUM_MODE4_ENDPOINTS] = { 0 }; + uint8_t trial_weight_vals0[ASTC_LDR_MAX_BLOCK_PIXELS] = { 0 }; + uint8_t trial_weight_vals1[ASTC_LDR_MAX_BLOCK_PIXELS] = { 0 }; + + encode_cem0_4(cem_index, lum_l, lum_h, a_l, a_h, endpoint_ise_range, trial_endpoint_vals); + + uint64_t trial_err = eval_solution_dp( + pixel_stats, cem_index, 3, + trial_endpoint_vals, endpoint_ise_range, + trial_weight_vals0, trial_weight_vals1, weight_ise_range, + enc_params); + + bool improved_flag = false; + if (trial_err < trial_blk_error) + { + trial_blk_error = trial_err; + memcpy(pTrial_endpoint_vals, trial_endpoint_vals, astc_helpers::get_num_cem_values(cem_index)); + memcpy(pTrial_weight_vals0, trial_weight_vals0, pixel_stats.m_num_pixels); + memcpy(pTrial_weight_vals1, trial_weight_vals1, pixel_stats.m_num_pixels); + improved_flag = true; + } + + bool any_degen = false; + if ((trial_endpoint_vals[0] == trial_endpoint_vals[1]) && (lum_l != lum_h)) + any_degen = true; + + if (cem_has_alpha) + { + if ((trial_endpoint_vals[2] == trial_endpoint_vals[3]) && (a_l != a_h)) + any_degen = true; + } + + if (any_degen) + { + const int l_delta = (lum_l < lum_h) ? -1 : 1; + const int a_delta = (a_l < a_h) ? -1 : 1; + + for (uint32_t t = 1; t <= 3; t++) + { + uint8_t fixed_endpoint_vals[astc_helpers::NUM_MODE4_ENDPOINTS]; + memcpy(fixed_endpoint_vals, trial_endpoint_vals, num_endpoint_vals); + + if (t & 1) + { + if ((trial_endpoint_vals[0] == trial_endpoint_vals[1]) && (lum_l != lum_h)) + fixed_endpoint_vals[0] = (uint8_t)astc_helpers::apply_delta_to_bise_endpoint_val(endpoint_ise_range, trial_endpoint_vals[0], l_delta); + + if (cem_has_alpha) + { + if ((trial_endpoint_vals[2] == trial_endpoint_vals[3]) && (a_l != a_h)) + fixed_endpoint_vals[2] = (uint8_t)astc_helpers::apply_delta_to_bise_endpoint_val(endpoint_ise_range, trial_endpoint_vals[2], a_delta); + } + } + + if (t & 2) + { + if ((trial_endpoint_vals[0] == trial_endpoint_vals[1]) && (lum_l != lum_h)) + fixed_endpoint_vals[1] = (uint8_t)astc_helpers::apply_delta_to_bise_endpoint_val(endpoint_ise_range, trial_endpoint_vals[1], -l_delta); + + if (cem_has_alpha) + { + if ((trial_endpoint_vals[2] == trial_endpoint_vals[3]) && (a_l != a_h)) + fixed_endpoint_vals[3] = (uint8_t)astc_helpers::apply_delta_to_bise_endpoint_val(endpoint_ise_range, trial_endpoint_vals[3], -a_delta); + } + } + + trial_err = eval_solution_dp( + pixel_stats, cem_index, 3, + fixed_endpoint_vals, endpoint_ise_range, + trial_weight_vals0, trial_weight_vals1, weight_ise_range, + enc_params); + + if (trial_err < trial_blk_error) + { + trial_blk_error = trial_err; + memcpy(pTrial_endpoint_vals, fixed_endpoint_vals, astc_helpers::get_num_cem_values(cem_index)); + memcpy(pTrial_weight_vals0, trial_weight_vals0, pixel_stats.m_num_pixels); + memcpy(pTrial_weight_vals1, trial_weight_vals1, pixel_stats.m_num_pixels); + improved_flag = true; + } + + } // t + } + + return improved_flag; + } + + // Direct RGB/RGBA + // Cannot fail, but may have to fall back to non-blue-contracted + // Returns false if trial solution not improved + static bool try_cem8_12( + uint32_t cem_index, + const pixel_stats_t& pixel_stats, const cem_encode_params& enc_params, + uint32_t endpoint_ise_range, uint32_t weight_ise_range, + const vec4F& low_color_f, const vec4F& high_color_f, + uint8_t* pTrial_endpoint_vals, uint8_t* pTrial_weight_vals, uint64_t& trial_blk_error, bool& trial_used_blue_contraction, + bool try_blue_contract, bool& tried_used_blue_contraction) + { + assert(g_initialized); + assert((cem_index == astc_helpers::CEM_LDR_RGB_DIRECT) || (cem_index == astc_helpers::CEM_LDR_RGBA_DIRECT)); + + const uint32_t num_endpoint_vals = astc_helpers::get_num_cem_values(cem_index); + const uint32_t num_comps = (cem_index == astc_helpers::CEM_LDR_RGB_DIRECT) ? 3 : 4; + + color_rgba low_color, high_color; + for (uint32_t c = 0; c < 4; c++) + { + low_color[c] = (uint8_t)basisu::clamp((int)std::round(low_color_f[c] * 255.0f), 0, 255); + high_color[c] = (uint8_t)basisu::clamp((int)std::round(high_color_f[c] * 255.0f), 0, 255); + } + + uint8_t trial_endpoint_vals[astc_helpers::NUM_MODE12_ENDPOINTS] = { 0 }; + uint8_t trial_weight_vals[ASTC_LDR_MAX_BLOCK_PIXELS] = { 0 }; + + // Cannot fail, but may have to fall back to non-blue-contracted + cem_encode_ldr_rgb_or_rgba_direct_result res = cem_encode_ldr_rgb_or_rgba_direct(cem_index, endpoint_ise_range, low_color, high_color, trial_endpoint_vals, try_blue_contract); + + // Let caller know if we tried blue contraction + tried_used_blue_contraction = res.m_is_blue_contracted; + + if (endpoint_ise_range < astc_helpers::BISE_256_LEVELS) + { + refine_cem8_or_12_endpoints(cem_index, endpoint_ise_range, trial_endpoint_vals, low_color_f, high_color_f, res.m_endpoints_are_swapped); + } + + uint64_t trial_err = eval_solution( + pixel_stats, cem_index, + trial_endpoint_vals, endpoint_ise_range, + trial_weight_vals, weight_ise_range, + enc_params); + + bool improved_flag = false; + if (trial_err < trial_blk_error) + { + trial_blk_error = trial_err; + memcpy(pTrial_endpoint_vals, trial_endpoint_vals, astc_helpers::get_num_cem_values(cem_index)); + memcpy(pTrial_weight_vals, trial_weight_vals, pixel_stats.m_num_pixels); + trial_used_blue_contraction = res.m_is_blue_contracted; + improved_flag = true; + } + + if (res.m_any_degen) + { + color_rgba dec_l(0), dec_h(0); + decode_endpoints(cem_index, trial_endpoint_vals, endpoint_ise_range, dec_l, dec_h); + + uint32_t s0 = dec_l.r + dec_l.g + dec_l.b + dec_l.a; + uint32_t s1 = dec_h.r + dec_h.g + dec_h.b + dec_h.a; + if (astc_helpers::cem8_or_12_used_blue_contraction(cem_index, trial_endpoint_vals, endpoint_ise_range)) + std::swap(s0, s1); + + for (uint32_t t = 1; t <= 3; t++) + { + uint8_t fixed_endpoint_vals[astc_helpers::NUM_MODE12_ENDPOINTS]; + memcpy(fixed_endpoint_vals, trial_endpoint_vals, num_endpoint_vals); + + if (t & 1) + { + for (uint32_t c = 0; c < num_comps; c++) + { + uint32_t l_idx = c * 2 + 0; + uint32_t h_idx = c * 2 + 1; + + if ((trial_endpoint_vals[l_idx] == trial_endpoint_vals[h_idx]) && (low_color[c] != high_color[c])) + { + int delta = (s0 <= s1) ? -1 : 1; + + fixed_endpoint_vals[l_idx] = (uint8_t)astc_helpers::apply_delta_to_bise_endpoint_val(endpoint_ise_range, trial_endpoint_vals[l_idx], delta); + } + } + } + + if (t & 2) + { + for (uint32_t c = 0; c < num_comps; c++) + { + uint32_t l_idx = c * 2 + 0; + uint32_t h_idx = c * 2 + 1; + + if ((trial_endpoint_vals[l_idx] == trial_endpoint_vals[h_idx]) && (low_color[c] != high_color[c])) + { + int delta = (s0 <= s1) ? 1 : -1; + + fixed_endpoint_vals[h_idx] = (uint8_t)astc_helpers::apply_delta_to_bise_endpoint_val(endpoint_ise_range, trial_endpoint_vals[h_idx], delta); + } + } + } + + bool fixed_used_blue_contraction = astc_helpers::cem8_or_12_used_blue_contraction(cem_index, fixed_endpoint_vals, endpoint_ise_range); + if (fixed_used_blue_contraction != res.m_is_blue_contracted) + continue; + + trial_err = eval_solution( + pixel_stats, + cem_index, fixed_endpoint_vals, endpoint_ise_range, + trial_weight_vals, weight_ise_range, + enc_params); + + if (trial_err < trial_blk_error) + { + trial_blk_error = trial_err; + memcpy(pTrial_endpoint_vals, fixed_endpoint_vals, astc_helpers::get_num_cem_values(cem_index)); + memcpy(pTrial_weight_vals, trial_weight_vals, pixel_stats.m_num_pixels); + trial_used_blue_contraction = res.m_is_blue_contracted; + improved_flag = true; + } + + } // t + + } // if (res.m_any_degen) + + return improved_flag; + } + + static bool try_cem8_12_dp( + uint32_t cem_index, uint32_t ccs_index, + const pixel_stats_t& pixel_stats, const cem_encode_params& enc_params, + uint32_t endpoint_ise_range, uint32_t weight_ise_range, + const vec4F& low_color_f, const vec4F& high_color_f, + uint8_t* pTrial_endpoint_vals, uint8_t* pTrial_weight_vals0, uint8_t* pTrial_weight_vals1, uint64_t& trial_blk_error, bool& trial_used_blue_contraction, + bool try_blue_contract, bool& tried_used_blue_contraction) + { + assert(g_initialized); + assert((cem_index == astc_helpers::CEM_LDR_RGB_DIRECT) || (cem_index == astc_helpers::CEM_LDR_RGBA_DIRECT)); + + bool improved_flag = false; + + const uint32_t num_endpoint_vals = astc_helpers::get_num_cem_values(cem_index); + const uint32_t num_comps = (cem_index == astc_helpers::CEM_LDR_RGB_DIRECT) ? 3 : 4; + + color_rgba low_color, high_color; + for (uint32_t c = 0; c < 4; c++) + { + low_color[c] = (uint8_t)basisu::clamp((int)std::round(low_color_f[c] * 255.0f), 0, 255); + high_color[c] = (uint8_t)basisu::clamp((int)std::round(high_color_f[c] * 255.0f), 0, 255); + } + + uint8_t trial_endpoint_vals[astc_helpers::NUM_MODE12_ENDPOINTS] = { 0 }; + uint8_t trial_weight_vals0[ASTC_LDR_MAX_BLOCK_PIXELS] = { 0 }; + uint8_t trial_weight_vals1[ASTC_LDR_MAX_BLOCK_PIXELS] = { 0 }; + + // Cannot fail, but may have to fall back to non-blue-contracted + cem_encode_ldr_rgb_or_rgba_direct_result res = cem_encode_ldr_rgb_or_rgba_direct(cem_index, endpoint_ise_range, low_color, high_color, trial_endpoint_vals, try_blue_contract); + + // Let caller know if we tried blue contraction + tried_used_blue_contraction = res.m_is_blue_contracted; + + if (endpoint_ise_range < astc_helpers::BISE_256_LEVELS) + { + refine_cem8_or_12_endpoints(cem_index, endpoint_ise_range, trial_endpoint_vals, low_color_f, high_color_f, res.m_endpoints_are_swapped); + } + + uint64_t trial_err = eval_solution_dp(pixel_stats, cem_index, ccs_index, trial_endpoint_vals, endpoint_ise_range, trial_weight_vals0, trial_weight_vals1, weight_ise_range, enc_params); + + if (trial_err < trial_blk_error) + { + trial_blk_error = trial_err; + memcpy(pTrial_endpoint_vals, trial_endpoint_vals, astc_helpers::get_num_cem_values(cem_index)); + memcpy(pTrial_weight_vals0, trial_weight_vals0, pixel_stats.m_num_pixels); + memcpy(pTrial_weight_vals1, trial_weight_vals1, pixel_stats.m_num_pixels); + trial_used_blue_contraction = res.m_is_blue_contracted; + improved_flag = true; + } + + if (res.m_any_degen) + { + color_rgba dec_l(0), dec_h(0); + decode_endpoints(cem_index, trial_endpoint_vals, endpoint_ise_range, dec_l, dec_h); + + uint32_t s0 = dec_l.r + dec_l.g + dec_l.b + dec_l.a; + uint32_t s1 = dec_h.r + dec_h.g + dec_h.b + dec_h.a; + if (astc_helpers::cem8_or_12_used_blue_contraction(cem_index, trial_endpoint_vals, endpoint_ise_range)) + std::swap(s0, s1); + + for (uint32_t t = 1; t <= 3; t++) + { + uint8_t fixed_endpoint_vals[astc_helpers::NUM_MODE12_ENDPOINTS]; + memcpy(fixed_endpoint_vals, trial_endpoint_vals, num_endpoint_vals); + + if (t & 1) + { + for (uint32_t c = 0; c < num_comps; c++) + { + uint32_t l_idx = c * 2 + 0; + uint32_t h_idx = c * 2 + 1; + + if ((trial_endpoint_vals[l_idx] == trial_endpoint_vals[h_idx]) && (low_color[c] != high_color[c])) + { + int delta = (s0 <= s1) ? -1 : 1; + + fixed_endpoint_vals[l_idx] = (uint8_t)astc_helpers::apply_delta_to_bise_endpoint_val(endpoint_ise_range, trial_endpoint_vals[l_idx], delta); + } + } + } + + if (t & 2) + { + for (uint32_t c = 0; c < num_comps; c++) + { + uint32_t l_idx = c * 2 + 0; + uint32_t h_idx = c * 2 + 1; + + if ((trial_endpoint_vals[l_idx] == trial_endpoint_vals[h_idx]) && (low_color[c] != high_color[c])) + { + int delta = (s0 <= s1) ? 1 : -1; + + fixed_endpoint_vals[h_idx] = (uint8_t)astc_helpers::apply_delta_to_bise_endpoint_val(endpoint_ise_range, trial_endpoint_vals[h_idx], delta); + } + } + } + + bool fixed_used_blue_contraction = astc_helpers::cem8_or_12_used_blue_contraction(cem_index, fixed_endpoint_vals, endpoint_ise_range); + if (fixed_used_blue_contraction != res.m_is_blue_contracted) + continue; + + trial_err = eval_solution_dp(pixel_stats, cem_index, ccs_index, fixed_endpoint_vals, endpoint_ise_range, trial_weight_vals0, trial_weight_vals1, weight_ise_range, enc_params); + + if (trial_err < trial_blk_error) + { + trial_blk_error = trial_err; + memcpy(pTrial_endpoint_vals, fixed_endpoint_vals, astc_helpers::get_num_cem_values(cem_index)); + memcpy(pTrial_weight_vals0, trial_weight_vals0, pixel_stats.m_num_pixels); + memcpy(pTrial_weight_vals1, trial_weight_vals1, pixel_stats.m_num_pixels); + improved_flag = true; + } + + } // t + + } // if (res.m_any_degen) + + return improved_flag; + } + + // base+offset rgb/rgba, single or dual plane + static bool try_cem9_13_sp_or_dp( + uint32_t cem_index, int ccs_index, + const pixel_stats_t& pixel_stats, const cem_encode_params& enc_params, + uint32_t endpoint_ise_range, uint32_t weight_ise_range, + const vec4F& low_color_f, const vec4F& high_color_f, + uint8_t* pTrial_endpoint_vals, uint8_t* pTrial_weight_vals0, uint8_t* pTrial_weight_vals1, uint64_t& trial_blk_error, bool& trial_used_blue_contraction, + bool try_blue_contract, bool& tried_used_blue_contraction, bool &tried_base_ofs_clamped) + { + assert(g_initialized); + assert((cem_index == astc_helpers::CEM_LDR_RGB_BASE_PLUS_OFFSET) || (cem_index == astc_helpers::CEM_LDR_RGBA_BASE_PLUS_OFFSET)); + assert((ccs_index >= -1) && (ccs_index <= 3)); + assert((pixel_stats.m_num_pixels) && (pixel_stats.m_num_pixels <= ASTC_LDR_MAX_BLOCK_PIXELS)); + assert((endpoint_ise_range >= astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE) && (endpoint_ise_range <= astc_helpers::LAST_VALID_ENDPOINT_ISE_RANGE)); + assert(((weight_ise_range >= astc_helpers::FIRST_VALID_WEIGHT_ISE_RANGE) && (weight_ise_range <= astc_helpers::LAST_VALID_WEIGHT_ISE_RANGE)) || (weight_ise_range == astc_helpers::BISE_64_LEVELS)); + + assert(pTrial_weight_vals0); + assert((ccs_index == -1) || (pTrial_weight_vals1)); + + //const uint32_t num_endpoint_vals = astc_helpers::get_num_cem_values(cem_index); + const uint32_t num_comps = (cem_index == astc_helpers::CEM_LDR_RGB_BASE_PLUS_OFFSET) ? 3 : 4; + + color_rgba low_color, high_color; + for (uint32_t c = 0; c < 4; c++) + { + low_color[c] = (uint8_t)basisu::clamp((int)std::round(low_color_f[c] * 255.0f), 0, 255); + high_color[c] = (uint8_t)basisu::clamp((int)std::round(high_color_f[c] * 255.0f), 0, 255); + } + + uint8_t trial_endpoint_vals[astc_helpers::NUM_MODE13_ENDPOINTS] = { 0 }; + uint8_t trial_weight_vals0[ASTC_LDR_MAX_BLOCK_PIXELS] = { 0 }; + uint8_t trial_weight_vals1[ASTC_LDR_MAX_BLOCK_PIXELS] = { 0 }; + + rgb_base_offset_res res = cem_encode_ldr_rgb_or_rgba_base_offset(cem_index, endpoint_ise_range, low_color, high_color, trial_endpoint_vals, try_blue_contract); + + tried_used_blue_contraction = res.m_used_blue_contraction; + tried_base_ofs_clamped = res.m_delta_clamped; + + if (res.m_failed_flag) + return false; + + bool improved_flag = false; + + if (ccs_index == -1) + { + uint64_t trial_err = eval_solution( + pixel_stats, + cem_index, trial_endpoint_vals, endpoint_ise_range, + trial_weight_vals0, weight_ise_range, + enc_params); + + if (trial_err < trial_blk_error) + { + trial_blk_error = trial_err; + memcpy(pTrial_endpoint_vals, trial_endpoint_vals, astc_helpers::get_num_cem_values(cem_index)); + memcpy(pTrial_weight_vals0, trial_weight_vals0, pixel_stats.m_num_pixels); + if (pTrial_weight_vals1) + memset(pTrial_weight_vals1, 0, pixel_stats.m_num_pixels); + trial_used_blue_contraction = res.m_used_blue_contraction; + improved_flag = true; + } + } + else + { + uint64_t trial_err = eval_solution_dp( + pixel_stats, + cem_index, ccs_index, trial_endpoint_vals, endpoint_ise_range, + trial_weight_vals0, trial_weight_vals1, weight_ise_range, + enc_params); + + if (trial_err < trial_blk_error) + { + trial_blk_error = trial_err; + memcpy(pTrial_endpoint_vals, trial_endpoint_vals, astc_helpers::get_num_cem_values(cem_index)); + memcpy(pTrial_weight_vals0, trial_weight_vals0, pixel_stats.m_num_pixels); + memcpy(pTrial_weight_vals1, trial_weight_vals1, pixel_stats.m_num_pixels); + trial_used_blue_contraction = res.m_used_blue_contraction; + improved_flag = true; + } + } + + if (res.m_any_degen) + { + color_rgba dec_l(0), dec_h(0); + decode_endpoints(cem_index, trial_endpoint_vals, endpoint_ise_range, dec_l, dec_h); + + // The packing in these modes is so complex that we're going to approximate the biasing, and hope for the best. + const uint32_t num_ise_levels = astc_helpers::get_ise_levels(endpoint_ise_range); + int vals_per_ise_level = (256 + num_ise_levels - 1) / num_ise_levels; + + // TODO: There is potential cross-talk between RGB and A with the way this is done. + for (uint32_t p = 1; p <= 3; p++) + { + color_rgba trial_low_color(low_color), trial_high_color(high_color); + + for (uint32_t c = 0; c < num_comps; c++) + { + if (low_color[c] == high_color[c]) + continue; + + if (dec_l[c] != dec_h[c]) + continue; + + int delta = (low_color[c] < high_color[c]) ? -1 : 1; + if (p & 1) + trial_low_color[c] = (uint8_t)basisu::clamp((int)trial_low_color[c] + vals_per_ise_level * delta, 0, 255); + + if (p & 2) + trial_high_color[c] = (uint8_t)basisu::clamp((int)trial_high_color[c] + vals_per_ise_level * -delta, 0, 255); + } // c + + res = cem_encode_ldr_rgb_or_rgba_base_offset(cem_index, endpoint_ise_range, trial_low_color, trial_high_color, trial_endpoint_vals, try_blue_contract); + + if (res.m_failed_flag) + continue; + + if (ccs_index == -1) + { + uint64_t trial_err = eval_solution( + pixel_stats, + cem_index, trial_endpoint_vals, endpoint_ise_range, + trial_weight_vals0, weight_ise_range, + enc_params); + + if (trial_err < trial_blk_error) + { + trial_blk_error = trial_err; + memcpy(pTrial_endpoint_vals, trial_endpoint_vals, astc_helpers::get_num_cem_values(cem_index)); + memcpy(pTrial_weight_vals0, trial_weight_vals0, pixel_stats.m_num_pixels); + if (pTrial_weight_vals1) + memset(pTrial_weight_vals1, 0, pixel_stats.m_num_pixels); + trial_used_blue_contraction = res.m_used_blue_contraction; + if (res.m_delta_clamped) + tried_base_ofs_clamped = true; + improved_flag = true; + } + } + else + { + uint64_t trial_err = eval_solution_dp( + pixel_stats, + cem_index, ccs_index, trial_endpoint_vals, endpoint_ise_range, + trial_weight_vals0, trial_weight_vals1, weight_ise_range, + enc_params); + + if (trial_err < trial_blk_error) + { + trial_blk_error = trial_err; + memcpy(pTrial_endpoint_vals, trial_endpoint_vals, astc_helpers::get_num_cem_values(cem_index)); + memcpy(pTrial_weight_vals0, trial_weight_vals0, pixel_stats.m_num_pixels); + memcpy(pTrial_weight_vals1, trial_weight_vals1, pixel_stats.m_num_pixels); + trial_used_blue_contraction = res.m_used_blue_contraction; + if (res.m_delta_clamped) + tried_base_ofs_clamped = true; + improved_flag = true; + } + } + + } // p + } + else + { + // Now factor in the quantization introduced into the low (base) color, and apply this to the offset, for gain. + color_rgba dec_l(0), dec_h(0); + decode_endpoints(cem_index, trial_endpoint_vals, endpoint_ise_range, dec_l, dec_h); + + if (res.m_endpoints_swapped) + dec_l = low_color; // high color is the quantized base + else + dec_h = high_color; // low color is the quantized base + + res = cem_encode_ldr_rgb_or_rgba_base_offset(cem_index, endpoint_ise_range, dec_l, dec_h, trial_endpoint_vals, try_blue_contract); + + if (!res.m_failed_flag) + { + if (ccs_index == -1) + { + uint64_t trial_err = eval_solution( + pixel_stats, + cem_index, trial_endpoint_vals, endpoint_ise_range, + trial_weight_vals0, weight_ise_range, + enc_params); + + if (trial_err < trial_blk_error) + { + trial_blk_error = trial_err; + memcpy(pTrial_endpoint_vals, trial_endpoint_vals, astc_helpers::get_num_cem_values(cem_index)); + memcpy(pTrial_weight_vals0, trial_weight_vals0, pixel_stats.m_num_pixels); + if (pTrial_weight_vals1) + memset(pTrial_weight_vals1, 0, pixel_stats.m_num_pixels); + trial_used_blue_contraction = res.m_used_blue_contraction; + if (res.m_delta_clamped) + tried_base_ofs_clamped = true; + improved_flag = true; + } + } + else + { + uint64_t trial_err = eval_solution_dp( + pixel_stats, + cem_index, ccs_index, trial_endpoint_vals, endpoint_ise_range, + trial_weight_vals0, trial_weight_vals1, weight_ise_range, + enc_params); + + if (trial_err < trial_blk_error) + { + trial_blk_error = trial_err; + memcpy(pTrial_endpoint_vals, trial_endpoint_vals, astc_helpers::get_num_cem_values(cem_index)); + memcpy(pTrial_weight_vals0, trial_weight_vals0, pixel_stats.m_num_pixels); + memcpy(pTrial_weight_vals1, trial_weight_vals1, pixel_stats.m_num_pixels); + trial_used_blue_contraction = res.m_used_blue_contraction; + if (res.m_delta_clamped) + tried_base_ofs_clamped = true; + improved_flag = true; + } + } + } + } + + return improved_flag; + } + + // l/la direct, single plane + static uint64_t encode_cem0_4( + uint32_t cem_index, + const pixel_stats_t& pixel_stats, const cem_encode_params& enc_params, + uint32_t endpoint_ise_range, uint32_t weight_ise_range, + uint8_t* pEndpoint_vals, uint8_t* pWeight_vals, uint64_t cur_blk_error) + { + assert(g_initialized); + assert((cem_index == astc_helpers::CEM_LDR_LUM_DIRECT) || (cem_index == astc_helpers::CEM_LDR_LUM_ALPHA_DIRECT)); + assert((pixel_stats.m_num_pixels) && (pixel_stats.m_num_pixels <= ASTC_LDR_MAX_BLOCK_PIXELS)); + assert((endpoint_ise_range >= astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE) && (endpoint_ise_range <= astc_helpers::LAST_VALID_ENDPOINT_ISE_RANGE)); + assert(((weight_ise_range >= astc_helpers::FIRST_VALID_WEIGHT_ISE_RANGE) && (weight_ise_range <= astc_helpers::LAST_VALID_WEIGHT_ISE_RANGE)) || (weight_ise_range == astc_helpers::BISE_64_LEVELS)); + + const bool cem_has_alpha = (cem_index == astc_helpers::CEM_LDR_LUM_ALPHA_DIRECT); + + const uint32_t total_endpoint_vals = astc_helpers::get_num_cem_values(cem_index); + const uint32_t total_weights = pixel_stats.m_num_pixels; + + float lum_l = BIG_FLOAT_VAL, lum_h = -BIG_FLOAT_VAL; + + float pixel1F[ASTC_LDR_MAX_BLOCK_PIXELS]; + vec2F pixel2F[ASTC_LDR_MAX_BLOCK_PIXELS]; + + for (uint32_t i = 0; i < pixel_stats.m_num_pixels; i++) + { + const vec4F& px = pixel_stats.m_pixels_f[i]; + + float l = (px[0] + px[1] + px[2]) * (1.0f / 3.0f); + + pixel1F[i] = l; + + pixel2F[i][0] = l; + pixel2F[i][1] = px[3]; + + lum_l = minimum(lum_l, l); + lum_h = maximum(lum_h, l); + } + + const float a_l = pixel_stats.m_min_f[3]; + const float a_h = pixel_stats.m_max_f[3]; + + const vec2F min_pixel2F(lum_l, a_l), max_pixel2F(lum_h, a_h); + + uint8_t trial_blk_endpoints[astc_helpers::MAX_CEM_ENDPOINT_VALS] = { 0 }; + uint8_t trial_blk_weights[ASTC_LDR_MAX_BLOCK_PIXELS] = { 0 }; + uint64_t trial_blk_error = UINT64_MAX; + + bool did_improve = try_cem0_or_4( + cem_index, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + lum_l, lum_h, a_l, a_h, + trial_blk_endpoints, trial_blk_weights, trial_blk_error); + BASISU_NOTE_UNUSED(did_improve); + + if (trial_blk_error == UINT64_MAX) + return cur_blk_error; + + if (trial_blk_error < cur_blk_error) + { + cur_blk_error = trial_blk_error; + memcpy(pEndpoint_vals, trial_blk_endpoints, total_endpoint_vals); + memcpy(pWeight_vals, trial_blk_weights, total_weights); + } + + const uint32_t NUM_LS_OPT_PASSES = 3; + + for (uint32_t pass = 0; pass < NUM_LS_OPT_PASSES; pass++) + { + vec2F xl(lum_l, a_l), xh(lum_h, a_h); + + bool ls_res; + if (cem_has_alpha) + { + ls_res = compute_least_squares_endpoints_2D( + pixel_stats.m_num_pixels, trial_blk_weights, get_ls_weights_ise(weight_ise_range), + &xl, &xh, pixel2F, min_pixel2F, max_pixel2F); + + } + else + { + ls_res = compute_least_squares_endpoints_1D( + pixel_stats.m_num_pixels, trial_blk_weights, get_ls_weights_ise(weight_ise_range), + &xl[0], &xh[0], pixel1F, lum_l, lum_h); + } + if (!ls_res) + break; + + bool did_improve_res = false; + + did_improve_res = try_cem0_or_4( + cem_index, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + xl[0], xh[0], xl[1], xh[1], + trial_blk_endpoints, trial_blk_weights, trial_blk_error); + + BASISU_NOTE_UNUSED(did_improve_res); + + if (trial_blk_error >= cur_blk_error) + break; + + cur_blk_error = trial_blk_error; + memcpy(pEndpoint_vals, trial_blk_endpoints, total_endpoint_vals); + memcpy(pWeight_vals, trial_blk_weights, total_weights); + + } // pass + + return cur_blk_error; + } + + // lum+alpha direct, dual plane + static uint64_t encode_cem4_dp_a( + uint32_t cem_index, + const pixel_stats_t& pixel_stats, const cem_encode_params& enc_params, + uint32_t endpoint_ise_range, uint32_t weight_ise_range, + uint8_t* pEndpoint_vals, uint8_t* pWeight_vals0, uint8_t* pWeight_vals1, uint64_t cur_blk_error) + { + assert(g_initialized); + assert(cem_index == astc_helpers::CEM_LDR_LUM_ALPHA_DIRECT); + assert((pixel_stats.m_num_pixels) && (pixel_stats.m_num_pixels <= ASTC_LDR_MAX_BLOCK_PIXELS)); + assert((endpoint_ise_range >= astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE) && (endpoint_ise_range <= astc_helpers::LAST_VALID_ENDPOINT_ISE_RANGE)); + assert(((weight_ise_range >= astc_helpers::FIRST_VALID_WEIGHT_ISE_RANGE) && (weight_ise_range <= astc_helpers::LAST_VALID_WEIGHT_ISE_RANGE)) || (weight_ise_range == astc_helpers::BISE_64_LEVELS)); + + const uint32_t total_endpoint_vals = astc_helpers::get_num_cem_values(cem_index); + const uint32_t total_weights = pixel_stats.m_num_pixels; + + float alpha_vals[ASTC_LDR_MAX_BLOCK_PIXELS]; + + for (uint32_t i = 0; i < pixel_stats.m_num_pixels; i++) + { + const vec4F& px = pixel_stats.m_pixels_f[i]; + + alpha_vals[i] = px[3]; + } + + // First get plane0's low/high (lum) + uint8_t lum_endpoints[astc_helpers::MAX_CEM_ENDPOINT_VALS]; + uint8_t lum_weights0[ASTC_LDR_MAX_BLOCK_PIXELS]; + + uint64_t lum_blk_error = encode_cem0_4( + astc_helpers::CEM_LDR_LUM_DIRECT, + pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + lum_endpoints, lum_weights0, UINT64_MAX); + + if (lum_blk_error == UINT64_MAX) + return cur_blk_error; + + const auto& dequant_endpoints_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(endpoint_ise_range).m_ISE_to_val; + + float lum_l = (float)dequant_endpoints_tab[lum_endpoints[0]] * (1.0f / 255.0f); + float lum_h = (float)dequant_endpoints_tab[lum_endpoints[1]] * (1.0f / 255.0f); + float a_l = pixel_stats.m_min_f[3]; + float a_h = pixel_stats.m_max_f[3]; + + uint8_t trial_endpoints[astc_helpers::MAX_CEM_ENDPOINT_VALS]; + uint8_t trial_weights0[ASTC_LDR_MAX_BLOCK_PIXELS]; + uint8_t trial_weights1[ASTC_LDR_MAX_BLOCK_PIXELS]; + uint64_t trial_blk_error = UINT64_MAX; + + bool did_improve = try_cem4_dp_a( + cem_index, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + lum_l, lum_h, a_l, a_h, + trial_endpoints, trial_weights0, trial_weights1, trial_blk_error); + + if (!did_improve) + { + assert(0); + return cur_blk_error; + } + + if (trial_blk_error < cur_blk_error) + { + cur_blk_error = trial_blk_error; + memcpy(pEndpoint_vals, trial_endpoints, total_endpoint_vals); + memcpy(pWeight_vals0, trial_weights0, total_weights); + memcpy(pWeight_vals1, trial_weights1, total_weights); + } + + const uint32_t NUM_LS_OPT_PASSES = 3; + + for (uint32_t pass = 0; pass < NUM_LS_OPT_PASSES; pass++) + { + float xl = pixel_stats.m_min_f[3], xh = pixel_stats.m_max_f[3]; + + bool ls_res = compute_least_squares_endpoints_1D( + pixel_stats.m_num_pixels, trial_weights1, get_ls_weights_ise(weight_ise_range), + &xl, &xh, alpha_vals, pixel_stats.m_min_f[3], pixel_stats.m_max_f[3]); + if (!ls_res) + break; + + did_improve = try_cem4_dp_a( + cem_index, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + lum_l, lum_h, xl, xh, + trial_endpoints, trial_weights0, trial_weights1, trial_blk_error); + + if (!did_improve) + break; + + cur_blk_error = trial_blk_error; + memcpy(pEndpoint_vals, trial_endpoints, total_endpoint_vals); + memcpy(pWeight_vals0, trial_weights0, total_weights); + memcpy(pWeight_vals1, trial_weights1, total_weights); + + } // pass + + return cur_blk_error; + } + + struct weight_refiner + { + void init(uint32_t weight_ise_range, uint32_t total_pixels, const uint8_t *pInitial_ise_weights) + { + m_weight_ise_range = weight_ise_range; + m_total_pixels = total_pixels; + m_pISE_to_rank = &astc_helpers::g_dequant_tables.get_weight_tab(weight_ise_range).m_ISE_to_rank; + m_pRank_to_ise = &astc_helpers::g_dequant_tables.get_weight_tab(weight_ise_range).m_rank_to_ISE; + m_num_weight_levels = astc_helpers::get_ise_levels(weight_ise_range); + + for (uint32_t i = 0; i < total_pixels; i++) + m_start_weights[i] = (*m_pISE_to_rank)[pInitial_ise_weights[i]]; + + m_min_weight = UINT32_MAX; + m_max_weight = 0; + m_sum_weight = 0; + + for (uint32_t i = 0; i < total_pixels; i++) + { + const uint32_t weight = m_start_weights[i]; + m_sum_weight += weight; + m_min_weight = minimumu(m_min_weight, weight); + m_max_weight = maximumu(m_max_weight, weight); + } + } + + void refine(uint32_t pass_index, uint8_t* pTrial_ise_weights) + { + switch (pass_index) + { + case 0: + { + for (uint32_t i = 0; i < m_total_pixels; i++) + { + uint32_t v = m_start_weights[i]; + if ((v == m_min_weight) && (v < (m_num_weight_levels - 1))) + v++; + + pTrial_ise_weights[i] = (*m_pRank_to_ise)[v]; + } + break; + } + case 1: + { + for (uint32_t i = 0; i < m_total_pixels; i++) + { + uint32_t v = m_start_weights[i]; + if ((v == m_max_weight) && (v > 0)) + v--; + + pTrial_ise_weights[i] = (*m_pRank_to_ise)[v]; + } + break; + } + case 2: + { + for (uint32_t i = 0; i < m_total_pixels; i++) + { + uint32_t v = m_start_weights[i]; + if ((v == m_min_weight) && (v < (m_num_weight_levels - 1))) + v++; + else if ((v == m_max_weight) && (v > 0)) + v--; + + pTrial_ise_weights[i] = (*m_pRank_to_ise)[v]; + } + break; + } + case 3: + { + const int max_weight_rank_index = m_num_weight_levels - 1; + int ly = -1, hy = max_weight_rank_index + 1; + + for (uint32_t i = 0; i < m_total_pixels; i++) + { + int s = (int)clampf(floor((float)max_weight_rank_index * ((float)m_start_weights[i] - (float)ly) / ((float)hy - (float)ly) + .5f), 0, (float)max_weight_rank_index); + pTrial_ise_weights[i] = (*m_pRank_to_ise)[s]; + } + + break; + } + case 4: + { + const int max_weight_rank_index = m_num_weight_levels - 1; + int ly = -2, hy = max_weight_rank_index + 2; + + for (uint32_t i = 0; i < m_total_pixels; i++) + { + int s = (int)clampf(floor((float)max_weight_rank_index * ((float)m_start_weights[i] - (float)ly) / ((float)hy - (float)ly) + .5f), 0, (float)max_weight_rank_index); + pTrial_ise_weights[i] = (*m_pRank_to_ise)[s]; + } + + break; + } + case 5: + { + const int max_weight_rank_index = m_num_weight_levels - 1; + int ly = -1, hy = max_weight_rank_index + 2; + + for (uint32_t i = 0; i < m_total_pixels; i++) + { + int s = (int)clampf(floor((float)max_weight_rank_index * ((float)m_start_weights[i] - (float)ly) / ((float)hy - (float)ly) + .5f), 0, (float)max_weight_rank_index); + pTrial_ise_weights[i] = (*m_pRank_to_ise)[s]; + } + + break; + } + case 6: + { + const int max_weight_rank_index = m_num_weight_levels - 1; + int ly = -2, hy = max_weight_rank_index + 1; + + for (uint32_t i = 0; i < m_total_pixels; i++) + { + int s = (int)clampf(floor((float)max_weight_rank_index * ((float)m_start_weights[i] - (float)ly) / ((float)hy - (float)ly) + .5f), 0, (float)max_weight_rank_index); + pTrial_ise_weights[i] = (*m_pRank_to_ise)[s]; + } + + break; + } + case 7: + { + for (uint32_t i = 0; i < m_total_pixels; i++) + { + uint32_t v = m_start_weights[i]; + if ((v == m_min_weight) && (v < (m_num_weight_levels - 1))) + { + v++; + if (v < (m_num_weight_levels - 1)) + v++; + } + + pTrial_ise_weights[i] = (*m_pRank_to_ise)[v]; + } + break; + + break; + } + case 8: + { + for (uint32_t i = 0; i < m_total_pixels; i++) + { + uint32_t v = m_start_weights[i]; + if ((v == m_max_weight) && (v > 0)) + { + v--; + if (v > 0) + v--; + } + + pTrial_ise_weights[i] = (*m_pRank_to_ise)[v]; + } + break; + } + case 9: + { + for (uint32_t i = 0; i < m_total_pixels; i++) + { + uint32_t v = m_start_weights[i]; + if ((v == m_min_weight) && (v < (m_num_weight_levels - 1))) + { + v++; + if (v < (m_num_weight_levels - 1)) + v++; + } + else if ((v == m_max_weight) && (v > 0)) + { + v--; + if (v > 0) + v--; + } + + pTrial_ise_weights[i] = (*m_pRank_to_ise)[v]; + } + break; + } + case 10: + { + float mid_weight = (float)m_sum_weight / (float)m_total_pixels; + + for (uint32_t i = 0; i < m_total_pixels; i++) + { + int v = m_start_weights[i]; + + float fv = ((float)v - mid_weight) * .8f + ((float)m_num_weight_levels * .5f); + + v = clamp((int)std::round(fv), 0, m_num_weight_levels - 1); + + pTrial_ise_weights[i] = (*m_pRank_to_ise)[v]; + } + break; + } + case 11: + { + float mid_weight = (float)m_sum_weight / (float)m_total_pixels; + + for (uint32_t i = 0; i < m_total_pixels; i++) + { + int v = m_start_weights[i]; + + float fv = ((float)v - mid_weight) * .9f + ((float)m_num_weight_levels * .5f); + + v = clamp((int)std::round(fv), 0, m_num_weight_levels - 1); + + pTrial_ise_weights[i] = (*m_pRank_to_ise)[v]; + } + break; + } + case 12: + { + float mid_weight = (float)m_sum_weight / (float)m_total_pixels; + + for (uint32_t i = 0; i < m_total_pixels; i++) + { + int v = m_start_weights[i]; + + float fv = ((float)v - mid_weight) * 1.1f + ((float)m_num_weight_levels * .5f); + + v = clamp((int)std::round(fv), 0, m_num_weight_levels - 1); + + pTrial_ise_weights[i] = (*m_pRank_to_ise)[v]; + } + break; + } + case 13: + { + float mid_weight = (float)m_sum_weight / (float)m_total_pixels; + + for (uint32_t i = 0; i < m_total_pixels; i++) + { + int v = m_start_weights[i]; + + float fv; + if (v < mid_weight) + fv = ((float)v - mid_weight) * .8f + ((float)m_num_weight_levels * .5f); + else + fv = (float)v; + + v = clamp((int)std::round(fv), 0, m_num_weight_levels - 1); + + pTrial_ise_weights[i] = (*m_pRank_to_ise)[v]; + } + break; + } + case 14: + { + float mid_weight = (float)m_sum_weight / (float)m_total_pixels; + + for (uint32_t i = 0; i < m_total_pixels; i++) + { + int v = m_start_weights[i]; + + float fv; + if (v >= mid_weight) + fv = ((float)v - mid_weight) * .8f + ((float)m_num_weight_levels * .5f); + else + fv = (float)v; + + v = clamp((int)std::round(fv), 0, m_num_weight_levels - 1); + + pTrial_ise_weights[i] = (*m_pRank_to_ise)[v]; + } + break; + } + case 15: + { + for (uint32_t i = 0; i < m_total_pixels; i++) + { + uint32_t v = m_start_weights[i]; + if (v < (m_num_weight_levels - 1)) + v++; + + pTrial_ise_weights[i] = (*m_pRank_to_ise)[v]; + } + break; + } + case 16: + { + for (uint32_t i = 0; i < m_total_pixels; i++) + { + uint32_t v = m_start_weights[i]; + if (v) + v--; + + pTrial_ise_weights[i] = (*m_pRank_to_ise)[v]; + } + break; + } + default: + { + assert(0); + memset(pTrial_ise_weights, 0, m_total_pixels); + break; + } + } + } + + uint32_t m_total_pixels; + uint32_t m_weight_ise_range; + uint32_t m_num_weight_levels; + uint8_t m_start_weights[ASTC_LDR_MAX_BLOCK_PIXELS]; // ranks, not ISE + + uint32_t m_min_weight, m_max_weight, m_sum_weight; + + const basisu::vector* m_pISE_to_rank; + const basisu::vector* m_pRank_to_ise; + }; + + // rgb/rgba direct or rgb/rgba base+offset, single plane + static uint64_t encode_cem8_12_9_13( + uint32_t cem_index, + const pixel_stats_t& pixel_stats, const cem_encode_params& enc_params, + uint32_t endpoint_ise_range, uint32_t weight_ise_range, + uint8_t* pEndpoint_vals, uint8_t* pWeight_vals, uint64_t cur_blk_error, bool use_blue_contraction, bool* pBase_ofs_clamped_flag) + { + assert(g_initialized); + assert((cem_index == astc_helpers::CEM_LDR_RGB_DIRECT) || (cem_index == astc_helpers::CEM_LDR_RGBA_DIRECT) || + (cem_index == astc_helpers::CEM_LDR_RGB_BASE_PLUS_OFFSET) || (cem_index == astc_helpers::CEM_LDR_RGBA_BASE_PLUS_OFFSET)); + + assert((pixel_stats.m_num_pixels) && (pixel_stats.m_num_pixels <= ASTC_LDR_MAX_BLOCK_PIXELS)); + assert((endpoint_ise_range >= astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE) && (endpoint_ise_range <= astc_helpers::LAST_VALID_ENDPOINT_ISE_RANGE)); + assert(((weight_ise_range >= astc_helpers::FIRST_VALID_WEIGHT_ISE_RANGE) && (weight_ise_range <= astc_helpers::LAST_VALID_WEIGHT_ISE_RANGE)) || (weight_ise_range == astc_helpers::BISE_64_LEVELS)); + + if (pBase_ofs_clamped_flag) + *pBase_ofs_clamped_flag = false; + + const bool cem_has_alpha = (cem_index == astc_helpers::CEM_LDR_RGBA_DIRECT) || (cem_index == astc_helpers::CEM_LDR_RGBA_BASE_PLUS_OFFSET); + const bool cem_is_base_offset = (cem_index == astc_helpers::CEM_LDR_RGB_BASE_PLUS_OFFSET) || (cem_index == astc_helpers::CEM_LDR_RGBA_BASE_PLUS_OFFSET); + + const uint32_t total_endpoint_vals = astc_helpers::get_num_cem_values(cem_index); + const uint32_t total_weights = pixel_stats.m_num_pixels; + + float best_l = BIG_FLOAT_VAL, best_h = -BIG_FLOAT_VAL; + //int best_l_index = 0, best_h_index = 0; + + for (uint32_t c = 0; c < pixel_stats.m_num_pixels; c++) + { + const vec4F px(pixel_stats.m_pixels_f[c] - pixel_stats.m_mean_f); + + float p = cem_has_alpha ? px.dot(pixel_stats.m_mean_rel_axis4) : px.dot3(pixel_stats.m_mean_rel_axis3); + if (p < best_l) + { + best_l = p; + //best_l_index = c; + } + + if (p > best_h) + { + best_h = p; + //best_h_index = c; + } + } // c + +#if 0 + vec4F low_color_f(pixel_stats.m_pixels_f[best_l_index]), high_color_f(pixel_stats.m_pixels_f[best_h_index]); +#else + vec4F low_color_f, high_color_f; + if (cem_has_alpha) + { + low_color_f = pixel_stats.m_mean_rel_axis4 * best_l + pixel_stats.m_mean_f; + high_color_f = pixel_stats.m_mean_rel_axis4 * best_h + pixel_stats.m_mean_f; + } + else + { + low_color_f = vec4F(pixel_stats.m_mean_rel_axis3) * best_l + pixel_stats.m_mean_f; + high_color_f = vec4F(pixel_stats.m_mean_rel_axis3) * best_h + pixel_stats.m_mean_f; + } + + low_color_f.clamp(0.0f, 1.0f); + high_color_f.clamp(0.0f, 1.0f); +#endif + + uint8_t trial_blk_endpoints[astc_helpers::MAX_CEM_ENDPOINT_VALS] = { 0 }; + uint8_t trial_blk_weights[ASTC_LDR_MAX_BLOCK_PIXELS] = { 0 }; + uint64_t trial_blk_error = UINT64_MAX; + bool trial_used_blue_contraction = false; + + bool tried_used_blue_contraction = false; + + if (cem_is_base_offset) + { + bool tried_base_ofs_clamped = false; + + try_cem9_13_sp_or_dp( + cem_index, -1, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + low_color_f, high_color_f, + trial_blk_endpoints, trial_blk_weights, nullptr, trial_blk_error, trial_used_blue_contraction, use_blue_contraction, + tried_used_blue_contraction, tried_base_ofs_clamped); + + if ((pBase_ofs_clamped_flag) && (tried_base_ofs_clamped)) + *pBase_ofs_clamped_flag = true; + + if (tried_used_blue_contraction) + { + try_cem9_13_sp_or_dp( + cem_index, -1, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + low_color_f, high_color_f, + trial_blk_endpoints, trial_blk_weights, nullptr, trial_blk_error, trial_used_blue_contraction, false, + tried_used_blue_contraction, tried_base_ofs_clamped); + + if ((pBase_ofs_clamped_flag) && (tried_base_ofs_clamped)) + *pBase_ofs_clamped_flag = true; + } + } + else + { + try_cem8_12( + cem_index, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + low_color_f, high_color_f, + trial_blk_endpoints, trial_blk_weights, trial_blk_error, trial_used_blue_contraction, use_blue_contraction, tried_used_blue_contraction); + + if (tried_used_blue_contraction) + { + try_cem8_12( + cem_index, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + low_color_f, high_color_f, + trial_blk_endpoints, trial_blk_weights, trial_blk_error, trial_used_blue_contraction, false, tried_used_blue_contraction); + } + } + + if (trial_blk_error == UINT64_MAX) + return cur_blk_error; + + if (trial_blk_error < cur_blk_error) + { + cur_blk_error = trial_blk_error; + memcpy(pEndpoint_vals, trial_blk_endpoints, total_endpoint_vals); + memcpy(pWeight_vals, trial_blk_weights, total_weights); + } + + for (uint32_t pass = 0; pass < enc_params.m_max_ls_passes; pass++) + { + vec4F xl, xh; + + bool ls_res; + if (cem_has_alpha) + { + ls_res = compute_least_squares_endpoints_4D( + pixel_stats.m_num_pixels, trial_blk_weights, get_ls_weights_ise(weight_ise_range), + &xl, &xh, pixel_stats.m_pixels_f, pixel_stats.m_min_f, pixel_stats.m_max_f); + } + else + { + ls_res = compute_least_squares_endpoints_3D( + pixel_stats.m_num_pixels, trial_blk_weights, get_ls_weights_ise(weight_ise_range), + &xl, &xh, pixel_stats.m_pixels_f, pixel_stats.m_min_f, pixel_stats.m_max_f); + } + if (!ls_res) + break; + + if (cem_is_base_offset) + { + bool tried_base_ofs_clamped = false; + + try_cem9_13_sp_or_dp( + cem_index, -1, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + xl, xh, + trial_blk_endpoints, trial_blk_weights, nullptr, trial_blk_error, trial_used_blue_contraction, use_blue_contraction, tried_used_blue_contraction, tried_base_ofs_clamped); + + if ((pBase_ofs_clamped_flag) && (tried_base_ofs_clamped)) + *pBase_ofs_clamped_flag = true; + + if (tried_used_blue_contraction) + { + // Try without blue contraction for a minor gain. + try_cem9_13_sp_or_dp( + cem_index, -1, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + xl, xh, + trial_blk_endpoints, trial_blk_weights, nullptr, trial_blk_error, trial_used_blue_contraction, false, tried_used_blue_contraction, tried_base_ofs_clamped); + + if ((pBase_ofs_clamped_flag) && (tried_base_ofs_clamped)) + *pBase_ofs_clamped_flag = true; + } + } + else + { + try_cem8_12( + cem_index, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + xl, xh, + trial_blk_endpoints, trial_blk_weights, trial_blk_error, trial_used_blue_contraction, use_blue_contraction, tried_used_blue_contraction); + + if (tried_used_blue_contraction) + { + // Try without blue contraction for a minor gain. + try_cem8_12( + cem_index, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + xl, xh, + trial_blk_endpoints, trial_blk_weights, trial_blk_error, trial_used_blue_contraction, false, tried_used_blue_contraction); + } + } + + if (trial_blk_error >= cur_blk_error) + break; + + cur_blk_error = trial_blk_error; + memcpy(pEndpoint_vals, trial_blk_endpoints, total_endpoint_vals); + memcpy(pWeight_vals, trial_blk_weights, total_weights); + + } // pass + + if ((enc_params.m_total_weight_refine_passes) && ((weight_ise_range != astc_helpers::BISE_2_LEVELS) && (weight_ise_range != astc_helpers::BISE_64_LEVELS))) + { + weight_refiner refiner; + refiner.init(weight_ise_range, pixel_stats.m_num_pixels, pWeight_vals); + + for (uint32_t pass = 0; pass < enc_params.m_total_weight_refine_passes; pass++) + { + refiner.refine(pass, trial_blk_weights); + + vec4F xl, xh; + + bool ls_res; + if (cem_has_alpha) + { + ls_res = compute_least_squares_endpoints_4D( + pixel_stats.m_num_pixels, trial_blk_weights, get_ls_weights_ise(weight_ise_range), + &xl, &xh, pixel_stats.m_pixels_f, pixel_stats.m_min_f, pixel_stats.m_max_f); + } + else + { + ls_res = compute_least_squares_endpoints_3D( + pixel_stats.m_num_pixels, trial_blk_weights, get_ls_weights_ise(weight_ise_range), + &xl, &xh, pixel_stats.m_pixels_f, pixel_stats.m_min_f, pixel_stats.m_max_f); + } + if (!ls_res) + continue; + + if (cem_is_base_offset) + { + bool tried_base_ofs_clamped = false; + + try_cem9_13_sp_or_dp( + cem_index, -1, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + xl, xh, + trial_blk_endpoints, trial_blk_weights, nullptr, trial_blk_error, trial_used_blue_contraction, use_blue_contraction, tried_used_blue_contraction, tried_base_ofs_clamped); + + if ((pBase_ofs_clamped_flag) && (tried_base_ofs_clamped)) + *pBase_ofs_clamped_flag = true; + + if (tried_used_blue_contraction) + { + // Try without blue contraction for a minor gain. + try_cem9_13_sp_or_dp( + cem_index, -1, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + xl, xh, + trial_blk_endpoints, trial_blk_weights, nullptr, trial_blk_error, trial_used_blue_contraction, false, tried_used_blue_contraction, tried_base_ofs_clamped); + + if ((pBase_ofs_clamped_flag) && (tried_base_ofs_clamped)) + *pBase_ofs_clamped_flag = true; + } + } + else + { + try_cem8_12( + cem_index, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + xl, xh, + trial_blk_endpoints, trial_blk_weights, trial_blk_error, trial_used_blue_contraction, use_blue_contraction, tried_used_blue_contraction); + + if (tried_used_blue_contraction) + { + // Try without blue contraction for a minor gain. + try_cem8_12( + cem_index, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + xl, xh, + trial_blk_endpoints, trial_blk_weights, trial_blk_error, trial_used_blue_contraction, false, tried_used_blue_contraction); + } + } + + if (trial_blk_error < cur_blk_error) + { + cur_blk_error = trial_blk_error; + memcpy(pEndpoint_vals, trial_blk_endpoints, total_endpoint_vals); + memcpy(pWeight_vals, trial_blk_weights, total_weights); + } + + } // pass + } + + const uint32_t N = 4; + if ((enc_params.m_worst_weight_nudging_flag) && + (pixel_stats.m_num_pixels > N) && + ((weight_ise_range != astc_helpers::BISE_2_LEVELS) && (weight_ise_range != astc_helpers::BISE_64_LEVELS))) + { + const uint32_t NUM_NUDGING_PASSES = 1; + for (uint32_t pass = 0; pass < NUM_NUDGING_PASSES; pass++) + { + color_rgba l, h; + decode_endpoints(cem_index, pEndpoint_vals, endpoint_ise_range, l, h); + + vec4F dir; + dir[0] = (float)(h[0] - l[0]); + dir[1] = (float)(h[1] - l[1]); + dir[2] = (float)(h[2] - l[2]); + dir[3] = cem_has_alpha ? (float)(h[3] - l[3]) : 0.0f; + + dir.normalize_in_place(); + + float errs[ASTC_LDR_MAX_BLOCK_PIXELS]; + float delta_dots[ASTC_LDR_MAX_BLOCK_PIXELS]; + for (uint32_t i = 0; i < pixel_stats.m_num_pixels; i++) + { + vec4F ofs(pixel_stats.m_pixels_f[i] - pixel_stats.m_mean_f); + + float proj = dir.dot(ofs); + + vec4F proj_vec(pixel_stats.m_mean_f + proj * dir); + + vec4F delta_vec(pixel_stats.m_pixels_f[i] - proj_vec); + + delta_dots[i] = dir.dot(delta_vec); + + errs[i] = cem_has_alpha ? vec4F::dot_product(delta_vec, delta_vec) : vec4F::dot_product3(delta_vec, delta_vec); + } + + uint32_t errs_indices[ASTC_LDR_MAX_BLOCK_PIXELS]; + indirect_sort(pixel_stats.m_num_pixels, errs_indices, errs); + + memcpy(trial_blk_weights, pWeight_vals, total_weights); + + for (uint32_t i = 0; i < N; i++) + { + const uint32_t idx = errs_indices[pixel_stats.m_num_pixels - 1 - i]; + + int delta_to_apply = (delta_dots[idx] > 0.0f) ? 1 : -1; + + trial_blk_weights[idx] = (uint8_t)apply_delta_to_bise_weight_val(weight_ise_range, trial_blk_weights[idx], delta_to_apply); + } // i + + vec4F xl, xh; + + bool ls_res; + if (cem_has_alpha) + { + ls_res = compute_least_squares_endpoints_4D( + pixel_stats.m_num_pixels, trial_blk_weights, get_ls_weights_ise(weight_ise_range), + &xl, &xh, pixel_stats.m_pixels_f, pixel_stats.m_min_f, pixel_stats.m_max_f); + } + else + { + ls_res = compute_least_squares_endpoints_3D( + pixel_stats.m_num_pixels, trial_blk_weights, get_ls_weights_ise(weight_ise_range), + &xl, &xh, pixel_stats.m_pixels_f, pixel_stats.m_min_f, pixel_stats.m_max_f); + } + if (!ls_res) + break; + + if (cem_is_base_offset) + { + bool tried_base_ofs_clamped = false; + + try_cem9_13_sp_or_dp( + cem_index, -1, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + xl, xh, + trial_blk_endpoints, trial_blk_weights, nullptr, trial_blk_error, trial_used_blue_contraction, use_blue_contraction, tried_used_blue_contraction, tried_base_ofs_clamped); + + if ((pBase_ofs_clamped_flag) && (tried_base_ofs_clamped)) + *pBase_ofs_clamped_flag = true; + + if (tried_used_blue_contraction) + { + // Try without blue contraction for a minor gain. + try_cem9_13_sp_or_dp( + cem_index, -1, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + xl, xh, + trial_blk_endpoints, trial_blk_weights, nullptr, trial_blk_error, trial_used_blue_contraction, false, tried_used_blue_contraction, tried_base_ofs_clamped); + + if ((pBase_ofs_clamped_flag) && (tried_base_ofs_clamped)) + *pBase_ofs_clamped_flag = true; + } + } + else + { + try_cem8_12( + cem_index, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + xl, xh, + trial_blk_endpoints, trial_blk_weights, trial_blk_error, trial_used_blue_contraction, use_blue_contraction, tried_used_blue_contraction); + + if (tried_used_blue_contraction) + { + // Try without blue contraction for a minor gain. + try_cem8_12( + cem_index, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + xl, xh, + trial_blk_endpoints, trial_blk_weights, trial_blk_error, trial_used_blue_contraction, false, tried_used_blue_contraction); + } + } + + if (trial_blk_error < cur_blk_error) + { + cur_blk_error = trial_blk_error; + memcpy(pEndpoint_vals, trial_blk_endpoints, total_endpoint_vals); + memcpy(pWeight_vals, trial_blk_weights, total_weights); + } + else + { + break; + } + } // pass + } + + if (enc_params.m_endpoint_refinement_flag) + { + const uint32_t num_comps = cem_has_alpha ? 4 : 3; + + for (uint32_t c = 0; c < num_comps; c++) + { + uint8_t base_endpoint_vals[astc_helpers::MAX_CEM_ENDPOINT_VALS]; + memcpy(base_endpoint_vals, pEndpoint_vals, total_endpoint_vals); + + for (int dl = -1; dl <= 1; dl++) + { + for (int dh = -1; dh <= 1; dh++) + { + if (!dl && !dh) + continue; + + memcpy(trial_blk_endpoints, base_endpoint_vals, total_endpoint_vals); + + trial_blk_endpoints[c * 2 + 0] = (uint8_t)astc_helpers::apply_delta_to_bise_endpoint_val(endpoint_ise_range, trial_blk_endpoints[c * 2 + 0], dl); + trial_blk_endpoints[c * 2 + 1] = (uint8_t)astc_helpers::apply_delta_to_bise_endpoint_val(endpoint_ise_range, trial_blk_endpoints[c * 2 + 1], dh); + + if (!use_blue_contraction) + { + const bool uses_blue_contraction = astc_helpers::used_blue_contraction(cem_index, trial_blk_endpoints, endpoint_ise_range); + if (uses_blue_contraction) + continue; + } + + trial_blk_error = eval_solution( + pixel_stats, + cem_index, trial_blk_endpoints, endpoint_ise_range, + trial_blk_weights, weight_ise_range, + enc_params); + + if (trial_blk_error < cur_blk_error) + { + cur_blk_error = trial_blk_error; + memcpy(pEndpoint_vals, trial_blk_endpoints, total_endpoint_vals); + memcpy(pWeight_vals, trial_blk_weights, total_weights); + } + + } // dh + + } // dl + } + } + + return cur_blk_error; + } + + // rgb/rgba direct, or rgb/rgba base+offset, dual plane + static uint64_t encode_cem8_12_9_13_dp( + uint32_t cem_index, uint32_t ccs_index, + const pixel_stats_t& pixel_stats, const cem_encode_params& enc_params, + uint32_t endpoint_ise_range, uint32_t weight_ise_range, + uint8_t* pEndpoint_vals, uint8_t* pWeight_vals0, uint8_t* pWeight_vals1, + uint64_t cur_blk_error, bool use_blue_contraction, bool *pBase_ofs_clamped_flag) + { + assert(g_initialized); + assert(ccs_index <= 3); + assert((pixel_stats.m_num_pixels) && (pixel_stats.m_num_pixels <= ASTC_LDR_MAX_BLOCK_PIXELS)); + assert((endpoint_ise_range >= astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE) && (endpoint_ise_range <= astc_helpers::LAST_VALID_ENDPOINT_ISE_RANGE)); + assert(((weight_ise_range >= astc_helpers::FIRST_VALID_WEIGHT_ISE_RANGE) && (weight_ise_range <= astc_helpers::LAST_VALID_WEIGHT_ISE_RANGE)) || (weight_ise_range == astc_helpers::BISE_64_LEVELS)); + + if (pBase_ofs_clamped_flag) + *pBase_ofs_clamped_flag = false; + + bool cem_has_alpha = false, cem_is_base_offset = false; + switch (cem_index) + { + case astc_helpers::CEM_LDR_RGB_DIRECT: break; + case astc_helpers::CEM_LDR_RGBA_DIRECT: cem_has_alpha = true; break; + case astc_helpers::CEM_LDR_RGB_BASE_PLUS_OFFSET: cem_is_base_offset = true; break; + case astc_helpers::CEM_LDR_RGBA_BASE_PLUS_OFFSET: cem_is_base_offset = true; cem_has_alpha = true; break; + default: + assert(0); + return false; + } + + assert((ccs_index <= 2) || cem_has_alpha); + + const uint32_t total_endpoint_vals = astc_helpers::get_num_cem_values(cem_index); + const uint32_t total_weights = pixel_stats.m_num_pixels; + + // Remove influence of the 2nd plane's values, recalc principle axis on other values. + vec4F flattened_pixels[ASTC_LDR_MAX_BLOCK_PIXELS]; + for (uint32_t i = 0; i < pixel_stats.m_num_pixels; i++) + { + flattened_pixels[i] = pixel_stats.m_pixels_f[i]; + flattened_pixels[i][ccs_index] = 0.0f; + + if (!cem_has_alpha) + flattened_pixels[i][3] = 0.0f; + } + + vec4F flattened_pixels_mean(pixel_stats.m_mean_f); + flattened_pixels_mean[ccs_index] = 0.0f; + + if (!cem_has_alpha) + flattened_pixels_mean[3] = 0.0f; + + vec4F flattened_axis; + if (!cem_has_alpha) + flattened_axis = calc_pca_3D(pixel_stats.m_num_pixels, flattened_pixels, flattened_pixels_mean); + else + flattened_axis = calc_pca_4D(pixel_stats.m_num_pixels, flattened_pixels, flattened_pixels_mean); + + float best_l = BIG_FLOAT_VAL, best_h = -BIG_FLOAT_VAL; + //int best_l_index = 0, best_h_index = 0; + + for (uint32_t c = 0; c < pixel_stats.m_num_pixels; c++) + { + const vec4F px(flattened_pixels[c] - flattened_pixels_mean); + + float p = px.dot(flattened_axis); + if (p < best_l) + { + best_l = p; + //best_l_index = c; + } + + if (p > best_h) + { + best_h = p; + //best_h_index = c; + } + } // c + +#if 0 + vec4F low_color_f(pixel_stats.m_pixels_f[best_l_index]), high_color_f(pixel_stats.m_pixels_f[best_h_index]); +#else + vec4F low_color_f, high_color_f; + low_color_f = flattened_pixels_mean + flattened_axis * best_l; + high_color_f = flattened_pixels_mean + flattened_axis * best_h; + + low_color_f.clamp(0.0f, 1.0f); + high_color_f.clamp(0.0f, 1.0f); +#endif + + low_color_f[ccs_index] = pixel_stats.m_min_f[ccs_index]; + high_color_f[ccs_index] = pixel_stats.m_max_f[ccs_index]; + + uint8_t trial_blk_endpoints[astc_helpers::MAX_CEM_ENDPOINT_VALS] = { 0 }; + uint8_t trial_blk_weights0[ASTC_LDR_MAX_BLOCK_PIXELS] = { 0 }; + uint8_t trial_blk_weights1[ASTC_LDR_MAX_BLOCK_PIXELS] = { 0 }; + uint64_t trial_blk_error = UINT64_MAX; + bool trial_used_blue_contraction = false; + + bool tried_used_blue_contraction = false; + + if (cem_is_base_offset) + { + bool tried_base_ofs_clamped = false; + + try_cem9_13_sp_or_dp( + cem_index, ccs_index, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + low_color_f, high_color_f, + trial_blk_endpoints, trial_blk_weights0, trial_blk_weights1, + trial_blk_error, trial_used_blue_contraction, use_blue_contraction, tried_used_blue_contraction, tried_base_ofs_clamped); + + if ((pBase_ofs_clamped_flag) && (tried_base_ofs_clamped)) + *pBase_ofs_clamped_flag = true; + + if (tried_used_blue_contraction) + { + try_cem9_13_sp_or_dp( + cem_index, ccs_index, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + low_color_f, high_color_f, + trial_blk_endpoints, trial_blk_weights0, trial_blk_weights1, trial_blk_error, trial_used_blue_contraction, false, tried_used_blue_contraction, tried_base_ofs_clamped); + + if ((pBase_ofs_clamped_flag) && (tried_base_ofs_clamped)) + *pBase_ofs_clamped_flag = true; + } + } + else + { + try_cem8_12_dp( + cem_index, ccs_index, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + low_color_f, high_color_f, + trial_blk_endpoints, trial_blk_weights0, trial_blk_weights1, + trial_blk_error, trial_used_blue_contraction, use_blue_contraction, tried_used_blue_contraction); + + if (tried_used_blue_contraction) + { + try_cem8_12_dp( + cem_index, ccs_index, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + low_color_f, high_color_f, + trial_blk_endpoints, trial_blk_weights0, trial_blk_weights1, trial_blk_error, trial_used_blue_contraction, false, tried_used_blue_contraction); + } + } + + if (trial_blk_error == UINT64_MAX) + return cur_blk_error; + + if (trial_blk_error < cur_blk_error) + { + cur_blk_error = trial_blk_error; + memcpy(pEndpoint_vals, trial_blk_endpoints, total_endpoint_vals); + memcpy(pWeight_vals0, trial_blk_weights0, total_weights); + memcpy(pWeight_vals1, trial_blk_weights1, total_weights); + } + + vec4F flattened_pixels_min_f(pixel_stats.m_min_f); + flattened_pixels_min_f[ccs_index] = 0; + + vec4F flattened_pixels_max_f(pixel_stats.m_max_f); + flattened_pixels_max_f[ccs_index] = 0; + + for (uint32_t pass = 0; pass < enc_params.m_max_ls_passes; pass++) + { + vec4F xl, xh; + + // TODO: Switch between 4D or 3D + if (!compute_least_squares_endpoints_4D( + pixel_stats.m_num_pixels, trial_blk_weights0, get_ls_weights_ise(weight_ise_range), + &xl, &xh, flattened_pixels, flattened_pixels_min_f, flattened_pixels_max_f)) + { + break; + } + + color_rgba dec_l(0), dec_h(0); + decode_endpoints(cem_index, trial_blk_endpoints, endpoint_ise_range, dec_l, dec_h); + + xl[ccs_index] = dec_l[ccs_index] * (1.0f / 255.0f); + xh[ccs_index] = dec_h[ccs_index] * (1.0f / 255.0f); + + if (cem_is_base_offset) + { + bool tried_base_ofs_clamped = false; + + try_cem9_13_sp_or_dp( + cem_index, ccs_index, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + xl, xh, + trial_blk_endpoints, trial_blk_weights0, trial_blk_weights1, trial_blk_error, trial_used_blue_contraction, + use_blue_contraction, tried_used_blue_contraction, tried_base_ofs_clamped); + + if ((pBase_ofs_clamped_flag) && (tried_base_ofs_clamped)) + *pBase_ofs_clamped_flag = true; + + if (tried_used_blue_contraction) + { + // Try without blue contraction for a minor gain. + try_cem9_13_sp_or_dp( + cem_index, ccs_index, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + xl, xh, + trial_blk_endpoints, trial_blk_weights0, trial_blk_weights1, trial_blk_error, trial_used_blue_contraction, + false, tried_used_blue_contraction, tried_base_ofs_clamped); + + if ((pBase_ofs_clamped_flag) && (tried_base_ofs_clamped)) + *pBase_ofs_clamped_flag = true; + } + } + else + { + try_cem8_12_dp( + cem_index, ccs_index, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + xl, xh, + trial_blk_endpoints, trial_blk_weights0, trial_blk_weights1, trial_blk_error, trial_used_blue_contraction, + use_blue_contraction, tried_used_blue_contraction); + + if (tried_used_blue_contraction) + { + // Try without blue contraction for a minor gain. + try_cem8_12_dp( + cem_index, ccs_index, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + xl, xh, + trial_blk_endpoints, trial_blk_weights0, trial_blk_weights1, trial_blk_error, trial_used_blue_contraction, + false, tried_used_blue_contraction); + } + } + + if (trial_blk_error >= cur_blk_error) + break; + + cur_blk_error = trial_blk_error; + memcpy(pEndpoint_vals, trial_blk_endpoints, total_endpoint_vals); + memcpy(pWeight_vals0, trial_blk_weights0, total_weights); + memcpy(pWeight_vals1, trial_blk_weights1, total_weights); + + } // pass + + const float ccs_bounds_min = pixel_stats.m_min_f[ccs_index]; + const float ccs_bounds_max = pixel_stats.m_max_f[ccs_index]; + float ccs_vals[ASTC_LDR_MAX_BLOCK_PIXELS]; + + if (ccs_bounds_min != ccs_bounds_max) + { + for (uint32_t i = 0; i < pixel_stats.m_num_pixels; i++) + ccs_vals[i] = pixel_stats.m_pixels_f[i][ccs_index]; + + for (uint32_t pass = 0; pass < enc_params.m_max_ls_passes; pass++) + { + float xl = 0.0f, xh = 0.0f; + + if (!compute_least_squares_endpoints_1D( + pixel_stats.m_num_pixels, trial_blk_weights1, get_ls_weights_ise(weight_ise_range), + &xl, &xh, ccs_vals, ccs_bounds_min, ccs_bounds_max)) + { + break; + } + + color_rgba dec_l(0), dec_h(0); + decode_endpoints(cem_index, trial_blk_endpoints, endpoint_ise_range, dec_l, dec_h); + + vec4F vl, vh; + for (uint32_t c = 0; c < 4; c++) + { + if (c == ccs_index) + { + vl[c] = xl; + vh[c] = xh; + } + else + { + vl[c] = (float)dec_l[c] * (1.0f / 255.0f); + vh[c] = (float)dec_h[c] * (1.0f / 255.0f); + } + } + + if (cem_is_base_offset) + { + bool tried_base_ofs_clamped = false; + + try_cem9_13_sp_or_dp( + cem_index, ccs_index, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + vl, vh, + trial_blk_endpoints, trial_blk_weights0, trial_blk_weights1, trial_blk_error, trial_used_blue_contraction, + use_blue_contraction, tried_used_blue_contraction, tried_base_ofs_clamped); + + if ((pBase_ofs_clamped_flag) && (tried_base_ofs_clamped)) + *pBase_ofs_clamped_flag = true; + + if (tried_used_blue_contraction) + { + // Try without blue contraction for a minor gain. + try_cem9_13_sp_or_dp( + cem_index, ccs_index, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + vl, vh, + trial_blk_endpoints, trial_blk_weights0, trial_blk_weights1, trial_blk_error, trial_used_blue_contraction, + false, tried_used_blue_contraction, tried_base_ofs_clamped); + + if ((pBase_ofs_clamped_flag) && (tried_base_ofs_clamped)) + *pBase_ofs_clamped_flag = true; + } + } + else + { + try_cem8_12_dp( + cem_index, ccs_index, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + vl, vh, + trial_blk_endpoints, trial_blk_weights0, trial_blk_weights1, trial_blk_error, trial_used_blue_contraction, + use_blue_contraction, tried_used_blue_contraction); + + if (tried_used_blue_contraction) + { + // Try without blue contraction for a minor gain. + try_cem8_12_dp( + cem_index, ccs_index, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + vl, vh, + trial_blk_endpoints, trial_blk_weights0, trial_blk_weights1, trial_blk_error, trial_used_blue_contraction, + false, tried_used_blue_contraction); + } + } + + if (trial_blk_error >= cur_blk_error) + break; + + cur_blk_error = trial_blk_error; + memcpy(pEndpoint_vals, trial_blk_endpoints, total_endpoint_vals); + memcpy(pWeight_vals0, trial_blk_weights0, total_weights); + memcpy(pWeight_vals1, trial_blk_weights1, total_weights); + + } // pass + } + + if ((enc_params.m_total_weight_refine_passes) && ((weight_ise_range != astc_helpers::BISE_2_LEVELS) && (weight_ise_range != astc_helpers::BISE_64_LEVELS))) + { + weight_refiner refiner; + refiner.init(weight_ise_range, pixel_stats.m_num_pixels, pWeight_vals0); + + for (uint32_t pass = 0; pass < enc_params.m_total_weight_refine_passes; pass++) + { + refiner.refine(pass, trial_blk_weights0); + + vec4F xl, xh; + + if (!compute_least_squares_endpoints_4D( + pixel_stats.m_num_pixels, trial_blk_weights0, get_ls_weights_ise(weight_ise_range), + &xl, &xh, flattened_pixels, flattened_pixels_min_f, flattened_pixels_max_f)) + { + break; + } + + color_rgba dec_l(0), dec_h(0); + decode_endpoints(cem_index, trial_blk_endpoints, endpoint_ise_range, dec_l, dec_h); + + xl[ccs_index] = dec_l[ccs_index] * (1.0f / 255.0f); + xh[ccs_index] = dec_h[ccs_index] * (1.0f / 255.0f); + + if (cem_is_base_offset) + { + bool tried_base_ofs_clamped = false; + + try_cem9_13_sp_or_dp( + cem_index, ccs_index, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + xl, xh, + trial_blk_endpoints, trial_blk_weights0, trial_blk_weights1, trial_blk_error, trial_used_blue_contraction, + use_blue_contraction, tried_used_blue_contraction, tried_base_ofs_clamped); + + if ((pBase_ofs_clamped_flag) && (tried_base_ofs_clamped)) + *pBase_ofs_clamped_flag = true; + + if (tried_used_blue_contraction) + { + // Try without blue contraction for a minor gain. + try_cem9_13_sp_or_dp( + cem_index, ccs_index, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + xl, xh, + trial_blk_endpoints, trial_blk_weights0, trial_blk_weights1, trial_blk_error, trial_used_blue_contraction, + false, tried_used_blue_contraction, tried_base_ofs_clamped); + + if ((pBase_ofs_clamped_flag) && (tried_base_ofs_clamped)) + *pBase_ofs_clamped_flag = true; + } + } + else + { + try_cem8_12_dp( + cem_index, ccs_index, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + xl, xh, + trial_blk_endpoints, trial_blk_weights0, trial_blk_weights1, trial_blk_error, trial_used_blue_contraction, + use_blue_contraction, tried_used_blue_contraction); + + if (tried_used_blue_contraction) + { + // Try without blue contraction for a minor gain. + try_cem8_12_dp( + cem_index, ccs_index, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + xl, xh, + trial_blk_endpoints, trial_blk_weights0, trial_blk_weights1, trial_blk_error, trial_used_blue_contraction, + false, tried_used_blue_contraction); + } + } + + if (trial_blk_error >= cur_blk_error) + continue; + + cur_blk_error = trial_blk_error; + memcpy(pEndpoint_vals, trial_blk_endpoints, total_endpoint_vals); + memcpy(pWeight_vals0, trial_blk_weights0, total_weights); + memcpy(pWeight_vals1, trial_blk_weights1, total_weights); + + } // pass + + if (ccs_bounds_min != ccs_bounds_max) + { + refiner.init(weight_ise_range, pixel_stats.m_num_pixels, pWeight_vals1); + + for (uint32_t pass = 0; pass < WEIGHT_REFINER_MAX_PASSES; pass++) + { + refiner.refine(pass, trial_blk_weights1); + + float xl = 0.0f, xh = 0.0f; + + if (!compute_least_squares_endpoints_1D( + pixel_stats.m_num_pixels, trial_blk_weights1, get_ls_weights_ise(weight_ise_range), + &xl, &xh, ccs_vals, ccs_bounds_min, ccs_bounds_max)) + { + break; + } + + color_rgba dec_l(0), dec_h(0); + decode_endpoints(cem_index, trial_blk_endpoints, endpoint_ise_range, dec_l, dec_h); + + vec4F vl, vh; + for (uint32_t c = 0; c < 4; c++) + { + if (c == ccs_index) + { + vl[c] = xl; + vh[c] = xh; + } + else + { + vl[c] = (float)dec_l[c] * (1.0f / 255.0f); + vh[c] = (float)dec_h[c] * (1.0f / 255.0f); + } + } + + bool did_improve_res = false; + + if (cem_is_base_offset) + { + bool tried_base_ofs_clamped = false; + + did_improve_res = try_cem9_13_sp_or_dp( + cem_index, ccs_index, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + vl, vh, + trial_blk_endpoints, trial_blk_weights0, trial_blk_weights1, trial_blk_error, trial_used_blue_contraction, + use_blue_contraction, tried_used_blue_contraction, tried_base_ofs_clamped); + BASISU_NOTE_UNUSED(did_improve_res); + + if ((pBase_ofs_clamped_flag) && (tried_base_ofs_clamped)) + *pBase_ofs_clamped_flag = true; + + if (tried_used_blue_contraction) + { + // Try without blue contraction for a minor gain. + did_improve_res = try_cem9_13_sp_or_dp( + cem_index, ccs_index, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + vl, vh, + trial_blk_endpoints, trial_blk_weights0, trial_blk_weights1, trial_blk_error, trial_used_blue_contraction, + false, tried_used_blue_contraction, tried_base_ofs_clamped); + + if ((pBase_ofs_clamped_flag) && (tried_base_ofs_clamped)) + *pBase_ofs_clamped_flag = true; + } + } + else + { + did_improve_res = try_cem8_12_dp( + cem_index, ccs_index, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + vl, vh, + trial_blk_endpoints, trial_blk_weights0, trial_blk_weights1, trial_blk_error, trial_used_blue_contraction, + use_blue_contraction, tried_used_blue_contraction); + + if (tried_used_blue_contraction) + { + // Try without blue contraction for a minor gain. + did_improve_res = try_cem8_12_dp( + cem_index, ccs_index, pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + vl, vh, + trial_blk_endpoints, trial_blk_weights0, trial_blk_weights1, trial_blk_error, trial_used_blue_contraction, + false, tried_used_blue_contraction); + } + } + + if (trial_blk_error >= cur_blk_error) + continue; + + cur_blk_error = trial_blk_error; + memcpy(pEndpoint_vals, trial_blk_endpoints, total_endpoint_vals); + memcpy(pWeight_vals0, trial_blk_weights0, total_weights); + memcpy(pWeight_vals1, trial_blk_weights1, total_weights); + + } // pass + } + } + + return cur_blk_error; + } + + // base scale rgb/rgba + // returns true if improved + static bool try_cem6_10( + uint32_t cem_index, + const pixel_stats_t& pixel_stats, const cem_encode_params& enc_params, + uint32_t endpoint_ise_range, uint32_t weight_ise_range, + float scale, float low_a_f, const vec4F& high_color_f, + uint8_t* pTrial_endpoint_vals, uint8_t* pTrial_weight_vals, uint64_t& trial_blk_error) + { + assert(g_initialized); + assert((cem_index == astc_helpers::CEM_LDR_RGB_BASE_SCALE) || (cem_index == astc_helpers::CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A)); + assert((pixel_stats.m_num_pixels) && (pixel_stats.m_num_pixels <= ASTC_LDR_MAX_BLOCK_PIXELS)); + assert((endpoint_ise_range >= astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE) && (endpoint_ise_range <= astc_helpers::LAST_VALID_ENDPOINT_ISE_RANGE)); + assert(((weight_ise_range >= astc_helpers::FIRST_VALID_WEIGHT_ISE_RANGE) && (weight_ise_range <= astc_helpers::LAST_VALID_WEIGHT_ISE_RANGE)) || (weight_ise_range == astc_helpers::BISE_64_LEVELS)); + + uint8_t trial_endpoint_vals[astc_helpers::NUM_MODE10_ENDPOINTS] = { 0 }; + uint8_t trial_weight_vals[ASTC_LDR_MAX_BLOCK_PIXELS] = { 0 }; + + cem_encode_ldr_rgb_or_rgba_base_scale(cem_index, endpoint_ise_range, scale, low_a_f, high_color_f, trial_endpoint_vals); + + uint64_t trial_err = eval_solution( + pixel_stats, cem_index, trial_endpoint_vals, endpoint_ise_range, + trial_weight_vals, weight_ise_range, + enc_params); + + bool improved_flag = false; + if (trial_err < trial_blk_error) + { + trial_blk_error = trial_err; + memcpy(pTrial_endpoint_vals, trial_endpoint_vals, astc_helpers::get_num_cem_values(cem_index)); + memcpy(pTrial_weight_vals, trial_weight_vals, pixel_stats.m_num_pixels); + improved_flag = true; + } + + const uint32_t num_endpoint_vals = astc_helpers::get_num_cem_values(cem_index); + + // TODO + for (int delta = -1; delta <= 1; delta += 1) + { + if (!delta) + continue; + + uint8_t fixed_endpoint_vals[astc_helpers::NUM_MODE10_ENDPOINTS]; + memcpy(fixed_endpoint_vals, trial_endpoint_vals, num_endpoint_vals); + + fixed_endpoint_vals[3] = (uint8_t)astc_helpers::apply_delta_to_bise_endpoint_val(endpoint_ise_range, fixed_endpoint_vals[3], delta); + + trial_err = eval_solution( + pixel_stats, cem_index, fixed_endpoint_vals, endpoint_ise_range, + trial_weight_vals, weight_ise_range, + enc_params); + + if (trial_err < trial_blk_error) + { + trial_blk_error = trial_err; + memcpy(pTrial_endpoint_vals, fixed_endpoint_vals, astc_helpers::get_num_cem_values(cem_index)); + memcpy(pTrial_weight_vals, trial_weight_vals, pixel_stats.m_num_pixels); + improved_flag = true; + } + } + + return improved_flag; + } + + static bool try_cem6_10_dp( + uint32_t cem_index, uint32_t ccs_index, + const pixel_stats_t& pixel_stats, const cem_encode_params& enc_params, + uint32_t endpoint_ise_range, uint32_t weight_ise_range, + float scale, float low_a_f, const vec4F& high_color_f, + uint8_t* pTrial_endpoint_vals, uint8_t* pTrial_weight_vals0, uint8_t* pTrial_weight_vals1, uint64_t& trial_blk_error) + { + assert(g_initialized); + assert((cem_index == astc_helpers::CEM_LDR_RGB_BASE_SCALE) || (cem_index == astc_helpers::CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A)); + assert(ccs_index <= 3); + assert((pixel_stats.m_num_pixels) && (pixel_stats.m_num_pixels <= ASTC_LDR_MAX_BLOCK_PIXELS)); + assert((endpoint_ise_range >= astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE) && (endpoint_ise_range <= astc_helpers::LAST_VALID_ENDPOINT_ISE_RANGE)); + assert(((weight_ise_range >= astc_helpers::FIRST_VALID_WEIGHT_ISE_RANGE) && (weight_ise_range <= astc_helpers::LAST_VALID_WEIGHT_ISE_RANGE)) || (weight_ise_range == astc_helpers::BISE_64_LEVELS)); + assert(pTrial_weight_vals0 && pTrial_weight_vals1); + + uint8_t trial_endpoint_vals[astc_helpers::NUM_MODE10_ENDPOINTS] = { 0 }; + uint8_t trial_weight_vals0[ASTC_LDR_MAX_BLOCK_PIXELS] = { 0 }; + uint8_t trial_weight_vals1[ASTC_LDR_MAX_BLOCK_PIXELS] = { 0 }; + + cem_encode_ldr_rgb_or_rgba_base_scale(cem_index, endpoint_ise_range, scale, low_a_f, high_color_f, trial_endpoint_vals); + + uint64_t trial_err = eval_solution_dp( + pixel_stats, cem_index, ccs_index, + trial_endpoint_vals, endpoint_ise_range, + trial_weight_vals0, trial_weight_vals1, weight_ise_range, + enc_params); + + bool improved_flag = false; + if (trial_err < trial_blk_error) + { + trial_blk_error = trial_err; + memcpy(pTrial_endpoint_vals, trial_endpoint_vals, astc_helpers::get_num_cem_values(cem_index)); + memcpy(pTrial_weight_vals0, trial_weight_vals0, pixel_stats.m_num_pixels); + memcpy(pTrial_weight_vals1, trial_weight_vals1, pixel_stats.m_num_pixels); + improved_flag = true; + } + + const uint32_t num_endpoint_vals = astc_helpers::get_num_cem_values(cem_index); + + for (int delta = -1; delta <= 1; delta += 1) + { + if (!delta) + continue; + + uint8_t fixed_endpoint_vals[astc_helpers::NUM_MODE10_ENDPOINTS]; + memcpy(fixed_endpoint_vals, trial_endpoint_vals, num_endpoint_vals); + + fixed_endpoint_vals[3] = (uint8_t)astc_helpers::apply_delta_to_bise_endpoint_val(endpoint_ise_range, fixed_endpoint_vals[3], delta); + + trial_err = eval_solution_dp( + pixel_stats, cem_index, ccs_index, + fixed_endpoint_vals, endpoint_ise_range, + trial_weight_vals0, trial_weight_vals1, weight_ise_range, + enc_params); + + if (trial_err < trial_blk_error) + { + trial_blk_error = trial_err; + memcpy(pTrial_endpoint_vals, fixed_endpoint_vals, astc_helpers::get_num_cem_values(cem_index)); + memcpy(pTrial_weight_vals0, trial_weight_vals0, pixel_stats.m_num_pixels); + memcpy(pTrial_weight_vals1, trial_weight_vals1, pixel_stats.m_num_pixels); + improved_flag = true; + } + } + + return improved_flag; + } + + // rgb/rgba base+scale + static uint64_t encode_cem6_10( + uint32_t cem_index, + const pixel_stats_t& pixel_stats, const cem_encode_params& enc_params, + uint32_t endpoint_ise_range, uint32_t weight_ise_range, + uint8_t* pEndpoint_vals, uint8_t* pWeight_vals, uint64_t cur_blk_error) + { + assert(g_initialized); + assert((cem_index == astc_helpers::CEM_LDR_RGB_BASE_SCALE) || (cem_index == astc_helpers::CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A)); + assert((pixel_stats.m_num_pixels) && (pixel_stats.m_num_pixels <= ASTC_LDR_MAX_BLOCK_PIXELS)); + assert((endpoint_ise_range >= astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE) && (endpoint_ise_range <= astc_helpers::LAST_VALID_ENDPOINT_ISE_RANGE)); + assert(((weight_ise_range >= astc_helpers::FIRST_VALID_WEIGHT_ISE_RANGE) && (weight_ise_range <= astc_helpers::LAST_VALID_WEIGHT_ISE_RANGE)) || (weight_ise_range == astc_helpers::BISE_64_LEVELS)); + + const bool cem_has_alpha = (cem_index == astc_helpers::CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A); + + const uint32_t total_endpoint_vals = astc_helpers::get_num_cem_values(cem_index); + const uint32_t total_weights = pixel_stats.m_num_pixels; + + float best_l = BIG_FLOAT_VAL, best_h = -BIG_FLOAT_VAL; + //int best_l_index = 0, best_h_index = 0; + + for (uint32_t c = 0; c < pixel_stats.m_num_pixels; c++) + { + const vec3F px(pixel_stats.m_pixels_f[c]); + + float p = px.dot(pixel_stats.m_zero_rel_axis3); + + if (p < best_l) + { + best_l = p; + //best_l_index = c; + } + + if (p > best_h) + { + best_h = p; + //best_h_index = c; + } + } // c + + const float MAX_S = 255.0f / 256.0f; + const float EPS = 1e-6f; + + uint64_t trial_blk_error = UINT64_MAX; + uint8_t trial_blk_endpoints[astc_helpers::NUM_MODE10_ENDPOINTS] = { 0 }; + uint8_t trial_blk_weights[ASTC_LDR_MAX_BLOCK_PIXELS] = { 0 }; + + uint64_t best_blk_error = UINT64_MAX; + uint8_t best_blk_endpoints[astc_helpers::NUM_MODE10_ENDPOINTS] = { 0 }; + uint8_t best_blk_weights[ASTC_LDR_MAX_BLOCK_PIXELS] = { 0 }; + + vec3F low_color3_f(best_l * pixel_stats.m_zero_rel_axis3); + low_color3_f.clamp(0.0f, 1.0f); + + vec3F high_color3_f(best_h * pixel_stats.m_zero_rel_axis3); + high_color3_f.clamp(0.0f, 1.0f); + + float scale = MAX_S; + + float d = low_color3_f.dot(high_color3_f); + float nrm = high_color3_f.norm(); + if (nrm > 0.0f) + scale = saturate(d / nrm); + scale = minimum(scale, MAX_S); + + vec4F low_color_f(low_color3_f[0], low_color3_f[1], low_color3_f[2], pixel_stats.m_min_f[3]); + vec4F high_color_f(high_color3_f[0], high_color3_f[1], high_color3_f[2], pixel_stats.m_max_f[3]); + + try_cem6_10( + cem_index, + pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + scale, low_color_f[3], high_color_f, + trial_blk_endpoints, trial_blk_weights, trial_blk_error); + + best_blk_error = trial_blk_error; + memcpy(best_blk_endpoints, trial_blk_endpoints, total_endpoint_vals); + memcpy(best_blk_weights, trial_blk_weights, total_weights); + + const uint32_t NUM_PASSES = 2; + for (uint32_t pass = 0; pass < NUM_PASSES; pass++) + { + color_rgba actual_l(0), actual_h(0); + float actual_scale = 0; + decode_endpoints(cem_index, trial_blk_endpoints, endpoint_ise_range, actual_l, actual_h, &actual_scale); + + vec3F actual_high_f((float)actual_h[0], (float)actual_h[1], (float)actual_h[2]); + actual_high_f *= (1.0f / 255.0f); + + // invalid on raw weights + const auto& dequant_weights_tab = astc_helpers::g_dequant_tables.get_weight_tab(minimum(astc_helpers::BISE_32_LEVELS, weight_ise_range)).m_ISE_to_val; + + vec3F Pa(0.0f), Pb(0.0f); + float A = 0.0f, B = 0.0f, C = 0.0f; + + for (uint32_t i = 0; i < pixel_stats.m_num_pixels; i++) + { + const vec3F px(pixel_stats.m_pixels_f[i]); + + const int iw = (weight_ise_range == astc_helpers::BISE_64_LEVELS) ? trial_blk_weights[i] : dequant_weights_tab[trial_blk_weights[i]]; + float t = (float)iw * (1.0f / 64.0f); + float bi = t, ai = 1.0f - t; + + Pa += px * ai; + Pb += px * bi; + + A += ai * ai; + B += ai * bi; + C += bi * bi; + } + + vec3F new_high = actual_high_f; + float new_scale = actual_scale; + + float h2 = actual_high_f.norm(); + if ((h2 > EPS) && (A > EPS)) + { + new_scale = (Pa.dot(actual_high_f) / h2 - B) / A; + new_scale = clamp(new_scale, 0.0f, MAX_S); + } + + const float den = A * new_scale * new_scale + 2.0f * B * new_scale + C; + if (den > EPS) + { + new_high = (Pb + Pa * new_scale) / den; + } + + h2 = new_high.norm(); + if ((h2 > EPS) && (A > EPS)) + { + new_scale = (Pa.dot(new_high) / h2 - B) / A; + new_scale = clamp(new_scale, 0.0f, MAX_S); + } + + try_cem6_10( + cem_index, + pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + new_scale, (float)actual_l[3] * (1.0f / 255.0f), vec4F(new_high[0], new_high[1], new_high[2], (float)actual_h[3] * (1.0f / 255.0f)), + trial_blk_endpoints, trial_blk_weights, trial_blk_error); + + if (trial_blk_error >= best_blk_error) + break; + + best_blk_error = trial_blk_error; + memcpy(best_blk_endpoints, trial_blk_endpoints, total_endpoint_vals); + memcpy(best_blk_weights, trial_blk_weights, total_weights); + + } // pass + + if (cem_has_alpha) + { + // Try to refine low a/high given the current selectors. + float bounds_min = pixel_stats.m_min_f[3]; + float bounds_max = pixel_stats.m_max_f[3]; + if (bounds_min != bounds_max) + { + float a_vals[ASTC_LDR_MAX_BLOCK_PIXELS]; + for (uint32_t i = 0; i < pixel_stats.m_num_pixels; i++) + a_vals[i] = pixel_stats.m_pixels_f[i][3]; + + const uint32_t TOTAL_PASSES = 1; + for (uint32_t pass = 0; pass < TOTAL_PASSES; pass++) + { + float xl = 0.0f, xh = 0.0f; + + if (compute_least_squares_endpoints_1D( + pixel_stats.m_num_pixels, best_blk_weights, get_ls_weights_ise(weight_ise_range), + &xl, &xh, a_vals, bounds_min, bounds_max)) + { + color_rgba actual_l(0), actual_h(0); + float actual_scale = 0; + decode_endpoints(cem_index, trial_blk_endpoints, endpoint_ise_range, actual_l, actual_h, &actual_scale); + + try_cem6_10( + cem_index, + pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + actual_scale, xl, vec4F(actual_h[0], actual_h[1], actual_h[2], xh), + trial_blk_endpoints, trial_blk_weights, trial_blk_error); + + if (trial_blk_error < best_blk_error) + { + best_blk_error = trial_blk_error; + memcpy(best_blk_endpoints, trial_blk_endpoints, total_endpoint_vals); + memcpy(best_blk_weights, trial_blk_weights, total_weights); + } + else + { + break; + } + } + else + { + break; + } + } // pass + } + } + + if (best_blk_error < cur_blk_error) + { + cur_blk_error = best_blk_error; + memcpy(pEndpoint_vals, trial_blk_endpoints, total_endpoint_vals); + memcpy(pWeight_vals, trial_blk_weights, total_weights); + } + + return cur_blk_error; + } + + // rgba base+scale, dual plane a, ccs_index must be 3 + static uint64_t encode_cem10_dp_a( + uint32_t cem_index, + const pixel_stats_t& pixel_stats, const cem_encode_params& enc_params, + uint32_t endpoint_ise_range, uint32_t weight_ise_range, + uint8_t* pEndpoint_vals, uint8_t* pWeight_vals0, uint8_t* pWeight_vals1, uint64_t cur_blk_error) + { + assert(g_initialized); + assert(cem_index == astc_helpers::CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A); + assert((pixel_stats.m_num_pixels) && (pixel_stats.m_num_pixels <= ASTC_LDR_MAX_BLOCK_PIXELS)); + assert((endpoint_ise_range >= astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE) && (endpoint_ise_range <= astc_helpers::LAST_VALID_ENDPOINT_ISE_RANGE)); + assert(((weight_ise_range >= astc_helpers::FIRST_VALID_WEIGHT_ISE_RANGE) && (weight_ise_range <= astc_helpers::LAST_VALID_WEIGHT_ISE_RANGE)) || (weight_ise_range == astc_helpers::BISE_64_LEVELS)); + + // RGB uses plane0, alpha plane1. So solve RGB first. + uint8_t rgba_endpoint_vals[astc_helpers::NUM_MODE10_ENDPOINTS] = { 0 }; + uint8_t rgb_weight_vals[ASTC_LDR_MAX_BLOCK_PIXELS] = { 0 }; + uint8_t a_weight_vals[ASTC_LDR_MAX_BLOCK_PIXELS] = { 0 }; + + // First just solve RGB, single plane. + uint64_t rgb_blk_error = encode_cem6_10( + astc_helpers::CEM_LDR_RGB_BASE_SCALE, + pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + rgba_endpoint_vals, rgb_weight_vals, UINT64_MAX); + + assert(rgb_blk_error != UINT64_MAX); + + if (rgb_blk_error == UINT64_MAX) + return cur_blk_error; + + const auto& endpoint_quant_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(endpoint_ise_range).m_val_to_ise; + + rgba_endpoint_vals[4] = endpoint_quant_tab[pixel_stats.m_min[3]]; + rgba_endpoint_vals[5] = endpoint_quant_tab[pixel_stats.m_max[3]]; + + uint64_t rgba_blk_error = eval_solution_dp( + pixel_stats, + cem_index, 3, + rgba_endpoint_vals, endpoint_ise_range, + rgb_weight_vals, a_weight_vals, weight_ise_range, + enc_params); + + assert(rgba_blk_error != UINT64_MAX); + + if (rgba_blk_error < cur_blk_error) + { + cur_blk_error = rgba_blk_error; + memcpy(pEndpoint_vals, rgba_endpoint_vals, astc_helpers::NUM_MODE10_ENDPOINTS); + memcpy(pWeight_vals0, rgb_weight_vals, pixel_stats.m_num_pixels); + memcpy(pWeight_vals1, a_weight_vals, pixel_stats.m_num_pixels); + + if (!cur_blk_error) + return cur_blk_error; + } + + float bounds_min = pixel_stats.m_min_f[3], bounds_max = pixel_stats.m_max_f[3]; + if (bounds_min != bounds_max) + { + float a_vals[ASTC_LDR_MAX_BLOCK_PIXELS]; + for (uint32_t i = 0; i < pixel_stats.m_num_pixels; i++) + a_vals[i] = pixel_stats.m_pixels_f[i][3]; + + const uint32_t TOTAL_PASSES = 2; + + uint8_t trial_rgba_endpoint_vals[astc_helpers::NUM_MODE10_ENDPOINTS] = { 0 }; + uint8_t trial_rgb_weight_vals[ASTC_LDR_MAX_BLOCK_PIXELS] = { 0 }; + uint8_t trial_a_weight_vals[ASTC_LDR_MAX_BLOCK_PIXELS] = { 0 }; + + for (uint32_t pass = 0; pass < TOTAL_PASSES; pass++) + { + float xl = 0.0f, xh = 0.0f; + + if (compute_least_squares_endpoints_1D( + pixel_stats.m_num_pixels, pass ? trial_a_weight_vals : a_weight_vals, get_ls_weights_ise(weight_ise_range), + &xl, &xh, a_vals, bounds_min, bounds_max)) + { + memcpy(trial_rgba_endpoint_vals, rgba_endpoint_vals, astc_helpers::NUM_MODE10_ENDPOINTS); + + trial_rgba_endpoint_vals[4] = precise_round_bise_endpoint_val(xl, endpoint_ise_range); + trial_rgba_endpoint_vals[5] = precise_round_bise_endpoint_val(xh, endpoint_ise_range); + + uint64_t trial_rgba_blk_error = eval_solution_dp( + pixel_stats, + cem_index, 3, + trial_rgba_endpoint_vals, endpoint_ise_range, + trial_rgb_weight_vals, trial_a_weight_vals, weight_ise_range, + enc_params); + + assert(trial_rgba_blk_error != UINT64_MAX); + + if (trial_rgba_blk_error < cur_blk_error) + { + cur_blk_error = trial_rgba_blk_error; + memcpy(pEndpoint_vals, trial_rgba_endpoint_vals, astc_helpers::NUM_MODE10_ENDPOINTS); + memcpy(pWeight_vals0, trial_rgb_weight_vals, pixel_stats.m_num_pixels); + memcpy(pWeight_vals1, trial_a_weight_vals, pixel_stats.m_num_pixels); + } + else + { + break; + } + } + else + { + break; + } + } // pass + } + + return cur_blk_error; + } + + // rgb/rgba base+scale, dual plane rgb (not a!) + static uint64_t encode_cem6_10_dp_rgb( + uint32_t cem_index, uint32_t ccs_index, + const pixel_stats_t& pixel_stats, const cem_encode_params& enc_params, + uint32_t endpoint_ise_range, uint32_t weight_ise_range, + uint8_t* pEndpoint_vals, uint8_t* pWeight_vals0, uint8_t* pWeight_vals1, uint64_t cur_blk_error) + { + assert(g_initialized); + assert((cem_index == astc_helpers::CEM_LDR_RGB_BASE_SCALE) || (cem_index == astc_helpers::CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A)); + assert(ccs_index <= 2); + assert((pixel_stats.m_num_pixels) && (pixel_stats.m_num_pixels <= ASTC_LDR_MAX_BLOCK_PIXELS)); + assert((endpoint_ise_range >= astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE) && (endpoint_ise_range <= astc_helpers::LAST_VALID_ENDPOINT_ISE_RANGE)); + assert(((weight_ise_range >= astc_helpers::FIRST_VALID_WEIGHT_ISE_RANGE) && (weight_ise_range <= astc_helpers::LAST_VALID_WEIGHT_ISE_RANGE)) || (weight_ise_range == astc_helpers::BISE_64_LEVELS)); + assert(pWeight_vals0 && pWeight_vals1); + + // First solve using a single plane, then we'll introduce the other plane's weights and tune the encoded H/s values + uint8_t sp_endpoint_vals[astc_helpers::NUM_MODE10_ENDPOINTS] = { 0 }; + uint8_t sp_weight_vals[ASTC_LDR_MAX_BLOCK_PIXELS] = { 0 }; + + uint64_t sp_block_err = encode_cem6_10( + cem_index, + pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + sp_endpoint_vals, sp_weight_vals, UINT64_MAX); + + assert(sp_block_err != UINT64_MAX); + BASISU_NOTE_UNUSED(sp_block_err); + + // Now compute both plane's weights using the initial H/s values + uint8_t trial_weights0_vals[ASTC_LDR_MAX_BLOCK_PIXELS] = { 0 }; + uint8_t trial_weights1_vals[ASTC_LDR_MAX_BLOCK_PIXELS] = { 0 }; + uint64_t dp_blk_error = eval_solution_dp( + pixel_stats, + cem_index, ccs_index, + sp_endpoint_vals, endpoint_ise_range, + trial_weights0_vals, trial_weights1_vals, weight_ise_range, + enc_params); + + if (dp_blk_error < cur_blk_error) + { + cur_blk_error = dp_blk_error; + memcpy(pEndpoint_vals, sp_endpoint_vals, astc_helpers::NUM_MODE10_ENDPOINTS); + memcpy(pWeight_vals0, trial_weights0_vals, pixel_stats.m_num_pixels); + memcpy(pWeight_vals1, trial_weights1_vals, pixel_stats.m_num_pixels); + + if (!cur_blk_error) + return cur_blk_error; + } + + // Compute refined H/s values using the current weights. + const float MAX_S = 255.0f / 256.0f; + const float EPS = 1e-6f; + + vec3F Pa(0.0f); // (Pa_r,Pa_g,Pa_b) + vec3F Pb(0.0f); // (Pb_r,Pb_g,Pb_b) + float A[3] = { 0 }, B[3] = { 0 }, C[3] = { 0 }; // per-channel + + // invalid on raw weights + const auto& dequant_weights_tab = astc_helpers::g_dequant_tables.get_weight_tab(minimum(astc_helpers::BISE_32_LEVELS, weight_ise_range)).m_ISE_to_val; + + for (uint32_t i = 0; i < pixel_stats.m_num_pixels; i++) + { + float w0, w1; + if (weight_ise_range == astc_helpers::BISE_64_LEVELS) + { + w0 = (float)trial_weights0_vals[i] * (1.0f / 64.0f); + w1 = (float)trial_weights1_vals[i] * (1.0f / 64.0f); + } + else + { + w0 = dequant_weights_tab[trial_weights0_vals[i]] * (1.0f / 64.0f); + w1 = dequant_weights_tab[trial_weights1_vals[i]] * (1.0f / 64.0f); + } + + float w[3] = { w0, w0, w0 }; + w[ccs_index] = w1; + + const vec3F& p = pixel_stats.m_pixels_f[i]; + + for (int c = 0; c < 3; ++c) + { + const float a = 1.0f - w[c]; + const float b = w[c]; + + Pa[c] += a * p[c]; + Pb[c] += b * p[c]; + A[c] += a * a; + B[c] += a * b; + C[c] += b * b; + } // c + } // i + + color_rgba actual_l(0), actual_h(0); + float actual_scale = 0; + decode_endpoints(cem_index, sp_endpoint_vals, endpoint_ise_range, actual_l, actual_h, &actual_scale); + + vec3F H((float)actual_h[0], (float)actual_h[1], (float)actual_h[2]); + H *= (1.0f / 255.0f); + + const float S1 = H[0] * Pa[0] + H[1] * Pa[1] + H[2] * Pa[2]; + float S2 = 0.0f, S3 = 0.0f; + for (int c = 0; c < 3; c++) + { + const float H2 = H[c] * H[c]; + S2 += H2 * A[c]; + S3 += H2 * B[c]; + } + + float new_s = actual_scale; + if (S2 > EPS) + new_s = (S1 - S3) / S2; + + new_s = clamp(new_s, 0.0f, MAX_S); + + vec3F new_H(0.0f); + for (int c = 0; c < 3; ++c) + { + const float den = A[c] * new_s * new_s + 2.0f * B[c] * new_s + C[c]; + + float Hc = 0.0f; + if (den > EPS) + { + const float num = Pb[c] + new_s * Pa[c]; + Hc = num / den; + } + new_H[c] = Hc; + } + + bool improved_flag = try_cem6_10_dp( + cem_index, ccs_index, + pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + new_s, (float)actual_l[3] * (1.0f / 255.0f), vec4F(new_H[0], new_H[1], new_H[2], (float)actual_h[3] * (1.0f / 255.0f)), + pEndpoint_vals, pWeight_vals0, pWeight_vals1, cur_blk_error); + (void)improved_flag; + + return cur_blk_error; + } + + // dispatcher + uint64_t cem_encode_pixels( + uint32_t cem_index, int ccs_index, + const pixel_stats_t& pixel_stats, const cem_encode_params& enc_params, + uint32_t endpoint_ise_range, uint32_t weight_ise_range, + uint8_t* pEndpoint_vals, uint8_t* pWeight_vals0, uint8_t* pWeight_vals1, uint64_t cur_blk_error, + bool use_blue_contraction, bool *pBase_ofs_clamped_flag) + { + assert(g_initialized); + assert((ccs_index >= -1) && (ccs_index <= 3)); + assert(astc_helpers::is_cem_ldr(cem_index)); + assert(pEndpoint_vals); + assert(pWeight_vals0); + + const bool dual_plane = (ccs_index >= 0); + + if (pBase_ofs_clamped_flag) + *pBase_ofs_clamped_flag = false; + + uint64_t blk_error = UINT64_MAX; + + switch (cem_index) + { + case astc_helpers::CEM_LDR_LUM_DIRECT: + { + assert(!dual_plane); + + blk_error = encode_cem0_4( + cem_index, + pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + pEndpoint_vals, pWeight_vals0, cur_blk_error); + + break; + } + case astc_helpers::CEM_LDR_LUM_ALPHA_DIRECT: + { + if (dual_plane) + { + assert(ccs_index == 3); + assert(pWeight_vals1); + + blk_error = encode_cem4_dp_a( + cem_index, + pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + pEndpoint_vals, pWeight_vals0, pWeight_vals1, cur_blk_error); + } + else + { + blk_error = encode_cem0_4( + cem_index, + pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + pEndpoint_vals, pWeight_vals0, cur_blk_error); + } + break; + } + + case astc_helpers::CEM_LDR_RGB_DIRECT: + case astc_helpers::CEM_LDR_RGBA_DIRECT: + case astc_helpers::CEM_LDR_RGB_BASE_PLUS_OFFSET: + case astc_helpers::CEM_LDR_RGBA_BASE_PLUS_OFFSET: + { + if (dual_plane) + { + assert(pWeight_vals1); + blk_error = encode_cem8_12_9_13_dp( + cem_index, ccs_index, + pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + pEndpoint_vals, pWeight_vals0, pWeight_vals1, cur_blk_error, use_blue_contraction, pBase_ofs_clamped_flag); + } + else + { + blk_error = encode_cem8_12_9_13( + cem_index, + pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + pEndpoint_vals, pWeight_vals0, cur_blk_error, use_blue_contraction, pBase_ofs_clamped_flag); + } + break; + } + case astc_helpers::CEM_LDR_RGB_BASE_SCALE: + { + if (dual_plane) + { + assert(ccs_index <= 2); + assert(pWeight_vals1); + + blk_error = encode_cem6_10_dp_rgb( + cem_index, ccs_index, + pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + pEndpoint_vals, pWeight_vals0, pWeight_vals1, cur_blk_error); + } + else + { + blk_error = encode_cem6_10( + cem_index, + pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + pEndpoint_vals, pWeight_vals0, cur_blk_error); + } + break; + } + case astc_helpers::CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A: + { + if (dual_plane) + { + assert(pWeight_vals1); + + if (ccs_index == 3) + { + blk_error = encode_cem10_dp_a( + cem_index, + pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + pEndpoint_vals, pWeight_vals0, pWeight_vals1, cur_blk_error); + } + else + { + blk_error = encode_cem6_10_dp_rgb( + cem_index, ccs_index, + pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + pEndpoint_vals, pWeight_vals0, pWeight_vals1, cur_blk_error); + } + } + else + { + blk_error = encode_cem6_10( + cem_index, + pixel_stats, enc_params, + endpoint_ise_range, weight_ise_range, + pEndpoint_vals, pWeight_vals0, cur_blk_error); + } + break; + } + default: + { + assert(0); + break; + } + } + + return blk_error; + } + + //--------------------------------------------------------------------------------------------- + + float surrogate_evaluate_rgba_sp(const pixel_stats_t& ps, const vec4F& l, const vec4F& h, float* pWeights0, uint32_t num_weight_levels, + const cem_encode_params& enc_params, uint32_t flags) + { + assert(g_initialized); + assert((ps.m_num_pixels) && (ps.m_num_pixels <= ASTC_LDR_MAX_BLOCK_PIXELS)); + assert(pWeights0); + + const float wr = (float)enc_params.m_comp_weights[0], wg = (float)enc_params.m_comp_weights[1], + wb = (float)enc_params.m_comp_weights[2], wa = (float)enc_params.m_comp_weights[3]; + + float total_err = 0; + + const bool compute_error = ((flags & cFlagNoError) == 0); + + float lr = l[0], lg = l[1], lb = l[2], la = l[3]; + float dr = h[0] - lr, dg = h[1] - lg, db = h[2] - lb, da = h[3] - la; + float delta_col_nrm = dr * dr + dg * dg + db * db + da * da; + + if (flags & cFlagDisableQuant) + { + float f = (float)1.0f / (delta_col_nrm + REALLY_SMALL_FLOAT_VAL); + + lr *= -dr; lg *= -dg; lb *= -db; la *= -da; + + dr *= f; dg *= f; db *= f; da *= f; + float l_sum = (lr + lg + lb + la) * f; + + for (uint32_t i = 0; i < ps.m_num_pixels; i++) + { + const vec4F& p = ps.m_pixels_f[i]; + const float r = p[0], g = p[1], b = p[2], a = p[3]; + + float w = r * dr + g * dg + b * db + a * da + l_sum; + + if (w < 0.0f) + w = 0.0f; + else if (w > 1.0f) + w = 1.0f; + + pWeights0[i] = w; + + if (compute_error) + { + float one_minus_w = 1.0f - w; + + float dec_r = l[0] * one_minus_w + h[0] * w; + float dec_g = l[1] * one_minus_w + h[1] * w; + float dec_b = l[2] * one_minus_w + h[2] * w; + float dec_a = l[3] * one_minus_w + h[3] * w; + + float diff_r = r - dec_r; + float diff_g = g - dec_g; + float diff_b = b - dec_b; + float diff_a = a - dec_a; + + total_err += (wr * diff_r * diff_r) + (wg * diff_g * diff_g) + (wb * diff_b * diff_b) + (wa * diff_a * diff_a); + } + + } // i + } + else + { + const float inv_weight_levels = 1.0f / (float)(num_weight_levels - 1); + + float f = (float)(num_weight_levels - 1) / (delta_col_nrm + REALLY_SMALL_FLOAT_VAL); + + lr *= -dr; lg *= -dg; lb *= -db; la *= -da; + + dr *= f; dg *= f; db *= f; da *= f; + float l_sum_biased = (lr + lg + lb + la) * f + .5f; + + for (uint32_t i = 0; i < ps.m_num_pixels; i++) + { + const vec4F& p = ps.m_pixels_f[i]; + const float r = p[0], g = p[1], b = p[2], a = p[3]; + + float w = (float)fast_floorf_int(r * dr + g * dg + b * db + a * da + l_sum_biased) * inv_weight_levels; + + if (w < 0.0f) + w = 0.0f; + else if (w > 1.0f) + w = 1.0f; + + pWeights0[i] = w; + + if (compute_error) + { + float one_minus_w = 1.0f - w; + + float dec_r = l[0] * one_minus_w + h[0] * w; + float dec_g = l[1] * one_minus_w + h[1] * w; + float dec_b = l[2] * one_minus_w + h[2] * w; + float dec_a = l[3] * one_minus_w + h[3] * w; + + float diff_r = r - dec_r; + float diff_g = g - dec_g; + float diff_b = b - dec_b; + float diff_a = a - dec_a; + + total_err += (wr * diff_r * diff_r) + (wg * diff_g * diff_g) + (wb * diff_b * diff_b) + (wa * diff_a * diff_a); + } + + } // i + } + + return total_err; + + } + + float surrogate_evaluate_rgba_dp(uint32_t ccs_index, const pixel_stats_t& ps, const vec4F& l, const vec4F& h, float* pWeights0, float* pWeights1, uint32_t num_weight_levels, + const cem_encode_params& enc_params, uint32_t flags) + { + assert(g_initialized); + assert(ccs_index <= 3); + assert((ps.m_num_pixels) && (ps.m_num_pixels <= ASTC_LDR_MAX_BLOCK_PIXELS)); + assert(pWeights0 && pWeights1); + + const float inv_weight_levels = 1.0f / (float)(num_weight_levels - 1); + + const uint32_t c0 = (ccs_index + 1) & 3, c1 = (ccs_index + 2) & 3, c2 = (ccs_index + 3) & 3; + + const float orig_lx = l[c0], orig_ly = l[c1], orig_lz = l[c2], orig_lw = l[ccs_index]; + const float orig_hx = h[c0], orig_hy = h[c1], orig_hz = h[c2], orig_hw = h[ccs_index]; + + const float wx = (float)enc_params.m_comp_weights[c0], wy = (float)enc_params.m_comp_weights[c1], + wz = (float)enc_params.m_comp_weights[c2], ww = (float)enc_params.m_comp_weights[ccs_index]; + + float total_err = 0; + + const bool compute_error = ((flags & cFlagNoError) == 0); + + if (flags & cFlagDisableQuant) + { + // Plane 0 + { + float dx = orig_hx - orig_lx, dy = orig_hy - orig_ly, dz = orig_hz - orig_lz; + + float delta_col_nrm = dx * dx + dy * dy + dz * dz; + + float f = (float)1.0f / (delta_col_nrm + REALLY_SMALL_FLOAT_VAL); + + float lx = orig_lx, ly = orig_ly, lz = orig_lz; + lx *= -dx; ly *= -dy; lz *= -dz; + + dx *= f; dy *= f; dz *= f; + float l_sum = (lx + ly + lz) * f; + + for (uint32_t i = 0; i < ps.m_num_pixels; i++) + { + const vec4F& p = ps.m_pixels_f[i]; + const float x = p[c0], y = p[c1], z = p[c2]; + + float weight = x * dx + y * dy + z * dz + l_sum; + + if (weight < 0.0f) + weight = 0.0f; + else if (weight > 1.0f) + weight = 1.0f; + + pWeights0[i] = weight; + + if (compute_error) + { + float one_minus_weight = 1.0f - weight; + + float dec_x = orig_lx * one_minus_weight + orig_hx * weight; + float dec_y = orig_ly * one_minus_weight + orig_hy * weight; + float dec_z = orig_lz * one_minus_weight + orig_hz * weight; + + float diff_x = x - dec_x; + float diff_y = y - dec_y; + float diff_z = z - dec_z; + + total_err += (wx * diff_x * diff_x) + (wy * diff_y * diff_y) + (wz * diff_z * diff_z); + } + + } // i + } + + // Plane 1 + { + const float delta_w = orig_hw - orig_lw; + const float f = (fabsf(delta_w) > REALLY_SMALL_FLOAT_VAL) ? (1.0f / delta_w) : 0.0f; + + for (uint32_t i = 0; i < ps.m_num_pixels; i++) + { + const vec4F& p = ps.m_pixels_f[i]; + const float w = p[ccs_index]; + + float weight = (w - orig_lw) * f; + + if (weight < 0.0f) + weight = 0.0f; + else if (weight > 1.0f) + weight = 1.0f; + + pWeights1[i] = weight; + + if (compute_error) + { + // Error for DP here is 0 if there's no quant and L/H are sufficient to cover the entire span. + if ((w < orig_lw) || (w > orig_hw)) + { + float one_minus_weight = 1.0f - weight; + + float dec_w = orig_lw * one_minus_weight + orig_hw * weight; + + float diff_w = w - dec_w; + + total_err += (ww * diff_w * diff_w); + } + } + + } // i + } + } + else + { + // Plane 0 + { + float dx = orig_hx - orig_lx, dy = orig_hy - orig_ly, dz = orig_hz - orig_lz; + + float delta_col_nrm = dx * dx + dy * dy + dz * dz; + + float f = (float)(num_weight_levels - 1) / (delta_col_nrm + REALLY_SMALL_FLOAT_VAL); + + float lx = orig_lx, ly = orig_ly, lz = orig_lz; + lx *= -dx; ly *= -dy; lz *= -dz; + + dx *= f; dy *= f; dz *= f; + float l_sum_biased = (lx + ly + lz) * f + .5f; + + for (uint32_t i = 0; i < ps.m_num_pixels; i++) + { + const vec4F& p = ps.m_pixels_f[i]; + const float x = p[c0], y = p[c1], z = p[c2]; + + float weight = (float)fast_floorf_int(x * dx + y * dy + z * dz + l_sum_biased) * inv_weight_levels; + + if (weight < 0.0f) + weight = 0.0f; + else if (weight > 1.0f) + weight = 1.0f; + + pWeights0[i] = weight; + + if (compute_error) + { + float one_minus_weight = 1.0f - weight; + + float dec_x = orig_lx * one_minus_weight + orig_hx * weight; + float dec_y = orig_ly * one_minus_weight + orig_hy * weight; + float dec_z = orig_lz * one_minus_weight + orig_hz * weight; + + float diff_x = x - dec_x; + float diff_y = y - dec_y; + float diff_z = z - dec_z; + + total_err += (wx * diff_x * diff_x) + (wy * diff_y * diff_y) + (wz * diff_z * diff_z); + } + + } // i + } + + // Plane 1 + { + const float delta_w = orig_hw - orig_lw; + const float f = (fabs(delta_w) > REALLY_SMALL_FLOAT_VAL) ? ((float)(num_weight_levels - 1) / delta_w) : 0.0f; + + for (uint32_t i = 0; i < ps.m_num_pixels; i++) + { + const vec4F& p = ps.m_pixels_f[i]; + const float w = p[ccs_index]; + + float weight = (float)fast_floorf_int((w - orig_lw) * f + .5f) * inv_weight_levels; + + if (weight < 0.0f) + weight = 0.0f; + else if (weight > 1.0f) + weight = 1.0f; + + pWeights1[i] = weight; + + if (compute_error) + { + float one_minus_weight = 1.0f - weight; + + float dec_w = orig_lw * one_minus_weight + orig_hw * weight; + + float diff_w = w - dec_w; + + total_err += (ww * diff_w * diff_w); + } + + } // i + } + } + + return total_err; + } + + //--------------------------------------------------------------------------------------------- + + float surrogate_quant_endpoint_val(float e, uint32_t num_endpoint_levels, uint32_t flags) + { + assert((e >= 0.0f) && (e <= 1.0f)); + + if (flags & cFlagDisableQuant) + return e; + + const float endpoint_levels_minus_1 = (float)(num_endpoint_levels - 1); + const float inv_endpoint_levels = 1.0f / endpoint_levels_minus_1; + return (float)fast_roundf_pos_int(e * endpoint_levels_minus_1) * inv_endpoint_levels; + } + + vec4F surrogate_quant_endpoint(const vec4F& e, uint32_t num_endpoint_levels, uint32_t flags) + { + if (flags & cFlagDisableQuant) + return e; + + const float endpoint_levels_minus_1 = (float)(num_endpoint_levels - 1); + const float inv_endpoint_levels = 1.0f / endpoint_levels_minus_1; + + assert((e[0] >= 0.0f) && (e[0] <= 1.0f)); + assert((e[1] >= 0.0f) && (e[1] <= 1.0f)); + assert((e[2] >= 0.0f) && (e[2] <= 1.0f)); + assert((e[3] >= 0.0f) && (e[3] <= 1.0f)); + + vec4F res; + res[0] = (float)fast_roundf_pos_int(e[0] * endpoint_levels_minus_1) * inv_endpoint_levels; + res[1] = (float)fast_roundf_pos_int(e[1] * endpoint_levels_minus_1) * inv_endpoint_levels; + res[2] = (float)fast_roundf_pos_int(e[2] * endpoint_levels_minus_1) * inv_endpoint_levels; + res[3] = (float)fast_roundf_pos_int(e[3] * endpoint_levels_minus_1) * inv_endpoint_levels; + + return res; + } + + static uint32_t get_num_weight_levels(uint32_t weight_ise_range) + { + // astc_helpers::BISE_64_LEVELS=raw weights ([0,64], NOT [0,63]) + const uint32_t num_weight_levels = (weight_ise_range == astc_helpers::BISE_64_LEVELS) ? 65 : astc_helpers::get_ise_levels(weight_ise_range); + return num_weight_levels; + } + + //--------------------------------------------------------------------------------------------- + + static float cem_surrogate_encode_cem6_10_sp( + uint32_t cem_index, + const pixel_stats_t& ps, const cem_encode_params& enc_params, + uint32_t endpoint_ise_range, uint32_t weight_ise_range, + vec4F& low_endpoint, vec4F& high_endpoint, float &s, float* pWeights0, uint32_t flags) + { + const uint32_t num_endpoint_levels = astc_helpers::get_ise_levels(endpoint_ise_range); + + // astc_helpers::BISE_64_LEVELS=raw weights ([0,64], NOT [0,63]) + const uint32_t num_weight_levels = get_num_weight_levels(weight_ise_range); + + const bool cem_has_alpha = (cem_index == astc_helpers::CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A); + + float d_min = BIG_FLOAT_VAL, d_max = -BIG_FLOAT_VAL; + + for (uint32_t i = 0; i < ps.m_num_pixels; i++) + { + const vec4F p(ps.m_pixels_f[i]); + + float dot = p.dot3(ps.m_zero_rel_axis3); + + if (dot < d_min) + d_min = dot; + + if (dot > d_max) + d_max = dot; + } + + vec3F low_color3_f(d_min * ps.m_zero_rel_axis3); + low_color3_f.clamp(0.0f, 1.0f); + + vec3F high_color3_f(d_max * ps.m_zero_rel_axis3); + high_color3_f.clamp(0.0f, 1.0f); + + const float MAX_S = 255.0f / 256.0f; + + float scale = MAX_S; + + float d = low_color3_f.dot(high_color3_f); + float nrm = high_color3_f.norm(); + if (nrm > 0.0f) + scale = d / nrm; + + scale = clamp(scale, 0.0f, MAX_S); + + scale = surrogate_quant_endpoint_val(scale * (256.0f / 255.0f), num_endpoint_levels, flags); + + s = scale; + + high_endpoint = surrogate_quant_endpoint(vec4F(high_color3_f[0], high_color3_f[1], high_color3_f[2], cem_has_alpha ? ps.m_max_f[3] : 1.0f), num_endpoint_levels, flags); + + low_endpoint = vec4F(high_endpoint[0] * scale, high_endpoint[1] * scale, high_endpoint[2] * scale, cem_has_alpha ? ps.m_min_f[3] : 1.0f); + + return surrogate_evaluate_rgba_sp(ps, low_endpoint, high_endpoint, pWeights0, num_weight_levels, enc_params, flags); + } + + static float cem_surrogate_encode_cem6_10_dp( + uint32_t cem_index, uint32_t ccs_index, + const pixel_stats_t& ps, const cem_encode_params& enc_params, + uint32_t endpoint_ise_range, uint32_t weight_ise_range, + vec4F& low_endpoint, vec4F& high_endpoint, float& s, float* pWeights0, float* pWeights1, uint32_t flags) + { + const bool cem_has_alpha = (cem_index == astc_helpers::CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A); + BASISU_NOTE_UNUSED(cem_has_alpha); + + // astc_helpers::BISE_64_LEVELS=raw weights ([0,64], NOT [0,63]) + const uint32_t num_weight_levels = get_num_weight_levels(weight_ise_range); + + assert(cem_has_alpha || (ccs_index <= 2)); + + float temp_weights[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + cem_surrogate_encode_cem6_10_sp( + (ccs_index == 3) ? (uint32_t)astc_helpers::CEM_LDR_RGB_BASE_SCALE : cem_index, + ps, enc_params, endpoint_ise_range, weight_ise_range, low_endpoint, high_endpoint, s, temp_weights, flags); + + if (ccs_index == 3) + { + low_endpoint[3] = ps.m_min_f[3]; + high_endpoint[3] = ps.m_max_f[3]; + } + + return surrogate_evaluate_rgba_dp(ccs_index, ps, low_endpoint, high_endpoint, pWeights0, pWeights1, num_weight_levels, enc_params, flags); + } + + static float cem_surrogate_encode_cem8_12_sp( + uint32_t cem_index, + const pixel_stats_t& ps, const cem_encode_params& enc_params, + uint32_t endpoint_ise_range, uint32_t weight_ise_range, + vec4F& low_endpoint, vec4F& high_endpoint, float* pWeights0, uint32_t flags) + { + const uint32_t num_endpoint_levels = astc_helpers::get_ise_levels(endpoint_ise_range); + + // astc_helpers::BISE_64_LEVELS=raw weights ([0,64], NOT [0,63]) + const uint32_t num_weight_levels = get_num_weight_levels(weight_ise_range); + + const bool cem_has_alpha = (cem_index == astc_helpers::CEM_LDR_RGBA_DIRECT); + const uint32_t num_comps = cem_has_alpha ? 4 : 3; + + float d_min = BIG_FLOAT_VAL, d_max = -BIG_FLOAT_VAL; + uint32_t l_idx = 0, h_idx = 0; + + for (uint32_t i = 0; i < ps.m_num_pixels; i++) + { + const vec4F p(ps.m_pixels_f[i] - ps.m_mean_f); + + float dot = cem_has_alpha ? p.dot(ps.m_mean_rel_axis4) : p.dot3(ps.m_mean_rel_axis3); + + if (dot < d_min) + { + d_min = dot; + l_idx = i; + } + + if (dot > d_max) + { + d_max = dot; + h_idx = i; + } + } + + low_endpoint = surrogate_quant_endpoint(ps.m_pixels_f[l_idx], num_endpoint_levels, flags); + high_endpoint = surrogate_quant_endpoint(ps.m_pixels_f[h_idx], num_endpoint_levels, flags); + + if (!cem_has_alpha) + { + low_endpoint[3] = 1.0f; + high_endpoint[3] = 1.0f; + } + + if (low_endpoint.dot(vec4F(1.0f)) > high_endpoint.dot(vec4F(1.0f))) + std::swap(low_endpoint, high_endpoint); + + if ((flags & cFlagDisableQuant) == 0) + { + for (uint32_t i = 0; i < num_comps; i++) + { + if ((low_endpoint[i] == high_endpoint[i]) && (ps.m_min_f[i] != ps.m_max_f[i])) + { + const float inv_endpoint_levels = 1.0f / (float)(num_endpoint_levels - 1); + + float best_dist = BIG_FLOAT_VAL; + float best_l = 0.0f, best_h = 0.0f; + + for (int ld = -2; ld <= 0; ld++) + { + float actual_l = saturate(low_endpoint[i] + (float)ld * inv_endpoint_levels); + + for (int hd = 0; hd <= 2; hd++) + { + float actual_h = saturate(high_endpoint[i] + (float)hd * inv_endpoint_levels); + + float v0 = lerp(actual_l, actual_h, 1.0f / 3.0f); + float v1 = lerp(actual_l, actual_h, 2.0f / 3.0f); + assert(v0 <= v1); + + float dist0 = v0 - ps.m_min_f[0]; + float dist1 = v1 - ps.m_max_f[0]; + + float total_dist = dist0 * dist0 + dist1 * dist1; + if (total_dist < best_dist) + { + best_dist = total_dist; + best_l = actual_l; + best_h = actual_h; + } + } // hd + } // ld + + low_endpoint[i] = best_l; + high_endpoint[i] = best_h; + } + } + } + + return surrogate_evaluate_rgba_sp(ps, low_endpoint, high_endpoint, pWeights0, num_weight_levels, enc_params, flags); + } + + static float cem_surrogate_encode_cem8_12_dp( + uint32_t cem_index, uint32_t ccs_index, + const pixel_stats_t& ps, const cem_encode_params& enc_params, + uint32_t endpoint_ise_range, uint32_t weight_ise_range, + vec4F& low_endpoint, vec4F& high_endpoint, float* pWeights0, float *pWeights1, uint32_t flags) + { + assert(ccs_index <= 3); + const uint32_t num_endpoint_levels = astc_helpers::get_ise_levels(endpoint_ise_range); + + // astc_helpers::BISE_64_LEVELS=raw weights ([0,64], NOT [0,63]) + const uint32_t num_weight_levels = get_num_weight_levels(weight_ise_range); + + const bool cem_has_alpha = (cem_index == astc_helpers::CEM_LDR_RGBA_DIRECT); + const uint32_t num_comps = cem_has_alpha ? 4 : 3; + + assert(cem_has_alpha || (ccs_index <= 2)); + + vec4F flattened_pixels[ASTC_LDR_MAX_BLOCK_PIXELS]; + for (uint32_t i = 0; i < ps.m_num_pixels; i++) + { + flattened_pixels[i] = ps.m_pixels_f[i]; + + flattened_pixels[i][ccs_index] = 0.0f; + + if (!cem_has_alpha) + flattened_pixels[i][3] = 0.0f; + } + + vec4F flattened_pixels_mean(ps.m_mean_f); + flattened_pixels_mean[ccs_index] = 0.0f; + + if (!cem_has_alpha) + flattened_pixels_mean[3] = 0.0f; + + // suppress bogus gcc warning on flattened_pixels +#ifndef __clang__ +#if defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif +#endif + const vec4F flattened_axis(calc_pca_4D(ps.m_num_pixels, flattened_pixels, flattened_pixels_mean)); + +#ifndef __clang__ +#if defined(__GNUC__) +#pragma GCC diagnostic pop +#endif +#endif + + float best_dl = BIG_FLOAT_VAL, best_dh = -BIG_FLOAT_VAL; + int best_l_index = 0, best_h_index = 0; + + for (uint32_t c = 0; c < ps.m_num_pixels; c++) + { + const vec4F px(flattened_pixels[c] - flattened_pixels_mean); + + float p = px.dot(flattened_axis); + if (p < best_dl) + { + best_dl = p; + best_l_index = c; + } + + if (p > best_dh) + { + best_dh = p; + best_h_index = c; + } + } // c + + vec4F low_color_f(ps.m_pixels_f[best_l_index]), high_color_f(ps.m_pixels_f[best_h_index]); + + low_color_f[ccs_index] = 0.0f; + high_color_f[ccs_index] = 0.0f; + + if (!cem_has_alpha) + { + low_color_f[3] = 1.0f; + high_color_f[3] = 1.0f; + } + + if (low_color_f.dot(vec4F(1.0f)) > high_color_f.dot(vec4F(1.0f))) + std::swap(low_color_f, high_color_f); + + low_color_f[ccs_index] = ps.m_min_f[ccs_index]; + high_color_f[ccs_index] = ps.m_max_f[ccs_index]; + + if (!cem_has_alpha) + { + low_color_f[3] = 1.0f; + high_color_f[3] = 1.0f; + } + + low_endpoint = surrogate_quant_endpoint(low_color_f, num_endpoint_levels, flags); + high_endpoint = surrogate_quant_endpoint(high_color_f, num_endpoint_levels, flags); + + if ((flags & cFlagDisableQuant) == 0) + { + for (uint32_t i = 0; i < num_comps; i++) + { + if ((low_endpoint[i] == high_endpoint[i]) && (ps.m_min_f[i] != ps.m_max_f[i])) + { + const float inv_endpoint_levels = 1.0f / (float)(num_endpoint_levels - 1); + + float best_dist = BIG_FLOAT_VAL; + float best_l = 0.0f, best_h = 0.0f; + + for (int ld = -2; ld <= 0; ld++) + { + float actual_l = saturate(low_endpoint[i] + (float)ld * inv_endpoint_levels); + + for (int hd = 0; hd <= 2; hd++) + { + float actual_h = saturate(high_endpoint[i] + (float)hd * inv_endpoint_levels); + + float v0 = lerp(actual_l, actual_h, 1.0f / 3.0f); + float v1 = lerp(actual_l, actual_h, 2.0f / 3.0f); + assert(v0 <= v1); + + //if (v0 > v1) + // std::swap(v0, v1); + + float dist0 = v0 - ps.m_min_f[0]; + float dist1 = v1 - ps.m_max_f[0]; + + float total_dist = dist0 * dist0 + dist1 * dist1; + if (total_dist < best_dist) + { + best_dist = total_dist; + best_l = actual_l; + best_h = actual_h; + } + } // hd + } // ld + + low_endpoint[i] = best_l; + high_endpoint[i] = best_h; + } + } + } + + return surrogate_evaluate_rgba_dp(ccs_index, ps, low_endpoint, high_endpoint, pWeights0, pWeights1, num_weight_levels, enc_params, flags); + } + + static float cem_surrogate_encode_cem0_4_sp_or_dp( + uint32_t cem_index, int ccs_index, + const pixel_stats_t& ps, const cem_encode_params& enc_params, + uint32_t endpoint_ise_range, uint32_t weight_ise_range, + vec4F& low_endpoint, vec4F& high_endpoint, float* pWeights0, float *pWeights1, uint32_t flags) + { + const bool cem_has_alpha = (cem_index == astc_helpers::CEM_LDR_LUM_ALPHA_DIRECT); + const bool dual_plane = (ccs_index == 3); + + if (cem_index == astc_helpers::CEM_LDR_LUM_ALPHA_DIRECT) + { + assert((ccs_index == -1) || (ccs_index == 3)); + } + else + { + assert(cem_index == astc_helpers::CEM_LDR_LUM_DIRECT); + assert(ccs_index == -1); + } + + const uint32_t num_endpoint_levels = astc_helpers::get_ise_levels(endpoint_ise_range); + const uint32_t num_weight_levels = get_num_weight_levels(weight_ise_range); + + float lum_l = BIG_FLOAT_VAL, lum_h = -BIG_FLOAT_VAL; + + for (uint32_t i = 0; i < ps.m_num_pixels; i++) + { + const vec4F& px = ps.m_pixels_f[i]; + + float l = (px[0] + px[1] + px[2]) * (1.0f / 3.0f); + + lum_l = minimum(lum_l, l); + lum_h = maximum(lum_h, l); + } + + const float a_l = cem_has_alpha ? ps.m_min_f[3] : 1.0f; + const float a_h = cem_has_alpha ? ps.m_max_f[3] : 1.0f; + + low_endpoint.set(lum_l, lum_l, lum_l, a_l); + high_endpoint.set(lum_h, lum_h, lum_h, a_h); + + low_endpoint = surrogate_quant_endpoint(low_endpoint, num_endpoint_levels, flags); + high_endpoint = surrogate_quant_endpoint(high_endpoint, num_endpoint_levels, flags); + + if (dual_plane) + return surrogate_evaluate_rgba_dp(ccs_index, ps, low_endpoint, high_endpoint, pWeights0, pWeights1, num_weight_levels, enc_params, flags); + else + return surrogate_evaluate_rgba_sp(ps, low_endpoint, high_endpoint, pWeights0, num_weight_levels, enc_params, flags); + } + + float cem_surrogate_encode_pixels( + uint32_t cem_index, int ccs_index, + const pixel_stats_t& ps, const cem_encode_params& enc_params, + uint32_t endpoint_ise_range, uint32_t weight_ise_range, + vec4F &low_endpoint, vec4F &high_endpoint, float &s, float* pWeights0, float* pWeights1, uint32_t flags) + { + assert(g_initialized); + assert((ccs_index >= -1) && (ccs_index <= 3)); + assert(astc_helpers::is_cem_ldr(cem_index)); + assert(pWeights0 && pWeights1); + + const bool dual_plane = (ccs_index >= 0); + + switch (cem_index) + { + case astc_helpers::CEM_LDR_LUM_DIRECT: + case astc_helpers::CEM_LDR_LUM_ALPHA_DIRECT: + { + return cem_surrogate_encode_cem0_4_sp_or_dp( + cem_index, ccs_index, + ps, enc_params, + endpoint_ise_range, weight_ise_range, + low_endpoint, high_endpoint, pWeights0, pWeights1, flags); + } + case astc_helpers::CEM_LDR_RGB_BASE_SCALE: + case astc_helpers::CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A: + { + if (dual_plane) + { + return cem_surrogate_encode_cem6_10_dp( + cem_index, ccs_index, + ps, enc_params, + endpoint_ise_range, weight_ise_range, + low_endpoint, high_endpoint, s, pWeights0, pWeights1, flags); + } + else + { + return cem_surrogate_encode_cem6_10_sp( + cem_index, + ps, enc_params, + endpoint_ise_range, weight_ise_range, + low_endpoint, high_endpoint, s, pWeights0, flags); + } + break; + } + case astc_helpers::CEM_LDR_RGB_DIRECT: + case astc_helpers::CEM_LDR_RGBA_DIRECT: + { + if (dual_plane) + { + return cem_surrogate_encode_cem8_12_dp( + cem_index, ccs_index, + ps, enc_params, + endpoint_ise_range, weight_ise_range, + low_endpoint, high_endpoint, pWeights0, pWeights1, flags); + } + else + { + return cem_surrogate_encode_cem8_12_sp( + cem_index, + ps, enc_params, + endpoint_ise_range, weight_ise_range, + low_endpoint, high_endpoint, pWeights0, flags); + } + + break; + } + default: + assert(0); + break; + } + + return BIG_FLOAT_VAL; + } + + //--------------------------------------------------------------------------------------------- + + uint8_t g_part3_mapping[NUM_PART3_MAPPINGS][3] = + { + { 0, 1, 2 }, + { 1, 2, 0 }, + { 2, 0, 1 }, + { 0, 2, 1 }, + { 1, 0, 2 }, + { 2, 1, 0 } + }; + + partition_pattern_vec::partition_pattern_vec() + { + clear(); + } + + partition_pattern_vec::partition_pattern_vec(const partition_pattern_vec& other) + { + *this = other; + } + + partition_pattern_vec::partition_pattern_vec(uint32_t width, uint32_t height, const uint8_t *pParts) : + m_width(width), m_height(height) + { + if (pParts) + { + memcpy(m_parts, pParts, get_total()); + } + } + + void partition_pattern_vec::init(uint32_t width, uint32_t height, const uint8_t* pParts) + { + m_width = width; + m_height = height; + if (pParts) + { + const uint32_t num_texels = get_total(); + memcpy(m_parts, pParts, num_texels); + } + } + + void partition_pattern_vec::clear() + { + m_width = 0; + m_height = 0; + memset(m_parts, 0, sizeof(m_parts)); + } + + partition_pattern_vec& partition_pattern_vec::operator= (const partition_pattern_vec& rhs) + { + if (this == &rhs) + return *this; + + m_width = rhs.m_width; + m_height = rhs.m_height; + memcpy(m_parts, rhs.m_parts, get_total()); + + return *this; + } + + // misnamed- just SAD distance, not square + int partition_pattern_vec::get_squared_distance(const partition_pattern_vec& other) const + { + const uint32_t total_pixels = get_total(); + + int total_dist = 0; + for (uint32_t i = 0; i < total_pixels; i++) + total_dist += iabs((int)m_parts[i] - (int)other.m_parts[i]); + + return total_dist; + } + + partition_pattern_vec partition_pattern_vec::get_permuted2(uint32_t permute_index) const + { + assert(permute_index <= 1); + const uint32_t total_pixels = get_total(); + + partition_pattern_vec res(m_width, m_height); + for (uint32_t i = 0; i < total_pixels; i++) + { + assert(m_parts[i] <= 1); + res.m_parts[i] = (uint8_t)(m_parts[i] ^ permute_index); + } + + return res; + } + + partition_pattern_vec partition_pattern_vec::get_permuted3(uint32_t permute_index) const + { + assert(permute_index <= 5); + const uint32_t total_pixels = get_total(); + + partition_pattern_vec res(m_width, m_height); + for (uint32_t i = 0; i < total_pixels; i++) + { + assert(m_parts[i] <= 2); + res.m_parts[i] = g_part3_mapping[permute_index][m_parts[i]]; + } + + return res; + } + + partition_pattern_vec partition_pattern_vec::get_canonicalized() const + { + partition_pattern_vec res(m_width, m_height); + + const uint32_t total_pixels = get_total(); + + int new_labels[4] = { -1, -1, -1, -1 }; + + uint32_t next_index = 0; + for (uint32_t i = 0; i < total_pixels; i++) + { + uint32_t p = m_parts[i]; + assert(p <= 3); + + if (new_labels[p] == -1) + new_labels[p] = next_index++; + + res.m_parts[i] = (uint8_t)new_labels[p]; + } + + return res; + } + + // This requires no redundant patterns, i.e. all must be unique. + bool vp_tree::init(uint32_t n, const partition_pattern_vec* pUnique_pats) + { + clear(); + + uint_vec pat_indices(n); + for (uint32_t i = 0; i < n; i++) + pat_indices[i] = i; + + std::pair root_idx = find_best_vantage_point(n, pUnique_pats, pat_indices); + + if (root_idx.first == -1) + return false; + + m_nodes.resize(1); + m_nodes[0].m_vantage_point = pUnique_pats[root_idx.first]; + m_nodes[0].m_point_index = root_idx.first; + m_nodes[0].m_dist = root_idx.second; + m_nodes[0].m_inner_node = -1; + m_nodes[0].m_outer_node = -1; + + uint_vec inner_list, outer_list; + + inner_list.reserve(n / 2); + outer_list.reserve(n / 2); + + for (uint32_t pat_index = 0; pat_index < n; pat_index++) + { + if ((int)pat_index == root_idx.first) + continue; + + const float dist = m_nodes[0].m_vantage_point.get_distance(pUnique_pats[pat_index]); + + if (dist <= root_idx.second) + inner_list.push_back(pat_index); + else + outer_list.push_back(pat_index); + } + + if (inner_list.size()) + { + m_nodes[0].m_inner_node = create_node(n, pUnique_pats, inner_list); + if (m_nodes[0].m_inner_node < 0) + return false; + } + + if (outer_list.size()) + { + m_nodes[0].m_outer_node = create_node(n, pUnique_pats, outer_list); + if (m_nodes[0].m_outer_node < 0) + return false; + } + + return true; + } + + void vp_tree::find_nearest(uint32_t num_subsets, const partition_pattern_vec& desired_pat, result_queue& results, uint32_t max_results) const + { + assert((num_subsets >= 2) && (num_subsets <= 3)); + + results.clear(); + + if (!m_nodes.size()) + return; + + uint32_t num_desired_pats; + partition_pattern_vec desired_pats[NUM_PART3_MAPPINGS]; + + if (num_subsets == 2) + { + num_desired_pats = 2; + for (uint32_t i = 0; i < 2; i++) + desired_pats[i] = desired_pat.get_permuted2(i); + } + else + { + num_desired_pats = NUM_PART3_MAPPINGS; + for (uint32_t i = 0; i < NUM_PART3_MAPPINGS; i++) + desired_pats[i] = desired_pat.get_permuted3(i); + } + +#if 0 + find_nearest_at_node(0, num_desired_pats, desired_pats, results, max_results); +#else + find_nearest_at_node_non_recursive(0, num_desired_pats, desired_pats, results, max_results); +#endif + } + + void vp_tree::find_nearest_at_node(int node_index, uint32_t num_desired_pats, const partition_pattern_vec* pDesired_pats, result_queue& results, uint32_t max_results) const + { + float best_dist_to_vantage = BIG_FLOAT_VAL; + uint32_t best_mapping = 0; + for (uint32_t i = 0; i < num_desired_pats; i++) + { + float dist = pDesired_pats[i].get_distance(m_nodes[node_index].m_vantage_point); + if (dist < best_dist_to_vantage) + { + best_dist_to_vantage = dist; + best_mapping = i; + } + } + + result r; + r.m_dist = best_dist_to_vantage; + r.m_mapping_index = best_mapping; + r.m_pat_index = m_nodes[node_index].m_point_index; + + results.insert(r, max_results); + + if (best_dist_to_vantage <= m_nodes[node_index].m_dist) + { + // inner first + if (m_nodes[node_index].m_inner_node >= 0) + find_nearest_at_node(m_nodes[node_index].m_inner_node, num_desired_pats, pDesired_pats, results, max_results); + + if (m_nodes[node_index].m_outer_node >= 0) + { + if ((results.get_size() < max_results) || + ((m_nodes[node_index].m_dist - best_dist_to_vantage) <= results.get_highest_dist()) + ) + { + find_nearest_at_node(m_nodes[node_index].m_outer_node, num_desired_pats, pDesired_pats, results, max_results); + } + } + } + else + { + // outer first + if (m_nodes[node_index].m_outer_node >= 0) + find_nearest_at_node(m_nodes[node_index].m_outer_node, num_desired_pats, pDesired_pats, results, max_results); + + if (m_nodes[node_index].m_inner_node >= 0) + { + if ((results.get_size() < max_results) || + ((best_dist_to_vantage - m_nodes[node_index].m_dist) <= results.get_highest_dist()) + ) + { + find_nearest_at_node(m_nodes[node_index].m_inner_node, num_desired_pats, pDesired_pats, results, max_results); + } + } + } + } + + void vp_tree::find_nearest_at_node_non_recursive(int init_node_index, uint32_t num_desired_pats, const partition_pattern_vec* pDesired_pats, result_queue& results, uint32_t max_results) const + { + uint_vec node_stack; + node_stack.reserve(16); + node_stack.push_back(init_node_index); + + do + { + const uint32_t node_index = node_stack.back(); + node_stack.pop_back(); + + float best_dist_to_vantage = BIG_FLOAT_VAL; + uint32_t best_mapping = 0; + for (uint32_t i = 0; i < num_desired_pats; i++) + { + float dist = pDesired_pats[i].get_distance(m_nodes[node_index].m_vantage_point); + if (dist < best_dist_to_vantage) + { + best_dist_to_vantage = dist; + best_mapping = i; + } + } + + result r; + r.m_dist = best_dist_to_vantage; + r.m_mapping_index = best_mapping; + r.m_pat_index = m_nodes[node_index].m_point_index; + + results.insert(r, max_results); + + if (best_dist_to_vantage <= m_nodes[node_index].m_dist) + { + if (m_nodes[node_index].m_outer_node >= 0) + { + if ((results.get_size() < max_results) || + ((m_nodes[node_index].m_dist - best_dist_to_vantage) <= results.get_highest_dist()) + ) + { + node_stack.push_back(m_nodes[node_index].m_outer_node); + } + } + + // inner first + if (m_nodes[node_index].m_inner_node >= 0) + { + node_stack.push_back(m_nodes[node_index].m_inner_node); + } + } + else + { + if (m_nodes[node_index].m_inner_node >= 0) + { + if ((results.get_size() < max_results) || + ((best_dist_to_vantage - m_nodes[node_index].m_dist) <= results.get_highest_dist()) + ) + { + node_stack.push_back(m_nodes[node_index].m_inner_node); + } + } + + // outer first + if (m_nodes[node_index].m_outer_node >= 0) + { + node_stack.push_back(m_nodes[node_index].m_outer_node); + } + } + + } while (!node_stack.empty()); + } + + // returns the index of the new node, or -1 on error + int vp_tree::create_node(uint32_t n, const partition_pattern_vec* pUnique_pats, const uint_vec& pat_indices) + { + std::pair root_idx = find_best_vantage_point(n, pUnique_pats, pat_indices); + + if (root_idx.first < 0) + return -1; + + m_nodes.resize(m_nodes.size() + 1); + const uint32_t new_node_index = m_nodes.size_u32() - 1; + + m_nodes[new_node_index].m_vantage_point = pUnique_pats[root_idx.first]; + m_nodes[new_node_index].m_point_index = root_idx.first; + m_nodes[new_node_index].m_dist = root_idx.second; + m_nodes[new_node_index].m_inner_node = -1; + m_nodes[new_node_index].m_outer_node = -1; + + uint_vec inner_list, outer_list; + + inner_list.reserve(pat_indices.size_u32() / 2); + outer_list.reserve(pat_indices.size_u32() / 2); + + for (uint32_t pat_indices_iter = 0; pat_indices_iter < pat_indices.size(); pat_indices_iter++) + { + const uint32_t pat_index = pat_indices[pat_indices_iter]; + + if ((int)pat_index == root_idx.first) + continue; + + const float dist = m_nodes[new_node_index].m_vantage_point.get_distance(pUnique_pats[pat_index]); + + if (dist <= root_idx.second) + inner_list.push_back(pat_index); + else + outer_list.push_back(pat_index); + } + + if (inner_list.size()) + m_nodes[new_node_index].m_inner_node = create_node(n, pUnique_pats, inner_list); + + if (outer_list.size()) + m_nodes[new_node_index].m_outer_node = create_node(n, pUnique_pats, outer_list); + + return new_node_index; + } + + // returns the pattern index of the vantage point (-1 on error), and the optimal split distance + std::pair vp_tree::find_best_vantage_point(uint32_t num_unique_pats, const partition_pattern_vec* pUnique_pats, const uint_vec& pat_indices) + { + BASISU_NOTE_UNUSED(num_unique_pats); + + const uint32_t n = pat_indices.size_u32(); + + assert(n); + if (n == 1) + return std::pair(pat_indices[0], 0.0f); + + float best_split_metric = -1.0f; + int best_split_pat = -1; + float best_split_dist = 0.0f; + float best_split_var = 0.0f; + + basisu::vector< std::pair > dists; + dists.reserve(n); + + float_vec float_dists; + float_dists.reserve(n); + + for (uint32_t pat_indices_iter = 0; pat_indices_iter < n; pat_indices_iter++) + { + const uint32_t split_pat_index = pat_indices[pat_indices_iter]; + assert(split_pat_index < num_unique_pats); + + const partition_pattern_vec& trial_vantage = pUnique_pats[split_pat_index]; + + dists.resize(0); + float_dists.resize(0); + + for (uint32_t j = 0; j < n; j++) + { + const uint32_t pat_index = pat_indices[j]; + assert(pat_index < num_unique_pats); + + if (pat_index == split_pat_index) + continue; + + float dist = trial_vantage.get_distance(pUnique_pats[pat_index]); + dists.emplace_back(std::pair(dist, pat_index)); + + float_dists.push_back(dist); + } + + stats s; + s.calc(float_dists.size_u32(), float_dists.data()); + + std::sort(dists.begin(), dists.end(), [](const auto& a, const auto& b) { + return a.first < b.first; + }); + + const uint32_t num_dists = dists.size_u32(); + float split_dist = dists[num_dists / 2].first; + if ((num_dists & 1) == 0) + split_dist = (split_dist + dists[(num_dists / 2) - 1].first) * .5f; + + uint32_t total_inner = 0, total_outer = 0; + + for (uint32_t j = 0; j < n; j++) + { + const uint32_t pat_index = pat_indices[j]; + if (pat_index == split_pat_index) + continue; + + float dist = trial_vantage.get_distance(pUnique_pats[pat_index]); + + if (dist <= split_dist) + total_inner++; + else + total_outer++; + } + + float split_metric = (float)minimum(total_inner, total_outer) / (float)maximum(total_inner, total_outer); + + if ((split_metric > best_split_metric) || + ((split_metric == best_split_metric) && (s.m_var > best_split_var))) + { + best_split_metric = split_metric; + best_split_dist = split_dist; + best_split_pat = split_pat_index; + best_split_var = (float)s.m_var; + } + } + + return std::pair(best_split_pat, best_split_dist); + } + + void partitions_data::init(uint32_t num_partitions, uint32_t block_width, uint32_t block_height, bool init_vp_tree) + { + assert((num_partitions >= 2) && (num_partitions <= 4)); + + //const uint32_t total_texels = block_width * block_height; + + m_width = block_width; + m_height = block_height; + m_num_partitions = num_partitions; + + m_part_vp_tree.clear(); + + for (uint32_t i = 0; i < 1024; i++) + { + m_part_seed_to_unique_index[i] = -1; + m_unique_index_to_part_seed[i] = -1; + } + + //const bool is_small_block = astc_helpers::is_small_block(block_width, block_height); + + partition_hash_map part_hash; + part_hash.reserve(1024); + m_total_unique_patterns = 0; + + clear_obj(m_partition_pat_histograms); + + for (uint32_t seed_index = 0; seed_index < astc_helpers::NUM_PARTITION_PATTERNS; seed_index++) + { + partition_pattern_vec pat; + uint32_t part_hist[4] = { 0 }; + + pat.init(block_width, block_height); + + for (uint32_t y = 0; y < block_height; y++) + { + for (uint32_t x = 0; x < block_width; x++) + { + //const uint8_t p = (uint8_t)astc_helpers::compute_texel_partition(seed_index, x, y, 0, m_num_partitions, is_small_block); + const uint8_t p = (uint8_t)astc_helpers::get_precomputed_texel_partition(block_width, block_height, seed_index, x, y, num_partitions); + + assert((p < m_num_partitions) && (p < 4)); + + pat(x, y) = p; + + part_hist[p]++; + } // x + } // y + + bool skip_pat = false; + for (uint32_t i = 0; i < m_num_partitions; i++) + { + if (!part_hist[i]) + { + skip_pat = true; + break; + } + } + if (skip_pat) + continue; + + partition_pattern_vec std_pat(pat.get_canonicalized()); + + if (part_hash.contains(std_pat)) + continue; + + if (num_partitions == 2) + { + assert(!part_hash.contains(pat)); + assert(!part_hash.contains(pat.get_permuted2(1))); + } + else if (num_partitions == 3) + { + for (uint32_t i = 0; i < partition_pattern_vec::cMaxPermute3Index; i++) + { + assert(!part_hash.contains(pat.get_permuted3(i))); + } + } + + for (uint32_t c = 0; c < 4; c++) + m_partition_pat_histograms[m_total_unique_patterns].m_hist[c] = (uint8_t)part_hist[c]; + + part_hash.insert(std_pat, std::make_pair(seed_index, m_total_unique_patterns)); + + m_part_seed_to_unique_index[seed_index] = (int16_t)m_total_unique_patterns; + m_unique_index_to_part_seed[m_total_unique_patterns] = (int16_t)seed_index; + + m_partition_pats[m_total_unique_patterns] = pat; + + m_total_unique_patterns++; + + } // seed_index + + if (init_vp_tree) + m_part_vp_tree.init(m_total_unique_patterns, m_partition_pats); + } + +} // namespace astc_ldr + +} // namespace basisu diff --git a/external/basis_universal/encoder/basisu_astc_ldr_common.h b/external/basis_universal/encoder/basisu_astc_ldr_common.h new file mode 100644 index 0000000000..76e7e3f1ff --- /dev/null +++ b/external/basis_universal/encoder/basisu_astc_ldr_common.h @@ -0,0 +1,445 @@ +// File: basisu_astc_ldr_common.h +#pragma once +#include "basisu_enc.h" +#include "basisu_gpu_texture.h" +#include + +namespace basisu +{ + +namespace astc_ldr +{ + const uint32_t ASTC_LDR_MAX_BLOCK_WIDTH = astc_helpers::MAX_BLOCK_DIM; // 12 + const uint32_t ASTC_LDR_MAX_BLOCK_HEIGHT = astc_helpers::MAX_BLOCK_DIM; // 12 + const uint32_t ASTC_LDR_MAX_BLOCK_PIXELS = astc_helpers::MAX_BLOCK_PIXELS; // 144 + const uint32_t ASTC_LDR_MAX_RAW_WEIGHTS = astc_helpers::MAX_WEIGHT_INTERPOLANT_VALUE + 1; // 65 + + const uint32_t WEIGHT_REFINER_MAX_PASSES = 17; + + inline basist::color_rgba convert_to_basist_color_rgba(const color_rgba& c) + { + return basist::color_rgba(c.r, c.g, c.b, c.a); + } + + struct cem_encode_params + { + uint32_t m_comp_weights[4]; + bool m_decode_mode_srgb; // todo: store astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8 instead, also the alpha mode for srgb because the decoders are broken + + const uint8_t* m_pForced_weight_vals0; + const uint8_t* m_pForced_weight_vals1; + + uint32_t m_max_ls_passes, m_total_weight_refine_passes; + bool m_worst_weight_nudging_flag; + bool m_endpoint_refinement_flag; + + cem_encode_params() + { + init(); + } + + void init() + { + m_comp_weights[0] = 1; + m_comp_weights[1] = 1; + m_comp_weights[2] = 1; + m_comp_weights[3] = 1; + + m_decode_mode_srgb = true; + + m_pForced_weight_vals0 = nullptr; + m_pForced_weight_vals1 = nullptr; + + m_max_ls_passes = 3; + m_total_weight_refine_passes = 0; + m_worst_weight_nudging_flag = false; + m_endpoint_refinement_flag = false; + } + + float get_total_comp_weights() const + { + return (float)(m_comp_weights[0] + m_comp_weights[1] + m_comp_weights[2] + m_comp_weights[3]); + } + }; + + struct pixel_stats_t + { + uint32_t m_num_pixels; + + color_rgba m_pixels[ASTC_LDR_MAX_BLOCK_PIXELS]; + vec4F m_pixels_f[ASTC_LDR_MAX_BLOCK_PIXELS]; + + color_rgba m_min, m_max; + + vec4F m_min_f, m_max_f; + vec4F m_mean_f; + + // Always 3D, ignoring alpha + vec3F m_mean_rel_axis3; + vec3F m_zero_rel_axis3; + + // Always 4D + vec4F m_mean_rel_axis4; + + bool m_has_alpha; + + stats m_rgba_stats[4]; + + void clear() + { + clear_obj(*this); + } + + void init(uint32_t num_pixels, const color_rgba* pPixels); + + }; // struct struct pixel_stats + + void global_init(); + + void bit_transfer_signed_enc(int& a, int& b); + void bit_transfer_signed_dec(int& a, int& b); // transfers MSB from a to b, a is then [-32,31] + color_rgba blue_contract_enc(color_rgba orig, bool& did_clamp, int encoded_b); + int quant_preserve2(uint32_t ise_range, uint32_t v); + + uint32_t get_colors(const color_rgba& l, const color_rgba& h, uint32_t weight_ise_index, color_rgba* pColors, bool decode_mode_srgb); + uint32_t get_colors_raw_weights(const color_rgba& l, const color_rgba& h, color_rgba* pColors, bool decode_mode_srgb); + void decode_endpoints_ise20(uint32_t cem_index, const uint8_t* pEndpoint_vals, color_rgba& l, color_rgba& h); // assume BISE 20 + void decode_endpoints(uint32_t cem_index, const uint8_t* pEndpoint_vals, uint32_t endpoint_ise_index, color_rgba& l, color_rgba& h, float* pScale = nullptr); + uint32_t get_colors(uint32_t cem_index, const uint8_t* pEndpoint_vals, uint32_t endpoint_ise_index, uint32_t weight_ise_index, color_rgba* pColors, bool decode_mode_srgb); + uint32_t get_colors_raw_weights(uint32_t cem_index, const uint8_t* pEndpoint_vals, uint32_t endpoint_ise_index, color_rgba* pColors, bool decode_mode_srgb); + + //int apply_delta_to_bise_endpoint_val(uint32_t endpoint_ise_range, int ise_val, int delta); + int apply_delta_to_bise_weight_val(uint32_t weight_ise_range, int ise_val, int delta); + + uint64_t eval_solution( + const pixel_stats_t& pixel_stats, + uint32_t total_weights, const color_rgba* pWeight_colors, + uint8_t* pWeight_vals, uint32_t weight_ise_index, + const cem_encode_params& params); + + uint64_t eval_solution( + const pixel_stats_t& pixel_stats, + uint32_t cem_index, + const uint8_t* pEndpoint_vals, uint32_t endpoint_ise_index, + uint8_t* pWeight_vals, uint32_t weight_ise_index, + const cem_encode_params& params); + + uint64_t eval_solution_dp( + uint32_t ccs_index, + const pixel_stats_t& pixel_stats, + uint32_t total_weights, const color_rgba* pWeight_colors, + uint8_t* pWeight_vals0, uint8_t* pWeight_vals1, uint32_t weight_ise_index, + const cem_encode_params& params); + + uint64_t eval_solution_dp( + const pixel_stats_t& pixel_stats, + uint32_t cem_index, uint32_t ccs_index, + const uint8_t* pEndpoint_vals, uint32_t endpoint_ise_index, + uint8_t* pWeight_vals0, uint8_t* pWeight_vals1, uint32_t weight_ise_index, + const cem_encode_params& params); + + //bool cem8_or_12_used_blue_contraction(uint32_t cem_index, const uint8_t* pEndpoint_vals, uint32_t endpoint_ise_index); + //bool cem9_or_13_used_blue_contraction(uint32_t cem_index, const uint8_t* pEndpoint_vals, uint32_t endpoint_ise_index); + //bool used_blue_contraction(uint32_t cem_index, const uint8_t* pEndpoint_vals, uint32_t endpoint_ise_index); + + uint64_t cem_encode_pixels( + uint32_t cem_index, int ccs_index, + const pixel_stats_t& pixel_stats, const cem_encode_params& enc_params, + uint32_t endpoint_ise_range, uint32_t weight_ise_range, + uint8_t* pEndpoint_vals, uint8_t* pWeight_vals0, uint8_t* pWeight_vals1, uint64_t cur_blk_error, + bool use_blue_contraction, bool* pBase_ofs_clamped_flag); + + // TODO: Rename, confusing vs. std::vector or basisu::vector or vec4F etc. + struct partition_pattern_vec + { + uint32_t m_width, m_height; + uint8_t m_parts[ASTC_LDR_MAX_BLOCK_PIXELS]; + + partition_pattern_vec(); + + partition_pattern_vec(const partition_pattern_vec& other); + + partition_pattern_vec(uint32_t width, uint32_t height, const uint8_t* pParts = nullptr); + + void init(uint32_t width, uint32_t height, const uint8_t* pParts = nullptr); + + void init_part_hist(); + + void clear(); + + partition_pattern_vec& operator= (const partition_pattern_vec& rhs); + + uint32_t get_width() const { return m_width; } + uint32_t get_height() const { return m_height; } + uint32_t get_total() const { return m_width * m_height; } + + uint8_t operator[] (uint32_t i) const { assert(i < get_total()); return m_parts[i]; } + uint8_t& operator[] (uint32_t i) { assert(i < get_total()); return m_parts[i]; } + + uint8_t operator() (uint32_t x, uint32_t y) const { assert((x < m_width) && (y < m_height)); return m_parts[x + y * m_width]; } + uint8_t& operator() (uint32_t x, uint32_t y) { assert((x < m_width) && (y < m_height)); return m_parts[x + y * m_width]; } + + int get_squared_distance(const partition_pattern_vec& other) const; + + float get_distance(const partition_pattern_vec& other) const + { + return sqrtf((float)get_squared_distance(other)); + } + + enum { cMaxPermute2Index = 1 }; + partition_pattern_vec get_permuted2(uint32_t permute_index) const; + + enum { cMaxPermute3Index = 5 }; + partition_pattern_vec get_permuted3(uint32_t permute_index) const; + + partition_pattern_vec get_canonicalized() const; + + bool operator== (const partition_pattern_vec& rhs) const + { + if ((m_width != rhs.m_width) || (m_height != rhs.m_height)) + return false; + + return memcmp(m_parts, rhs.m_parts, get_total()) == 0; + } + + operator size_t() const + { + return basist::hash_hsieh(m_parts, get_total()); + } + }; + + struct vp_tree_node + { + partition_pattern_vec m_vantage_point; + uint32_t m_point_index; + float m_dist; + + int m_inner_node, m_outer_node; + }; + + const uint32_t NUM_PART3_MAPPINGS = 6; + extern uint8_t g_part3_mapping[NUM_PART3_MAPPINGS][3]; + + class vp_tree + { + public: + vp_tree() + { + } + + void clear() + { + m_nodes.clear(); + } + + // This requires no redundant patterns, i.e. all must be unique. + bool init(uint32_t n, const partition_pattern_vec* pUnique_pats); + + struct result + { + uint32_t m_pat_index; + uint32_t m_mapping_index; + float m_dist; + + bool operator< (const result& rhs) const { return m_dist < rhs.m_dist; } + bool operator> (const result& rhs) const { return m_dist > rhs.m_dist; } + }; + + class result_queue + { + enum { MaxSupportedSize = 512 + 1 }; + + public: + result_queue() : + m_cur_size(0) + { + } + + size_t get_size() const + { + return m_cur_size; + } + + bool empty() const + { + return !m_cur_size; + } + + typedef std::array result_array_type; + + const result_array_type& get_elements() const { return m_elements; } + result_array_type& get_elements() { return m_elements; } + + void clear() + { + m_cur_size = 0; + } + + void reserve(uint32_t n) + { + BASISU_NOTE_UNUSED(n); + } + + const result& top() const + { + assert(m_cur_size); + return m_elements[1]; + } + + bool insert(const result& val, uint32_t max_size) + { + assert(max_size < MaxSupportedSize); + + if (m_cur_size >= MaxSupportedSize) + return false; + + m_elements[++m_cur_size] = val; + up_heap(m_cur_size); + + if (m_cur_size > max_size) + pop(); + + return true; + } + + bool pop() + { + if (m_cur_size == 0) + return false; + + m_elements[1] = m_elements[m_cur_size--]; + down_heap(1); + return true; + } + + float get_highest_dist() const + { + if (!m_cur_size) + return 0.0f; + + return top().m_dist; + } + + private: + result_array_type m_elements; + size_t m_cur_size; + + void up_heap(size_t index) + { + while ((index > 1) && (m_elements[index] > m_elements[index >> 1])) + { + std::swap(m_elements[index], m_elements[index >> 1]); + index >>= 1; + } + } + + void down_heap(size_t index) + { + for (; ; ) + { + size_t largest = index, left_child = 2 * index, right_child = 2 * index + 1; + + if ((left_child <= m_cur_size) && (m_elements[left_child] > m_elements[largest])) + largest = left_child; + + if ((right_child <= m_cur_size) && (m_elements[right_child] > m_elements[largest])) + largest = right_child; + + if (largest == index) + break; + + std::swap(m_elements[index], m_elements[largest]); + index = largest; + } + } + }; + + void find_nearest(uint32_t num_subsets, const partition_pattern_vec& desired_pat, result_queue& results, uint32_t max_results) const; + + private: + basisu::vector m_nodes; + + void find_nearest_at_node(int node_index, uint32_t num_desired_pats, const partition_pattern_vec* pDesired_pats, result_queue& results, uint32_t max_results) const; + + void find_nearest_at_node_non_recursive(int init_node_index, uint32_t num_desired_pats, const partition_pattern_vec* pDesired_pats, result_queue& results, uint32_t max_results) const; + + // returns the index of the new node, or -1 on error + int create_node(uint32_t n, const partition_pattern_vec* pUnique_pats, const uint_vec& pat_indices); + + // returns the pattern index of the vantage point (-1 on error), and the optimal split distance + std::pair find_best_vantage_point(uint32_t num_unique_pats, const partition_pattern_vec* pUnique_pats, const uint_vec& pat_indices); + }; + + typedef basisu::hash_map > partition_hash_map; + + struct partition_pattern_hist + { + uint8_t m_hist[4]; + + partition_pattern_hist() { clear(); } + + void clear() { clear_obj(m_hist); } + }; + + struct partitions_data + { + uint32_t m_width, m_height, m_num_partitions; + partition_pattern_vec m_partition_pats[astc_helpers::NUM_PARTITION_PATTERNS]; // indexed by unique index, NOT the 10-bit ASTC seed/pattern index + + partition_pattern_hist m_partition_pat_histograms[astc_helpers::NUM_PARTITION_PATTERNS]; // indexed by unique index, histograms of each pattern + + // ASTC seed to unique index and vice versa + int16_t m_part_seed_to_unique_index[astc_helpers::NUM_PARTITION_PATTERNS]; + int16_t m_unique_index_to_part_seed[astc_helpers::NUM_PARTITION_PATTERNS]; + + // Total number of unique patterns + uint32_t m_total_unique_patterns; + + // VP tree used to rapidly find nearby/similar patterns. + vp_tree m_part_vp_tree; + + void init(uint32_t num_partitions, uint32_t block_width, uint32_t block_height, bool init_vp_tree = true); + }; + + float surrogate_quant_endpoint_val(float e, uint32_t num_endpoint_levels, uint32_t flags); + vec4F surrogate_quant_endpoint(const vec4F& e, uint32_t num_endpoint_levels, uint32_t flags); + + float surrogate_evaluate_rgba_sp(const pixel_stats_t& ps, const vec4F& l, const vec4F& h, float* pWeights0, uint32_t num_weight_levels, const cem_encode_params& enc_params, uint32_t flags); + float surrogate_evaluate_rgba_dp(uint32_t ccs_index, const pixel_stats_t& ps, const vec4F& l, const vec4F& h, float* pWeights0, float* pWeights1, uint32_t num_weight_levels, const cem_encode_params& enc_params, uint32_t flags); + + enum + { + cFlagDisableQuant = 1, + cFlagNoError = 2 + } + ; + float cem_surrogate_encode_pixels( + uint32_t cem_index, int ccs_index, + const pixel_stats_t& pixel_stats, const cem_encode_params& enc_params, + uint32_t endpoint_ise_range, uint32_t weight_ise_range, + vec4F& low_endpoint, vec4F& high_endpoint, float& s, float* pWeights0, float* pWeights1, uint32_t flags = 0); + +#if 0 + bool requantize_ise_endpoints(uint32_t cem, + uint32_t src_ise_endpoint_range, const uint8_t* pSrc_endpoints, + uint32_t dst_ise_endpoint_range, uint8_t* pDst_endpoints); + + uint32_t get_base_cem_without_alpha(uint32_t cem); + + bool pack_base_offset( + uint32_t cem_index, uint32_t dst_ise_endpoint_range, uint8_t* pPacked_endpoints, + const color_rgba& l, const color_rgba& h, + bool use_blue_contraction, bool auto_disable_blue_contraction_if_clamped, + bool& blue_contraction_clamped_flag, bool& base_ofs_clamped_flag, bool& endpoints_swapped); + + bool convert_endpoints_across_cems( + uint32_t prev_cem, uint32_t prev_endpoint_ise_range, const uint8_t* pPrev_endpoints, + uint32_t dst_cem, uint32_t dst_endpoint_ise_range, uint8_t* pDst_endpoints, + bool always_repack, + bool use_blue_contraction, bool auto_disable_blue_contraction_if_clamped, + bool& blue_contraction_clamped_flag, bool& base_ofs_clamped_flag); +#endif + +} // namespace astc_ldr + +} // namespace basisu diff --git a/external/basis_universal/encoder/basisu_astc_ldr_encode.cpp b/external/basis_universal/encoder/basisu_astc_ldr_encode.cpp new file mode 100644 index 0000000000..e1424ba38a --- /dev/null +++ b/external/basis_universal/encoder/basisu_astc_ldr_encode.cpp @@ -0,0 +1,11095 @@ +// File: basisu_astc_ldr_encode.cpp +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "basisu_enc.h" +#include "basisu_astc_ldr_encode.h" +#include "basisu_astc_hdr_common.h" +#include "basisu_astc_ldr_common.h" +#include "3rdparty/android_astc_decomp.h" + +// pick up BASISD_SUPPORT_KTX2_ZSTD macro (this defines it automatically and sets to 1 if not defined) +#include "../transcoder/basisu_transcoder.h" + +#include + +#ifndef BASISD_SUPPORT_KTX2_ZSTD +#error BASISD_SUPPORT_KTX2_ZSTD must be defined here +#endif + +#if BASISD_SUPPORT_KTX2_ZSTD +#include "../zstd/zstd.h" +#endif + +namespace basisu { +namespace astc_ldr { + +const bool g_devel_messages = true; +const bool ASTC_LDR_CONSISTENCY_CHECKING = true; + +bool g_initialized; + +const uint32_t EXPECTED_SUPERBUCKET_HASH_SIZE = 8192; +const uint32_t EXPECTED_SHORTLIST_HASH_SIZE = 4096; + +const uint32_t MAX_BASE_PARTS2 = 128; +const uint32_t MAX_BASE_PARTS3 = 128; + +const uint32_t PART_ESTIMATE_STAGE1_MULTIPLIER = 4; + +const uint32_t MAX_WIDTH = 65535, MAX_HEIGHT = 65535; + +void code_block_weights( + basist::astc_ldr_t::grid_weight_dct &gw_dct, + float q, uint32_t plane_index, + const astc_helpers::log_astc_block& log_blk, + const basist::astc_ldr_t::astc_block_grid_data* pGrid_data, + basisu::bitwise_coder& c, + basist::astc_ldr_t::dct_syms& syms) +{ + assert(q > 0.0f); + + syms.clear(); + + const uint32_t grid_width = log_blk.m_grid_width, grid_height = log_blk.m_grid_height; + const uint32_t total_grid_samples = grid_width * grid_height; + const uint32_t num_planes = log_blk.m_dual_plane ? 2 : 1; + + //const auto& dequant_tab = astc_helpers::g_dequant_tables.get_weight_tab(log_blk.m_weight_ise_range).m_ISE_to_val; + //const auto& quant_tab = astc_helpers::g_dequant_tables.get_weight_tab(log_blk.m_weight_ise_range).m_val_to_ise; + + uint8_t dequantized_raw_weights0[astc_helpers::MAX_BLOCK_PIXELS]; + + for (uint32_t i = 0; i < grid_width * grid_height; i++) + dequantized_raw_weights0[i] = astc_helpers::g_dequant_tables.get_weight_tab(log_blk.m_weight_ise_range).m_ISE_to_val[log_blk.m_weights[i * num_planes + plane_index]]; + + auto grid_dim_vals_iter = gw_dct.m_grid_dim_key_vals.find(basist::astc_ldr_t::grid_dim_key(grid_width, grid_height)); + assert(grid_dim_vals_iter != gw_dct.m_grid_dim_key_vals.end()); + + auto& grid_dim_vals = grid_dim_vals_iter->second; + + float orig_weights[astc_helpers::MAX_BLOCK_PIXELS]; + float weight_sum = 0; + for (uint32_t y = 0; y < grid_height; y++) + { + for (uint32_t x = 0; x < grid_width; x++) + { + orig_weights[x + y * grid_width] = dequantized_raw_weights0[x + y * grid_width]; + weight_sum += orig_weights[x + y * grid_width]; + } + } + + float scaled_weight_coding_scale = basist::astc_ldr_t::SCALED_WEIGHT_BASE_CODING_SCALE; + if (log_blk.m_weight_ise_range <= astc_helpers::BISE_8_LEVELS) + scaled_weight_coding_scale = 1.0f / 8.0f; + + float scaled_mean_weight = std::round((float)scaled_weight_coding_scale * (weight_sum / total_grid_samples)); + scaled_mean_weight = basisu::clamp(scaled_mean_weight, 0.0f, 64.0f * (float)scaled_weight_coding_scale); + + float mean_weight = scaled_mean_weight / (float)scaled_weight_coding_scale; + + for (uint32_t y = 0; y < grid_height; y++) + for (uint32_t x = 0; x < grid_width; x++) + orig_weights[x + y * grid_width] -= mean_weight; + + const float span_len = gw_dct.get_max_span_len(log_blk, plane_index); + + float dct_weights[astc_helpers::MAX_BLOCK_PIXELS]; + + // TODO - temp alloc + basist::astc_ldr_t::fvec dct_work; + grid_dim_vals.m_dct.forward(orig_weights, dct_weights, dct_work); + + const float level_scale = gw_dct.compute_level_scale(q, span_len, pGrid_data->m_weight_gamma, grid_width, grid_height, log_blk.m_weight_ise_range); + + int dct_quant_tab[astc_helpers::MAX_BLOCK_PIXELS]; + gw_dct.compute_quant_table(q, grid_width, grid_height, level_scale, dct_quant_tab); + +#if defined(DEBUG) || defined(_DEBUG) + // sanity checking + basist::astc_ldr_t::sample_quant_table_state quant_state; + quant_state.init(q, gw_dct.m_block_width, gw_dct.m_block_height, level_scale); +#endif + + c.put_truncated_binary((int)scaled_mean_weight, (uint32_t)(64.0f * scaled_weight_coding_scale) + 1); + + syms.m_dc_sym = (int)scaled_mean_weight; + syms.m_num_dc_levels = (uint32_t)(64.0f * scaled_weight_coding_scale) + 1; + assert(syms.m_num_dc_levels == gw_dct.get_num_weight_dc_levels(log_blk.m_weight_ise_range)); + + int dct_coeffs[astc_helpers::MAX_BLOCK_PIXELS]; + + for (uint32_t y = 0; y < grid_height; y++) + { + for (uint32_t x = 0; x < grid_width; x++) + { + if (!x && !y) + { + dct_coeffs[0] = 0; + continue; + } + + const int levels = dct_quant_tab[x + y * grid_width]; + +#if defined(DEBUG) || defined(_DEBUG) + // sanity checking + assert(levels == gw_dct.sample_quant_table(quant_state, x, y)); +#endif + + float d = dct_weights[x + y * grid_width]; + + int id = gw_dct.quantize_deadzone(d, levels, basist::astc_ldr_t::DEADZONE_ALPHA, x, y); + + dct_coeffs[x + y * grid_width] = id; + + } // x + + } // y + + const basisu::int_vec& zigzag = grid_dim_vals.m_zigzag; + assert(zigzag.size() == total_grid_samples); + + int total_zeros = 0; + for (uint32_t i = 0; i < total_grid_samples; i++) + { + uint32_t dct_idx = zigzag[i]; + if (!dct_idx) + continue; + + int coeff = dct_coeffs[dct_idx]; + if (!coeff) + { + total_zeros++; + continue; + } + + basist::astc_ldr_t::dct_syms::coeff cf; + cf.m_num_zeros = basisu::safe_cast_uint16(total_zeros); + cf.m_coeff = basisu::safe_cast_int16(coeff); + syms.m_coeffs.push_back(cf); + syms.m_max_coeff_mag = basisu::maximum(syms.m_max_coeff_mag, basisu::iabs(coeff)); + syms.m_max_zigzag_index = basisu::maximum(syms.m_max_zigzag_index, i); + + c.put_rice(total_zeros, gw_dct.m_zero_run); + total_zeros = 0; + + c.put_bits(coeff < 0 ? 1 : 0, 1); + + if (coeff < 0) + coeff = -coeff; + + c.put_rice(coeff, gw_dct.m_coeff); + } + + if (total_zeros) + { + basist::astc_ldr_t::dct_syms::coeff cf; + cf.m_num_zeros = basisu::safe_cast_uint16(total_zeros); + cf.m_coeff = INT16_MAX; + syms.m_coeffs.push_back(cf); + + c.put_rice(total_zeros, gw_dct.m_zero_run); + } +} + +void astc_ldr_requantize_astc_weights(uint32_t n, const uint8_t* pSrc_ise_vals, uint32_t from_ise_range, uint8_t* pDst_ise_vals, uint32_t to_ise_range) +{ + if (from_ise_range == to_ise_range) + { + if (pDst_ise_vals != pSrc_ise_vals) + memcpy(pDst_ise_vals, pSrc_ise_vals, n); + return; + } + + // from/to BISE ranges not equal + if (from_ise_range == astc_helpers::BISE_64_LEVELS) + { + // from [0,64] + const auto& quant_tab = astc_helpers::g_dequant_tables.get_weight_tab(to_ise_range).m_val_to_ise; + + for (uint32_t i = 0; i < n; i++) + pDst_ise_vals[i] = quant_tab[pSrc_ise_vals[i]]; + } + else if (to_ise_range == astc_helpers::BISE_64_LEVELS) + { + // to [0,64] + const auto& dequant_tab = astc_helpers::g_dequant_tables.get_weight_tab(from_ise_range).m_ISE_to_val; + + for (uint32_t i = 0; i < n; i++) + pDst_ise_vals[i] = dequant_tab[pSrc_ise_vals[i]]; + } + else + { + // from/to any other + const auto& dequant_tab = astc_helpers::g_dequant_tables.get_weight_tab(from_ise_range).m_ISE_to_val; + const auto& quant_tab = astc_helpers::g_dequant_tables.get_weight_tab(to_ise_range).m_val_to_ise; + + for (uint32_t i = 0; i < n; i++) + pDst_ise_vals[i] = quant_tab[dequant_tab[pSrc_ise_vals[i]]]; + } +} + +void astc_ldr_downsample_ise_weights( + uint32_t dequant_weight_ise_range, uint32_t quant_weight_ise_range, + uint32_t block_w, uint32_t block_h, + uint32_t grid_w, uint32_t grid_h, + const uint8_t* pSrc_weights, uint8_t* pDst_weights, + const float* pDownsample_matrix) +{ + assert((block_w <= astc_ldr::ASTC_LDR_MAX_BLOCK_WIDTH) && (block_h <= astc_ldr::ASTC_LDR_MAX_BLOCK_HEIGHT)); + assert((grid_w >= 2) && (grid_w <= block_w)); + assert((grid_h >= 2) && (grid_h <= block_h)); + + assert(((dequant_weight_ise_range >= astc_helpers::FIRST_VALID_WEIGHT_ISE_RANGE) && (dequant_weight_ise_range <= astc_helpers::LAST_VALID_WEIGHT_ISE_RANGE)) || + (dequant_weight_ise_range == astc_helpers::BISE_64_LEVELS)); + + assert(((quant_weight_ise_range >= astc_helpers::FIRST_VALID_WEIGHT_ISE_RANGE) && (quant_weight_ise_range <= astc_helpers::LAST_VALID_WEIGHT_ISE_RANGE)) || + (quant_weight_ise_range == astc_helpers::BISE_64_LEVELS)); + + assert(pDownsample_matrix); + + if ((block_w == grid_w) && (block_h == grid_h)) + { + if (dequant_weight_ise_range != quant_weight_ise_range) + { + astc_ldr_requantize_astc_weights(block_w * block_h, pSrc_weights, dequant_weight_ise_range, pDst_weights, quant_weight_ise_range); + } + else + { + if (pDst_weights != pSrc_weights) + memcpy(pDst_weights, pSrc_weights, block_w * block_h); + } + + return; + } + + uint8_t desired_weights[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + + if (dequant_weight_ise_range == astc_helpers::BISE_64_LEVELS) + { + memcpy(desired_weights, pSrc_weights, block_w * block_h); + } + else + { + const auto& dequant_tab = astc_helpers::g_dequant_tables.get_weight_tab(dequant_weight_ise_range).m_ISE_to_val; + + for (uint32_t by = 0; by < block_h; by++) + for (uint32_t bx = 0; bx < block_w; bx++) + desired_weights[bx + by * block_w] = dequant_tab[pSrc_weights[bx + by * block_w]]; + } + + if (quant_weight_ise_range == astc_helpers::BISE_64_LEVELS) + { + downsample_weight_grid( + pDownsample_matrix, + block_w, block_h, // source/from dimension (block size) + grid_w, grid_h, // dest/to dimension (grid size) + desired_weights, // these are dequantized weights, NOT ISE symbols, [by][bx] + pDst_weights); // [wy][wx] + } + else + { + uint8_t raw_downsampled_weights[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + + downsample_weight_grid( + pDownsample_matrix, + block_w, block_h, // source/from dimension (block size) + grid_w, grid_h, // dest/to dimension (grid size) + desired_weights, // these are dequantized weights, NOT ISE symbols, [by][bx] + raw_downsampled_weights); // [wy][wx] + + const auto& weight_quant_tab = astc_helpers::g_dequant_tables.get_weight_tab(quant_weight_ise_range).m_val_to_ise; + + for (uint32_t gy = 0; gy < grid_h; gy++) + for (uint32_t gx = 0; gx < grid_w; gx++) + pDst_weights[gx + gy * grid_w] = weight_quant_tab[raw_downsampled_weights[gx + gy * grid_w]]; + } +} + +void downsample_weight_residual_grid( + const float* pMatrix_weights, + uint32_t bx, uint32_t by, // source/from dimension (block size) + uint32_t wx, uint32_t wy, // dest/to dimension (grid size) + const int* pSrc_weights, // these are dequantized weights, NOT ISE symbols, [by][bx] + float* pDst_weights) // [wy][wx] +{ + const uint32_t total_block_samples = bx * by; + + for (uint32_t y = 0; y < wy; y++) + { + for (uint32_t x = 0; x < wx; x++) + { + float total = 0.0f; + + for (uint32_t i = 0; i < total_block_samples; i++) + if (pMatrix_weights[i]) + total += pMatrix_weights[i] * (float)pSrc_weights[i]; + + pDst_weights[x + y * wx] = total; + + pMatrix_weights += total_block_samples; + } + } +} + +void downsample_weightsf( + const float* pMatrix_weights, + uint32_t bx, uint32_t by, // source/from dimension (block size) + uint32_t wx, uint32_t wy, // dest/to dimension (grid size) + const float* pSrc_weights, // these are dequantized weights, NOT ISE symbols, [by][bx] + float* pDst_weights) // [wy][wx] +{ + const uint32_t total_block_samples = bx * by; + + for (uint32_t y = 0; y < wy; y++) + { + for (uint32_t x = 0; x < wx; x++) + { + float total = 0.0f; + + for (uint32_t i = 0; i < total_block_samples; i++) + if (pMatrix_weights[i]) + total += pMatrix_weights[i] * pSrc_weights[i]; + + pDst_weights[x + y * wx] = total; + + pMatrix_weights += total_block_samples; + } + } +} + +static inline uint32_t weighted_color_error(const color_rgba& a, const color_rgba& b, const astc_ldr::cem_encode_params& p) +{ + uint32_t total_e = 0; + for (uint32_t c = 0; c < 4; c++) + { + int av = a[c]; + int bv = b[c]; + int ev = av - bv; + total_e += (uint32_t)(ev * ev) * p.m_comp_weights[c]; + } + + return total_e; +} + +uint64_t eval_error( + uint32_t block_width, uint32_t block_height, + const astc_helpers::log_astc_block& enc_log_block, + const astc_ldr::pixel_stats_t& pixel_stats, + const astc_ldr::cem_encode_params& params) +{ + color_rgba dec_block_pixels[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + bool status = astc_helpers::decode_block_xuastc_ldr(enc_log_block, dec_block_pixels, block_width, block_height, params.m_decode_mode_srgb ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8); + if (!status) + { + // Shouldn't ever happen + assert(0); + return UINT64_MAX; + } + +#if defined(_DEBUG) || defined(DEBUG) + // Sanity check vs. unoptimized decoder + color_rgba dec_block_pixels_alt[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + bool alt_status = astc_helpers::decode_block(enc_log_block, dec_block_pixels_alt, block_width, block_height, params.m_decode_mode_srgb ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8); + if (!alt_status) + { + // Shouldn't ever happen + assert(0); + return UINT64_MAX; + } + + if (memcmp(dec_block_pixels, dec_block_pixels_alt, sizeof(color_rgba) * block_width * block_height) != 0) + { + // Very bad + assert(0); + return UINT64_MAX; + } +#endif + + uint64_t total_err = 0; + + const uint32_t total_block_pixels = block_width * block_height; + for (uint32_t i = 0; i < total_block_pixels; i++) + total_err += weighted_color_error(dec_block_pixels[i], pixel_stats.m_pixels[i], params); + + return total_err; +} + +uint64_t eval_error( + uint32_t block_width, uint32_t block_height, + const astc_ldr::pixel_stats_t& pixel_stats, + uint32_t cem_index, + bool dual_plane_flag, int ccs_index, + uint32_t endpoint_ise_range, uint32_t weight_ise_range, + uint32_t grid_width, uint32_t grid_height, + const uint8_t* pEndpoint_vals, const uint8_t* pWeight_grid_vals0, const uint8_t* pWeight_grid_vals1, + const astc_ldr::cem_encode_params& params) +{ + const uint32_t total_block_pixels = block_width * block_height; + const uint32_t total_grid_pixels = grid_width * grid_height; + + astc_helpers::log_astc_block enc_log_block; + + enc_log_block.clear(); + enc_log_block.m_grid_width = (uint8_t)grid_width; + enc_log_block.m_grid_height = (uint8_t)grid_height; + enc_log_block.m_weight_ise_range = (uint8_t)weight_ise_range; + enc_log_block.m_endpoint_ise_range = (uint8_t)endpoint_ise_range; + enc_log_block.m_color_endpoint_modes[0] = (uint8_t)cem_index; + enc_log_block.m_num_partitions = 1; + + memcpy(enc_log_block.m_endpoints, pEndpoint_vals, astc_helpers::get_num_cem_values(cem_index)); + + if (dual_plane_flag) + { + assert((ccs_index >= 0) && (ccs_index <= 3)); + + enc_log_block.m_dual_plane = true; + enc_log_block.m_color_component_selector = (uint8_t)ccs_index; + + for (uint32_t i = 0; i < total_grid_pixels; i++) + { + enc_log_block.m_weights[i * 2 + 0] = pWeight_grid_vals0[i]; + enc_log_block.m_weights[i * 2 + 1] = pWeight_grid_vals1[i]; + } + } + else + { + assert(ccs_index < 0); + + memcpy(enc_log_block.m_weights, pWeight_grid_vals0, total_grid_pixels); + } + + color_rgba decoded_pixels[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + bool status = astc_helpers::decode_block(enc_log_block, decoded_pixels, block_width, block_height, params.m_decode_mode_srgb ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8); + assert(status); + + if (!status) + return UINT64_MAX; + + uint64_t total_err = 0; + + for (uint32_t i = 0; i < total_block_pixels; i++) + total_err += weighted_color_error(pixel_stats.m_pixels[i], decoded_pixels[i], params); + + return total_err; +} + +float compute_psnr_from_wsse(uint32_t block_width, uint32_t block_height, uint64_t sse, float total_comp_weights) +{ + const uint32_t total_block_pixels = block_width * block_height; + const float wmse = (float)sse / (total_comp_weights * (float)total_block_pixels); + const float wpsnr = (wmse > 1e-5f) ? (20.0f * log10f(255.0f / sqrtf(wmse))) : 10000.0f; + return wpsnr; +} + +// quantized coordinate descent (QCD), quadratic objective +namespace qcd +{ + struct qcd_min_solver + { + // geometry / sizes + int m_N = 0; // texels + int m_K = 0; // controls + int m_Q = 0; // label count + + // inputs (not owned), (N x K) row-major + const float* m_pU = nullptr; // grid to texel upsample matrix + + // cached + float_vec m_ucols; // N*K, column k at &m_ucols[k*m_N] + float_vec m_alpha; // K, ||u_k||^2 (>= eps) + float_vec m_labels; // Q, sorted unique u-labels (ints in [0..64]), ASTC raw [0,64] weights + + bool m_ready_flag = false; + + // init: cache columns, norms, and label set + bool init(const float* pU_rowmajor, int N, int K, const int* pLabels_u, int Q) + { + if ((!pU_rowmajor) || (!pLabels_u) || (N <= 0) || (K <= 0) || (Q <= 0)) + return false; + + m_pU = pU_rowmajor; + m_N = N; + m_K = K; + m_Q = Q; + + // cache columns + m_ucols.assign(size_t(N) * K, 0.0f); + + for (int k = 0; k < K; ++k) + { + float* pDst = &m_ucols[size_t(k) * size_t(N)]; + const float* pSrc = m_pU + k; // first element of column k + for (int t = 0; t < N; ++t) + pDst[t] = pSrc[size_t(t) * size_t(K)]; + } + + // column norms + m_alpha.resize(K); + + for (int k = 0; k < K; ++k) + { + const float* pUK = &m_ucols[size_t(k) * size_t(N)]; + + float a = 0.0f; + for (int t = 0; t < N; ++t) + a += pUK[t] * pUK[t]; + + if (!(a > 0.0f)) + a = 1e-8f; + + m_alpha[k] = a; + } + + m_labels.assign(pLabels_u, pLabels_u + Q); + +#if defined(_DEBUG) || defined(DEBUG) + for (size_t i = 1; i < m_labels.size(); ++i) + { + assert(m_labels[i] > m_labels[i - 1]); // strictly increasing + assert((m_labels[i] >= 0) && (m_labels[i] <= 64)); + } +#endif + + m_Q = (int)m_labels.size(); + if (m_Q <= 0) + return false; + + m_ready_flag = true; + return true; + } + + // compute residual r = U*g - w* (uses label IDs -> u-values) + void build_residual(const int* pG_idx, const float* pW_star, float* pR_out) const + { + assert(m_ready_flag && pG_idx && pW_star && pR_out); + + // r = sum_k (u_label[pG_idx[k]] * ucol_k) - pW_star + std::fill(pR_out, pR_out + m_N, 0.0f); + + for (int k = 0; k < m_K; ++k) + { + const float* pUK = &m_ucols[size_t(k) * size_t(m_N)]; + const float s = m_labels[pG_idx[k]]; + + for (int t = 0; t < m_N; ++t) + pR_out[t] += s * pUK[t]; + } + + for (int t = 0; t < m_N; ++t) + pR_out[t] -= pW_star[t]; + } + + // one QCD sweep: returns num moves accepted (strict dE < -eps) + int sweep(int* pG_idx, float* pR_io, float accept_eps = 1e-6f) const + { + assert(m_ready_flag && pG_idx && pR_io); + int num_moved = 0; + + for (int k = 0; k < m_K; ++k) + { + const float* pUK = &m_ucols[size_t(k) * size_t(m_N)]; + + // beta = + float beta = 0.0f; + for (int t = 0; t < m_N; ++t) + beta += pR_io[t] * pUK[t]; + + const float a = m_alpha[k]; // >= 1e-8 + + const float cur_u = m_labels[pG_idx[k]]; + const float s_star = cur_u - beta / a; // continuous minimizer (u-domain) + + // nearest label index to s_star (binary search) + const int j0 = nearest_label_idx(s_star); + + const int cand[3] = + { + j0, + (j0 + 1 < m_Q) ? (j0 + 1) : j0, + (j0 - 1 >= 0) ? (j0 - 1) : j0 + }; + + int best_j = pG_idx[k]; + float best_dE = 0.0f; + + for (int c = 0; c < 3; ++c) + { + const int j = cand[c]; + if (j == pG_idx[k]) + continue; + + const float s = m_labels[j]; + const float d = s - cur_u; // u-change at coord k + const float dE = 2.0f * d * beta + d * d * a; // exact delta E + + if ((best_j == pG_idx[k]) || (dE < best_dE)) + { + best_dE = dE; + best_j = j; + } + } + + if ((best_j != pG_idx[k]) && (best_dE < -accept_eps)) + { + // commit: update residual and label ID + const float d = m_labels[best_j] - cur_u; + + for (int t = 0; t < m_N; ++t) + pR_io[t] += d * pUK[t]; + + pG_idx[k] = best_j; + ++num_moved; + } + } // k + + return num_moved; + } + + // utility: energy from residual (sum r^2) + float residual_energy(const float* pR) const + { + assert(pR); + + float E = 0.0f; + for (int t = 0; t < m_N; ++t) + E += pR[t] * pR[t]; + + return E; + } + + private: + // nearest label index by u-value (handles non-uniform spacing) + int nearest_label_idx(float x) const + { + const int Q = m_Q; + + if (Q <= 1) + return 0; + if (x <= m_labels.front()) + return 0; + if (x >= m_labels.back()) + return Q - 1; + + int lo = 0, hi = Q - 1; + while (hi - lo > 1) + { + const int mid = (lo + hi) >> 1; + (x >= m_labels[mid]) ? lo = mid : hi = mid; + } + + const float dlo = std::fabs(x - m_labels[lo]); + const float dhi = std::fabs(x - m_labels[hi]); + return (dlo <= dhi) ? lo : hi; + } + }; + +} // namespace qcd + +// 1-3 subsets, requires initial weights +bool polish_block_weights( + uint32_t block_width, uint32_t block_height, + const astc_ldr::pixel_stats_t& pixel_stats, + astc_helpers::log_astc_block& enc_log_block, // assumes there is already a good encoding to improve here + const astc_ldr::cem_encode_params& params, + const astc_ldr::partition_pattern_vec* pPat, + bool& improved_flag, + bool gradient_descent_flag, bool polish_weights_flag, bool qcd_enabled_flag) +{ + improved_flag = false; + + if (!gradient_descent_flag && !polish_weights_flag && !qcd_enabled_flag) + return true; + + const uint32_t grid_width = enc_log_block.m_grid_width, grid_height = enc_log_block.m_grid_height; + const uint32_t cem_index = enc_log_block.m_color_endpoint_modes[0]; + const uint32_t num_subsets = enc_log_block.m_num_partitions; + const bool dual_plane_flag = enc_log_block.m_dual_plane; + //const uint32_t num_planes = dual_plane_flag ? 2 : 1; + const int ccs_index = dual_plane_flag ? enc_log_block.m_color_component_selector : -1; + + const uint32_t endpoint_ise_range = enc_log_block.m_endpoint_ise_range; + const uint32_t weight_ise_range = enc_log_block.m_weight_ise_range; + + const auto& dequant_tab = astc_helpers::g_dequant_tables.get_weight_tab(weight_ise_range).m_ISE_to_val; + const auto& quant_tab = astc_helpers::g_dequant_tables.get_weight_tab(weight_ise_range).m_val_to_ise; + + //const bool is_downsampling = (grid_width < block_width) || (grid_height < block_height); + +#if defined(_DEBUG) || defined(DEBUG) + if (num_subsets > 1) + { + for (uint32_t i = 1; i < num_subsets; i++) + { + assert(enc_log_block.m_color_endpoint_modes[i] == cem_index); + } + } +#endif + + //const astc_block_grid_data* pBlock_grid_data = find_astc_block_grid_data(block_width, block_height, grid_width, grid_height); + + const uint32_t total_block_pixels = block_width * block_height; + const uint32_t total_grid_pixels = grid_width * grid_height; + + uint64_t cur_err = eval_error(block_width, block_height, enc_log_block, pixel_stats, params); + + uint8_t weights0[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + uint8_t weights1[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + + astc_helpers::extract_weights(enc_log_block, weights0, 0); + + if (dual_plane_flag) + astc_helpers::extract_weights(enc_log_block, weights1, 1); + + const bool global_gradient_desc_enabled = true; + const bool global_qcd_enabled = true; + const bool global_polish_weights_enabled = true; + + const uint32_t NUM_WEIGHT_POLISH_PASSES = 1; + + // Gradient descent + if ((gradient_descent_flag) && (global_gradient_desc_enabled)) + { + // Downsample the residuals to grid res + vector2D upsample_matrix; + compute_upsample_matrix(upsample_matrix, block_width, block_height, grid_width, grid_height); + + // First compute the block's ideal raw weights given the current endpoints at full block/texel res + // TODO: Move to helper + uint8_t ideal_block_raw_weights0[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS], ideal_block_raw_weights1[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + + if (num_subsets == 1) + { + if (dual_plane_flag) + astc_ldr::eval_solution_dp(pixel_stats, cem_index, ccs_index, enc_log_block.m_endpoints, endpoint_ise_range, ideal_block_raw_weights0, ideal_block_raw_weights1, astc_helpers::BISE_64_LEVELS, params); + else + astc_ldr::eval_solution(pixel_stats, cem_index, enc_log_block.m_endpoints, endpoint_ise_range, ideal_block_raw_weights0, astc_helpers::BISE_64_LEVELS, params); + } + else + { + // Extract each subset's texels, compute the raw weights, place back into full res texel/block weight grid. + color_rgba part_pixels[astc_helpers::MAX_PARTITIONS][astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + uint32_t num_part_pixels[astc_helpers::MAX_PARTITIONS] = { 0 }; + + for (uint32_t y = 0; y < block_height; y++) + { + for (uint32_t x = 0; x < block_width; x++) + { + const color_rgba& px = pixel_stats.m_pixels[x + y * block_width]; + + const uint32_t part_index = (*pPat)(x, y); + assert(part_index < num_subsets); + + // Sanity check + assert(part_index == (uint32_t)astc_helpers::compute_texel_partition(enc_log_block.m_partition_id, x, y, 0, num_subsets, astc_helpers::is_small_block(block_width, block_height))); + + part_pixels[part_index][num_part_pixels[part_index]] = px; + num_part_pixels[part_index]++; + } // x + } // y + + astc_ldr::pixel_stats_t part_pixel_stats[astc_helpers::MAX_PARTITIONS]; + + for (uint32_t i = 0; i < num_subsets; i++) + part_pixel_stats[i].clear(); + + uint8_t part_raw_weights[astc_helpers::MAX_PARTITIONS][astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + + for (uint32_t part_index = 0; part_index < num_subsets; part_index++) + { + part_pixel_stats[part_index].init(num_part_pixels[part_index], &part_pixels[part_index][0]); + + const uint8_t* pPart_endpoints = astc_helpers::get_endpoints(enc_log_block, part_index); + + astc_ldr::eval_solution(part_pixel_stats[part_index], cem_index, pPart_endpoints, endpoint_ise_range, &part_raw_weights[part_index][0], astc_helpers::BISE_64_LEVELS, params); + + } // part_index + + clear_obj(num_part_pixels); + + for (uint32_t y = 0; y < block_height; y++) + { + for (uint32_t x = 0; x < block_width; x++) + { + const uint32_t part_index = (*pPat)(x, y); + assert(part_index < num_subsets); + + ideal_block_raw_weights0[x + y * block_width] = part_raw_weights[part_index][num_part_pixels[part_index]]; + num_part_pixels[part_index]++; + } // x + } // y + } + +#if 1 + // Now compute the current block/texel res (upsampled) raw [0,64] weights given the current quantized grid weights. Dequant then upsample. + // This is what an ASTC decoder would use during unpacking. + uint8_t dequantized_grid_weights0[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS], dequantized_grid_weights1[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + uint8_t dequantized_block_weights_upsampled0[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS], dequantized_block_weights_upsampled1[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + + astc_ldr_requantize_astc_weights(total_grid_pixels, weights0, weight_ise_range, dequantized_grid_weights0, astc_helpers::BISE_64_LEVELS); + + if (dual_plane_flag) + astc_ldr_requantize_astc_weights(total_grid_pixels, weights1, weight_ise_range, dequantized_grid_weights1, astc_helpers::BISE_64_LEVELS); + + astc_helpers::upsample_weight_grid( + block_width, block_height, // destination/to dimension + grid_width, grid_height, // source/from dimension + dequantized_grid_weights0, // these are dequantized [0,64] weights, NOT ISE symbols, [wy][wx] + dequantized_block_weights_upsampled0); // [by][bx] + + if (dual_plane_flag) + { + astc_helpers::upsample_weight_grid( + block_width, block_height, // destination/to dimension + grid_width, grid_height, // source/from dimension + dequantized_grid_weights1, // these are dequantized [0,64] weights, NOT ISE symbols, [wy][wx] + dequantized_block_weights_upsampled1); // [by][bx] + } + + // Now compute residuals at the block res + int weight_block_raw_residuals0[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS], weight_block_raw_residuals1[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + + for (uint32_t i = 0; i < total_block_pixels; i++) + weight_block_raw_residuals0[i] = ideal_block_raw_weights0[i] - dequantized_block_weights_upsampled0[i]; + + if (dual_plane_flag) + { + for (uint32_t i = 0; i < total_block_pixels; i++) + weight_block_raw_residuals1[i] = ideal_block_raw_weights1[i] - dequantized_block_weights_upsampled1[i]; + } + + float weight_grid_residuals_downsampled0[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS], weight_grid_residuals_downsampled1[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + + basisu::vector unweighted_downsample_matrix; + + // TODO: precompute, store in weight grid data + compute_upsample_matrix_transposed(unweighted_downsample_matrix, block_width, block_height, grid_width, grid_height); + + basisu::vector diag_AtA(total_grid_pixels); + compute_diag_AtA_vector(block_width, block_height, grid_width, grid_height, upsample_matrix, diag_AtA.get_ptr()); + + downsample_weight_residual_grid( + unweighted_downsample_matrix.get_ptr(), + block_width, block_height, // source/from dimension (block size) + grid_width, grid_height, // dest/to dimension (grid size) + weight_block_raw_residuals0, // these are dequantized weights, NOT ISE symbols, [by][bx] + weight_grid_residuals_downsampled0); // [wy][wx] + + for (uint32_t i = 0; i < total_grid_pixels; i++) + weight_grid_residuals_downsampled0[i] /= diag_AtA[i]; + + if (dual_plane_flag) + { + downsample_weight_residual_grid( + unweighted_downsample_matrix.get_ptr(), + block_width, block_height, // source/from dimension (block size) + grid_width, grid_height, // dest/to dimension (grid size) + weight_block_raw_residuals1, // these are dequantized weights, NOT ISE symbols, [by][bx] + weight_grid_residuals_downsampled1); // [wy][wx] + + for (uint32_t i = 0; i < total_grid_pixels; i++) + weight_grid_residuals_downsampled1[i] /= diag_AtA[i]; + } + + // Apply the residuals at grid res and quantize + const float Q = 1.0f; + + uint8_t refined_grid_weights0[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS], refined_grid_weights1[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + + for (uint32_t i = 0; i < total_grid_pixels; i++) + { + float v = (float)dequant_tab[weights0[i]] + weight_grid_residuals_downsampled0[i] * Q; + int iv = clamp((int)std::roundf(v), 0, 64); + refined_grid_weights0[i] = quant_tab[iv]; + } + + if (dual_plane_flag) + { + for (uint32_t i = 0; i < total_grid_pixels; i++) + { + float v = (float)dequant_tab[weights1[i]] + weight_grid_residuals_downsampled1[i] * Q; + int iv = clamp((int)std::roundf(v), 0, 64); + refined_grid_weights1[i] = quant_tab[iv]; + } + } +#else + uint8_t refined_grid_weights0[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS], refined_grid_weights1[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + + for (uint32_t i = 0; i < total_grid_pixels; i++) + refined_grid_weights0[i] = weights0[i]; + + if (dual_plane_flag) + { + for (uint32_t i = 0; i < total_grid_pixels; i++) + refined_grid_weights1[i] = weights1[i]; + } +#endif + + astc_helpers::log_astc_block refined_log_block(enc_log_block); + + // TODO: This refines both weight planes simultanously, probably not optimal, could do individually. + astc_helpers::set_weights(refined_log_block, refined_grid_weights0, 0); + + if (dual_plane_flag) + astc_helpers::set_weights(refined_log_block, refined_grid_weights1, 1); + + uint64_t refined_err = eval_error(block_width, block_height, refined_log_block, pixel_stats, params); + + if (refined_err < cur_err) + { + cur_err = refined_err; + + memcpy(weights0, refined_grid_weights0, total_grid_pixels); + + if (dual_plane_flag) + memcpy(weights1, refined_grid_weights1, total_grid_pixels); + + improved_flag = true; + } + + // QCD - not a huge boost (.05-.75 dB), but on the toughest blocks it does help. + if ((qcd_enabled_flag) && (global_qcd_enabled)) + { + float ideal_block_weights0[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS], ideal_block_weights1[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + for (uint32_t i = 0; i < total_block_pixels; i++) + { + ideal_block_weights0[i] = (float)ideal_block_raw_weights0[i]; + + if (dual_plane_flag) + ideal_block_weights1[i] = (float)ideal_block_raw_weights1[i]; + } + + const float* pUpsample_matrix = basist::astc_ldr_t::find_astc_block_grid_data(block_width, block_height, grid_width, grid_height)->m_upsample_matrix.get_ptr(); + + qcd::qcd_min_solver solver; + + const uint32_t num_weight_levels = astc_helpers::get_ise_levels(weight_ise_range); + + assert(num_weight_levels <= 32); + int labels[32 + 1]; + + for (uint32_t i = 0; i < num_weight_levels; i++) + labels[i] = astc_helpers::g_dequant_tables.get_weight_tab(weight_ise_range).get_rank_to_val(i); + + solver.init(pUpsample_matrix, total_block_pixels, total_grid_pixels, labels, num_weight_levels); + + int grid_idx0[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS], grid_idx1[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + + const auto& ise_to_rank = astc_helpers::g_dequant_tables.get_weight_tab(weight_ise_range).m_ISE_to_rank; + + for (uint32_t i = 0; i < total_grid_pixels; i++) + { + grid_idx0[i] = ise_to_rank[refined_grid_weights0[i]]; + + if (dual_plane_flag) + grid_idx1[i] = ise_to_rank[refined_grid_weights1[i]]; + } + + float resid0[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS], resid1[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + + solver.build_residual(grid_idx0, ideal_block_weights0, resid0); + + const uint32_t MAX_QCD_SWEEPS = 5; + for (uint32_t t = 0; t < MAX_QCD_SWEEPS; t++) + { + int moved0 = solver.sweep(grid_idx0, resid0); + if (!moved0) + break; + } + + if (dual_plane_flag) + { + solver.build_residual(grid_idx1, ideal_block_weights1, resid1); + + for (uint32_t t = 0; t < MAX_QCD_SWEEPS; t++) + { + int moved1 = solver.sweep(grid_idx1, resid1); + if (!moved1) + break; + } + } + + const auto& rank_to_ise = astc_helpers::g_dequant_tables.get_weight_tab(weight_ise_range).m_rank_to_ISE; + + for (uint32_t i = 0; i < total_grid_pixels; i++) + { + refined_grid_weights0[i] = rank_to_ise[grid_idx0[i]]; + + if (dual_plane_flag) + refined_grid_weights1[i] = rank_to_ise[grid_idx1[i]]; + } + + refined_log_block = enc_log_block; + + astc_helpers::set_weights(refined_log_block, refined_grid_weights0, 0); + + if (dual_plane_flag) + astc_helpers::set_weights(refined_log_block, refined_grid_weights1, 1); + + refined_err = eval_error(block_width, block_height, refined_log_block, pixel_stats, params); + + if (refined_err < cur_err) + { + cur_err = refined_err; + + memcpy(weights0, refined_grid_weights0, total_grid_pixels); + + if (dual_plane_flag) + memcpy(weights1, refined_grid_weights1, total_grid_pixels); + + improved_flag = true; + } + } + } // if (qcd_enabled) + + if ((polish_weights_flag) && (global_polish_weights_enabled)) + { + // Final, expensive, weight polish. Much can be done to improve this, but it's hopefully not ran much in the first place. + // TODO: The dB gain from this is large, must optimize. + for (uint32_t polish_pass = 0; polish_pass < NUM_WEIGHT_POLISH_PASSES; polish_pass++) + { + for (uint32_t y = 0; y < grid_height; y++) + { + for (uint32_t x = 0; x < grid_width; x++) + { + for (uint32_t plane_iter = 0; plane_iter < (dual_plane_flag ? 2u : 1u); plane_iter++) + { + uint8_t base_grid_weights0[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS], base_grid_weights1[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + + memcpy(base_grid_weights0, weights0, total_grid_pixels); + if (dual_plane_flag) + memcpy(base_grid_weights1, weights1, total_grid_pixels); + + for (int delta = -1; delta <= 1; delta += 2) + { + uint8_t trial_grid_weights0[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS], trial_grid_weights1[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + + memcpy(trial_grid_weights0, base_grid_weights0, total_grid_pixels); + + if (dual_plane_flag) + memcpy(trial_grid_weights1, base_grid_weights1, total_grid_pixels); + + if (plane_iter == 0) + trial_grid_weights0[x + y * grid_width] = (uint8_t)astc_ldr::apply_delta_to_bise_weight_val(weight_ise_range, base_grid_weights0[x + y * grid_width], delta); + else + trial_grid_weights1[x + y * grid_width] = (uint8_t)astc_ldr::apply_delta_to_bise_weight_val(weight_ise_range, base_grid_weights1[x + y * grid_width], delta); + + astc_helpers::log_astc_block trial_log_block(enc_log_block); + + astc_helpers::set_weights(trial_log_block, trial_grid_weights0, 0); + + if (dual_plane_flag) + astc_helpers::set_weights(trial_log_block, trial_grid_weights1, 1); + + uint64_t trial_err = eval_error(block_width, block_height, trial_log_block, pixel_stats, params); + + if (trial_err < cur_err) + { + cur_err = trial_err; + + memcpy(weights0, trial_grid_weights0, total_grid_pixels); + + if (dual_plane_flag) + memcpy(weights1, trial_grid_weights1, total_grid_pixels); + + improved_flag = true; + } + + } // delta + + } // plane_iter + + } // x + } // y + + } // polish_pass + + } // polish_flag + + astc_helpers::log_astc_block new_log_block(enc_log_block); + + astc_helpers::set_weights(new_log_block, weights0, 0); + + if (dual_plane_flag) + astc_helpers::set_weights(new_log_block, weights1, 1); + +#if defined(_DEBUG) || defined(DEBUG) + uint64_t new_err = eval_error(block_width, block_height, new_log_block, pixel_stats, params); + + assert(cur_err == new_err); + + if (improved_flag) + { + uint64_t orig_err = eval_error(block_width, block_height, enc_log_block, pixel_stats, params); + + assert(new_err < orig_err); + } +#endif + + enc_log_block = new_log_block; + + return true; +} + +bool encode_trial_subsets( + uint32_t block_width, uint32_t block_height, + const astc_ldr::pixel_stats_t& pixel_stats, + uint32_t cem_index, uint32_t num_parts, + uint32_t pat_seed_index, const astc_ldr::partition_pattern_vec* pPat, // seed index is a ASTC partition pattern index + uint32_t endpoint_ise_range, uint32_t weight_ise_range, + uint32_t grid_width, uint32_t grid_height, + astc_helpers::log_astc_block& enc_log_block, + const astc_ldr::cem_encode_params& params, + bool refine_only_flag = false, + bool gradient_descent_flag = true, bool polish_weights_flag = true, bool qcd_enabled_flag = true, + bool use_blue_contraction = true, + bool* pBase_ofs_clamped_flag = nullptr) +{ + assert((num_parts >= 2) && (num_parts <= astc_helpers::MAX_PARTITIONS)); + assert(pPat); + assert(pat_seed_index < astc_helpers::NUM_PARTITION_PATTERNS); + + if (pBase_ofs_clamped_flag) + *pBase_ofs_clamped_flag = false; + + const bool is_downsampling = (grid_width < block_width) || (grid_height < block_height); + //const uint32_t total_block_pixels = block_width * block_height; + const uint32_t total_grid_pixels = grid_width * grid_height; + + color_rgba part_pixels[astc_helpers::MAX_PARTITIONS][astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + uint32_t num_part_pixels[astc_helpers::MAX_PARTITIONS] = { 0 }; + + for (uint32_t y = 0; y < block_height; y++) + { + for (uint32_t x = 0; x < block_width; x++) + { + const color_rgba& px = pixel_stats.m_pixels[x + y * block_width]; + + const uint32_t part_index = (*pPat)(x, y); + assert(part_index < num_parts); + + part_pixels[part_index][num_part_pixels[part_index]] = px; + num_part_pixels[part_index]++; + } // x + } // y + +#if defined(_DEBUG) || defined(DEBUG) + for (uint32_t i = 0; i < num_parts; i++) + assert(num_part_pixels[i]); +#endif + + astc_ldr::pixel_stats_t part_pixel_stats[astc_helpers::MAX_PARTITIONS]; + + for (uint32_t i = 0; i < num_parts; i++) + part_pixel_stats[i].clear(); + + uint8_t part_endpoints[astc_helpers::MAX_PARTITIONS][astc_helpers::MAX_CEM_ENDPOINT_VALS]; + uint8_t part_weights[astc_helpers::MAX_PARTITIONS][astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + + for (uint32_t part_index = 0; part_index < num_parts; part_index++) + { + part_pixel_stats[part_index].init(num_part_pixels[part_index], &part_pixels[part_index][0]); + + if (!refine_only_flag) + { + bool base_ofs_clamped_flag = false; + + // Encode at block res, but with quantized weights + uint64_t block_err = astc_ldr::cem_encode_pixels(cem_index, -1, part_pixel_stats[part_index], params, + endpoint_ise_range, weight_ise_range, + &part_endpoints[part_index][0], &part_weights[part_index][0], nullptr, UINT64_MAX, use_blue_contraction, &base_ofs_clamped_flag); + + if (block_err == UINT64_MAX) + return false; + + if ((pBase_ofs_clamped_flag) && (base_ofs_clamped_flag)) + *pBase_ofs_clamped_flag = true; + } + + } // part_index + + const uint32_t num_endpoint_vals = astc_helpers::get_num_cem_values(cem_index); + + if (!refine_only_flag) + { + uint8_t block_weights[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + + clear_obj(num_part_pixels); + + for (uint32_t y = 0; y < block_height; y++) + { + for (uint32_t x = 0; x < block_width; x++) + { + const uint32_t part_index = (*pPat)(x, y); + assert(part_index < num_parts); + + block_weights[x + y * block_width] = part_weights[part_index][num_part_pixels[part_index]]; + num_part_pixels[part_index]++; + } // x + } // y + + enc_log_block.clear(); + + enc_log_block.m_grid_width = (uint8_t)grid_width; + enc_log_block.m_grid_height = (uint8_t)grid_height; + enc_log_block.m_weight_ise_range = (uint8_t)weight_ise_range; + enc_log_block.m_endpoint_ise_range = (uint8_t)endpoint_ise_range; + + enc_log_block.m_num_partitions = (uint8_t)num_parts; + for (uint32_t i = 0; i < num_parts; i++) + enc_log_block.m_color_endpoint_modes[i] = (uint8_t)cem_index; + enc_log_block.m_partition_id = (uint16_t)pat_seed_index; + + if (is_downsampling) + { + // TODO: Make the downsample step faster + const float* pDownsample_matrix = basist::astc_ldr_t::find_astc_block_grid_data(block_width, block_height, grid_width, grid_height)->m_downsample_matrix.get_ptr(); + + // Now downsample the weight grid (quantized to quantized) + astc_ldr_downsample_ise_weights( + weight_ise_range, weight_ise_range, + block_width, block_height, + grid_width, grid_height, + block_weights, enc_log_block.m_weights, + pDownsample_matrix); + } + else + { + memcpy(enc_log_block.m_weights, block_weights, total_grid_pixels); + } + + for (uint32_t p = 0; p < num_parts; p++) + memcpy(enc_log_block.m_endpoints + num_endpoint_vals * p, &part_endpoints[p][0], num_endpoint_vals); + } + + // attempt endpoint refinement given the current weights + // TODO: Expose to caller + const uint32_t NUM_REFINEMENT_PASSES = 3; + for (uint32_t refine_pass = 0; refine_pass < NUM_REFINEMENT_PASSES; refine_pass++) + { + uint8_t dequantized_raw_weights0[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + uint8_t upsampled_weights0[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; // raw weights, NOT ISE + + for (uint32_t i = 0; i < total_grid_pixels; i++) + dequantized_raw_weights0[i] = astc_helpers::g_dequant_tables.get_weight_tab(weight_ise_range).m_ISE_to_val[enc_log_block.m_weights[i]]; + + astc_helpers::upsample_weight_grid(block_width, block_height, grid_width, grid_height, dequantized_raw_weights0, upsampled_weights0); + + astc_helpers::log_astc_block alt_enc_log_block(enc_log_block); + + uint8_t raw_part_weights[astc_helpers::MAX_PARTITIONS][astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + + clear_obj(num_part_pixels); + + for (uint32_t y = 0; y < block_height; y++) + { + for (uint32_t x = 0; x < block_width; x++) + { + const uint32_t part_index = (*pPat)(x, y); + assert(part_index < num_parts); + + raw_part_weights[part_index][num_part_pixels[part_index]] = upsampled_weights0[x + y * block_width]; + num_part_pixels[part_index]++; + } // x + } // y + + for (uint32_t part_index = 0; part_index < num_parts; part_index++) + { + assert(num_part_pixels[part_index] == part_pixel_stats[part_index].m_num_pixels); + + astc_ldr::cem_encode_params temp_params(params); + temp_params.m_pForced_weight_vals0 = &raw_part_weights[part_index][0]; + + uint8_t temp_weights[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + + bool base_ofs_clamped_flag = false; + + // Encode at block res, but with quantized weights + uint64_t block_err = astc_ldr::cem_encode_pixels(cem_index, -1, part_pixel_stats[part_index], temp_params, + endpoint_ise_range, astc_helpers::BISE_64_LEVELS, + &alt_enc_log_block.m_endpoints[num_endpoint_vals * part_index], temp_weights, nullptr, UINT64_MAX, use_blue_contraction, &base_ofs_clamped_flag); + + if (block_err == UINT64_MAX) + return false; + + if ((pBase_ofs_clamped_flag) && (base_ofs_clamped_flag)) + *pBase_ofs_clamped_flag = true; + +#if defined(_DEBUG) || defined(DEBUG) + for (uint32_t i = 0; i < part_pixel_stats[part_index].m_num_pixels; i++) + { + assert(temp_weights[i] == temp_params.m_pForced_weight_vals0[i]); + } +#endif + + } // part_index + + uint64_t cur_err = eval_error(block_width, block_height, enc_log_block, pixel_stats, params); + uint64_t ref_err = eval_error(block_width, block_height, alt_enc_log_block, pixel_stats, params); + + if (ref_err < cur_err) + { + memcpy(&enc_log_block, &alt_enc_log_block, sizeof(astc_helpers::log_astc_block)); + } + + if (refine_pass == (NUM_REFINEMENT_PASSES - 1)) + break; + + if ((is_downsampling) && (gradient_descent_flag || polish_weights_flag)) + { + bool improved_flag = false; + bool status = polish_block_weights(block_width, block_height, pixel_stats, enc_log_block, params, pPat, improved_flag, gradient_descent_flag, polish_weights_flag, qcd_enabled_flag); + if (!status) + { + assert(0); + } + + if (!improved_flag) + break; + } + else + { + break; + } + } // refine_pass + + return true; +} + +bool encode_trial( + uint32_t block_width, uint32_t block_height, + const astc_ldr::pixel_stats_t& pixel_stats, + uint32_t cem_index, + bool dual_plane_flag, int ccs_index, + uint32_t endpoint_ise_range, uint32_t weight_ise_range, + uint32_t grid_width, uint32_t grid_height, + astc_helpers::log_astc_block& enc_log_block, + const astc_ldr::cem_encode_params& params, + bool gradient_descent_flag = true, bool polish_weights_flag = true, bool qcd_enabled_flag = true, + bool use_blue_contraction = true, + bool* pBase_ofs_clamped_flag = nullptr) +{ + assert(dual_plane_flag || (ccs_index == -1)); + + if (pBase_ofs_clamped_flag) + *pBase_ofs_clamped_flag = false; + + const bool is_downsampling = (grid_width < block_width) || (grid_height < block_height); + + const basist::astc_ldr_t::astc_block_grid_data* pBlock_grid_data = basist::astc_ldr_t::find_astc_block_grid_data(block_width, block_height, grid_width, grid_height); + + const float* pDownsample_matrix = nullptr; + if (is_downsampling) + pDownsample_matrix = pBlock_grid_data->m_downsample_matrix.get_ptr(); + + //const uint32_t total_block_pixels = block_width * block_height; + const uint32_t total_grid_pixels = grid_width * grid_height; + + const auto& dequant_tab = astc_helpers::g_dequant_tables.get_weight_tab(weight_ise_range).m_ISE_to_val; + //const auto& quant_tab = astc_helpers::g_dequant_tables.get_weight_tab(weight_ise_range).m_val_to_ise; + + enc_log_block.clear(); + + enc_log_block.m_grid_width = (uint8_t)grid_width; + enc_log_block.m_grid_height = (uint8_t)grid_height; + enc_log_block.m_weight_ise_range = (uint8_t)weight_ise_range; + enc_log_block.m_endpoint_ise_range = (uint8_t)endpoint_ise_range; + + enc_log_block.m_dual_plane = dual_plane_flag; + if (dual_plane_flag) + { + assert((ccs_index >= 0) && (ccs_index <= 3)); + enc_log_block.m_color_component_selector = (uint8_t)ccs_index; + } + else + { + assert(ccs_index == -1); + } + + enc_log_block.m_num_partitions = 1; + enc_log_block.m_color_endpoint_modes[0] = (uint8_t)cem_index; + + uint8_t fullres_endpoints[astc_helpers::MAX_CEM_ENDPOINT_VALS]; + uint8_t weights0[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + uint8_t weights1[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + + if ((grid_width == block_width) && (grid_height == block_height)) + { + bool base_ofs_clamped_flag = false; + + uint64_t block_err = astc_ldr::cem_encode_pixels(cem_index, ccs_index, pixel_stats, params, + endpoint_ise_range, weight_ise_range, + fullres_endpoints, weights0, weights1, UINT64_MAX, use_blue_contraction, &base_ofs_clamped_flag); + + if (block_err == UINT64_MAX) + return false; + + if ((pBase_ofs_clamped_flag) && (base_ofs_clamped_flag)) + *pBase_ofs_clamped_flag = base_ofs_clamped_flag; + + if (dual_plane_flag) + { + for (uint32_t i = 0; i < total_grid_pixels; i++) + { + enc_log_block.m_weights[i * 2 + 0] = weights0[i]; + enc_log_block.m_weights[i * 2 + 1] = weights1[i]; + } + } + else + { + memcpy(enc_log_block.m_weights, weights0, total_grid_pixels); + } + + memcpy(enc_log_block.m_endpoints, fullres_endpoints, astc_helpers::get_num_cem_values(cem_index)); + + return true; + } + + // Handle downsampled weight grids case + + uint8_t fullres_raw_weights0[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + uint8_t fullres_raw_weights1[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + + bool base_ofs_clamped_flag = false; + + // Encode at block res, but with quantized weights + uint64_t block_err = astc_ldr::cem_encode_pixels(cem_index, ccs_index, pixel_stats, params, + endpoint_ise_range, weight_ise_range, + fullres_endpoints, fullres_raw_weights0, fullres_raw_weights1, UINT64_MAX, use_blue_contraction, &base_ofs_clamped_flag); + + if (block_err == UINT64_MAX) + return false; + + if ((pBase_ofs_clamped_flag) && (base_ofs_clamped_flag)) + *pBase_ofs_clamped_flag = base_ofs_clamped_flag; + + // Now downsample the weight grid (quantized to quantized) + astc_ldr_downsample_ise_weights( + weight_ise_range, weight_ise_range, + block_width, block_height, + grid_width, grid_height, + fullres_raw_weights0, weights0, + pDownsample_matrix); + + astc_helpers::set_weights(enc_log_block, weights0, 0); + + if (dual_plane_flag) + { + astc_ldr_downsample_ise_weights( + weight_ise_range, weight_ise_range, + block_width, block_height, + grid_width, grid_height, + fullres_raw_weights1, weights1, + pDownsample_matrix); + } + + if (dual_plane_flag) + astc_helpers::set_weights(enc_log_block, weights1, 1); + + memcpy(enc_log_block.m_endpoints, fullres_endpoints, astc_helpers::get_num_cem_values(cem_index)); + + // TODO: Expose to caller + const uint32_t NUM_OUTER_PASSES = 3; + for (uint32_t outer_pass = 0; outer_pass < NUM_OUTER_PASSES; outer_pass++) + { + // endpoint refinement, given current upsampled weights + { + astc_helpers::extract_weights(enc_log_block, weights0, 0); + + if (dual_plane_flag) + astc_helpers::extract_weights(enc_log_block, weights1, 1); + + // Plane 0 + uint8_t dequantized_raw_weights0[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + uint8_t upsampled_weights0[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; // raw weights, NOT ISE + + for (uint32_t i = 0; i < total_grid_pixels; i++) + dequantized_raw_weights0[i] = dequant_tab[weights0[i]]; + + astc_helpers::upsample_weight_grid(block_width, block_height, grid_width, grid_height, dequantized_raw_weights0, upsampled_weights0); + + // Plane 1 + uint8_t dequantized_raw_weights1[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + uint8_t upsampled_weights1[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; // raw weights, NOT ISE + + if (dual_plane_flag) + { + for (uint32_t i = 0; i < total_grid_pixels; i++) + dequantized_raw_weights1[i] = dequant_tab[weights1[i]]; + astc_helpers::upsample_weight_grid(block_width, block_height, grid_width, grid_height, dequantized_raw_weights1, upsampled_weights1); + } + + // Jam in the weights to the actual raw [0,64] weights the decoder is going to use after upsampling the grid. + astc_ldr::cem_encode_params refine_params(params); + refine_params.m_pForced_weight_vals0 = upsampled_weights0; + if (dual_plane_flag) + refine_params.m_pForced_weight_vals1 = upsampled_weights1; + + uint8_t refined_endpoints[astc_helpers::MAX_CEM_ENDPOINT_VALS]; + uint8_t refined_weights0[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + uint8_t refined_weights1[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + + uint64_t refined_block_err = astc_ldr::cem_encode_pixels(cem_index, ccs_index, pixel_stats, refine_params, + endpoint_ise_range, astc_helpers::BISE_64_LEVELS, + refined_endpoints, refined_weights0, refined_weights1, UINT64_MAX, use_blue_contraction, &base_ofs_clamped_flag); + assert(refined_block_err != UINT64_MAX); + + if ((pBase_ofs_clamped_flag) && (base_ofs_clamped_flag)) + *pBase_ofs_clamped_flag = base_ofs_clamped_flag; + + if (refined_block_err != UINT64_MAX) + { + uint64_t cur_err = eval_error( + block_width, block_height, + pixel_stats, + cem_index, + dual_plane_flag, ccs_index, + endpoint_ise_range, weight_ise_range, + grid_width, grid_height, + enc_log_block.m_endpoints, weights0, weights1, + params); + + if (refined_block_err < cur_err) + { + memcpy(enc_log_block.m_endpoints, refined_endpoints, astc_helpers::get_num_cem_values(cem_index)); + } + } + } + + if (outer_pass == (NUM_OUTER_PASSES - 1)) + break; + + if ((!gradient_descent_flag) && (!polish_weights_flag)) + break; + + bool improved_flag = false; + + bool status = polish_block_weights( + block_width, block_height, + pixel_stats, + enc_log_block, // assumes there is already a good encoding to improve here + params, + nullptr, + improved_flag, + gradient_descent_flag, + polish_weights_flag, + qcd_enabled_flag); + + if (!status) + { + assert(0); + return false; + } + + if (!improved_flag) + break; + + } // outer_pass + + return true; +} + +// 1 part only, refines endpoints given current weights +bool encode_trial_refine_only( + uint32_t block_width, uint32_t block_height, + const astc_ldr::pixel_stats_t& pixel_stats, + astc_helpers::log_astc_block& enc_log_block, + const astc_ldr::cem_encode_params& params, + bool use_blue_contraction = true, + bool* pBase_ofs_clamped_flag = nullptr) +{ + assert(enc_log_block.m_num_partitions == 1); + + if (pBase_ofs_clamped_flag) + *pBase_ofs_clamped_flag = false; + + const uint32_t cem_index = enc_log_block.m_color_endpoint_modes[0]; + const bool dual_plane_flag = enc_log_block.m_dual_plane; + const int ccs_index = dual_plane_flag ? enc_log_block.m_color_component_selector : -1; + const uint32_t endpoint_ise_range = enc_log_block.m_endpoint_ise_range; + const uint32_t weight_ise_range = enc_log_block.m_weight_ise_range; + const uint32_t grid_width = enc_log_block.m_grid_width; + const uint32_t grid_height = enc_log_block.m_grid_height; + + //const bool is_downsampling = (grid_width < block_width) || (grid_height < block_height); + + //const uint32_t total_block_pixels = block_width * block_height; + const uint32_t total_grid_pixels = grid_width * grid_height; + + uint8_t dequantized_raw_weights0[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + uint8_t upsampled_weights0[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; // raw weights, NOT ISE + + for (uint32_t i = 0; i < total_grid_pixels; i++) + dequantized_raw_weights0[i] = astc_helpers::g_dequant_tables.get_weight_tab(weight_ise_range).m_ISE_to_val[astc_helpers::get_weight(enc_log_block, 0, i)]; + + // suppress bogus gcc warning on dequantized_raw_weights0 +#ifndef __clang__ +#if defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif +#endif + + astc_helpers::upsample_weight_grid(block_width, block_height, grid_width, grid_height, dequantized_raw_weights0, upsampled_weights0); + +#ifndef __clang__ +#if defined(__GNUC__) +#pragma GCC diagnostic pop +#endif +#endif + + uint8_t dequantized_raw_weights1[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + uint8_t upsampled_weights1[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; // raw weights, NOT ISE + + if (dual_plane_flag) + { + for (uint32_t i = 0; i < total_grid_pixels; i++) + dequantized_raw_weights1[i] = astc_helpers::g_dequant_tables.get_weight_tab(weight_ise_range).m_ISE_to_val[astc_helpers::get_weight(enc_log_block, 1, i)]; + astc_helpers::upsample_weight_grid(block_width, block_height, grid_width, grid_height, dequantized_raw_weights1, upsampled_weights1); + } + + astc_ldr::cem_encode_params refine_params(params); + refine_params.m_pForced_weight_vals0 = upsampled_weights0; + if (dual_plane_flag) + refine_params.m_pForced_weight_vals1 = upsampled_weights1; + + uint8_t refined_endpoints[astc_helpers::MAX_CEM_ENDPOINT_VALS]; + uint8_t refined_weights0[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + uint8_t refined_weights1[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + + //bool use_blue_contraction = true; + + bool base_ofs_clamped_flag = false; + + uint64_t refined_block_err = astc_ldr::cem_encode_pixels(cem_index, ccs_index, pixel_stats, refine_params, + endpoint_ise_range, astc_helpers::BISE_64_LEVELS, + refined_endpoints, refined_weights0, refined_weights1, UINT64_MAX, use_blue_contraction, &base_ofs_clamped_flag); + assert(refined_block_err != UINT64_MAX); + + if ((pBase_ofs_clamped_flag) && (base_ofs_clamped_flag)) + *pBase_ofs_clamped_flag = base_ofs_clamped_flag; + +#if defined(_DEBUG) || defined(DEBUG) + for (uint32_t i = 0; i < total_grid_pixels; i++) + { + assert(refined_weights0[i] == upsampled_weights0[i]); + + if (dual_plane_flag) + { + assert(refined_weights1[i] == upsampled_weights1[i]); + } + } +#endif + + if (refined_block_err != UINT64_MAX) + { + astc_helpers::log_astc_block alt_enc_log_block(enc_log_block); + memcpy(alt_enc_log_block.m_endpoints, refined_endpoints, astc_helpers::get_num_cem_values(cem_index)); + +#if defined(_DEBUG) || defined(DEBUG) + // refined_block_err was computed on the actual ASTC [0,64] upsampled weights the decoder would use. But double check this for sanity. + { + uint64_t ref_err = eval_error(block_width, block_height, alt_enc_log_block, pixel_stats, params); + assert(ref_err == refined_block_err); + } +#endif + + uint64_t cur_err = eval_error(block_width, block_height, enc_log_block, pixel_stats, params); + + if (refined_block_err < cur_err) + { + memcpy(enc_log_block.m_endpoints, refined_endpoints, astc_helpers::get_num_cem_values(cem_index)); + } + } + + return true; +} + +struct log_surrogate_astc_blk +{ + int m_grid_width, m_grid_height; + + uint32_t m_cem_index; // base+scale or direct variants only + int m_ccs_index; // -1 for single plane + + uint32_t m_num_endpoint_levels; + uint32_t m_num_weight_levels; + + uint32_t m_num_parts; // 1-3 + uint32_t m_seed_index; // ASTC seed index, 10-bits if m_num_parts > 1 + + vec4F m_endpoints[astc_helpers::MAX_PARTITIONS][2]; // [subset_index][l/h endpoint] + float m_scales[astc_helpers::MAX_PARTITIONS]; // scale factor used for each subset + + float m_weights0[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + float m_weights1[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + + void clear() + { + memset((void *)this, 0, sizeof(*this)); + } + + void decode(uint32_t block_width, uint32_t block_height, vec4F* pPixels, const astc_ldr::partition_pattern_vec* pPat) const; + void decode(uint32_t block_width, uint32_t block_height, vec4F* pPixels, const astc_ldr::partitions_data* pPat_data) const; +}; + +void upsample_surrogate_weights( + const astc_helpers::weighted_sample* pWeighted_samples, + const float* pSrc_weights, + float* pDst_weights, + uint32_t by, uint32_t bx, + uint32_t wx, uint32_t wy, + uint32_t num_weight_levels) +{ + const uint32_t total_src_weights = wx * wy; + const float weight_levels_minus_1 = (float)(num_weight_levels - 1) * (1.0f / 16.0f); + const float inv_weight_levels = 1.0f / (float)(num_weight_levels - 1); + + const astc_helpers::weighted_sample* pS = pWeighted_samples; + + for (uint32_t y = 0; y < by; y++) + { + for (uint32_t x = 0; x < bx; x++, ++pS) + { + const uint32_t w00 = pS->m_weights[0][0]; + const uint32_t w01 = pS->m_weights[0][1]; + const uint32_t w10 = pS->m_weights[1][0]; + const uint32_t w11 = pS->m_weights[1][1]; + + assert(w00 || w01 || w10 || w11); + + const uint32_t sx = pS->m_src_x, sy = pS->m_src_y; + + float total = 0.0f; + + if (w00) total += pSrc_weights[bounds_check(sx + sy * wx, 0U, total_src_weights)] * (float)w00; + if (w01) total += pSrc_weights[bounds_check(sx + 1 + sy * wx, 0U, total_src_weights)] * (float)w01; + if (w10) total += pSrc_weights[bounds_check(sx + (sy + 1) * wx, 0U, total_src_weights)] * (float)w10; + if (w11) total += pSrc_weights[bounds_check(sx + 1 + (sy + 1) * wx, 0U, total_src_weights)] * (float)w11; + + float w = (float)fast_roundf_pos_int(total * weight_levels_minus_1) * inv_weight_levels; + + pDst_weights[x + y * bx] = w; + } // x + } // y +} + +void log_surrogate_astc_blk::decode(uint32_t block_width, uint32_t block_height, vec4F* pPixels, const astc_ldr::partition_pattern_vec* pPat) const +{ + const bool dual_plane = (m_ccs_index >= 0); + + const uint32_t total_block_pixels = block_width * block_height; + const uint32_t total_grid_pixels = m_grid_width * m_grid_height; + + const bool needs_upsampling = total_grid_pixels < total_block_pixels; + + const bool is_small_block = total_block_pixels < 31; // astc_helpers::is_small_block(block_width, block_height); + BASISU_NOTE_UNUSED(is_small_block); + + float upsampled_weights0[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS], upsampled_weights1[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + + const float* pWeights0 = m_weights0; + const float* pWeights1 = m_weights1; + + if (needs_upsampling) + { + // TODO: Precompute these in tables + astc_helpers::weighted_sample up_weights[astc_helpers::MAX_BLOCK_DIM * astc_helpers::MAX_BLOCK_DIM]; + astc_helpers::compute_upsample_weights(block_width, block_height, m_grid_width, m_grid_height, up_weights); + + upsample_surrogate_weights(up_weights, m_weights0, upsampled_weights0, block_width, block_height, m_grid_width, m_grid_height, m_num_weight_levels); + pWeights0 = upsampled_weights0; + + if (dual_plane) + { + upsample_surrogate_weights(up_weights, m_weights1, upsampled_weights1, block_width, block_height, m_grid_width, m_grid_height, m_num_weight_levels); + pWeights1 = upsampled_weights1; + } + } + + for (uint32_t y = 0; y < block_height; y++) + { + for (uint32_t x = 0; x < block_width; x++) + { + uint32_t part_index = 0; + if (m_num_parts > 1) + { + part_index = (*pPat)(x, y); + assert(part_index < m_num_parts); + + assert(part_index == (uint32_t)astc_helpers::compute_texel_partition(m_seed_index, x, y, 0, m_num_parts, is_small_block)); + } + + const vec4F& l = m_endpoints[part_index][0]; + const vec4F& h = m_endpoints[part_index][1]; + + vec4F& dst = pPixels[x + y * block_width]; + + for (uint32_t c = 0; c < 4; c++) + { + float w = ((int)c == m_ccs_index) ? pWeights1[x + y * block_width] : pWeights0[x + y * block_width]; + + //dst[c] = lerp(l[c], h[c], w); + + const float one_minus_w = 1.0f - w; + dst[c] = l[c] * one_minus_w + h[c] * w; + } // c + + } // x + } // y +} + +void log_surrogate_astc_blk::decode(uint32_t block_width, uint32_t block_height, vec4F* pPixels, const astc_ldr::partitions_data* pPat_data) const +{ + if (m_num_parts == 1) + return decode(block_width, block_height, pPixels, (const astc_ldr::partition_pattern_vec*)nullptr); + + uint32_t unique_pat_index = pPat_data->m_part_seed_to_unique_index[m_seed_index]; + assert(unique_pat_index < pPat_data->m_total_unique_patterns); + + return decode(block_width, block_height, pPixels, &pPat_data->m_partition_pats[unique_pat_index]); +} + +void downsample_float_weight_grid( + const float* pMatrix_weights, + uint32_t bx, uint32_t by, // source/from dimension (block size) + uint32_t wx, uint32_t wy, // dest/to dimension (grid size) + const float* pSrc_weights, // these are dequantized weights, NOT ISE symbols, [by][bx] + float* pDst_weights, // [wy][wx] + uint32_t num_weight_levels) +{ + const uint32_t total_block_samples = bx * by; + const float weight_levels_minus_1 = (float)(num_weight_levels - 1); + const float inv_weight_levels = 1.0f / (float)(num_weight_levels - 1); + + for (uint32_t y = 0; y < wy; y++) + { + for (uint32_t x = 0; x < wx; x++) + { + float total = 0.0f; + + // TODO - optimize! + for (uint32_t i = 0; i < total_block_samples; i++) + if (pMatrix_weights[i]) + total += pMatrix_weights[i] * (float)pSrc_weights[i]; + + pDst_weights[x + y * wx] = (float)fast_roundf_pos_int(total * weight_levels_minus_1) * inv_weight_levels; + + pMatrix_weights += total_block_samples; + } + } +} + +float decode_surrogate_and_compute_error( + uint32_t block_width, uint32_t block_height, + const astc_ldr::pixel_stats_t& pixel_stats, + log_surrogate_astc_blk& log_block, + const astc_ldr::partition_pattern_vec* pPat, + const astc_ldr::cem_encode_params& params) +{ + vec4F dec_pixels[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + log_block.decode(block_width, block_height, dec_pixels, pPat); + + const float wr = (float)params.m_comp_weights[0]; + const float wg = (float)params.m_comp_weights[1]; + const float wb = (float)params.m_comp_weights[2]; + const float wa = (float)params.m_comp_weights[3]; + + float total_err = 0.0f; + for (uint32_t by = 0; by < block_height; by++) + { + for (uint32_t bx = 0; bx < block_width; bx++) + { + const vec4F& s = pixel_stats.m_pixels_f[bx + by * block_width]; + const vec4F& d = dec_pixels[bx + by * block_width]; + + float dr = s[0] - d[0]; + float dg = s[1] - d[1]; + float db = s[2] - d[2]; + float da = s[3] - d[3]; + + total_err += (wr * dr * dr) + (wg * dg * dg) + (wb * db * db) + (wa * da * da); + } // bx + + } // by + + return total_err; +} + +// Returns WSSE error +float encode_surrogate_trial( + uint32_t block_width, uint32_t block_height, + const astc_ldr::pixel_stats_t& pixel_stats, + uint32_t cem_index, + int ccs_index, + uint32_t endpoint_ise_range, uint32_t weight_ise_range, + uint32_t grid_width, uint32_t grid_height, + log_surrogate_astc_blk& log_block, + const astc_ldr::cem_encode_params& params, + uint32_t flags) +{ + const bool is_downsampling = (grid_width < block_width) || (grid_height < block_height); + const bool dual_plane_flag = (ccs_index >= 0); + + const basist::astc_ldr_t::astc_block_grid_data* pBlock_grid_data = basist::astc_ldr_t::find_astc_block_grid_data(block_width, block_height, grid_width, grid_height); + + const float* pDownsample_matrix = nullptr; + if (is_downsampling) + pDownsample_matrix = pBlock_grid_data->m_downsample_matrix.get_ptr(); + + //const uint32_t total_block_pixels = block_width * block_height; + //const uint32_t total_grid_pixels = grid_width * grid_height; + + log_block.m_cem_index = cem_index; + log_block.m_ccs_index = ccs_index; + log_block.m_grid_width = grid_width; + log_block.m_grid_height = grid_height; + log_block.m_num_parts = 1; + log_block.m_seed_index = 0; + clear_obj(log_block.m_scales); + log_block.m_num_endpoint_levels = astc_helpers::get_ise_levels(endpoint_ise_range); + log_block.m_num_weight_levels = astc_helpers::get_ise_levels(weight_ise_range); + + float wsse_err = 0.0f; + + if (is_downsampling) + { + float temp_weights0[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS], temp_weights1[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + + astc_ldr::cem_surrogate_encode_pixels( + cem_index, ccs_index, + pixel_stats, params, + endpoint_ise_range, weight_ise_range, + log_block.m_endpoints[0][0], log_block.m_endpoints[0][1], log_block.m_scales[0], temp_weights0, temp_weights1, + flags); + + downsample_float_weight_grid( + pDownsample_matrix, + block_width, block_height, + grid_width, grid_height, + temp_weights0, + log_block.m_weights0, + log_block.m_num_weight_levels); + + if (dual_plane_flag) + { + downsample_float_weight_grid( + pDownsample_matrix, + block_width, block_height, + grid_width, grid_height, + temp_weights1, + log_block.m_weights1, + log_block.m_num_weight_levels); + } + + wsse_err = decode_surrogate_and_compute_error(block_width, block_height, pixel_stats, log_block, nullptr, params); + } + else + { + wsse_err = astc_ldr::cem_surrogate_encode_pixels( + cem_index, ccs_index, + pixel_stats, params, + endpoint_ise_range, weight_ise_range, + log_block.m_endpoints[0][0], log_block.m_endpoints[0][1], log_block.m_scales[0], log_block.m_weights0, log_block.m_weights1, + flags); + +#if defined(_DEBUG) || defined(DEBUG) + { + float alt_wsse_err = decode_surrogate_and_compute_error(block_width, block_height, pixel_stats, log_block, nullptr, params); + assert(fabs(wsse_err - alt_wsse_err) < .00125f); + } +#endif + } + + return wsse_err; +} + +float encode_surrogate_trial_subsets( + uint32_t block_width, uint32_t block_height, + const astc_ldr::pixel_stats_t& pixel_stats, + uint32_t cem_index, + uint32_t num_subsets, uint32_t pat_seed_index, const astc_ldr::partition_pattern_vec* pPat, + uint32_t endpoint_ise_range, uint32_t weight_ise_range, + uint32_t grid_width, uint32_t grid_height, + log_surrogate_astc_blk& log_block, + const astc_ldr::cem_encode_params& params, + uint32_t flags) +{ + assert((num_subsets >= 2) && (num_subsets <= astc_helpers::MAX_PARTITIONS)); + + const bool is_downsampling = (grid_width < block_width) || (grid_height < block_height); + //const uint32_t total_block_pixels = block_width * block_height; + //const uint32_t total_grid_pixels = grid_width * grid_height; + + const uint32_t num_weight_levels = astc_helpers::get_ise_levels(weight_ise_range); + const uint32_t num_endpoint_levels = astc_helpers::get_ise_levels(endpoint_ise_range); + + const basist::astc_ldr_t::astc_block_grid_data* pBlock_grid_data = basist::astc_ldr_t::find_astc_block_grid_data(block_width, block_height, grid_width, grid_height); + + const float* pDownsample_matrix = nullptr; + if (is_downsampling) + pDownsample_matrix = pBlock_grid_data->m_downsample_matrix.get_ptr(); + + color_rgba part_pixels[astc_helpers::MAX_PARTITIONS][astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + uint32_t num_part_pixels[astc_helpers::MAX_PARTITIONS] = { 0 }; + + for (uint32_t y = 0; y < block_height; y++) + { + for (uint32_t x = 0; x < block_width; x++) + { + const color_rgba& px = pixel_stats.m_pixels[x + y * block_width]; + + const uint32_t part_index = (*pPat)(x, y); + assert(part_index < num_subsets); + + part_pixels[part_index][num_part_pixels[part_index]] = px; + num_part_pixels[part_index]++; + } // x + } // y + +#if defined(_DEBUG) || defined(DEBUG) + for (uint32_t i = 0; i < num_subsets; i++) + assert(num_part_pixels[i] > 0); +#endif + + astc_ldr::pixel_stats_t part_pixel_stats[astc_helpers::MAX_PARTITIONS]; + + for (uint32_t i = 0; i < num_subsets; i++) + part_pixel_stats[i].clear(); + + float part_weights[astc_helpers::MAX_PARTITIONS][astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + float temp_block_weights[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + + double total_subset_err = 0.0f; + for (uint32_t part_index = 0; part_index < num_subsets; part_index++) + { + part_pixel_stats[part_index].init(num_part_pixels[part_index], &part_pixels[part_index][0]); + + float subset_err = astc_ldr::cem_surrogate_encode_pixels( + cem_index, -1, + part_pixel_stats[part_index], params, + endpoint_ise_range, weight_ise_range, + log_block.m_endpoints[part_index][0], log_block.m_endpoints[part_index][1], + log_block.m_scales[part_index], part_weights[part_index], temp_block_weights, + flags); + + total_subset_err += subset_err; + + } // part_index + + float* pDst_weights = is_downsampling ? temp_block_weights : log_block.m_weights0; + + clear_obj(num_part_pixels); + + for (uint32_t y = 0; y < block_height; y++) + { + for (uint32_t x = 0; x < block_width; x++) + { + const uint32_t part_index = (*pPat)(x, y); + assert(part_index < num_subsets); + + pDst_weights[x + y * block_width] = part_weights[part_index][num_part_pixels[part_index]]; + num_part_pixels[part_index]++; + } // x + } // y + + log_block.m_cem_index = cem_index; + log_block.m_ccs_index = -1; + log_block.m_num_endpoint_levels = num_endpoint_levels; + log_block.m_num_weight_levels = num_weight_levels; + log_block.m_grid_width = grid_width; + log_block.m_grid_height = grid_height; + log_block.m_num_parts = num_subsets; + log_block.m_seed_index = pat_seed_index; + + if (is_downsampling) + { + downsample_float_weight_grid( + pDownsample_matrix, + block_width, block_height, + grid_width, grid_height, + temp_block_weights, + log_block.m_weights0, + astc_helpers::get_ise_levels(weight_ise_range)); + + total_subset_err = decode_surrogate_and_compute_error(block_width, block_height, pixel_stats, log_block, pPat, params); + } + +#if defined(_DEBUG) || defined(DEBUG) + if (!is_downsampling) + { + float alt_subset_err = decode_surrogate_and_compute_error(block_width, block_height, pixel_stats, log_block, pPat, params); + + assert(fabs(total_subset_err - alt_subset_err) < .00125f); + } +#endif + + return (float)total_subset_err; +} + +#if 0 +static inline vec4F vec4F_norm_approx(vec4F axis) +{ + float l = axis.norm(); + axis = (fabs(l) >= SMALL_FLOAT_VAL) ? (axis * bu_math::inv_sqrt(l)) : vec4F(.5f); + return axis; +} +#endif + +static bool estimate_partition2( + uint32_t block_width, uint32_t block_height, + const astc_ldr::pixel_stats_t& pixels, + int* pBest_parts, uint32_t num_best_parts, // unique indices, not ASTC seeds + const astc_ldr::partitions_data* pPart_data, bool brute_force_flag) +{ + assert(num_best_parts && (num_best_parts <= pPart_data->m_total_unique_patterns)); + + const uint32_t num_block_pixels = block_width * block_height; + + if (brute_force_flag) + { + int desired_parts[astc_ldr::ASTC_LDR_MAX_BLOCK_HEIGHT][astc_ldr::ASTC_LDR_MAX_BLOCK_WIDTH]; // [y][x] + + for (uint32_t i = 0; i < num_block_pixels; i++) + { + float proj = (pixels.m_pixels_f[i] - pixels.m_mean_f).dot(pixels.m_mean_rel_axis4); + + desired_parts[i / block_width][i % block_width] = proj < 0.0f; + } + + uint32_t part_similarity[astc_helpers::NUM_PARTITION_PATTERNS]; + + for (uint32_t part_index = 0; part_index < pPart_data->m_total_unique_patterns; part_index++) + { + const astc_ldr::partition_pattern_vec& pat_vec = pPart_data->m_partition_pats[part_index]; + + int total_sim_non_inv = 0; + int total_sim_inv = 0; + + for (uint32_t y = 0; y < block_height; y++) + { + for (uint32_t x = 0; x < block_width; x++) + { + int part = pat_vec[x + y * block_width]; + + if (part == desired_parts[y][x]) + total_sim_non_inv++; + + if ((part ^ 1) == desired_parts[y][x]) + total_sim_inv++; + } + } + + int total_sim = maximum(total_sim_non_inv, total_sim_inv); + + part_similarity[part_index] = (total_sim << 16) | part_index; + + } // part_index; + + std::sort(part_similarity, part_similarity + pPart_data->m_total_unique_patterns); + + for (uint32_t i = 0; i < num_best_parts; i++) + pBest_parts[i] = part_similarity[(pPart_data->m_total_unique_patterns - 1) - i] & 0xFFFF; + } + else + { + astc_ldr::partition_pattern_vec desired_part(block_width, block_height); + + for (uint32_t i = 0; i < num_block_pixels; i++) + { + float proj = (pixels.m_pixels_f[i] - pixels.m_mean_f).dot(pixels.m_mean_rel_axis4); + + desired_part.m_parts[i] = proj < 0.0f; + } + + astc_ldr::vp_tree::result_queue results; + results.reserve(num_best_parts); + + pPart_data->m_part_vp_tree.find_nearest(2, desired_part, results, num_best_parts); + + assert(results.get_size() == num_best_parts); + + const auto& elements = results.get_elements(); + + for (uint32_t i = 0; i < results.get_size(); i++) + pBest_parts[i] = elements[1 + i].m_pat_index; + } + + return true; +} + +static bool estimate_partition3( + uint32_t block_width, uint32_t block_height, + const astc_ldr::pixel_stats_t& pixels, + int* pBest_parts, uint32_t num_best_parts, + const astc_ldr::partitions_data* pPart_data, bool brute_force_flag) +{ + assert(num_best_parts && (num_best_parts <= pPart_data->m_total_unique_patterns)); + + vec4F training_vecs[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS], mean(0.0f); + + const uint32_t num_block_pixels = block_width * block_height, NUM_SUBSETS = 3; + + float brightest_inten = 0.0f, darkest_inten = BIG_FLOAT_VAL; + vec4F cluster_centroids[NUM_SUBSETS]; + clear_obj(cluster_centroids); + + for (uint32_t i = 0; i < num_block_pixels; i++) + { + vec4F& v = training_vecs[i]; + + v = pixels.m_pixels_f[i]; + + float inten = v.dot(vec4F(1.0f)); + if (inten < darkest_inten) + { + darkest_inten = inten; + cluster_centroids[0] = v; + } + + if (inten > brightest_inten) + { + brightest_inten = inten; + cluster_centroids[1] = v; + } + } + + if (cluster_centroids[0] == cluster_centroids[1]) + return false; + + float furthest_dist2 = 0.0f; + for (uint32_t i = 0; i < num_block_pixels; i++) + { + vec4F& v = training_vecs[i]; + + float dist_a = v.squared_distance(cluster_centroids[0]); + if (dist_a == 0.0f) + continue; + + float dist_b = v.squared_distance(cluster_centroids[1]); + if (dist_b == 0.0f) + continue; + + float dist2 = dist_a + dist_b; + if (dist2 > furthest_dist2) + { + furthest_dist2 = dist2; + cluster_centroids[2] = v; + } + } + + if ((cluster_centroids[0] == cluster_centroids[2]) || (cluster_centroids[1] == cluster_centroids[2])) + return false; + + uint32_t cluster_pixels[NUM_SUBSETS][astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + uint32_t num_cluster_pixels[NUM_SUBSETS]; + vec4F new_cluster_means[NUM_SUBSETS]; + + const uint32_t NUM_ITERS = 4; + + for (uint32_t s = 0; s < NUM_ITERS; s++) + { + memset(num_cluster_pixels, 0, sizeof(num_cluster_pixels)); + memset((void *)new_cluster_means, 0, sizeof(new_cluster_means)); + + for (uint32_t i = 0; i < num_block_pixels; i++) + { + float d[NUM_SUBSETS] = { + training_vecs[i].squared_distance(cluster_centroids[0]), + training_vecs[i].squared_distance(cluster_centroids[1]), + training_vecs[i].squared_distance(cluster_centroids[2]) }; + + float min_d = d[0]; + uint32_t min_idx = 0; + for (uint32_t j = 1; j < NUM_SUBSETS; j++) + { + if (d[j] < min_d) + { + min_d = d[j]; + min_idx = j; + } + } + + cluster_pixels[min_idx][num_cluster_pixels[min_idx]] = i; + new_cluster_means[min_idx] += training_vecs[i]; + num_cluster_pixels[min_idx]++; + } // i + + // Can skip updating the centroids on the last iteration - all we care about is the final partitioning. + if (s == (NUM_ITERS - 1)) + { + for (uint32_t j = 0; j < NUM_SUBSETS; j++) + { + if (!num_cluster_pixels[j]) + return false; + } + } + else + { + for (uint32_t j = 0; j < NUM_SUBSETS; j++) + { + if (!num_cluster_pixels[j]) + return false; + + cluster_centroids[j] = new_cluster_means[j] / (float)num_cluster_pixels[j]; + } // j + } + + } // s + + astc_ldr::partition_pattern_vec desired_part(block_width, block_height); + + for (uint32_t p = 0; p < NUM_SUBSETS; p++) + { + for (uint32_t i = 0; i < num_cluster_pixels[p]; i++) + { + const uint32_t pix_index = cluster_pixels[p][i]; + desired_part[pix_index] = (uint8_t)p; + } // i + } // p + + if (brute_force_flag) + { + astc_ldr::partition_pattern_vec desired_parts[astc_ldr::NUM_PART3_MAPPINGS]; + for (uint32_t j = 0; j < astc_ldr::NUM_PART3_MAPPINGS; j++) + desired_parts[j] = desired_part.get_permuted3(j); + + uint32_t part_similarity[astc_helpers::NUM_PARTITION_PATTERNS]; + + for (uint32_t part_index = 0; part_index < pPart_data->m_total_unique_patterns; part_index++) + { + const astc_ldr::partition_pattern_vec& pat = pPart_data->m_partition_pats[part_index]; + + uint32_t lowest_pat_dist = UINT32_MAX; + for (uint32_t p = 0; p < astc_ldr::NUM_PART3_MAPPINGS; p++) + { + uint32_t dist = pat.get_squared_distance(desired_parts[p]); + if (dist < lowest_pat_dist) + lowest_pat_dist = dist; + } + + part_similarity[part_index] = (lowest_pat_dist << 16) | part_index; + + } // part_index; + + std::sort(part_similarity, part_similarity + pPart_data->m_total_unique_patterns); + + for (uint32_t i = 0; i < num_best_parts; i++) + pBest_parts[i] = part_similarity[i] & 0xFFFF; + } + else + { + astc_ldr::vp_tree::result_queue results; + results.reserve(num_best_parts); + + pPart_data->m_part_vp_tree.find_nearest(3, desired_part, results, num_best_parts); + + assert(results.get_size() == num_best_parts); + + const auto& elements = results.get_elements(); + + for (uint32_t i = 0; i < results.get_size(); i++) + pBest_parts[i] = elements[1 + i].m_pat_index; + } + + return true; +} + +//--------------------------------------------------------------------- + +static const float g_sobel_x[3][3] = // [y][x] +{ + { -1.0f, 0.0f, 1.0f }, + { -2.0f, 0.0f, 2.0f }, + { -1.0f, 0.0f, 1.0f } +}; + +static const float g_sobel_y[3][3] = // [y][x] +{ + { -1.0f, -2.0f, -1.0f }, + { 0.0f, 0.0f, 0.0f }, + { 1.0f, 2.0f, 1.0f } +}; + +void compute_sobel(const image& orig, image& dest, const float* pMatrix_3x3) +{ + const uint32_t width = orig.get_width(); + const uint32_t height = orig.get_height(); + + dest.resize(width, height); + + for (int y = 0; y < (int)height; y++) + { + for (int x = 0; x < (int)width; x++) + { + vec4F d(128.0f); + + for (int my = -1; my <= 1; my++) + { + for (int mx = -1; mx <= 1; mx++) + { + float w = pMatrix_3x3[(my + 1) * 3 + (mx + 1)]; + if (w == 0.0f) + continue; + + const color_rgba& s = orig.get_clamped(x + mx, y + my); + + for (uint32_t c = 0; c < 4; c++) + d[c] += w * (float)s[c]; + + } // mx + + } // my + + dest(x, y).set(fast_roundf_int(d[0]), fast_roundf_int(d[1]), fast_roundf_int(d[2]), fast_roundf_int(d[3])); + + } // x + } // y +} + +void compute_energy_from_dct(uint32_t block_width, uint32_t block_height, float* pDCT) +{ + const uint32_t num_texels = block_width * block_height; + + for (uint32_t i = 1; i < num_texels; i++) + pDCT[i] = square(pDCT[i]); + + pDCT[0] = 0.0f; +} + +// Results scaled by # block texels (block-SSE in weight space) +float compute_preserved_dct_energy(uint32_t block_width, uint32_t block_height, const float* pEnergy, uint32_t grid_w, uint32_t grid_h) +{ + float tot = 0.0f; + + for (uint32_t y = 0; y < block_height; y++) + { + for (uint32_t x = 0; x < block_width; x++) + { + if ((x < grid_w) && (y < grid_h)) + tot += pEnergy[x + y * block_width]; + } + } + + return tot; +} + +// Results scaled by # block texels (block-SSE in weight space) +inline float compute_lost_dct_energy(uint32_t block_width, uint32_t block_height, const float* pEnergy, uint32_t grid_w, uint32_t grid_h) +{ + float tot = 0.0f; + + for (uint32_t y = 0; y < block_height; y++) + { + for (uint32_t x = 0; x < block_width; x++) + { + if ((x < grid_w) && (y < grid_h)) + continue; + + tot += pEnergy[x + y * block_width]; + } + } + + return tot; +} + +struct ldr_astc_lowlevel_block_encoder_params +{ + ldr_astc_lowlevel_block_encoder_params() + { + clear(); + } + + void clear() + { + clear_obj(*this); + + for (uint32_t i = 0; i < 4; i++) + m_dp_active_chans[i] = true; + + m_subsets_edge_filtering = true; + + m_use_superbuckets = true; + m_bucket_pruning_passes = true; + m_use_dual_planes = true; + + m_superbucket_max_to_retain[0] = 4; + m_superbucket_max_to_retain[1] = 8; + m_superbucket_max_to_retain[2] = 16; + + m_shortlist_buckets_to_examine_fract = 1.0f; // after high-level bucket surrogate encoding and pruning stages, 1.0=effectively disabled + m_shortlist_buckets_to_examine_min = 1; + m_shortlist_buckets_to_examine_max = 1024; + + // TODO: Expose these at a higher level. Add alpha specific? + m_num_similar_modes_in_bucket_to_shortlist_fract = .33f; + m_num_similar_modes_in_bucket_to_shortlist_fract_min = 2; + m_num_similar_modes_in_bucket_to_shortlist_fract_max = 4096; + + m_final_shortlist_fraction[0] = .2f; + m_final_shortlist_fraction[1] = .3f; + m_final_shortlist_fraction[2] = .5f; + m_final_shortlist_min_size[0] = 1; + m_final_shortlist_min_size[1] = 1; + m_final_shortlist_min_size[2] = 1; + m_final_shortlist_max_size[0] = 4096; + m_final_shortlist_max_size[1] = 4096; + m_final_shortlist_max_size[2] = 4096; + + m_gradient_descent_flag = true; + m_polish_weights_flag = true; + m_qcd_enabled_flag = true; + + m_final_encode_try_base_ofs = true; + m_final_encode_always_try_rgb_direct = false; // if true, even if base_ofs succeeds, we try RGB/RGBA direct too + + m_use_parts_std_dev_thresh = (8.0f / 255.0f); + m_use_parts_std_dev_thresh2 = (40.0f / 255.0f); + m_sobel_energy_thresh1 = 3200.0f; + m_sobel_energy_thresh2 = 30000.0f; + m_sobel_energy_thresh3 = 50000.0f; + + m_part2_fraction_to_keep = 2; + m_part3_fraction_to_keep = 2; + m_base_parts2 = 32; + m_base_parts3 = 32; + + // TODO: Prehaps expose this at a higher level. + m_use_blue_contraction = true; + } + + uint32_t m_bx, m_by, m_block_width, m_block_height, m_total_block_pixels; + + const image* m_pOrig_img_sobel_xy_t; + + const astc_ldr::partitions_data* m_pPart_data_p2; + const astc_ldr::partitions_data* m_pPart_data_p3; + + const astc_ldr::cem_encode_params* m_pEnc_params; + + // RGB or alpha trial lists (shouldn't have both in same lists) + uint32_t m_num_trial_modes; + const basist::astc_ldr_t::trial_mode* m_pTrial_modes; + + const basist::astc_ldr_t::grouped_trial_modes* m_pGrouped_trial_modes; + + uint32_t m_superbucket_max_to_retain[3]; // [block_complexity_index] + + float m_shortlist_buckets_to_examine_fract; + uint32_t m_shortlist_buckets_to_examine_min; + uint32_t m_shortlist_buckets_to_examine_max; + + float m_num_similar_modes_in_bucket_to_shortlist_fract; + uint32_t m_num_similar_modes_in_bucket_to_shortlist_fract_min; + uint32_t m_num_similar_modes_in_bucket_to_shortlist_fract_max; + + float m_final_shortlist_fraction[3]; + uint32_t m_final_shortlist_min_size[3]; + uint32_t m_final_shortlist_max_size[3]; + + bool m_use_superbuckets; + bool m_bucket_pruning_passes; + + // true if this is a trial mode list containing alpha + bool m_alpha_cems; + + bool m_use_alpha_or_opaque_modes; // true for only alpha cems, false of only opaque cems; + bool m_use_lum_direct_modes; + bool m_use_base_scale_modes; + bool m_use_direct_modes; + bool m_use_dual_planes; + + bool m_grid_hv_filtering; + bool m_filter_horizontally_flag; // = h_energy_lost < v_energy_lost, if true it's visually better to resample the block on the X axis vs. Y + bool m_use_small_grids_only; + + bool m_dp_active_chans[4]; + + bool m_subsets_enabled; + bool m_subsets_edge_filtering; + + // TODO: Make polishing controllable per superpass. + bool m_gradient_descent_flag; + bool m_polish_weights_flag; + bool m_qcd_enabled_flag; + + bool m_final_encode_try_base_ofs; + bool m_final_encode_always_try_rgb_direct; + + bool m_brute_force_est_parts; + bool m_disable_part_est_stage2; // only use single stage partition estimation + + bool m_use_blue_contraction; // currently global enable/disable + + float m_use_parts_std_dev_thresh; + float m_use_parts_std_dev_thresh2; + float m_sobel_energy_thresh1; + float m_sobel_energy_thresh2; + float m_sobel_energy_thresh3; + + uint32_t m_part2_fraction_to_keep; + uint32_t m_part3_fraction_to_keep; + uint32_t m_base_parts2; + uint32_t m_base_parts3; + + float m_early_stop_wpsnr; + float m_early_stop2_wpsnr; + + basist::astc_ldr_t::dct2f* m_pDCT2F; // at block size +}; + +struct trial_surrogate +{ + uint32_t m_trial_mode_index; + float m_err; + + log_surrogate_astc_blk m_log_blk; + + void clear() + { + m_trial_mode_index = 0; + m_err = 0; + m_log_blk.clear(); + } + + bool operator < (const trial_surrogate& rhs) const + { + return m_err < rhs.m_err; + } +}; + +struct encode_block_output +{ + int16_t m_trial_mode_index; // -1 = solid, no trial mode + uint16_t m_blur_id; // blur index + + astc_helpers::log_astc_block m_log_blk; + + // Packed per-plane DCT data + basist::astc_ldr_t::dct_syms m_packed_dct_plane_data[2]; + + uint64_t m_sse; + + void clear() + { + m_trial_mode_index = -1; + m_blur_id = 0; + m_log_blk.clear(); + m_sse = 0; + } +}; + +struct encode_block_stats +{ + uint32_t m_total_superbuckets_created; + uint32_t m_total_buckets_created; + uint32_t m_total_surrogate_encodes; + uint32_t m_total_shortlist_candidates; + uint32_t m_total_full_encodes; + + encode_block_stats() { clear(); } + + void clear() + { + clear_obj(*this); + } +}; + +struct chan_mse_est +{ + float m_ep; + float m_wp; + + chan_mse_est() {} + chan_mse_est(float ep, float wp) : m_ep(ep), m_wp(wp) {} +}; + +struct weight_terms +{ + float m_mean; + float m_var; + float m_endpoint_factor; + float m_weight_spread_scale; + + void calc(uint32_t n, const float* pWeights) + { + assert(n); + + float weight_total = 0.0f; + for (uint32_t i = 0; i < n; i++) + { + assert(is_in_range(pWeights[i], 0.0f, 1.0f)); + weight_total += pWeights[i]; + } + m_mean = weight_total / (float)n; + + float weight_var = 0.0f; + for (uint32_t i = 0; i < n; i++) + weight_var += squaref(pWeights[i] - m_mean); + m_var = weight_var / (float)n; + + // drops below 2/3 on smooth blocks and tends to 2/3 when weights are well spread + m_endpoint_factor = (1.0f + 2.0f * m_var + 2.0f * m_mean * m_mean - 2.0f * m_mean) / (2.0f / 3.0f); + m_endpoint_factor = clamp(m_endpoint_factor, .25f, 1.50f); + + const float UNIFORM_VAR = 1.0f / 12.0f; + float s = m_var / UNIFORM_VAR; + + // shrinks the weight term on smooth blocks and is ~1 when weights are spread. + m_weight_spread_scale = saturate(s); + } +}; + +// weight_gamma is block size/grid size specific factor (0,1] (the amount of MSE quant error remaining taking into account bilinear smoothing) +inline chan_mse_est compute_quantized_channel_mse_estimates(uint32_t num_endpoint_levels, uint32_t num_weight_levels, float span_size, float weight_gamma, const weight_terms* pWeight_terms = nullptr) +{ + assert(num_endpoint_levels >= 2); + assert(num_weight_levels >= 2); + + const float Dep = 1.0f / (float)(num_endpoint_levels - 1); // endpoint quant step + const float Dw = 1.0f / (float)(num_weight_levels - 1); // weight quant step + + // Endpoint quant MSE estimate is not span dependent + float ep_lower = (Dep * Dep) / 12.0f * (2.0f / 3.0f); + + // Weight quant MSE estimate is span dependent + float wq_lower = (Dw * Dw) / 12.0f * weight_gamma * (span_size * span_size); + + if (pWeight_terms) + { + ep_lower *= pWeight_terms->m_endpoint_factor; + wq_lower *= pWeight_terms->m_weight_spread_scale; + } + + return chan_mse_est(ep_lower, wq_lower); +} + +inline float compute_quantized_channel_endpoint_mse_estimate(uint32_t num_endpoint_levels, const weight_terms* pWeight_terms = nullptr) +{ + assert(num_endpoint_levels >= 2); + + const float Dep = 1.0f / (float)(num_endpoint_levels - 1); // endpoint quant step + + // Endpoint quant MSE estimate is not span dependent + float ep_lower = (Dep * Dep) / 12.0f * (2.0f / 3.0f); + + if (pWeight_terms) + ep_lower *= pWeight_terms->m_endpoint_factor; + + return ep_lower; +} + +inline float compute_quantized_channel_weight_mse_estimate(uint32_t num_weight_levels, float span_size, float weight_gamma, const weight_terms* pWeight_terms = nullptr) +{ + assert(num_weight_levels >= 2); + + const float Dw = 1.0f / (float)(num_weight_levels - 1); // weight quant step + + // Weight quant MSE estimate is span dependent + float wq_lower = (Dw * Dw) / 12.0f * weight_gamma * (span_size * span_size); + + if (pWeight_terms) + wq_lower *= pWeight_terms->m_weight_spread_scale; + + return wq_lower; +} + +const float BLUE_CONTRACTION_BASE_OFS_DISCOUNT = .9f; +const float SKIP_IF_BUCKET_WORSE_MULTIPLIER = 5.0f; + +struct shortlist_bucket +{ + bool m_examined_flag; + int8_t m_grid_width, m_grid_height; + int8_t m_ccs_index; + + uint8_t m_cem_index; + uint8_t m_num_parts; + uint16_t m_unique_seed_index; + + log_surrogate_astc_blk m_surrogate_log_blk; + float m_sse; + + shortlist_bucket() + { + } + + shortlist_bucket(int grid_width, int grid_height, uint32_t cem_index, int ccs_index, uint32_t num_parts, uint32_t unique_seed_index) : + m_grid_width((int8_t)grid_width), m_grid_height((int8_t)grid_height), + m_ccs_index((int8_t)ccs_index), + m_cem_index((uint8_t)cem_index), + m_num_parts((uint8_t)num_parts), + m_unique_seed_index((uint16_t)unique_seed_index) + { + m_surrogate_log_blk.clear(); + m_sse = 0.0f; + m_examined_flag = false; + } + + operator size_t() const + { +#define ADD_HASH(H) h ^= basist::hash_hsieh((uint8_t*)&(H), sizeof(H)); + size_t h = 0; + ADD_HASH(m_grid_width); + ADD_HASH(m_grid_height); + ADD_HASH(m_ccs_index); + ADD_HASH(m_cem_index); + ADD_HASH(m_num_parts); + ADD_HASH(m_unique_seed_index); +#undef ADD_HASH + return h; + } + + // equality for hashing + bool operator== (const shortlist_bucket& rhs) const + { + return (m_grid_width == rhs.m_grid_width) && (m_grid_height == rhs.m_grid_height) && (m_cem_index == rhs.m_cem_index) && (m_ccs_index == rhs.m_ccs_index) && + (m_num_parts == rhs.m_num_parts) && (m_unique_seed_index == rhs.m_unique_seed_index); + } +}; + +typedef static_vector trial_mode_index_vec; +typedef basisu::hash_map shortlist_bucket_hash_t; + +#pragma pack(push, 1) +struct trial_mode_estimate_superbucket_key +{ + // All member vars from beginning to m_last will be hashed. Be careful of alignment. + uint8_t m_cem_index; + int8_t m_ccs_index; + uint16_t m_subset_unique_index; + + uint8_t m_num_subsets; + uint8_t m_last; + uint8_t m_unused[2]; + + trial_mode_estimate_superbucket_key() + { + static_assert((sizeof(*this) % 4) == 0, "struct size must be divisible by 4"); + } + + void clear() + { + clear_obj(*this); + } + + operator size_t() const + { + return basist::hash_hsieh((const uint8_t*)this, BASISU_OFFSETOF(trial_mode_estimate_superbucket_key, m_last)); + } + + bool operator== (const trial_mode_estimate_superbucket_key& rhs) const + { +#define COMP(e) if (e != rhs.e) return false; + COMP(m_cem_index); + COMP(m_ccs_index); + COMP(m_subset_unique_index); + COMP(m_num_subsets); +#undef COMP + return true; + } +}; +#pragma pack(pop) + +struct trial_mode_estimate_superbucket_value +{ + basisu::vector m_trial_mode_list; +}; + +typedef hash_map trial_mode_estimate_superbucket_hash; + +struct trial_mode_estimate +{ + trial_mode_estimate_superbucket_key m_superbucket_key; + + uint32_t m_trial_mode_index; + float m_wsse; + + bool operator< (const trial_mode_estimate& rhs) const + { + return m_wsse < rhs.m_wsse; + } +}; + +struct ranked_shortlist_bucket +{ + shortlist_bucket m_bucket; + trial_mode_index_vec m_trial_mode_indices; + + bool operator < (const ranked_shortlist_bucket& rhs) const { return m_bucket.m_sse < rhs.m_bucket.m_sse; } +}; + +struct ldr_astc_lowlevel_block_encoder +{ + ldr_astc_lowlevel_block_encoder() : + m_used_flag(false) + { + clear(); + } + + // Warning: These objects can migrate between threads (be cautious of determinism issues with containers/hash tables!) + bool m_used_flag; + + // Thread-local data follows + uint_vec m_trial_modes_to_estimate; + + trial_mode_estimate_superbucket_hash m_superbucket_hash; + + std::priority_queue m_trial_mode_estimate_priority_queue; + + basist::astc_ldr_t::fvec m_dct_work; + + shortlist_bucket_hash_t m_shortlist_hash0; + shortlist_bucket_hash_t m_shortlist_hash1; + + basisu::vector m_trial_surrogates; + + float m_sobel_energy; + float m_max_std_dev; + + uint32_t m_block_complexity_index; // [0,2] + bool m_strong_edges; + bool m_very_strong_edges; + bool m_super_strong_edges; + + bool m_used_superbuckets; + + int m_best_parts2[2][MAX_BASE_PARTS2 * PART_ESTIMATE_STAGE1_MULTIPLIER]; // [rgb[a]direct/rgbs][est_part] + int m_num_est_parts2[2]; + + int m_best_parts3[2][MAX_BASE_PARTS3 * PART_ESTIMATE_STAGE1_MULTIPLIER]; // [rgb[a]direct/rgbs][est_part] + int m_num_est_parts3[2]; + + basisu::vector m_ranked_buckets; + + void clear() + { + m_trial_modes_to_estimate.resize(0); + m_superbucket_hash.reset(); + + m_trial_surrogates.resize(0); + + m_sobel_energy = 0; + m_max_std_dev = 0; + m_block_complexity_index = 0; + m_strong_edges = false; + m_very_strong_edges = false; + m_super_strong_edges = false; + + m_used_superbuckets = false; + + clear_obj(m_best_parts2); + clear_obj(m_num_est_parts2); + + clear_obj(m_best_parts3); + clear_obj(m_num_est_parts3); + + m_ranked_buckets.resize(0); + } + + bool init( + const ldr_astc_lowlevel_block_encoder_params& p, + const astc_ldr::pixel_stats_t& pixel_stats, + basisu::vector& out_blocks, + uint32_t blur_id, + encode_block_stats& stats) + { + BASISU_NOTE_UNUSED(blur_id); + BASISU_NOTE_UNUSED(out_blocks); + BASISU_NOTE_UNUSED(stats); + + // TODO: This sums the *original* (not blurred) block's energy - precompute this? Replace with DCT? + m_sobel_energy = 0.0f; + for (uint32_t y = 0; y < p.m_block_height; y++) + { + for (uint32_t x = 0; x < p.m_block_width; x++) + { + const color_rgba& s = p.m_pOrig_img_sobel_xy_t->get_clamped(p.m_bx * p.m_block_width + x, p.m_by * p.m_block_height + y); + + // TODO: sum max of all channels instead? + m_sobel_energy += s[0] * s[0] + s[1] * s[1] + s[2] * s[2] + s[3] * s[3]; + } // x + } // y + + m_sobel_energy /= (float)p.m_total_block_pixels; + + m_max_std_dev = 0.0f; + for (uint32_t i = 0; i < 4; i++) + m_max_std_dev = maximum(m_max_std_dev, pixel_stats.m_rgba_stats[i].m_std_dev); + + m_strong_edges = (m_max_std_dev > p.m_use_parts_std_dev_thresh) && (m_sobel_energy > p.m_sobel_energy_thresh1); + m_very_strong_edges = (m_max_std_dev > p.m_use_parts_std_dev_thresh2) && (m_sobel_energy > p.m_sobel_energy_thresh2); + m_super_strong_edges = (m_max_std_dev > p.m_use_parts_std_dev_thresh2) && (m_sobel_energy > p.m_sobel_energy_thresh3); + + m_block_complexity_index = m_super_strong_edges ? 2 : (m_very_strong_edges ? 1 : 0); + + return true; + } + + bool partition_triage( + const ldr_astc_lowlevel_block_encoder_params& p, + const astc_ldr::pixel_stats_t& pixel_stats, + basisu::vector& out_blocks, + uint32_t blur_id, + encode_block_stats& stats) + { + BASISU_NOTE_UNUSED(blur_id); + BASISU_NOTE_UNUSED(out_blocks); + + clear_obj(m_num_est_parts2); + clear_obj(m_num_est_parts3); + + if (!p.m_subsets_enabled) + return true; + + if (p.m_subsets_edge_filtering) + { + if (!m_strong_edges) + return true; + } + + assert(p.m_base_parts2 <= MAX_BASE_PARTS2); + assert(p.m_base_parts3 <= MAX_BASE_PARTS3); + + // 2 subsets + int total_parts2 = m_super_strong_edges ? (p.m_base_parts2 * PART_ESTIMATE_STAGE1_MULTIPLIER) : (m_very_strong_edges ? (p.m_base_parts2 * 2) : p.m_base_parts2); + total_parts2 = minimum(total_parts2, MAX_BASE_PARTS2 * PART_ESTIMATE_STAGE1_MULTIPLIER); + total_parts2 = minimum(total_parts2, p.m_pPart_data_p2->m_total_unique_patterns); + + const uint32_t surrogate_encode_flags = 0; + + if (total_parts2) + { + int best_parts2_temp[MAX_BASE_PARTS2 * PART_ESTIMATE_STAGE1_MULTIPLIER]; + assert(total_parts2 <= (int)std::size(best_parts2_temp)); + + // Stage 1: kmeans+vptree + const bool has_est_parts2 = estimate_partition2( + p.m_block_width, p.m_block_height, + pixel_stats, + best_parts2_temp, total_parts2, + p.m_pPart_data_p2, p.m_brute_force_est_parts); + + if (has_est_parts2) + { + // Always try direct, optionally base+scale cem's + for (uint32_t s = 0; s < 2; s++) + { + if ((s) && (!p.m_use_base_scale_modes)) + continue; + + if (p.m_disable_part_est_stage2) + { + m_num_est_parts2[s] = total_parts2; + memcpy(m_best_parts2[s], best_parts2_temp, m_num_est_parts2[s] * sizeof(int)); + continue; + } + + uint32_t cem_to_surrogate_encode = p.m_alpha_cems ? astc_helpers::CEM_LDR_RGBA_DIRECT : astc_helpers::CEM_LDR_RGB_DIRECT; + if (s) + cem_to_surrogate_encode = p.m_alpha_cems ? astc_helpers::CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A : astc_helpers::CEM_LDR_RGB_BASE_SCALE; + + // Stage 2: Analytic surrogate WSSE + basisu::vector part_sses(total_parts2); + + for (int i = 0; i < total_parts2; i++) + { + const astc_ldr::partitions_data* pPart_data = p.m_pPart_data_p2; + + const uint32_t unique_seed_index = best_parts2_temp[i]; + const uint32_t part_seed_index = pPart_data->m_unique_index_to_part_seed[unique_seed_index]; + + const astc_ldr::partition_pattern_vec* pPat = &pPart_data->m_partition_pats[unique_seed_index]; + + log_surrogate_astc_blk surrogate_log_blk; + float sse = encode_surrogate_trial_subsets( + p.m_block_width, p.m_block_height, + pixel_stats, + cem_to_surrogate_encode, 2, part_seed_index, pPat, + astc_helpers::BISE_256_LEVELS, astc_helpers::BISE_64_LEVELS, + p.m_block_width, p.m_block_height, + surrogate_log_blk, + *p.m_pEnc_params, surrogate_encode_flags); + + stats.m_total_surrogate_encodes++; + + part_sses[i] = sse; + } // i + + basisu::vector part_sses_ranks(total_parts2); + + indirect_sort(total_parts2, part_sses_ranks.get_ptr(), part_sses.get_ptr()); + + m_num_est_parts2[s] = maximum(1, (total_parts2 + p.m_part2_fraction_to_keep - 1) / p.m_part2_fraction_to_keep); + + for (int i = 0; i < m_num_est_parts2[s]; i++) + { + const uint32_t rank_index = part_sses_ranks[i]; + const uint32_t unique_seed_unique = best_parts2_temp[rank_index]; + m_best_parts2[s][i] = unique_seed_unique; + } // i + + } // s + + } // if (has_est_parts2) + + } // if (total_parts2) + + // 3 subsets + int total_parts3 = m_super_strong_edges ? (p.m_base_parts3 * PART_ESTIMATE_STAGE1_MULTIPLIER) : (m_very_strong_edges ? (p.m_base_parts3 * 2) : p.m_base_parts3); + total_parts3 = minimum(total_parts3, MAX_BASE_PARTS3 * PART_ESTIMATE_STAGE1_MULTIPLIER); + total_parts3 = minimum(total_parts3, p.m_pPart_data_p3->m_total_unique_patterns); + + if (total_parts3) + { + int best_parts3_temp[MAX_BASE_PARTS3 * PART_ESTIMATE_STAGE1_MULTIPLIER]; + assert(total_parts3 <= (int)std::size(best_parts3_temp)); + + // Stage 1: kmeans+vptree + const bool has_est_parts3 = estimate_partition3( + p.m_block_width, p.m_block_height, + pixel_stats, + best_parts3_temp, total_parts3, + p.m_pPart_data_p3, p.m_brute_force_est_parts); + + if (has_est_parts3) + { + // Always try direct, optionally base+scale cem's + for (uint32_t s = 0; s < 2; s++) + { + if ((s) && (!p.m_use_base_scale_modes)) + continue; + + if (p.m_disable_part_est_stage2) + { + m_num_est_parts3[s] = total_parts3; + memcpy(m_best_parts3[s], best_parts3_temp, m_num_est_parts3[s] * sizeof(int)); + continue; + } + + uint32_t cem_to_surrogate_encode = p.m_alpha_cems ? astc_helpers::CEM_LDR_RGBA_DIRECT : astc_helpers::CEM_LDR_RGB_DIRECT; + if (s) + cem_to_surrogate_encode = p.m_alpha_cems ? astc_helpers::CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A : astc_helpers::CEM_LDR_RGB_BASE_SCALE; + + // Stage 2: Analytic surrogate WSSE + basisu::vector part_sses(total_parts3); + for (int i = 0; i < total_parts3; i++) + { + const astc_ldr::partitions_data* pPart_data = p.m_pPart_data_p3; + + const uint32_t unique_seed_index = best_parts3_temp[i]; + const uint32_t part_seed_index = pPart_data->m_unique_index_to_part_seed[unique_seed_index]; + + const astc_ldr::partition_pattern_vec* pPat = &pPart_data->m_partition_pats[unique_seed_index]; + + log_surrogate_astc_blk surrogate_log_blk; + float sse = encode_surrogate_trial_subsets( + p.m_block_width, p.m_block_height, + pixel_stats, + cem_to_surrogate_encode, 3, part_seed_index, pPat, + astc_helpers::BISE_256_LEVELS, astc_helpers::BISE_64_LEVELS, + p.m_block_width, p.m_block_height, + surrogate_log_blk, + *p.m_pEnc_params, surrogate_encode_flags); + + stats.m_total_surrogate_encodes++; + + part_sses[i] = sse; + } // i + + basisu::vector part_sses_ranks(total_parts3); + + indirect_sort(total_parts3, part_sses_ranks.get_ptr(), part_sses.get_ptr()); + + m_num_est_parts3[s] = maximum(1, (total_parts3 + p.m_part3_fraction_to_keep - 1) / p.m_part3_fraction_to_keep); + + for (int i = 0; i < m_num_est_parts3[s]; i++) + { + const uint32_t rank_index = part_sses_ranks[i]; + const uint32_t unique_seed_unique = best_parts3_temp[rank_index]; + m_best_parts3[s][i] = unique_seed_unique; + } // i + + } // s + + } // if (has_est_parts3) + + } // if (total_parts3) + + return true; + } + + bool trivial_triage( + const ldr_astc_lowlevel_block_encoder_params& p, + const astc_ldr::pixel_stats_t& pixel_stats, + basisu::vector& out_blocks, + uint32_t blur_id, + encode_block_stats& stats) + { + BASISU_NOTE_UNUSED(pixel_stats); + BASISU_NOTE_UNUSED(stats); + BASISU_NOTE_UNUSED(out_blocks); + BASISU_NOTE_UNUSED(blur_id); + + if (m_trial_modes_to_estimate.capacity() < 1024) + m_trial_modes_to_estimate.reserve(1024); + m_trial_modes_to_estimate.resize(0); + + assert((astc_helpers::CEM_LDR_RGBA_BASE_PLUS_OFFSET + 1) == basist::astc_ldr_t::OTM_NUM_CEMS); + + for (uint32_t cem_index = astc_helpers::CEM_LDR_LUM_DIRECT; cem_index < basist::astc_ldr_t::OTM_NUM_CEMS; cem_index++) + { + if (astc_helpers::does_cem_have_alpha(cem_index) != p.m_alpha_cems) + continue; + + const bool cem_has_alpha = astc_helpers::does_cem_have_alpha(cem_index); + if (cem_has_alpha != p.m_use_alpha_or_opaque_modes) + continue; + + bool accept_flag = false; + switch (cem_index) + { + case astc_helpers::CEM_LDR_LUM_DIRECT: + case astc_helpers::CEM_LDR_LUM_ALPHA_DIRECT: + { + accept_flag = p.m_use_lum_direct_modes; + break; + } + case astc_helpers::CEM_LDR_RGB_DIRECT: + case astc_helpers::CEM_LDR_RGBA_DIRECT: + { + accept_flag = p.m_use_direct_modes; + break; + } + case astc_helpers::CEM_LDR_RGB_BASE_SCALE: + case astc_helpers::CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A: + { + accept_flag = p.m_use_base_scale_modes; + break; + } + default: + break; + } + + if (!accept_flag) + continue; + + const uint32_t s = astc_helpers::cem_is_ldr_base_scale(cem_index) ? 1 : 0; + + for (uint32_t subsets_index = 0; subsets_index < basist::astc_ldr_t::OTM_NUM_SUBSETS; subsets_index++) + { + if (subsets_index == 1) + { + if (!m_num_est_parts2[s]) + continue; + } + else if (subsets_index == 2) + { + if (!m_num_est_parts3[s]) + continue; + } + + const uint32_t ccs_max_index = (p.m_use_dual_planes ? basist::astc_ldr_t::OTM_NUM_CCS : 1); + for (uint32_t ccs_index = 0; ccs_index < ccs_max_index; ccs_index++) + { + if (ccs_index) + { + if (!p.m_dp_active_chans[ccs_index - 1]) + continue; + } + + for (uint32_t grid_size_index = 0; grid_size_index < basist::astc_ldr_t::OTM_NUM_GRID_SIZES; grid_size_index++) + { + if (grid_size_index) // if large grid + { + if (p.m_use_small_grids_only) + continue; + } + + for (uint32_t grid_anisos_index = 0; grid_anisos_index < basist::astc_ldr_t::OTM_NUM_GRID_ANISOS; grid_anisos_index++) + { + if (p.m_grid_hv_filtering) + { + if (grid_anisos_index == 1) + { + // W>=H + if (p.m_filter_horizontally_flag) + continue; + } + else if (grid_anisos_index == 2) + { + // Wm_tm_groups[cem_index][subsets_index][ccs_index][grid_size_index][grid_anisos_index]); + + } // grid_aniso_index + + } // grid_size_index + + } // ccs_index + + } // subsets_index + + } // cem_iter + + if (!m_trial_modes_to_estimate.size()) + { + assert(0); + return false; + } + + return true; + } + + bool analytic_triage( + const ldr_astc_lowlevel_block_encoder_params& p, + const astc_ldr::pixel_stats_t& pixel_stats, + basisu::vector& out_blocks, + uint32_t blur_id, + encode_block_stats& stats) + { + BASISU_NOTE_UNUSED(blur_id); + BASISU_NOTE_UNUSED(out_blocks); + + //--------------------------------- superbucket analytical estimation + + shortlist_bucket_hash_t& shortlist_buckets = m_shortlist_hash0; + + if (m_shortlist_hash0.get_table_size() != EXPECTED_SHORTLIST_HASH_SIZE) + { + const bool was_allocated = m_shortlist_hash0.get_table_size() > 0; + + m_shortlist_hash0.clear(); + m_shortlist_hash0.reserve(EXPECTED_SHORTLIST_HASH_SIZE / 2); + + if ((g_devel_messages) && (was_allocated)) + fmt_debug_printf("shortlist hash0 thrash\n"); + } + else + { + m_shortlist_hash0.reset(); + } + + m_used_superbuckets = false; + + if (p.m_use_superbuckets) + { + m_used_superbuckets = true; + + // This may thrash if it grows larger on another thread, but we must avoid determinism issues. + if (m_superbucket_hash.get_table_size() != EXPECTED_SUPERBUCKET_HASH_SIZE) + { + const bool was_allocated = m_superbucket_hash.get_table_size() > 0; + + m_superbucket_hash.clear(); + m_superbucket_hash.reserve(EXPECTED_SUPERBUCKET_HASH_SIZE >> 1); + + if ((g_devel_messages) && (was_allocated)) + fmt_debug_printf("superbucket hash thrash\n"); + } + else + { + m_superbucket_hash.reset(); + } + + trial_mode_estimate_superbucket_key new_key; + new_key.clear(); + + trial_mode_estimate_superbucket_value new_val; + + // Create superbuckets + uint32_t max_superbucket_tm_indices = 0; + for (uint32_t j = 0; j < m_trial_modes_to_estimate.size(); j++) + { + const uint32_t trial_mode_iter = m_trial_modes_to_estimate[j]; + + assert(trial_mode_iter < p.m_num_trial_modes); + const basist::astc_ldr_t::trial_mode& tm = p.m_pTrial_modes[trial_mode_iter]; + + new_key.m_cem_index = safe_cast_uint8(tm.m_cem); + new_key.m_ccs_index = safe_cast_int8(tm.m_ccs_index); + + new_key.m_subset_unique_index = 0; + new_key.m_num_subsets = (uint8_t)tm.m_num_parts; + + if (tm.m_num_parts == 1) + { + auto ins_res = m_superbucket_hash.insert(new_key, new_val); + const bool created_flag = ins_res.second; + + assert(ins_res.first->first.m_cem_index == tm.m_cem); + assert(ins_res.first->first.m_ccs_index == tm.m_ccs_index); + assert(ins_res.first->first.m_num_subsets == tm.m_num_parts); + + trial_mode_estimate_superbucket_value& v = (ins_res.first)->second; + + if (created_flag) + v.m_trial_mode_list.reserve(256); + + v.m_trial_mode_list.push_back(trial_mode_iter); + + max_superbucket_tm_indices = maximum(max_superbucket_tm_indices, v.m_trial_mode_list.size_u32()); + } + else + { + //const astc_ldr::partitions_data* pPart_data = (tm.m_num_parts == 2) ? p.m_pPart_data_p2 : p.m_pPart_data_p3; + + const uint32_t s = astc_helpers::cem_is_ldr_base_scale(tm.m_cem) ? 1 : 0; + const uint32_t num_est_parts_to_try = (tm.m_num_parts == 2) ? m_num_est_parts2[s] : m_num_est_parts3[s]; + + for (uint32_t est_part_iter = 0; est_part_iter < num_est_parts_to_try; est_part_iter++) + { + const uint32_t part_unique_index = (tm.m_num_parts == 2) ? m_best_parts2[s][est_part_iter] : m_best_parts3[s][est_part_iter]; + + new_key.m_subset_unique_index = safe_cast_uint16(part_unique_index); + + auto ins_res = m_superbucket_hash.insert(new_key, new_val); + const bool created_flag = ins_res.second; + + assert(ins_res.first->first.m_cem_index == tm.m_cem); + assert(ins_res.first->first.m_ccs_index == tm.m_ccs_index); + assert(ins_res.first->first.m_num_subsets == tm.m_num_parts); + + trial_mode_estimate_superbucket_value& v = (ins_res.first)->second; + if (created_flag) + v.m_trial_mode_list.reserve(256); + + v.m_trial_mode_list.push_back(trial_mode_iter); + + max_superbucket_tm_indices = maximum(max_superbucket_tm_indices, v.m_trial_mode_list.size_u32()); + + } // est_part_iter + } + + } // j + + //fmt_debug_printf("Total superbucket entries: {}\n", m_superbucket_hash.size()); + //fmt_debug_printf("Max superbucket tm indices: {}\n", max_superbucket_tm_indices); + + const uint32_t total_block_texels = p.m_total_block_pixels; + const float inv_total_block_texels = 1.0f / (float)total_block_texels; + + while (m_trial_mode_estimate_priority_queue.size()) + m_trial_mode_estimate_priority_queue.pop(); + + const uint32_t max_priority_queue_size = p.m_superbucket_max_to_retain[m_block_complexity_index]; + + // purposely downscale lost scale energy relative to the other error sources + // this biased the encoder towards smaller grids + const float SLAM_TO_LINE_WEIGHT = 1.5f; // upweight STL relative to other errors to give the estimator more of a signal especially for dual plane + const float QUANT_ERROR_WEIGHT = 1.0f; // quant error is naturally quite pessimistic + const float SCALE_ERROR_WEIGHT = 3.0f; // weight grid downsample (scale) error + + // Discount for blue contraction encoding and base+offset CEM's. + const float BLUE_CONTRACTION_ENDPOINT_QUANT_DISCOUNT = .5f; + + // Iterate over all superbuckets, surrogate encode to compute slam to line error, DCT of weight grid(s) to estimate energy lost during weight grid downsampling. + // TODO: priority queue and aggressive early outs + for (auto superbucket_iter = m_superbucket_hash.begin(); superbucket_iter != m_superbucket_hash.end(); ++superbucket_iter) + { + const trial_mode_estimate_superbucket_key& key = superbucket_iter->first; + const trial_mode_estimate_superbucket_value& val = superbucket_iter->second; + + //const bool cem_has_alpha = astc_helpers::does_cem_have_alpha(key.m_cem_index); + + log_surrogate_astc_blk log_blk; + + const astc_ldr::partitions_data* pPart_data = nullptr; + const astc_ldr::partition_pattern_vec* pPat = nullptr; + + //const uint32_t num_planes = (key.m_ccs_index >= 0) ? 2 : 1; + + const float worst_wsse_found_so_far = (m_trial_mode_estimate_priority_queue.size() >= max_priority_queue_size) ? m_trial_mode_estimate_priority_queue.top().m_wsse : 1e+9f; + + float slam_to_line_wsse = 0; + if (key.m_num_subsets == 1) + { + slam_to_line_wsse = encode_surrogate_trial( + p.m_block_width, p.m_block_height, + pixel_stats, + key.m_cem_index, + key.m_ccs_index, + astc_helpers::BISE_256_LEVELS, astc_helpers::BISE_64_LEVELS, + p.m_block_width, p.m_block_height, + log_blk, + *p.m_pEnc_params, + astc_ldr::cFlagDisableQuant); + } + else + { + pPart_data = (key.m_num_subsets == 3) ? p.m_pPart_data_p3 : p.m_pPart_data_p2; + + const uint32_t unique_seed_index = key.m_subset_unique_index; + const uint32_t part_seed_index = pPart_data->m_unique_index_to_part_seed[unique_seed_index]; + + pPat = &pPart_data->m_partition_pats[unique_seed_index]; + + slam_to_line_wsse = encode_surrogate_trial_subsets( + p.m_block_width, p.m_block_height, + pixel_stats, + key.m_cem_index, key.m_num_subsets, part_seed_index, pPat, + astc_helpers::BISE_256_LEVELS, astc_helpers::BISE_64_LEVELS, + p.m_block_width, p.m_block_height, + log_blk, + *p.m_pEnc_params, + astc_ldr::cFlagDisableQuant); + } + + stats.m_total_surrogate_encodes++; + + // Early out: Slam to line error is so high it's impossible for any blocks in this bucket to win. + if ((SLAM_TO_LINE_WEIGHT * slam_to_line_wsse) >= worst_wsse_found_so_far) + continue; + + bool can_use_base_ofs = false; + if ((key.m_cem_index == astc_helpers::CEM_LDR_RGB_DIRECT) || (key.m_cem_index == astc_helpers::CEM_LDR_RGBA_DIRECT)) + { + float max_span_size = 0.0f; + + for (uint32_t subset_index = 0; subset_index < key.m_num_subsets; subset_index++) + { + const vec4F subset_chan_spans(log_blk.m_endpoints[subset_index][1] - log_blk.m_endpoints[subset_index][0]); + for (uint32_t c = 0; c < 4; c++) + { + float span_size = fabs(subset_chan_spans[c]); + max_span_size = maximum(max_span_size, span_size); + } + } + + can_use_base_ofs = (max_span_size < .25f); + } + + assert(p.m_pDCT2F); + + assert((p.m_pDCT2F->rows() == p.m_block_height) && (p.m_pDCT2F->cols() == p.m_block_width)); + + float weight0_energy[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + float weight1_energy[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + + basist::astc_ldr_t::fvec& dct_work = m_dct_work; + + // Forward DCT in normalized weight (surrogate) space + p.m_pDCT2F->forward(log_blk.m_weights0, weight0_energy, dct_work); + compute_energy_from_dct(p.m_block_width, p.m_block_height, weight0_energy); + + if (key.m_ccs_index >= 0) + { + p.m_pDCT2F->forward(log_blk.m_weights1, weight1_energy, dct_work); + compute_energy_from_dct(p.m_block_width, p.m_block_height, weight1_energy); + } + + weight_terms weight0_terms, weight1_terms; + weight_terms* pWeight0_terms = &weight0_terms; + weight_terms* pWeight1_terms = nullptr; + weight0_terms.calc(total_block_texels, log_blk.m_weights0); + if (key.m_ccs_index >= 0) + { + weight1_terms.calc(total_block_texels, log_blk.m_weights1); + pWeight1_terms = &weight1_terms; + } + + // Precompute subset span and total pixels info + vec4F subset_spans[astc_helpers::MAX_PARTITIONS]; + uint32_t subset_pixels[astc_helpers::MAX_PARTITIONS]; + + for (uint32_t subset_index = 0; subset_index < key.m_num_subsets; subset_index++) + { + subset_spans[subset_index] = log_blk.m_endpoints[subset_index][1] - log_blk.m_endpoints[subset_index][0]; + + uint32_t total_subset_pixels = p.m_total_block_pixels; + if (key.m_num_subsets > 1) + total_subset_pixels = pPart_data->m_partition_pat_histograms[key.m_subset_unique_index].m_hist[subset_index]; + + subset_pixels[subset_index] = total_subset_pixels; + } + + // Loop through all trial modes in this sueprbucket. TODO: Sort by endpoint levels? + for (uint32_t k = 0; k < val.m_trial_mode_list.size(); k++) + { + const uint32_t trial_mode_index = val.m_trial_mode_list[k]; + assert(trial_mode_index < p.m_num_trial_modes); + + const basist::astc_ldr_t::trial_mode& tm = p.m_pTrial_modes[trial_mode_index]; + + assert(tm.m_cem == key.m_cem_index); + assert(tm.m_ccs_index == key.m_ccs_index); + assert(tm.m_num_parts == key.m_num_subsets); + + const basist::astc_ldr_t::astc_block_grid_data* pGrid_data = basist::astc_ldr_t::find_astc_block_grid_data(p.m_block_width, p.m_block_height, tm.m_grid_width, tm.m_grid_height); + + const uint32_t total_endpoint_levels = astc_helpers::get_ise_levels(tm.m_endpoint_ise_range); + const uint32_t total_weight_levels = astc_helpers::get_ise_levels(tm.m_weight_ise_range); + + const uint32_t num_effective_e_levels = can_use_base_ofs ? minimum(total_endpoint_levels * 2, 256) : total_endpoint_levels; + float qe0 = compute_quantized_channel_endpoint_mse_estimate(num_effective_e_levels); + const float qe1 = (key.m_ccs_index >= 0) ? (qe0 * pWeight1_terms->m_endpoint_factor) : 0.0f; + qe0 *= pWeight0_terms->m_endpoint_factor; + + float total_e_quant_wsse = 0.0f; + + for (uint32_t subset_index = 0; subset_index < key.m_num_subsets; subset_index++) + { + const vec4F& subset_chan_spans = subset_spans[subset_index]; + const uint32_t total_subset_pixels = subset_pixels[subset_index]; + + for (uint32_t c = 0; c < 4; c++) + { + float span_size = fabs(subset_chan_spans[c]); + + if ((span_size == 0.0f) && ((log_blk.m_endpoints[subset_index][1][c] == 0.0f) || (log_blk.m_endpoints[subset_index][1][c] == 1.0f))) + continue; + + // Scale channel MSE by chan weight and the # of subset pixels to get weighted SSE + const float chan_N = (float)p.m_pEnc_params->m_comp_weights[c] * (float)total_subset_pixels; + + total_e_quant_wsse += ((key.m_ccs_index == (int)c) ? qe1 : qe0) * chan_N; + + } // chan_index + } + + if ((tm.m_cem == astc_helpers::CEM_LDR_RGB_DIRECT) || (tm.m_cem == astc_helpers::CEM_LDR_RGBA_DIRECT)) + total_e_quant_wsse *= BLUE_CONTRACTION_ENDPOINT_QUANT_DISCOUNT; + + float total_wsse_so_far = (SLAM_TO_LINE_WEIGHT * slam_to_line_wsse) + (QUANT_ERROR_WEIGHT * total_e_quant_wsse); + if (total_wsse_so_far >= worst_wsse_found_so_far) + continue; + + float lost_weight_energy0 = compute_lost_dct_energy(p.m_block_width, p.m_block_height, weight0_energy, tm.m_grid_width, tm.m_grid_height) * inv_total_block_texels; + + float lost_weight_energy1 = 0; + if (key.m_ccs_index >= 0) + lost_weight_energy1 = compute_lost_dct_energy(p.m_block_width, p.m_block_height, weight1_energy, tm.m_grid_width, tm.m_grid_height) * inv_total_block_texels; + + // Add up: + // slam to line error WSSE (weighted sum of squared errors) + // weight quant error WSSE + // endpoint quant error WSSE + // weight grid rescale error WSSE (scaled by span^2) + float total_scale_wsse = 0.0f; + + for (uint32_t subset_index = 0; subset_index < key.m_num_subsets; subset_index++) + { + const vec4F& subset_chan_spans = subset_spans[subset_index]; + const uint32_t total_subset_pixels = subset_pixels[subset_index]; + + for (uint32_t c = 0; c < 4; c++) + { + float span_size = fabs(subset_chan_spans[c]); + + if ((span_size == 0.0f) && ((log_blk.m_endpoints[subset_index][1][c] == 0.0f) || (log_blk.m_endpoints[subset_index][1][c] == 1.0f))) + { + // Won't have any E/W quant err at extremes (0.0 or 1.0 are always perfectly represented), no weight downsample error either. + //chan_mse.m_ep = 0.0f; + //chan_mse.m_wp = 0.0f; + } + else + { + // Scale channel MSE by chan weight and the # of subset pixels to get weighted SSE + const float chan_N = (float)p.m_pEnc_params->m_comp_weights[c] * (float)total_subset_pixels; + + // sum in the plane's lost weight energy, scaled by span_size^2 * chan_weight * num_texels_covered + if (key.m_ccs_index == (int)c) + total_scale_wsse += lost_weight_energy1 * square(span_size) * chan_N; + else + total_scale_wsse += lost_weight_energy0 * square(span_size) * chan_N; + } + + } // chan_index + } + + total_wsse_so_far += (SCALE_ERROR_WEIGHT * total_scale_wsse); + if (total_wsse_so_far >= worst_wsse_found_so_far) + continue; + + float total_w_quant_wsse = 0.0f; + for (uint32_t subset_index = 0; subset_index < key.m_num_subsets; subset_index++) + { + const vec4F& subset_chan_spans = subset_spans[subset_index]; + const uint32_t total_subset_pixels = subset_pixels[subset_index]; + + for (uint32_t c = 0; c < 4; c++) + { + float span_size = fabs(subset_chan_spans[c]); + + if ((span_size == 0.0f) && ((log_blk.m_endpoints[subset_index][1][c] == 0.0f) || (log_blk.m_endpoints[subset_index][1][c] == 1.0f))) + { + // Won't have any E/W quant err at extremes (0.0 or 1.0 are always perfectly represented), no weight downsample error either. + //chan_mse.m_ep = 0.0f; + //chan_mse.m_wp = 0.0f; + } + else + { + // span_size != 0 here - estimate weight/endpoint quantization errors + float chan_w_mse = compute_quantized_channel_weight_mse_estimate( + total_weight_levels, span_size, + pGrid_data->m_weight_gamma, (key.m_ccs_index == (int)c) ? pWeight1_terms : pWeight0_terms); + + // Scale channel MSE by chan weight and the # of subset pixels to get weighted SSE + const float chan_N = (float)p.m_pEnc_params->m_comp_weights[c] * (float)total_subset_pixels; + + total_w_quant_wsse += chan_w_mse * chan_N; + } + + } // chan_index + + } // subset_index + + const float total_wsse = total_wsse_so_far + (QUANT_ERROR_WEIGHT * total_w_quant_wsse); + + if (m_trial_mode_estimate_priority_queue.size() >= max_priority_queue_size) + { + if (total_wsse < m_trial_mode_estimate_priority_queue.top().m_wsse) + { + m_trial_mode_estimate_priority_queue.pop(); + + trial_mode_estimate est; + est.m_superbucket_key = key; + est.m_trial_mode_index = trial_mode_index; + est.m_wsse = total_wsse; + + m_trial_mode_estimate_priority_queue.push(est); + } + } + else + { + trial_mode_estimate est; + est.m_superbucket_key = key; + est.m_trial_mode_index = trial_mode_index; + est.m_wsse = total_wsse; + + m_trial_mode_estimate_priority_queue.push(est); + } + + } // k + + } // superbucket_iter + + stats.m_total_superbuckets_created += m_superbucket_hash.size_u32(); + + const uint32_t total_estimates_to_retain = (uint32_t)m_trial_mode_estimate_priority_queue.size(); + assert(total_estimates_to_retain); + + for (uint32_t i = 0; i < total_estimates_to_retain; i++) + { + const trial_mode_estimate &est = m_trial_mode_estimate_priority_queue.top(); + + const trial_mode_estimate_superbucket_key& key = est.m_superbucket_key; + const uint32_t trial_mode_iter = est.m_trial_mode_index; + + assert(trial_mode_iter < p.m_num_trial_modes); + const basist::astc_ldr_t::trial_mode& tm = p.m_pTrial_modes[trial_mode_iter]; + + assert(tm.m_cem == key.m_cem_index); + assert(tm.m_ccs_index == key.m_ccs_index); + assert(tm.m_num_parts == key.m_num_subsets); + + const uint32_t part_unique_index = key.m_subset_unique_index; + + auto ins_res = shortlist_buckets.insert(shortlist_bucket(tm.m_grid_width, tm.m_grid_height, tm.m_cem, tm.m_ccs_index, tm.m_num_parts, part_unique_index)); + + ins_res.first->second.push_back(safe_cast_uint16(trial_mode_iter)); + + m_trial_mode_estimate_priority_queue.pop(); + } + } + else + { + for (uint32_t j = 0; j < m_trial_modes_to_estimate.size(); j++) + { + const uint32_t trial_mode_iter = m_trial_modes_to_estimate[j]; + + assert(trial_mode_iter < p.m_num_trial_modes); + const basist::astc_ldr_t::trial_mode& tm = p.m_pTrial_modes[trial_mode_iter]; + + if (tm.m_num_parts > 1) + { + //const astc_ldr::partitions_data* pPart_data = (tm.m_num_parts == 2) ? p.m_pPart_data_p2 : p.m_pPart_data_p3; + + const uint32_t s = astc_helpers::cem_is_ldr_base_scale(tm.m_cem) ? 1 : 0; + const uint32_t num_est_parts_to_try = (tm.m_num_parts == 2) ? m_num_est_parts2[s] : m_num_est_parts3[s]; + + for (uint32_t est_part_iter = 0; est_part_iter < num_est_parts_to_try; est_part_iter++) + { + const uint32_t part_unique_index = (tm.m_num_parts == 2) ? m_best_parts2[s][est_part_iter] : m_best_parts3[s][est_part_iter]; + + auto ins_res = shortlist_buckets.insert(shortlist_bucket(tm.m_grid_width, tm.m_grid_height, tm.m_cem, tm.m_ccs_index, tm.m_num_parts, part_unique_index)); + + ins_res.first->second.push_back(safe_cast_uint16(trial_mode_iter)); + + } // est_part_iter + + } + else + { + auto ins_res = shortlist_buckets.insert(shortlist_bucket(tm.m_grid_width, tm.m_grid_height, tm.m_cem, tm.m_ccs_index, 1, 0)); + ins_res.first->second.push_back(safe_cast_uint16(trial_mode_iter)); + + } + } + } + + stats.m_total_buckets_created += (uint32_t)shortlist_buckets.size(); + +#if 0 + // TEMP + uint32_t max_bucket_tm_indices = 0; + for (auto it = shortlist_buckets.begin(); it != shortlist_buckets.end(); ++it) + { + shortlist_bucket& bucket = it->first; + trial_mode_index_vec& trial_mode_indices = it->second; + max_bucket_tm_indices = maximum(max_bucket_tm_indices, trial_mode_indices.size_u32()); + } + + fmt_debug_printf("max_bucket_tm_indices: {}\n", max_bucket_tm_indices); +#endif + + return true; + } + + bool surrogate_encode_shortlist_bucket_representatives( + const ldr_astc_lowlevel_block_encoder_params& p, + const astc_ldr::pixel_stats_t& pixel_stats, + basisu::vector& out_blocks, + uint32_t blur_id, + encode_block_stats& stats) + { + BASISU_NOTE_UNUSED(blur_id); + BASISU_NOTE_UNUSED(out_blocks); + + shortlist_bucket_hash_t& shortlist_buckets = m_shortlist_hash0; + + // Surrogate encode a representative for each bucket. + for (auto it = shortlist_buckets.begin(); it != shortlist_buckets.end(); ++it) + { + shortlist_bucket& bucket = it->first; + //const uint_vec& trial_mode_indices = it->second; + const trial_mode_index_vec& trial_mode_indices = it->second; + + // Choose bucket's largest endpoint/weight ise ranges (finest quant levels) - anything in the bucket will quite likely encode to worse SSE, which we can rapidly estimate. + uint32_t max_endpoint_ise_range = 0, max_weight_ise_range = 0; + for (uint32_t i = 0; i < trial_mode_indices.size(); i++) + { + const basist::astc_ldr_t::trial_mode& tm = p.m_pTrial_modes[trial_mode_indices[i]]; + + max_endpoint_ise_range = maximum(max_endpoint_ise_range, tm.m_endpoint_ise_range); + max_weight_ise_range = maximum(max_weight_ise_range, tm.m_weight_ise_range); + } + + log_surrogate_astc_blk& log_block = bucket.m_surrogate_log_blk; + + if (bucket.m_num_parts == 1) + { + bucket.m_sse = encode_surrogate_trial( + p.m_block_width, p.m_block_height, + pixel_stats, + bucket.m_cem_index, + bucket.m_ccs_index, + max_endpoint_ise_range, max_weight_ise_range, + bucket.m_grid_width, bucket.m_grid_height, + log_block, + *p.m_pEnc_params, 0); + + stats.m_total_surrogate_encodes++; + } + else + { + const astc_ldr::partitions_data* pPart_data = (bucket.m_num_parts == 2) ? p.m_pPart_data_p2 : p.m_pPart_data_p3; + + const uint32_t part_seed_index = pPart_data->m_unique_index_to_part_seed[bucket.m_unique_seed_index]; + + const astc_ldr::partition_pattern_vec* pPat = &pPart_data->m_partition_pats[bucket.m_unique_seed_index]; + + bucket.m_sse = encode_surrogate_trial_subsets( + p.m_block_width, p.m_block_height, + pixel_stats, + bucket.m_cem_index, bucket.m_num_parts, part_seed_index, pPat, + max_endpoint_ise_range, max_weight_ise_range, + bucket.m_grid_width, bucket.m_grid_height, + log_block, + *p.m_pEnc_params, 0); + + stats.m_total_surrogate_encodes++; + } + + if ((bucket.m_cem_index == astc_helpers::CEM_LDR_RGB_DIRECT) || (bucket.m_cem_index == astc_helpers::CEM_LDR_RGBA_DIRECT)) + { + // blue contraction/base+offset discount + bucket.m_sse *= BLUE_CONTRACTION_BASE_OFS_DISCOUNT; + } + + } // it + + return true; + } + + bool prune_shortlist_buckets( + const ldr_astc_lowlevel_block_encoder_params& p, + const astc_ldr::pixel_stats_t& pixel_stats, + basisu::vector& out_blocks, + uint32_t blur_id, + encode_block_stats& stats) + { + BASISU_NOTE_UNUSED(pixel_stats); + BASISU_NOTE_UNUSED(stats); + BASISU_NOTE_UNUSED(blur_id); + BASISU_NOTE_UNUSED(out_blocks); + + shortlist_bucket_hash_t& shortlist_buckets = m_shortlist_hash0; + + if (p.m_bucket_pruning_passes) + { + shortlist_bucket_hash_t& new_shortlist_buckets = m_shortlist_hash1; + + if (m_shortlist_hash1.get_table_size() != EXPECTED_SHORTLIST_HASH_SIZE) + { + const bool was_allocated = m_shortlist_hash1.get_table_size() > 0; + + m_shortlist_hash1.clear(); + m_shortlist_hash1.reserve(EXPECTED_SHORTLIST_HASH_SIZE / 2); + + if ((g_devel_messages) && (was_allocated)) + fmt_debug_printf("shortlist hash1 thrash\n"); + } + else + { + m_shortlist_hash1.reset(); + } + + const uint32_t NUM_PRUNE_PASSES = 3; + for (uint32_t prune_pass = 0; prune_pass < NUM_PRUNE_PASSES; prune_pass++) + { + for (auto it = shortlist_buckets.begin(); it != shortlist_buckets.end(); ++it) + it->first.m_examined_flag = false; + + new_shortlist_buckets.reset(); + + for (auto it = shortlist_buckets.begin(); it != shortlist_buckets.end(); ++it) + { + shortlist_bucket& bucket = it->first; + + if (bucket.m_examined_flag) + continue; + + if (prune_pass == 0) + { + // Prune pass 0: Dual plane groups: only accept best CCS index + if (bucket.m_ccs_index >= 0) + { + shortlist_bucket_hash_t::iterator ccs_buckets[4]; + + int best_ccs_index = -1; + float best_ccs_err = BIG_FLOAT_VAL; + + bool skip_bucket = false; + for (uint32_t c = 0; c < 4; c++) + { + auto ccs_res_it = shortlist_buckets.find(shortlist_bucket(bucket.m_grid_width, bucket.m_grid_height, bucket.m_cem_index, c, bucket.m_num_parts, bucket.m_unique_seed_index)); + ccs_buckets[c] = ccs_res_it; + + if (ccs_res_it == shortlist_buckets.end()) + continue; + + assert(!ccs_res_it->first.m_examined_flag); + + ccs_res_it->first.m_examined_flag = true; + + float ccs_sse_err = ccs_res_it->first.m_sse; + if (ccs_sse_err < best_ccs_err) + { + best_ccs_err = ccs_sse_err; + best_ccs_index = c; + } + } // c + + if (!skip_bucket) + { + assert(best_ccs_index >= 0); + + shortlist_bucket_hash_t::iterator best_ccs_it = ccs_buckets[best_ccs_index]; + assert(best_ccs_it != shortlist_buckets.end()); + + new_shortlist_buckets.insert(best_ccs_it->first, best_ccs_it->second); + } + } + else + { + new_shortlist_buckets.insert(it->first, it->second); + } + } + else if (prune_pass == 1) + { + // Prune pass 1: Same # of weight samples, compare WxH vs. HxW + if (bucket.m_grid_width != bucket.m_grid_height) + { + auto alt_res_it = shortlist_buckets.find(shortlist_bucket(bucket.m_grid_height, bucket.m_grid_width, bucket.m_cem_index, bucket.m_ccs_index, bucket.m_num_parts, bucket.m_unique_seed_index)); + if (alt_res_it == shortlist_buckets.end()) + { + new_shortlist_buckets.insert(it->first, it->second); + } + else + { + assert(!alt_res_it->first.m_examined_flag); + alt_res_it->first.m_examined_flag = true; + + const float fract = (bucket.m_sse > 0.0f) ? (alt_res_it->first.m_sse / bucket.m_sse) : 0.0f; + + const float ALT_RES_SSE_THRESH = .2f; + if (fract < (1.0f - ALT_RES_SSE_THRESH)) + new_shortlist_buckets.insert(alt_res_it->first, alt_res_it->second); + else if (fract > (1.0f + ALT_RES_SSE_THRESH)) + new_shortlist_buckets.insert(it->first, it->second); + else + { + new_shortlist_buckets.insert(alt_res_it->first, alt_res_it->second); + new_shortlist_buckets.insert(it->first, it->second); + } + } + } + else + { + new_shortlist_buckets.insert(it->first, it->second); + } + + } + else if (prune_pass == 2) + { + // Prune pass 2: RGB Direct vs. Scale bucket groups + + if ((bucket.m_cem_index == astc_helpers::CEM_LDR_RGB_DIRECT) || (bucket.m_cem_index == astc_helpers::CEM_LDR_RGB_BASE_SCALE) || + (bucket.m_cem_index == astc_helpers::CEM_LDR_RGBA_DIRECT) || (bucket.m_cem_index == astc_helpers::CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A)) + { + uint32_t alt_cem_index_to_find = astc_helpers::CEM_LDR_RGB_BASE_SCALE; + + // Check for pairs: CEM_LDR_RGB_DIRECT vs. CEM_LDR_RGB_BASE_SCALE, or CEM_LDR_RGBA_DIRECT vs. CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A. + switch (bucket.m_cem_index) + { + case astc_helpers::CEM_LDR_RGB_DIRECT: + alt_cem_index_to_find = astc_helpers::CEM_LDR_RGB_BASE_SCALE; + break; + case astc_helpers::CEM_LDR_RGB_BASE_SCALE: + alt_cem_index_to_find = astc_helpers::CEM_LDR_RGB_DIRECT; + break; + case astc_helpers::CEM_LDR_RGBA_DIRECT: + alt_cem_index_to_find = astc_helpers::CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A; + break; + case astc_helpers::CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A: + alt_cem_index_to_find = astc_helpers::CEM_LDR_RGBA_DIRECT; + break; + default: + assert(0); + break; + } + + auto alt_res_it = shortlist_buckets.find(shortlist_bucket(bucket.m_grid_width, bucket.m_grid_height, alt_cem_index_to_find, bucket.m_ccs_index, bucket.m_num_parts, bucket.m_unique_seed_index)); + + if (alt_res_it == shortlist_buckets.end()) + { + new_shortlist_buckets.insert(it->first, it->second); + } + else + { + assert(!alt_res_it->first.m_examined_flag); + + alt_res_it->first.m_examined_flag = true; + + // Compare the two buckets, decide if one or another can be tossed as not worth it. + const float fract = (bucket.m_sse > 0.0f) ? (alt_res_it->first.m_sse / bucket.m_sse) : 0.0f; + + const float ALT_RES_SSE_THRESH = .1f; + if (fract < (1.0f - ALT_RES_SSE_THRESH)) + new_shortlist_buckets.insert(alt_res_it->first, alt_res_it->second); + else if (fract > (1.0f + ALT_RES_SSE_THRESH)) + new_shortlist_buckets.insert(it->first, it->second); + else + { + new_shortlist_buckets.insert(alt_res_it->first, alt_res_it->second); + new_shortlist_buckets.insert(it->first, it->second); + } + } + } + else + { + new_shortlist_buckets.insert(it->first, it->second); + } + + } // if (prune_pass + + it->first.m_examined_flag = true; + } + + new_shortlist_buckets.swap(shortlist_buckets); + } // prune_pass + } // if (g_bucket_pruning_passes) + + assert(shortlist_buckets.size()); + + if (m_ranked_buckets.capacity() < shortlist_buckets.size()) + m_ranked_buckets.reserve(shortlist_buckets.size()); + + for (auto it = shortlist_buckets.begin(); it != shortlist_buckets.end(); ++it) + { + shortlist_bucket& bucket = it->first; + const trial_mode_index_vec& trial_mode_indices = it->second; + + ranked_shortlist_bucket* pDst = m_ranked_buckets.enlarge(1); + pDst->m_bucket = bucket; + pDst->m_trial_mode_indices = trial_mode_indices; + } + + assert(m_ranked_buckets.size()); + + // Sort the buckets by their surrogate encoded SSE to rank them. + std::sort(m_ranked_buckets.begin(), m_ranked_buckets.end()); + + return true; + } + + bool rank_and_sort_shortlist_buckets( + const ldr_astc_lowlevel_block_encoder_params& p, + const astc_ldr::pixel_stats_t& pixel_stats, + basisu::vector& out_blocks, + uint32_t blur_id, + encode_block_stats& stats) + { + BASISU_NOTE_UNUSED(blur_id); + BASISU_NOTE_UNUSED(out_blocks); + + basisu::vector& shortlist_trials = m_trial_surrogates; + + // TODO: Tune this further. Memory here adds up across all encoding threads. + { + //const float reserve_factor = (sizeof(void*) > 4) ? .5f : .25f; + const uint32_t reserve_size = 64;// maximum(256, (int)(p.m_num_trial_modes * reserve_factor)); + + if (shortlist_trials.capacity() < reserve_size) + shortlist_trials.reserve(reserve_size); + + shortlist_trials.resize(0); + } + + uint32_t num_buckets_to_examine = fast_roundf_int((float)m_ranked_buckets.size_u32() * p.m_shortlist_buckets_to_examine_fract); + num_buckets_to_examine = clamp(num_buckets_to_examine, p.m_shortlist_buckets_to_examine_min, p.m_shortlist_buckets_to_examine_max); + + num_buckets_to_examine = clamp(num_buckets_to_examine, 1, m_ranked_buckets.size_u32()); + + float best_err_so_far = BIG_FLOAT_VAL; + + for (uint32_t bucket_index = 0; bucket_index < num_buckets_to_examine; bucket_index++) + { + const shortlist_bucket& bucket = m_ranked_buckets[bucket_index].m_bucket; + const trial_mode_index_vec& bucket_trial_mode_indices = m_ranked_buckets[bucket_index].m_trial_mode_indices; + + if (best_err_so_far != BIG_FLOAT_VAL) + { + if (bucket.m_sse > best_err_so_far * SKIP_IF_BUCKET_WORSE_MULTIPLIER) + continue; + } + best_err_so_far = minimum(best_err_so_far, bucket.m_sse); + + if (bucket_trial_mode_indices.size() == 1) + { + // Bucket only contains 1 mode, so we've already encoded its surrogate. + trial_surrogate& s = *shortlist_trials.try_enlarge(1); + + s.m_trial_mode_index = bucket_trial_mode_indices[0]; + s.m_err = bucket.m_sse; + s.m_log_blk = bucket.m_surrogate_log_blk; + continue; + } + + //----- + // We have a bucket sharing all config except for ISE weight/endpoint levels. Decide how many to place on the shortlist using analytic weighted MSE/SSE estimates. + + const uint32_t num_modes_in_bucket = bucket_trial_mode_indices.size_u32(); + + uint32_t num_modes_in_bucket_to_shortlist = fast_roundf_pos_int(num_modes_in_bucket * p.m_num_similar_modes_in_bucket_to_shortlist_fract); + + num_modes_in_bucket_to_shortlist = clamp(num_modes_in_bucket_to_shortlist, p.m_num_similar_modes_in_bucket_to_shortlist_fract_min, p.m_num_similar_modes_in_bucket_to_shortlist_fract_max); + + num_modes_in_bucket_to_shortlist = clamp(num_modes_in_bucket_to_shortlist, 1, num_modes_in_bucket); + + basisu::vector bucket_indices(num_modes_in_bucket); + for (uint32_t i = 0; i < num_modes_in_bucket; i++) + bucket_indices[i] = i; + + if (num_modes_in_bucket_to_shortlist < num_modes_in_bucket) + { + basisu::vector sse_estimates(num_modes_in_bucket); + + const uint32_t bucket_surrogate_endpoint_levels = bucket.m_surrogate_log_blk.m_num_endpoint_levels; + const uint32_t bucket_surrogate_weight_levels = bucket.m_surrogate_log_blk.m_num_weight_levels; + const float bucket_surrogate_base_sse = bucket.m_sse; + + const basist::astc_ldr_t::astc_block_grid_data* pGrid_data = basist::astc_ldr_t::find_astc_block_grid_data(p.m_block_width, p.m_block_height, bucket.m_grid_width, bucket.m_grid_height); + const astc_ldr::partitions_data* pBucket_part_data = (bucket.m_num_parts == 1) ? nullptr : ((bucket.m_num_parts == 2) ? p.m_pPart_data_p2 : p.m_pPart_data_p3); + + bool can_use_base_ofs = false; + if ((bucket.m_cem_index == astc_helpers::CEM_LDR_RGB_DIRECT) || (bucket.m_cem_index == astc_helpers::CEM_LDR_RGBA_DIRECT)) + { + float max_span_size = 0.0f; + for (uint32_t part_iter = 0; part_iter < bucket.m_num_parts; part_iter++) + { + for (uint32_t c = 0; c < 4; c++) + { + float span_size = fabs(bucket.m_surrogate_log_blk.m_endpoints[part_iter][1][c] - bucket.m_surrogate_log_blk.m_endpoints[part_iter][0][c]); + max_span_size = maximum(max_span_size, span_size); + } + } + + can_use_base_ofs = max_span_size < .25f; + } + + chan_mse_est bucket_sse_est(0.0f, 0.0f); + for (uint32_t part_iter = 0; part_iter < bucket.m_num_parts; part_iter++) + { + uint32_t total_texels_in_part = p.m_block_width * p.m_block_height; + if (bucket.m_num_parts > 1) + { + total_texels_in_part = pBucket_part_data->m_partition_pat_histograms[bucket.m_unique_seed_index].m_hist[part_iter]; + assert(total_texels_in_part && total_texels_in_part < p.m_block_width * p.m_block_height); + } + + for (uint32_t c = 0; c < 4; c++) + { + float span_size = fabs(bucket.m_surrogate_log_blk.m_endpoints[part_iter][1][c] - bucket.m_surrogate_log_blk.m_endpoints[part_iter][0][c]); + + chan_mse_est chan_mse_est(compute_quantized_channel_mse_estimates( + can_use_base_ofs ? minimum(bucket_surrogate_endpoint_levels * 2, 256) : bucket_surrogate_endpoint_levels, + bucket_surrogate_weight_levels, + span_size, pGrid_data->m_weight_gamma)); + + if (span_size == 0.0f) + { + if ((bucket.m_surrogate_log_blk.m_endpoints[part_iter][1][c] == 1.0f) || (bucket.m_surrogate_log_blk.m_endpoints[part_iter][1][c] == 0.0f)) + { + chan_mse_est.m_ep = 0.0f; + chan_mse_est.m_wp = 0.0f; + } + } + + bucket_sse_est.m_ep += chan_mse_est.m_ep * (float)p.m_pEnc_params->m_comp_weights[c] * total_texels_in_part; + bucket_sse_est.m_wp += chan_mse_est.m_wp * (float)p.m_pEnc_params->m_comp_weights[c] * total_texels_in_part; + } // c + + } // part_iter + +#if 0 + fmt_debug_printf("----------------\n"); + + fmt_debug_printf("bucket endpoint levels: {}, weight levels: {}, surrogate sse: {}, ep_est: {}, wp_est: {}, avg RGB subset0 span: {}\n", + bucket_surrogate_endpoint_levels, bucket_surrogate_weight_levels, + bucket.m_sse, + bucket_sse_est.m_ep, bucket_sse_est.m_wp, + (fabs(bucket.m_surrogate_log_blk.m_endpoints[0][1][0] - bucket.m_surrogate_log_blk.m_endpoints[0][0][0]) + + fabs(bucket.m_surrogate_log_blk.m_endpoints[0][1][1] - bucket.m_surrogate_log_blk.m_endpoints[0][0][1]) + + fabs(bucket.m_surrogate_log_blk.m_endpoints[0][1][2] - bucket.m_surrogate_log_blk.m_endpoints[0][0][2])) / 3.0f); +#endif + + for (uint32_t j = 0; j < bucket_trial_mode_indices.size(); j++) + { + const uint32_t trial_mode_index = bucket_trial_mode_indices[j]; + const basist::astc_ldr_t::trial_mode& tm = p.m_pTrial_modes[trial_mode_index]; + + const uint32_t trial_mode_endpoint_levels = astc_helpers::get_ise_levels(tm.m_endpoint_ise_range); + const uint32_t trial_mode_weight_levels = astc_helpers::get_ise_levels(tm.m_weight_ise_range); + + assert(trial_mode_endpoint_levels <= bucket_surrogate_endpoint_levels); + assert(trial_mode_weight_levels <= bucket_surrogate_weight_levels); + + chan_mse_est mode_sse_est(0.0f, 0.0f); + for (uint32_t part_iter = 0; part_iter < bucket.m_num_parts; part_iter++) + { + uint32_t total_texels_in_part = p.m_block_width * p.m_block_height; + if (bucket.m_num_parts > 1) + { + total_texels_in_part = pBucket_part_data->m_partition_pat_histograms[bucket.m_unique_seed_index].m_hist[part_iter]; + assert(total_texels_in_part && total_texels_in_part < p.m_block_width * p.m_block_height); + } + + for (uint32_t c = 0; c < 4; c++) + { + float span_size = fabs(bucket.m_surrogate_log_blk.m_endpoints[part_iter][1][c] - bucket.m_surrogate_log_blk.m_endpoints[part_iter][0][c]); + + chan_mse_est chan_mse_est(compute_quantized_channel_mse_estimates( + can_use_base_ofs ? minimum(trial_mode_endpoint_levels * 2, 256) : trial_mode_endpoint_levels, + trial_mode_weight_levels, + span_size, pGrid_data->m_weight_gamma)); + + if (span_size == 0.0f) + { + if ((bucket.m_surrogate_log_blk.m_endpoints[part_iter][1][c] == 1.0f) || (bucket.m_surrogate_log_blk.m_endpoints[part_iter][1][c] == 0.0f)) + { + chan_mse_est.m_ep = 0.0f; + chan_mse_est.m_wp = 0.0f; + } + } + + mode_sse_est.m_ep += chan_mse_est.m_ep * (float)p.m_pEnc_params->m_comp_weights[c] * total_texels_in_part; + mode_sse_est.m_wp += chan_mse_est.m_wp * (float)p.m_pEnc_params->m_comp_weights[c] * total_texels_in_part; + } // c + + } // part_iter + + // Remove the bucket's base estimated endpoint/weight quant + if (trial_mode_endpoint_levels == bucket_surrogate_endpoint_levels) + { + mode_sse_est.m_ep = 0.0f; + } + else + { + mode_sse_est.m_ep -= bucket_sse_est.m_ep; + + if (mode_sse_est.m_ep < 0.0f) + mode_sse_est.m_ep = 0.0f; + } + + if (trial_mode_weight_levels == bucket_surrogate_weight_levels) + { + mode_sse_est.m_wp = 0.0f; + } + else + { + mode_sse_est.m_wp -= bucket_sse_est.m_wp; + + if (mode_sse_est.m_wp < 0.0f) + mode_sse_est.m_wp = 0.0f; + } + + float mode_total_sse_est = bucket_surrogate_base_sse + mode_sse_est.m_ep + mode_sse_est.m_wp; + + sse_estimates[j] = mode_total_sse_est; + +#if 0 + // TEMP comparison code + float actual_sse = 0.0f; + + { + log_surrogate_astc_blk temp_surrogate_log_blk; + if (bucket.m_num_parts == 1) + { + actual_sse = encode_surrogate_trial( + p.m_block_width, p.m_block_height, + pixel_stats, + bucket.m_cem_index, + bucket.m_ccs_index, + tm.m_endpoint_ise_range, tm.m_weight_ise_range, + bucket.m_grid_width, bucket.m_grid_height, + temp_surrogate_log_blk, + *p.m_pEnc_params); + } + else + { + const astc_ldr::partitions_data* pPart_data = (bucket.m_num_parts == 2) ? p.m_pPart_data_p2 : p.m_pPart_data_p3; + + const uint32_t part_seed_index = pPart_data->m_unique_index_to_part_seed[bucket.m_unique_seed_index]; + + const astc_ldr::partition_pattern_vec* pPat = &pPart_data->m_partition_pats[bucket.m_unique_seed_index]; + + actual_sse = encode_surrogate_trial_subsets( + p.m_block_width, p.m_block_height, + pixel_stats, + bucket.m_cem_index, bucket.m_num_parts, part_seed_index, pPat, + tm.m_endpoint_ise_range, tm.m_weight_ise_range, + bucket.m_grid_width, bucket.m_grid_height, + temp_surrogate_log_blk, + *p.m_pEnc_params, 0); + } + + stats.m_total_surrogate_encodes++; + } + + fmt_debug_printf("sse: {}, actual sse: {}, endpoint levels: {} weight levels: {}\n", sse_estimates[j], actual_sse, trial_mode_endpoint_levels, trial_mode_weight_levels); +#endif + + } // j + +#if 0 + fmt_debug_printf("\n"); +#endif + + indirect_sort(num_modes_in_bucket, bucket_indices.get_ptr(), sse_estimates.get_ptr()); + + } // if (num_modes_in_bucket_to_shortlist < num_modes_in_bucket) + + // Surrogate encode the best looking buckets after factoring in estimate SSE errors. + + for (uint32_t q = 0; q < num_modes_in_bucket_to_shortlist; q++) + { + const uint32_t j = bucket_indices[q]; + + trial_surrogate& s = *shortlist_trials.try_enlarge(1); + + const uint32_t trial_mode_index = bucket_trial_mode_indices[j]; + const basist::astc_ldr_t::trial_mode& tm = p.m_pTrial_modes[trial_mode_index]; + + s.m_trial_mode_index = trial_mode_index; + + if (bucket.m_num_parts == 1) + { + s.m_err = encode_surrogate_trial( + p.m_block_width, p.m_block_height, + pixel_stats, + bucket.m_cem_index, + bucket.m_ccs_index, + tm.m_endpoint_ise_range, tm.m_weight_ise_range, + bucket.m_grid_width, bucket.m_grid_height, + s.m_log_blk, + *p.m_pEnc_params, 0); + + stats.m_total_surrogate_encodes++; + } + else + { + const astc_ldr::partitions_data* pPart_data = (bucket.m_num_parts == 2) ? p.m_pPart_data_p2 : p.m_pPart_data_p3; + + const uint32_t part_seed_index = pPart_data->m_unique_index_to_part_seed[bucket.m_unique_seed_index]; + + const astc_ldr::partition_pattern_vec* pPat = &pPart_data->m_partition_pats[bucket.m_unique_seed_index]; + + s.m_err = encode_surrogate_trial_subsets( + p.m_block_width, p.m_block_height, + pixel_stats, + bucket.m_cem_index, bucket.m_num_parts, part_seed_index, pPat, + tm.m_endpoint_ise_range, tm.m_weight_ise_range, + bucket.m_grid_width, bucket.m_grid_height, + s.m_log_blk, + *p.m_pEnc_params, 0); + + stats.m_total_surrogate_encodes++; + } + + if ((bucket.m_cem_index == astc_helpers::CEM_LDR_RGB_DIRECT) || (bucket.m_cem_index == astc_helpers::CEM_LDR_RGBA_DIRECT)) + { + // blue contraction/base+offset discount + s.m_err *= BLUE_CONTRACTION_BASE_OFS_DISCOUNT; + } + + } // j + + } // bucket_index + + if (!shortlist_trials.size()) + return false; + + shortlist_trials.sort(); + + stats.m_total_shortlist_candidates += shortlist_trials.size_u32(); + + return true; + } + + bool final_polish_encode_from_shortlist( + const ldr_astc_lowlevel_block_encoder_params& p, + const astc_ldr::pixel_stats_t& pixel_stats, + basisu::vector& out_blocks, + uint32_t blur_id, + encode_block_stats& stats) + { + basisu::vector& shortlist_trials = m_trial_surrogates; + + // TODO: Diversity selection + const float shortlist_fract = p.m_final_shortlist_fraction[m_block_complexity_index]; + + uint32_t max_shortlist_trials = (uint32_t)std::roundf((float)shortlist_trials.size_u32() * shortlist_fract); + + max_shortlist_trials = clamp(max_shortlist_trials, p.m_final_shortlist_min_size[m_block_complexity_index], p.m_final_shortlist_max_size[m_block_complexity_index]); + + uint32_t total_shortlist_trials = clamp(max_shortlist_trials, 1, shortlist_trials.size_u32()); + + const uint32_t EARLY_STOP2_SHORTLIST_ITER_INDEX = 5; + + // Now do the real encodes on the top surrogate shortlist trials. + for (uint32_t shortlist_iter = 0; shortlist_iter < total_shortlist_trials; shortlist_iter++) + { + const uint32_t trial_mode_index = shortlist_trials[shortlist_iter].m_trial_mode_index; + const basist::astc_ldr_t::trial_mode& tm = p.m_pTrial_modes[trial_mode_index]; + + astc_helpers::log_astc_block log_astc_blk; + + bool base_ofs_succeeded_flag = false; + + if ((p.m_final_encode_try_base_ofs) && ((tm.m_cem == astc_helpers::CEM_LDR_RGB_DIRECT) || (tm.m_cem == astc_helpers::CEM_LDR_RGBA_DIRECT))) + { + // Add RGB/RGBA BASE PLUS OFFSET variant. + astc_helpers::log_astc_block log_astc_blk_alt; + + const uint32_t base_ofs_cem_index = (tm.m_cem == astc_helpers::CEM_LDR_RGB_DIRECT) ? astc_helpers::CEM_LDR_RGB_BASE_PLUS_OFFSET : astc_helpers::CEM_LDR_RGBA_BASE_PLUS_OFFSET; + + bool base_ofs_clamped_flag = false; + + bool alt_enc_trial_status; + if (tm.m_num_parts > 1) + { + const astc_ldr::partitions_data* pPart_data = (tm.m_num_parts == 2) ? p.m_pPart_data_p2 : p.m_pPart_data_p3; + + const uint32_t part_seed_index = shortlist_trials[shortlist_iter].m_log_blk.m_seed_index; + const uint32_t part_unique_index = pPart_data->m_part_seed_to_unique_index[part_seed_index]; + const astc_ldr::partition_pattern_vec* pPat = &pPart_data->m_partition_pats[part_unique_index]; + + alt_enc_trial_status = encode_trial_subsets( + p.m_block_width, p.m_block_height, pixel_stats, base_ofs_cem_index, tm.m_num_parts, + part_seed_index, pPat, + tm.m_endpoint_ise_range, tm.m_weight_ise_range, + tm.m_grid_width, tm.m_grid_height, log_astc_blk_alt, *p.m_pEnc_params, false, + p.m_gradient_descent_flag, p.m_polish_weights_flag, p.m_qcd_enabled_flag, + p.m_use_blue_contraction, &base_ofs_clamped_flag); + } + else + { + alt_enc_trial_status = encode_trial( + p.m_block_width, p.m_block_height, pixel_stats, base_ofs_cem_index, + tm.m_ccs_index != -1, tm.m_ccs_index, + tm.m_endpoint_ise_range, tm.m_weight_ise_range, + tm.m_grid_width, tm.m_grid_height, log_astc_blk_alt, *p.m_pEnc_params, + p.m_gradient_descent_flag, p.m_polish_weights_flag, p.m_qcd_enabled_flag, + p.m_use_blue_contraction, &base_ofs_clamped_flag); + } + + assert(alt_enc_trial_status); + + if (alt_enc_trial_status) + { + stats.m_total_full_encodes++; + + encode_block_output* pOut_block2 = out_blocks.enlarge(1); + pOut_block2->clear(); + pOut_block2->m_trial_mode_index = safe_cast_int16(trial_mode_index); + pOut_block2->m_log_blk = log_astc_blk_alt; + pOut_block2->m_blur_id = safe_cast_uint16(blur_id); + pOut_block2->m_sse = eval_error(p.m_block_width, p.m_block_height, log_astc_blk_alt, pixel_stats, *p.m_pEnc_params); + + if ((p.m_early_stop_wpsnr) || (p.m_early_stop2_wpsnr)) + { + const float wpsnr = compute_psnr_from_wsse(p.m_block_width, p.m_block_height, pOut_block2->m_sse, p.m_pEnc_params->get_total_comp_weights()); + + if ((p.m_early_stop_wpsnr) && (wpsnr >= p.m_early_stop_wpsnr)) + break; + + if (shortlist_iter >= EARLY_STOP2_SHORTLIST_ITER_INDEX) + { + if ((p.m_early_stop2_wpsnr) && (wpsnr >= p.m_early_stop2_wpsnr)) + break; + } + } + + base_ofs_succeeded_flag = !base_ofs_clamped_flag; + } + + } // (p.m_final_encode_try_base_ofs) + + if ((p.m_final_encode_always_try_rgb_direct) || (!base_ofs_succeeded_flag)) + { + bool enc_trial_status; + + if (tm.m_num_parts > 1) + { + const astc_ldr::partitions_data* pPart_data = (tm.m_num_parts == 2) ? p.m_pPart_data_p2 : p.m_pPart_data_p3; + + const uint32_t part_seed_index = shortlist_trials[shortlist_iter].m_log_blk.m_seed_index; + const uint32_t part_unique_index = pPart_data->m_part_seed_to_unique_index[part_seed_index]; + assert(part_unique_index < astc_helpers::NUM_PARTITION_PATTERNS); + const astc_ldr::partition_pattern_vec* pPat = &pPart_data->m_partition_pats[part_unique_index]; + + enc_trial_status = encode_trial_subsets( + p.m_block_width, p.m_block_height, pixel_stats, tm.m_cem, tm.m_num_parts, + part_seed_index, pPat, + tm.m_endpoint_ise_range, tm.m_weight_ise_range, + tm.m_grid_width, tm.m_grid_height, log_astc_blk, *p.m_pEnc_params, false, + p.m_gradient_descent_flag, p.m_polish_weights_flag, p.m_qcd_enabled_flag, + p.m_use_blue_contraction); + } + else + { + enc_trial_status = encode_trial( + p.m_block_width, p.m_block_height, pixel_stats, tm.m_cem, + tm.m_ccs_index != -1, tm.m_ccs_index, + tm.m_endpoint_ise_range, tm.m_weight_ise_range, + tm.m_grid_width, tm.m_grid_height, log_astc_blk, *p.m_pEnc_params, + p.m_gradient_descent_flag, p.m_polish_weights_flag, p.m_qcd_enabled_flag, + p.m_use_blue_contraction); + } + + assert(enc_trial_status); + + if (!enc_trial_status) + return false; + + stats.m_total_full_encodes++; + + { + encode_block_output* pOut_block1 = out_blocks.enlarge(1); + pOut_block1->clear(); + pOut_block1->m_trial_mode_index = safe_cast_int16(trial_mode_index); + pOut_block1->m_log_blk = log_astc_blk; + pOut_block1->m_blur_id = safe_cast_uint16(blur_id); + pOut_block1->m_sse = eval_error(p.m_block_width, p.m_block_height, log_astc_blk, pixel_stats, *p.m_pEnc_params); + + if ((p.m_early_stop_wpsnr) || (p.m_early_stop2_wpsnr)) + { + const float wpsnr = compute_psnr_from_wsse(p.m_block_width, p.m_block_height, pOut_block1->m_sse, p.m_pEnc_params->get_total_comp_weights()); + + if ((p.m_early_stop_wpsnr) && (wpsnr >= p.m_early_stop_wpsnr)) + break; + + if (shortlist_iter >= EARLY_STOP2_SHORTLIST_ITER_INDEX) + { + if ((p.m_early_stop2_wpsnr) && (wpsnr >= p.m_early_stop2_wpsnr)) + break; + } + } + } + + } // if (!skip_encode_flag) + + } // shortlist_iter + + return true; + } + + bool full_encode(const ldr_astc_lowlevel_block_encoder_params& p, + const astc_ldr::pixel_stats_t& pixel_stats, + basisu::vector& out_blocks, + uint32_t blur_id, + encode_block_stats& stats) + { + clear(); + + if (!init(p, pixel_stats, out_blocks, blur_id, stats)) + return false; + + if (!partition_triage(p, pixel_stats, out_blocks, blur_id, stats)) + return false; + + if (!trivial_triage(p, pixel_stats, out_blocks, blur_id, stats)) + return false; + + if (!analytic_triage(p, pixel_stats, out_blocks, blur_id, stats)) + return false; + + if (!surrogate_encode_shortlist_bucket_representatives(p, pixel_stats, out_blocks, blur_id, stats)) + return false; + + if (!prune_shortlist_buckets(p, pixel_stats, out_blocks, blur_id, stats)) + return false; + + if (!rank_and_sort_shortlist_buckets(p, pixel_stats, out_blocks, blur_id, stats)) + return false; + + if (!final_polish_encode_from_shortlist(p, pixel_stats, out_blocks, blur_id, stats)) + return false; + + return true; + } +}; + +class ldr_astc_lowlevel_block_encoder_pool +{ +public: + ldr_astc_lowlevel_block_encoder_pool() + { + } + + void init(uint32_t total_threads) + { + std::lock_guard g(m_mutex); + + m_pool.resize(total_threads); + + for (uint32_t i = 0; i < total_threads; i++) + m_pool[i].m_used_flag = false; + } + + void deinit() + { + std::lock_guard g(m_mutex); + + for (uint32_t i = 0; i < m_pool.size(); i++) + { + if (m_pool[i].m_used_flag) + { + assert(0); + debug_printf("ldr_astc_lowlevel_block_encoder_pool::deinit: Pool entry still marked as used\n"); + } + + m_pool[i].m_used_flag = false; + } + + m_pool.resize(0); + } + + ldr_astc_lowlevel_block_encoder* acquire() + { + std::lock_guard g(m_mutex); + + assert(m_pool.size()); + + ldr_astc_lowlevel_block_encoder* pRes = nullptr; + + for (uint32_t i = 0; i < m_pool.size(); i++) + { + if (!m_pool[i].m_used_flag) + { + pRes = &m_pool[i]; + pRes->m_used_flag = true; + + break; + } + } + + assert(pRes); + + return pRes; + } + + bool release(ldr_astc_lowlevel_block_encoder* pTemps) + { + std::lock_guard g(m_mutex); + + assert(m_pool.size()); + + if ((pTemps < m_pool.begin()) || (pTemps >= m_pool.end())) + { + assert(0); + return false; + } + + size_t idx = pTemps - m_pool.begin(); + if (idx >= m_pool.size()) + { + assert(0); + return false; + } + + m_pool[idx].m_used_flag = false; + + return true; + } + +private: + std::mutex m_mutex; + basisu::vector m_pool; +}; + +class scoped_ldr_astc_lowlevel_block_encoder +{ +public: + scoped_ldr_astc_lowlevel_block_encoder(ldr_astc_lowlevel_block_encoder_pool& pool) : + m_pool(pool) + { + m_pTemps = pool.acquire(); + } + + ~scoped_ldr_astc_lowlevel_block_encoder() + { + m_pool.release(m_pTemps); + } + + ldr_astc_lowlevel_block_encoder_pool& get_pool() const + { + return m_pool; + } + + ldr_astc_lowlevel_block_encoder* get_ptr() + { + return m_pTemps; + } + +private: + ldr_astc_lowlevel_block_encoder_pool& m_pool; + ldr_astc_lowlevel_block_encoder* m_pTemps; +}; + + +//------------------------------------------------------------------- + +#pragma pack(push, 1) +struct trial_mode_desc +{ + uint8_t m_unique_cem_index; // LDR base CEM's, 0-5 + uint8_t m_ccs; // 0 if SP, 1-4 for DP + uint8_t m_subsets; // 1-3 + uint8_t m_eise; // endpoint ise range, 4-20 + uint8_t m_wise; // weight ise range, 0-11 + uint8_t m_grid_w, m_grid_h; // grid resolution, 4-12 +}; +#pragma pack(pop) + +static const int s_astc_cem_to_unique_ldr_index[16] = +{ + 0, // CEM_LDR_LUM_DIRECT + -1, // CEM_LDR_LUM_BASE_PLUS_OFS + -1, // CEM_HDR_LUM_LARGE_RANGE + -1, // CEM_HDR_LUM_SMALL_RANGE + 1, // CEM_LDR_LUM_ALPHA_DIRECT + -1, // CEM_LDR_LUM_ALPHA_BASE_PLUS_OFS + 2, // CEM_LDR_RGB_BASE_SCALE + -1, // CEM_HDR_RGB_BASE_SCALE + 3, // CEM_LDR_RGB_DIRECT + -1, // CEM_LDR_RGB_BASE_PLUS_OFFSET + 4, // CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A + -1, // CEM_HDR_RGB + 5, // CEM_LDR_RGBA_DIRECT + -1, // CEM_LDR_RGBA_BASE_PLUS_OFFSET + -1, // CEM_HDR_RGB_LDR_ALPHA + -1, // CEM_HDR_RGB_HDR_ALPHA +}; + +#if 0 +static const int s_unique_ldr_index_to_astc_cem[6] = +{ + astc_helpers::CEM_LDR_LUM_DIRECT, + astc_helpers::CEM_LDR_LUM_ALPHA_DIRECT, + astc_helpers::CEM_LDR_RGB_BASE_SCALE, + astc_helpers::CEM_LDR_RGB_DIRECT, + astc_helpers::CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A, + astc_helpers::CEM_LDR_RGBA_DIRECT +}; +#endif + +static uint32_t pack_tm_desc( + uint32_t grid_width, uint32_t grid_height, + uint32_t cem_index, uint32_t ccs_index, uint32_t num_subsets, + uint32_t endpoint_ise_range, uint32_t weight_ise_range) +{ + assert((grid_width >= 2) && (grid_width <= 12)); + assert((grid_height >= 2) && (grid_height <= 12)); + assert((cem_index < 16) && astc_helpers::is_cem_ldr(cem_index)); + assert((num_subsets >= 1) && (num_subsets <= 3)); + assert(ccs_index <= 4); // 0 for SP, 1-4 for DP + assert((endpoint_ise_range >= astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE) && (endpoint_ise_range <= astc_helpers::LAST_VALID_ENDPOINT_ISE_RANGE)); + assert((weight_ise_range >= astc_helpers::FIRST_VALID_WEIGHT_ISE_RANGE) && (weight_ise_range <= astc_helpers::LAST_VALID_WEIGHT_ISE_RANGE)); + + grid_width -= 2; + grid_height -= 2; + assert((grid_width <= 10) && (grid_height <= 10)); + + const int unique_cem_index = s_astc_cem_to_unique_ldr_index[cem_index]; + assert((unique_cem_index >= 0) && (unique_cem_index <= 5)); + assert(basist::astc_ldr_t::s_unique_ldr_index_to_astc_cem[unique_cem_index] == (int)cem_index); + + num_subsets--; + + endpoint_ise_range -= astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE; + + uint32_t cur_bit_ofs = 0; + +#define BU_PACK_FIELD(val, bits) do { uint32_t v = (uint32_t)(val); assert(v < (1u << bits)); packed_id |= (v << cur_bit_ofs); cur_bit_ofs += (bits); } while(0) + + uint32_t packed_id = 0; + BU_PACK_FIELD(endpoint_ise_range, basist::astc_ldr_t::CFG_PACK_EISE_BITS); + BU_PACK_FIELD(weight_ise_range, basist::astc_ldr_t::CFG_PACK_WISE_BITS); + BU_PACK_FIELD(ccs_index, basist::astc_ldr_t::CFG_PACK_CCS_BITS); + BU_PACK_FIELD(num_subsets, basist::astc_ldr_t::CFG_PACK_SUBSETS_BITS); + BU_PACK_FIELD(unique_cem_index, basist::astc_ldr_t::CFG_PACK_CEM_BITS); + // must be at the top + BU_PACK_FIELD(grid_width * 11 + grid_height, basist::astc_ldr_t::CFG_PACK_GRID_BITS); +#undef BU_PACK_FIELD + + assert(cur_bit_ofs == 24); + + return packed_id; +} + +void create_encoder_trial_modes_full_eval(uint32_t block_width, uint32_t block_height, + basisu::vector& encoder_trial_modes, basist::astc_ldr_t::grouped_trial_modes& grouped_encoder_trial_modes, + bool print_debug_info = true, bool print_modes = false) +{ + interval_timer itm; + itm.start(); + + encoder_trial_modes.resize(0); + grouped_encoder_trial_modes.clear(); + + uint32_t max_grid_width = 0, max_grid_height = 0; + uint32_t total_evals = 0, total_partial_evals = 0, total_evals_succeeded = 0; + uint32_t mode_index = 0; + uint_vec packed_mode_ids; + + for (uint32_t alpha_iter = 0; alpha_iter < 2; alpha_iter++) + { + if (print_modes) + { + if (alpha_iter) + fmt_debug_printf("ALPHA TRIAL MODES\n"); + else + fmt_debug_printf("RGB TRIAL MODES\n"); + } + + astc_helpers::astc_block phys_block; + + for (uint32_t cem_mode_iter = 0; cem_mode_iter < 3; cem_mode_iter++) + { + const uint32_t s_rgb_cems[3] = { astc_helpers::CEM_LDR_LUM_DIRECT, astc_helpers::CEM_LDR_RGB_BASE_SCALE, astc_helpers::CEM_LDR_RGB_DIRECT }; + const uint32_t s_alpha_cems[3] = { astc_helpers::CEM_LDR_LUM_ALPHA_DIRECT, astc_helpers::CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A, astc_helpers::CEM_LDR_RGBA_DIRECT }; + + const uint32_t cem_index = alpha_iter ? s_alpha_cems[cem_mode_iter] : s_rgb_cems[cem_mode_iter]; + + uint32_t num_dp_chans = 0; + bool cem_supports_dual_plane = false; + bool cem_supports_subsets = false; + + // base+ofs variants are automatically used later as alternates to RGB/RGBA direct modes + switch (cem_index) + { + case astc_helpers::CEM_LDR_LUM_DIRECT: + num_dp_chans = 0; // only a single component, so only a single plane + cem_supports_dual_plane = false; + cem_supports_subsets = true; + break; + case astc_helpers::CEM_LDR_LUM_ALPHA_DIRECT: + num_dp_chans = 1; // CCS can only be 3 + cem_supports_dual_plane = true; + cem_supports_subsets = true; + break; + case astc_helpers::CEM_LDR_RGB_DIRECT: + num_dp_chans = 3; + cem_supports_dual_plane = true; + cem_supports_subsets = true; + break; + case astc_helpers::CEM_LDR_RGB_BASE_SCALE: + num_dp_chans = 3; + cem_supports_dual_plane = true; + cem_supports_subsets = true; + break; + case astc_helpers::CEM_LDR_RGBA_DIRECT: + num_dp_chans = 4; + cem_supports_dual_plane = true; + cem_supports_subsets = true; + break; + case astc_helpers::CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A: + num_dp_chans = 4; + cem_supports_dual_plane = true; + cem_supports_subsets = true; + break; + default: + assert(0); + break; + } + + for (int dp = 0; dp < (cem_supports_dual_plane ? 2 : 1); dp++) + { + const bool use_subsets = !dp && cem_supports_subsets; + + for (int subsets = 1; subsets <= (use_subsets ? 3 : 1); subsets++) + { + for (uint32_t grid_height = 2; grid_height <= block_height; grid_height++) + { + for (uint32_t grid_width = 2; grid_width <= block_width; grid_width++) + { + for (uint32_t dp_chan_index = 0; dp_chan_index < (dp ? num_dp_chans : 1); dp_chan_index++) + { + astc_helpers::log_astc_block log_block; + log_block.clear(); + + log_block.m_grid_width = (uint8_t)grid_width; + log_block.m_grid_height = (uint8_t)grid_height; + + log_block.m_num_partitions = (uint8_t)subsets; + + for (int i = 0; i < subsets; i++) + log_block.m_color_endpoint_modes[i] = (uint8_t)cem_index; + + log_block.m_dual_plane = dp > 0; + + if (log_block.m_dual_plane) + { + uint32_t ccs_index = dp_chan_index; + + if (cem_index == astc_helpers::CEM_LDR_LUM_ALPHA_DIRECT) + { + // must be 3 for LA if DP is enabled + ccs_index = 3; + } + + log_block.m_color_component_selector = (uint8_t)ccs_index; + } + + for (uint32_t weight_ise_range = astc_helpers::FIRST_VALID_WEIGHT_ISE_RANGE; weight_ise_range <= astc_helpers::LAST_VALID_WEIGHT_ISE_RANGE; weight_ise_range++) + { + log_block.m_weight_ise_range = (uint8_t)weight_ise_range; + log_block.m_endpoint_ise_range = astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE; // dummy value + + total_partial_evals++; + + bool success = astc_helpers::pack_astc_block(phys_block, log_block, nullptr, nullptr, astc_helpers::cValidateEarlyOutAtEndpointISEChecks); + if (!success) + continue; + + // in reality only 1 endpoint ISE range is valid here + for (uint32_t endpoint_ise_range = astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE; endpoint_ise_range <= astc_helpers::LAST_VALID_ENDPOINT_ISE_RANGE; endpoint_ise_range++) + { + log_block.m_endpoint_ise_range = (uint8_t)endpoint_ise_range; + + total_evals++; + + success = astc_helpers::pack_astc_block(phys_block, log_block, nullptr, nullptr, astc_helpers::cValidateSkipFinalEndpointWeightPacking); + if (!success) + continue; + + total_evals_succeeded++; + + if (print_modes) + { + fmt_debug_printf("{}: CEM: {} DP: {}, CCS: {}, SUBSETS: {}, GRID: {}x{}, ENDPOINTS: {}, WEIGHTS: {}\n", + mode_index, + log_block.m_color_endpoint_modes[0], + log_block.m_dual_plane, + log_block.m_color_component_selector, + log_block.m_num_partitions, + log_block.m_grid_width, log_block.m_grid_height, + astc_helpers::get_ise_levels(log_block.m_endpoint_ise_range), + astc_helpers::get_ise_levels(log_block.m_weight_ise_range)); + } + + basist::astc_ldr_t::trial_mode m; + m.m_ccs_index = log_block.m_dual_plane ? log_block.m_color_component_selector : -1; + m.m_cem = log_block.m_color_endpoint_modes[0]; + m.m_endpoint_ise_range = log_block.m_endpoint_ise_range; + m.m_weight_ise_range = log_block.m_weight_ise_range; + m.m_grid_width = grid_width; + m.m_grid_height = grid_height; + m.m_num_parts = log_block.m_num_partitions; + + uint32_t packed_index = pack_tm_desc( + log_block.m_grid_width, log_block.m_grid_height, + log_block.m_color_endpoint_modes[0], log_block.m_dual_plane ? (log_block.m_color_component_selector + 1) : 0, log_block.m_num_partitions, + log_block.m_endpoint_ise_range, log_block.m_weight_ise_range); + + assert(packed_index <= 0xFFFFFF); + packed_mode_ids.push_back(packed_index); + + grouped_encoder_trial_modes.add(block_width, block_height, m, encoder_trial_modes.size_u32()); + + encoder_trial_modes.push_back(m); + + max_grid_width = maximum(max_grid_width, grid_width); + max_grid_height = maximum(max_grid_height, grid_height); + + ++mode_index; + + } // weight_ise_range + } // endpoint_ise_range + + } // ccs_index + + } // grid_width + + } // grid_height + + } // subsets + + } // dp + + } // cem_mode_iter + + } // alpha_iter + +#if 0 + packed_mode_ids.sort(); + + for (uint32_t i = 0; i < packed_mode_ids.size(); i++) + { + uint32_t packed_index = packed_mode_ids[i]; + + fmt_debug_printf("{},{},{},", packed_index & 0xFF, (packed_index >> 8) & 0xFF, (packed_index >> 16) & 0xFF); + if ((i & 15) == 15) + fmt_debug_printf("\n"); + } +#endif + + if (print_debug_info) + { + fmt_debug_printf("create_encoder_trial_modes_full_eval() time: {} secs\n", itm.get_elapsed_secs()); + + fmt_debug_printf("create_encoder_trial_modes_full_eval() - ASTC {}x{} modes\n", block_width, block_height); + fmt_debug_printf("total_evals: {}, total_partial_evals: {}, total_evals_succeeded: {}\n", total_evals, total_partial_evals, total_evals_succeeded); + fmt_debug_printf("Total trial modes: {}\n", (uint32_t)encoder_trial_modes.size()); + fmt_debug_printf("Total used trial mode groups: {}\n", grouped_encoder_trial_modes.count_used_groups()); + fmt_debug_printf("Max ever grid dimensions: {}x{}\n", max_grid_width, max_grid_height); + } + + // sanity check + assert(encoder_trial_modes.size() < 11000); +} + +const uint32_t TOTAL_RGBA_CHAN_PAIRS = 6; +//const uint32_t TOTAL_RGB_CHAN_PAIRS = 3; +static const uint8_t g_rgba_chan_pairs[TOTAL_RGBA_CHAN_PAIRS][2] = +{ + { 0, 1 }, + { 0, 2 }, + { 1, 2 }, + { 0, 3 }, + { 1, 3 }, + { 2, 3 } +}; + +bool encoder_trial_mode_test() +{ + for (uint32_t w = 4; w <= 12; w++) + { + for (uint32_t h = 4; h <= 12; h++) + { + if (!astc_helpers::is_valid_block_size(w, h)) + continue; + + basisu::vector encoder_trial_modes_orig; + basist::astc_ldr_t::grouped_trial_modes grouped_encoder_trial_modes_orig; + + create_encoder_trial_modes_full_eval(w, h, + encoder_trial_modes_orig, grouped_encoder_trial_modes_orig, + false, false); + + fmt_debug_printf("Testing block size {}x{}, {} total modes\n", w, h, encoder_trial_modes_orig.size_u32()); + + basisu::hash_map trial_mode_hash; + for (uint32_t i = 0; i < encoder_trial_modes_orig.size(); i++) + { + trial_mode_hash.insert(encoder_trial_modes_orig[i]); + } + + basisu::vector encoder_trial_modes_new; + basist::astc_ldr_t::grouped_trial_modes grouped_encoder_trial_modes_new; + + basist::astc_ldr_t::create_encoder_trial_modes_table(w, h, + encoder_trial_modes_new, grouped_encoder_trial_modes_new, + false, false); + + if (encoder_trial_modes_new.size() != encoder_trial_modes_orig.size()) + { + fmt_error_printf("trial mode test failed!\n"); + + assert(0); + return false; + } + + for (uint32_t i = 0; i < encoder_trial_modes_new.size(); i++) + { + const basist::astc_ldr_t::trial_mode& tm = encoder_trial_modes_new[i]; + if (trial_mode_hash.find(tm) == trial_mode_hash.end()) + { + fmt_error_printf("trial mode test failed!\n"); + + assert(0); + return false; + } + } + + } // h + } // w + + fmt_debug_printf("trial mode test succeeded\n"); + return true; +} + +//---------------------------------------------------------------------------------- + +struct ldr_astc_block_encode_image_high_level_config +{ + uint32_t m_block_width = 6; + uint32_t m_block_height = 6; + + bool m_second_superpass_refinement = true; + float m_second_superpass_fract_to_recompress = .075f; + + bool m_third_superpass_try_neighbors = true; + + float m_base_q = 75.0f; + bool m_use_dct = false; + + bool m_subsets_enabled = true; + bool m_subsets_edge_filtering = true; + + bool m_filter_by_pca_angles_flag = true; + float m_use_direct_angle_thresh = 2.0f; + float m_use_base_scale_angle_thresh = 7.0f; + + bool m_force_all_dual_plane_chan_evals = false; // much slower, test on base + bool m_disable_rgb_dual_plane = false; // DP can be on alpha only, if block has alpha + float m_strong_dp_decorr_thresh_rgb = .998f; + + bool m_use_base_ofs = true; + bool m_use_blue_contraction = true; + + bool m_grid_hv_filtering = true; + bool m_low_freq_block_filtering = true; + + uint32_t m_superbucket_max_to_retain[3] = { 4, 8, 16 }; + + float m_final_shortlist_fraction[3] = { .25f, .33f, .5f }; + uint32_t m_final_shortlist_min_size[3] = { 1, 1, 1 }; + uint32_t m_final_shortlist_max_size[3] = { 4096, 4096, 4096 }; + + uint32_t m_part2_fraction_to_keep = 2; + uint32_t m_part3_fraction_to_keep = 2; + uint32_t m_base_parts2 = 32; + uint32_t m_base_parts3 = 32; + + float m_early_stop_wpsnr = 0.0f; + float m_early_stop2_wpsnr = 0.0f; + + bool m_blurring_enabled = false; + bool m_blurring_enabled_p2 = false; + + bool m_gradient_descent_flag = true; + bool m_polish_weights_flag = true; + bool m_qcd_enabled_flag = true; // gradient descent must be enabled too + bool m_bucket_pruning_passes = true; + + // 2nd superpass options + uint32_t m_base_parts2_p2 = 64; + uint32_t m_base_parts3_p2 = 64; + uint32_t m_superbucket_max_to_retain_p2[3] = { 16, 32, 256 }; + uint32_t m_final_shortlist_max_size_p2[3] = { 4096, 4096, 4096 }; + uint32_t m_second_pass_total_weight_refine_passes = astc_ldr::WEIGHT_REFINER_MAX_PASSES; + bool m_second_pass_force_subsets_enabled = true; + bool m_force_all_dp_chans_p2 = false; + bool m_final_encode_always_try_rgb_direct = false; + bool m_filter_by_pca_angles_flag_p2 = true; + + // only store the single best result per block + //bool m_save_single_result = false; + + bool m_debug_images = false; + bool m_debug_output = false; + + std::string m_debug_file_prefix; + + job_pool* m_pJob_pool; + + //saliency_map m_saliency_map; + + astc_ldr::cem_encode_params m_cem_enc_params; +}; + +struct ldr_astc_block_encode_image_output +{ + ldr_astc_block_encode_image_output() + { + } + + ~ldr_astc_block_encode_image_output() + { + interval_timer itm; + itm.start(); + + const int num_blocks_x = m_image_block_info.get_width(); + const int num_blocks_y = m_image_block_info.get_height(); + + for (int y = num_blocks_y - 1; y >= 0; --y) + { + for (int x = num_blocks_x - 1; x >= 0; --x) + { + auto& out_blocks = m_image_block_info(x, y).m_out_blocks; + out_blocks.clear(); + } + } // y + + //fmt_debug_printf("Cleared enc_out image block info: {3.3} secs\n", itm.get_elapsed_secs()); + } + + astc_ldr::partitions_data m_part_data_p2; + astc_ldr::partitions_data m_part_data_p3; + + basisu::vector m_encoder_trial_modes; + basist::astc_ldr_t::grouped_trial_modes m_grouped_encoder_trial_modes; + + vector2D m_packed_phys_blocks; + + struct block_info + { + block_info() + { + m_pixel_stats.clear(); + } + + astc_ldr::pixel_stats_t m_pixel_stats; // of original/input block + + basisu::vector m_out_blocks; + + uint32_t m_packed_out_block_index = 0; // index of best out block by WSSE + + bool m_low_freq_block_flag = false; + bool m_super_strong_edges = false; + bool m_very_strong_edges = false; + bool m_strong_edges = false; + }; + + vector2D m_image_block_info; + + struct block_info_superpass1 + { + int m_config_reuse_neighbor_out_block_indices[basist::astc_ldr_t::cMaxConfigReuseNeighbors] = { cInvalidIndex, cInvalidIndex, cInvalidIndex }; + + bool m_config_reuse_new_neighbor_out_block_flags[basist::astc_ldr_t::cMaxConfigReuseNeighbors] = { false, false, false }; + + basisu::vector m_new_out_config_reuse_blocks; + basisu::vector m_new_out_config_endpoint_reuse_blocks; + }; + + vector2D m_image_block_info_superpass2; + +private: + ldr_astc_block_encode_image_output(const ldr_astc_block_encode_image_output&); + ldr_astc_block_encode_image_output& operator= (const ldr_astc_block_encode_image_output&); +}; + +constexpr bool selective_blurring = true; + +bool ldr_astc_block_encode_image( + const image& orig_img, + const ldr_astc_block_encode_image_high_level_config& enc_cfg, + ldr_astc_block_encode_image_output& enc_out) +{ + if (enc_cfg.m_debug_output) + fmt_debug_printf("ldr_astc_block_encode_image:\n"); + + const uint32_t block_width = enc_cfg.m_block_width, block_height = enc_cfg.m_block_height; + const uint32_t width = orig_img.get_width(), height = orig_img.get_height(); + const uint32_t total_pixels = width * height; + const uint32_t total_block_pixels = enc_cfg.m_block_width * enc_cfg.m_block_height; + const uint32_t num_blocks_x = orig_img.get_block_width(enc_cfg.m_block_width); + const uint32_t num_blocks_y = orig_img.get_block_height(enc_cfg.m_block_height); + const uint32_t total_blocks = num_blocks_x * num_blocks_y; + + if (enc_cfg.m_debug_output) + { + fmt_debug_printf("ASTC base bitrate: {3.3} bpp\n", 128.0f / (float)(enc_cfg.m_block_width * enc_cfg.m_block_height)); + + fmt_debug_printf("ASTC block size: {}x{}\n", enc_cfg.m_block_width, enc_cfg.m_block_height); + } + + if (enc_cfg.m_debug_output) + fmt_debug_printf("Image has alpha: {}\n", orig_img.has_alpha()); + + astc_ldr::partitions_data* pPart_data_p2 = &enc_out.m_part_data_p2; + pPart_data_p2->init(2, enc_cfg.m_block_width, enc_cfg.m_block_height); + + astc_ldr::partitions_data* pPart_data_p3 = &enc_out.m_part_data_p3; + pPart_data_p3->init(3, enc_cfg.m_block_width, enc_cfg.m_block_height); + + // blurring coefficients + const float bw0 = 1.15f; + const float bw1 = 1.25f, bw1_a = 1.0f; + const float bw2 = 1.25f; + + // TODO: Make this optional/tune this, add only 2 level blurring support + image orig_img_blurred2, orig_img_blurred3, orig_img_blurred4, orig_img_blurred5; + + if ((enc_cfg.m_blurring_enabled) || (enc_cfg.m_blurring_enabled_p2)) + { + orig_img_blurred2.resize(orig_img.get_width(), orig_img.get_height()); + orig_img_blurred3.resize(orig_img.get_width(), orig_img.get_height()); + orig_img_blurred4.resize(orig_img.get_width(), orig_img.get_height()); + orig_img_blurred5.resize(orig_img.get_width(), orig_img.get_height()); + + image_resample(orig_img, orig_img_blurred2, true, "gaussian", bw0); + image_resample(orig_img, orig_img_blurred3, true, "gaussian", bw1, false, 0, 4, bw1_a); + image_resample(orig_img, orig_img_blurred4, true, "gaussian", bw1_a, false, 0, 4, bw1); + image_resample(orig_img, orig_img_blurred5, true, "gaussian", bw2, false); + } + + if (enc_cfg.m_debug_images) + { + save_png(enc_cfg.m_debug_file_prefix + "dbg_astc_ldr_orig_img.png", orig_img); + + if ((enc_cfg.m_blurring_enabled) || (enc_cfg.m_blurring_enabled_p2)) + { + save_png(enc_cfg.m_debug_file_prefix + "vis_orig_blurred2.png", orig_img_blurred2); + save_png(enc_cfg.m_debug_file_prefix + "vis_orig_blurred3.png", orig_img_blurred3); + save_png(enc_cfg.m_debug_file_prefix + "vis_orig_blurred4.png", orig_img_blurred4); + save_png(enc_cfg.m_debug_file_prefix + "vis_orig_blurred5.png", orig_img_blurred5); + } + } + + if (enc_cfg.m_debug_output) + fmt_debug_printf("Dimensions: {}x{}, Blocks: {}x{}, Total blocks: {}\n", width, height, num_blocks_x, num_blocks_y, total_blocks); + + image orig_img_sobel_x, orig_img_sobel_y; + compute_sobel(orig_img, orig_img_sobel_x, &g_sobel_x[0][0]); + compute_sobel(orig_img, orig_img_sobel_y, &g_sobel_y[0][0]); + + if (enc_cfg.m_debug_images) + { + save_png(enc_cfg.m_debug_file_prefix + "vis_orig_sobel_x.png", orig_img_sobel_x); + save_png(enc_cfg.m_debug_file_prefix + "vis_orig_sobel_y.png", orig_img_sobel_y); + } + + image orig_img_sobel_xy(width, height); + for (uint32_t y = 0; y < height; y++) + { + for (uint32_t x = 0; x < width; x++) + { + const color_rgba& sx = orig_img_sobel_x(x, y); + const color_rgba& sy = orig_img_sobel_y(x, y); + + orig_img_sobel_xy(x, y).set( + iabs((int)sx.r - 128) + iabs((int)sy.r - 128), + iabs((int)sx.g - 128) + iabs((int)sy.g - 128), + iabs((int)sx.b - 128) + iabs((int)sy.b - 128), + iabs((int)sx.a - 128) + iabs((int)sy.a - 128)); + } + } + + if (enc_cfg.m_debug_images) + save_png(enc_cfg.m_debug_file_prefix + "vis_orig_sobel_xy.png", orig_img_sobel_xy); + + vector2D& packed_blocks = enc_out.m_packed_phys_blocks; + packed_blocks.resize(num_blocks_x, num_blocks_y); + memset(packed_blocks.get_ptr(), 0, packed_blocks.size_in_bytes()); + + assert(enc_cfg.m_pJob_pool); + job_pool& job_pool = *enc_cfg.m_pJob_pool; + + std::atomic encoder_failed_flag; + encoder_failed_flag.store(false); + + std::mutex global_mutex; + + basisu::vector& encoder_trial_modes = enc_out.m_encoder_trial_modes; + encoder_trial_modes.reserve(4096); + + basist::astc_ldr_t::grouped_trial_modes& grouped_encoder_trial_modes = enc_out.m_grouped_encoder_trial_modes; + basist::astc_ldr_t::create_encoder_trial_modes_table(block_width, block_height, encoder_trial_modes, grouped_encoder_trial_modes, enc_cfg.m_debug_output, false); + + if (enc_cfg.m_debug_output) + { + uint32_t total_actual_modes = encoder_trial_modes.size_u32(); + + if (enc_cfg.m_use_base_ofs) + { + for (uint32_t i = 0; i < encoder_trial_modes.size(); i++) + { + const auto& tm = encoder_trial_modes[i]; + + switch (tm.m_cem) + { + case astc_helpers::CEM_LDR_RGBA_DIRECT: + case astc_helpers::CEM_LDR_RGB_DIRECT: + // add base+ofs variant + total_actual_modes++; + break; + default: + break; + } + } // i + } + + fmt_debug_printf("Base encoder trial modes: {}, grand total including base+ofs CEM's: {}\n", encoder_trial_modes.size_u32(), total_actual_modes); + } + + uint32_t total_used_bc = 0; + + uint_vec used_rgb_direct_count; + used_rgb_direct_count.resize(encoder_trial_modes.size()); + + uint_vec used_base_offset_count; + used_base_offset_count.resize(encoder_trial_modes.size()); + + uint32_t total_void_extent_blocks_skipped = 0; + + uint32_t total_superbuckets_created = 0; + uint32_t total_buckets_created = 0; + uint32_t total_surrogate_encodes = 0; + uint32_t total_full_encodes = 0; + uint32_t total_shortlist_candidates = 0; + uint32_t total_full_encodes_pass1 = 0; + uint32_t total_full_encodes_pass2 = 0; + + uint32_t total_blur_encodes = 0; + uint32_t total_blurred_blocks1 = 0; + uint32_t total_blurred_blocks2 = 0; + uint32_t total_blurred_blocks3 = 0; + uint32_t total_blurred_blocks4 = 0; + + basist::astc_ldr_t::dct2f dct; + dct.init(enc_cfg.m_block_height, enc_cfg.m_block_width); + + image vis_part_usage_img, vis_part_pat_img, vis_strong_edge, vis_dct_low_freq_block, vis_dp_img, vis_base_ofs_img; + if (enc_cfg.m_debug_images) + { + vis_part_usage_img.resize(block_width * num_blocks_x, block_height * num_blocks_y); + vis_part_pat_img.resize(block_width * num_blocks_x, block_height * num_blocks_y); + vis_strong_edge.resize(block_width * num_blocks_x, block_height * num_blocks_y); + vis_dct_low_freq_block.resize(block_width * num_blocks_x, block_height * num_blocks_y); + vis_dp_img.resize(block_width * num_blocks_x, block_height * num_blocks_y); + vis_base_ofs_img.resize(block_width * num_blocks_x, block_height * num_blocks_y); + } + + ldr_astc_lowlevel_block_encoder_pool encoder_pool; + assert(job_pool.get_total_threads()); + encoder_pool.init((uint32_t)job_pool.get_total_threads()); + + basist::astc_ldr_t::grid_weight_dct grid_coder; + grid_coder.init(block_width, block_height); + + struct output_block_devel_desc + { + const basist::astc_ldr_t::trial_mode* m_pTrial_modes; + int m_trial_mode_index; // this is the index of the mode it tried to encode, but the actual output/enc block could have used base+ofs + bool m_had_alpha; + + bool m_low_freq_block_flag; + bool m_super_strong_edges; + bool m_very_strong_edges; + bool m_strong_edges; + + void clear() + { + clear_obj(*this); + } + }; + + enc_out.m_image_block_info.resize(0, 0); + enc_out.m_image_block_info.resize(num_blocks_x, num_blocks_y); + +#if 0 + for (uint32_t y = 0; y < num_blocks_y; y++) + { + for (uint32_t x = 0; x < num_blocks_x; x++) + { + auto& out_blocks = enc_out.m_image_block_info(x, y).m_out_blocks; + out_blocks.reserve(16); + out_blocks.resize(0); + } + } // y +#endif + + vector2D superpass2_recompress_block_flags; + + if (enc_cfg.m_second_superpass_refinement) + superpass2_recompress_block_flags.resize(num_blocks_x, num_blocks_y); + + if (enc_cfg.m_third_superpass_try_neighbors) + enc_out.m_image_block_info_superpass2.resize(num_blocks_x, num_blocks_y); + + interval_timer itm; + itm.start(); + + //-------------------------------------------------------------------------------------- + // ASTC compression loop + + vector2D output_block_devel_info(num_blocks_x, num_blocks_y); + + uint32_t total_superpasses = 1; + if (enc_cfg.m_third_superpass_try_neighbors) + total_superpasses = 3; + else if (enc_cfg.m_second_superpass_refinement) + total_superpasses = 2; + + uint32_t total_blocks_to_recompress = 0; + + for (uint32_t superpass_index = 0; superpass_index < total_superpasses; superpass_index++) + { + if (superpass_index == 1) + { + if (!enc_cfg.m_second_superpass_refinement) + continue; + if (!total_blocks_to_recompress) + continue; + } + + if (enc_cfg.m_debug_output) + fmt_debug_printf("ASTC packing superpass: {}\n", 1 + superpass_index); + + uint32_t total_blocks_done = 0; + float last_printed_progress_val = -100.0f; + + for (uint32_t by = 0; by < num_blocks_y; by++) + { + for (uint32_t bx = 0; bx < num_blocks_x; bx++) + { + job_pool.add_job([superpass_index, + //width, height, + bx, by, + //num_blocks_x, num_blocks_y, + total_blocks, block_width, block_height, total_block_pixels, &packed_blocks, &global_mutex, + &orig_img, &orig_img_sobel_xy, &orig_img_blurred2, &orig_img_blurred3, &orig_img_blurred4, &orig_img_blurred5, + &enc_cfg, &encoder_failed_flag, pPart_data_p2, pPart_data_p3, + &total_blocks_done, &total_superbuckets_created, &total_buckets_created, &total_surrogate_encodes, &total_full_encodes, &total_shortlist_candidates, + &encoder_trial_modes, + &total_blur_encodes, &total_blurred_blocks1, + &total_full_encodes_pass1, &total_full_encodes_pass2, + &dct, &vis_dct_low_freq_block, + &encoder_pool, &grid_coder, &grouped_encoder_trial_modes, + &enc_out, &output_block_devel_info, &total_void_extent_blocks_skipped, &superpass2_recompress_block_flags, &total_blocks_to_recompress, &last_printed_progress_val] + { + if (encoder_failed_flag) + return; + + //const uint32_t base_x = bx * block_width, base_y = by * block_height; + + color_rgba block_pixels[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + orig_img.extract_block_clamped(block_pixels, bx * block_width, by * block_height, block_width, block_height); + + if (superpass_index == 2) + { + // Superpass 2: Encode to best neighbor configurations + const ldr_astc_block_encode_image_output::block_info& out_block_info = enc_out.m_image_block_info(bx, by); + + ldr_astc_block_encode_image_output::block_info_superpass1& out_block_info_superpass1 = enc_out.m_image_block_info_superpass2(bx, by); + + const astc_ldr::pixel_stats_t& pixel_stats = out_block_info.m_pixel_stats; + + const bool is_purely_solid_block = (pixel_stats.m_min == pixel_stats.m_max); + + // if void extent, just skip + if (is_purely_solid_block) + return; + + //const basisu::vector& out_blocks = out_block_info.m_out_blocks; + + for (uint32_t neighbor_index = 0; neighbor_index < basist::astc_ldr_t::cMaxConfigReuseNeighbors; neighbor_index++) + { + const ldr_astc_block_encode_image_output::block_info* pNeighbor_out_block_info = nullptr; + + if (neighbor_index == 0) + { + // Left + if (bx) + pNeighbor_out_block_info = &enc_out.m_image_block_info(bx - 1, by); + } + else if (neighbor_index == 1) + { + // Up + if (by) + pNeighbor_out_block_info = &enc_out.m_image_block_info(bx, by - 1); + } + else + { + assert(neighbor_index == 2); + + // Diagonal + if ((bx) && (by)) + pNeighbor_out_block_info = &enc_out.m_image_block_info(bx - 1, by - 1); + } + + if (!pNeighbor_out_block_info) + continue; + + const encode_block_output& neighbor_output = pNeighbor_out_block_info->m_out_blocks[pNeighbor_out_block_info->m_packed_out_block_index]; + + // Best neighbor was solid, skip it (TODO: reusing it is possible) + if (neighbor_output.m_log_blk.m_solid_color_flag_ldr) + continue; + + const uint32_t neighbor_tm_index = neighbor_output.m_trial_mode_index; + assert(neighbor_tm_index < encoder_trial_modes.size()); + + //const trial_mode& neighbor_tm = encoder_trial_modes[neighbor_tm_index]; // do not use the tm's cem, it may be base+ofs, use the log blk instead + + const astc_helpers::log_astc_block& neighbor_log_blk = neighbor_output.m_log_blk; + assert(!neighbor_log_blk.m_solid_color_flag_ldr); + + const uint32_t neighbor_actual_cem = neighbor_log_blk.m_color_endpoint_modes[0]; + const uint32_t neighbor_partition_id = neighbor_log_blk.m_partition_id; + + // See if we've already encoded this full config + int already_existing_out_block_index = cInvalidIndex; + for (uint32_t i = 0; i < out_block_info.m_out_blocks.size(); i++) + { + if ((out_block_info.m_out_blocks[i].m_trial_mode_index == (int)neighbor_tm_index) && + (out_block_info.m_out_blocks[i].m_log_blk.m_color_endpoint_modes[0] == neighbor_actual_cem) && + (out_block_info.m_out_blocks[i].m_log_blk.m_partition_id == neighbor_partition_id)) + { + already_existing_out_block_index = i; + break; + } + } + + if (already_existing_out_block_index != cInvalidIndex) + { + // We already have an output block using this neighbor trial mode, skip + out_block_info_superpass1.m_config_reuse_neighbor_out_block_indices[neighbor_index] = (uint32_t)already_existing_out_block_index; + out_block_info_superpass1.m_config_reuse_new_neighbor_out_block_flags[neighbor_index] = false; + } + else + { + // Re-encode using the neighbor's full config (tm, base+ofs, partition ID) + astc_helpers::log_astc_block new_log_block; + + bool status = false; + + if (neighbor_log_blk.m_num_partitions > 1) + { + const astc_ldr::partitions_data* pPart_data = (neighbor_log_blk.m_num_partitions == 2) ? pPart_data_p2 : pPart_data_p3; + + const uint32_t part_seed_index = neighbor_log_blk.m_partition_id; + const uint32_t part_unique_index = pPart_data->m_part_seed_to_unique_index[part_seed_index]; + + assert(part_unique_index < astc_helpers::NUM_PARTITION_PATTERNS); + const astc_ldr::partition_pattern_vec* pPat = &pPart_data->m_partition_pats[part_unique_index]; + + bool refine_only_flag = false; + + status = encode_trial_subsets( + block_width, block_height, + pixel_stats, + neighbor_log_blk.m_color_endpoint_modes[0], neighbor_log_blk.m_num_partitions, neighbor_log_blk.m_partition_id, pPat, + neighbor_log_blk.m_endpoint_ise_range, neighbor_log_blk.m_weight_ise_range, + neighbor_log_blk.m_grid_width, neighbor_log_blk.m_grid_height, + new_log_block, + enc_cfg.m_cem_enc_params, + refine_only_flag, + enc_cfg.m_gradient_descent_flag, enc_cfg.m_polish_weights_flag, enc_cfg.m_qcd_enabled_flag, + enc_cfg.m_use_blue_contraction); + } + else + { + status = encode_trial( + block_width, block_height, + pixel_stats, + neighbor_log_blk.m_color_endpoint_modes[0], + neighbor_log_blk.m_dual_plane, neighbor_log_blk.m_dual_plane ? neighbor_log_blk.m_color_component_selector : -1, + neighbor_log_blk.m_endpoint_ise_range, neighbor_log_blk.m_weight_ise_range, + neighbor_log_blk.m_grid_width, neighbor_log_blk.m_grid_height, + new_log_block, + enc_cfg.m_cem_enc_params, + enc_cfg.m_gradient_descent_flag, enc_cfg.m_polish_weights_flag, enc_cfg.m_qcd_enabled_flag, + enc_cfg.m_use_blue_contraction); + } + + if (!status) + { + fmt_debug_printf("encode_trial/encode_trial_subsets failed in superpass 1!\n"); + encoder_failed_flag.store(true); + return; + } + + out_block_info_superpass1.m_config_reuse_neighbor_out_block_indices[neighbor_index] = out_block_info_superpass1.m_new_out_config_reuse_blocks.size_u32(); + out_block_info_superpass1.m_config_reuse_new_neighbor_out_block_flags[neighbor_index] = true; + + encode_block_output& new_output_blk = *out_block_info_superpass1.m_new_out_config_reuse_blocks.enlarge(1); + + new_output_blk.clear(); + + if (enc_cfg.m_use_dct) + { + const basist::astc_ldr_t::astc_block_grid_data* pGrid_data = basist::astc_ldr_t::find_astc_block_grid_data(block_width, block_height, new_log_block.m_grid_width, new_log_block.m_grid_height); + + const uint32_t num_planes = (new_log_block.m_dual_plane ? 2 : 1); + + for (uint32_t plane_index = 0; plane_index < num_planes; plane_index++) + { + bitwise_coder c; + basist::astc_ldr_t::dct_syms syms; + code_block_weights(grid_coder, enc_cfg.m_base_q, plane_index, new_log_block, pGrid_data, c, syms); + + new_output_blk.m_packed_dct_plane_data[plane_index] = syms; + + c.flush(); + + basist::bitwise_decoder d; + d.init(c.get_bytes().data(), c.get_bytes().size_u32()); + + // ensure existing weights get blown away + for (uint32_t i = 0; i < (uint32_t)(new_log_block.m_grid_width * new_log_block.m_grid_height); i++) + new_log_block.m_weights[i * num_planes + plane_index] = 0; + + basist::astc_ldr_t::fvec dct_temp; + bool dec_status = grid_coder.decode_block_weights(enc_cfg.m_base_q, plane_index, new_log_block, &d, pGrid_data, nullptr, dct_temp, nullptr); + + assert(dec_status); + if (!dec_status) + { + error_printf("grid_coder.decode_block_weights() failed!\n"); + + encoder_failed_flag.store(true); + return; + } + } + } // if (enc_cfg.m_use_dct) + + new_output_blk.m_trial_mode_index = safe_cast_int16(neighbor_tm_index); + new_output_blk.m_log_blk = new_log_block; + //new_output_blk.m_trial_surrogate.clear(); + + new_output_blk.m_sse = eval_error(block_width, block_height, new_log_block, pixel_stats, enc_cfg.m_cem_enc_params); + + { + std::lock_guard g(global_mutex); + + total_full_encodes_pass2++; + } + } // if (already_existing_out_block_index != cInvalidIndex) + + { + // Re-encode using the neighbor's full config (tm, base+ofs, partition ID) AND its endpoints + astc_helpers::log_astc_block new_log_block(neighbor_log_blk); + + // Start with fresh 0 weights, then polish them. + clear_obj(new_log_block.m_weights); + + //const bool use_blue_contraction = enc_cfg.m_use_blue_contraction; + + bool improved_flag = false; + + const astc_ldr::partition_pattern_vec* pPat = nullptr; + if (neighbor_log_blk.m_num_partitions > 1) + { + const astc_ldr::partitions_data* pPart_data = (neighbor_log_blk.m_num_partitions == 2) ? pPart_data_p2 : pPart_data_p3; + + const uint32_t part_seed_index = neighbor_log_blk.m_partition_id; + const uint32_t part_unique_index = pPart_data->m_part_seed_to_unique_index[part_seed_index]; + + assert(part_unique_index < astc_helpers::NUM_PARTITION_PATTERNS); + pPat = &pPart_data->m_partition_pats[part_unique_index]; + } + + bool status = polish_block_weights( + block_width, block_height, + pixel_stats, + new_log_block, + enc_cfg.m_cem_enc_params, pPat, improved_flag, + enc_cfg.m_gradient_descent_flag, enc_cfg.m_polish_weights_flag, enc_cfg.m_qcd_enabled_flag); + + if (!status) + { + fmt_error_printf("polish_block_weights failed in superpass 1!\n"); + encoder_failed_flag.store(true); + return; + } + + encode_block_output& new_output_blk = *out_block_info_superpass1.m_new_out_config_endpoint_reuse_blocks.enlarge(1); + + new_output_blk.clear(); + + if (enc_cfg.m_use_dct) + { + const basist::astc_ldr_t::astc_block_grid_data* pGrid_data = basist::astc_ldr_t::find_astc_block_grid_data(block_width, block_height, new_log_block.m_grid_width, new_log_block.m_grid_height); + + const uint32_t num_planes = (new_log_block.m_dual_plane ? 2 : 1); + + for (uint32_t plane_index = 0; plane_index < num_planes; plane_index++) + { + bitwise_coder c; + basist::astc_ldr_t::dct_syms syms; + code_block_weights(grid_coder, enc_cfg.m_base_q, plane_index, new_log_block, pGrid_data, c, syms); + + new_output_blk.m_packed_dct_plane_data[plane_index] = syms; + + c.flush(); + + basist::bitwise_decoder d; + d.init(c.get_bytes().data(), c.get_bytes().size_u32()); + + // ensure existing weights get blown away + for (uint32_t i = 0; i < (uint32_t)(new_log_block.m_grid_width * new_log_block.m_grid_height); i++) + new_log_block.m_weights[i * num_planes + plane_index] = 0; + + basist::astc_ldr_t::fvec dct_temp; + bool dec_status = grid_coder.decode_block_weights(enc_cfg.m_base_q, plane_index, new_log_block, &d, pGrid_data, nullptr, dct_temp, nullptr); + + assert(dec_status); + if (!dec_status) + { + error_printf("grid_coder.decode_block_weights() failed!\n"); + + encoder_failed_flag.store(true); + return; + } + } + } // if (enc_cfg.m_use_dct) + + new_output_blk.m_trial_mode_index = safe_cast_int16(neighbor_tm_index); + new_output_blk.m_log_blk = new_log_block; + //new_output_blk.m_trial_surrogate.clear(); + + new_output_blk.m_sse = eval_error(block_width, block_height, new_log_block, pixel_stats, enc_cfg.m_cem_enc_params); + + { + std::lock_guard g(global_mutex); + + total_full_encodes_pass2++; + } + } + + } // neighbor_index + } + else + { + if (superpass_index == 1) + { + if (!superpass2_recompress_block_flags(bx, by)) + return; + } + + // Superpass 0/2: core ASTC encoding + basisu::vector& out_blocks = enc_out.m_image_block_info(bx, by).m_out_blocks; + out_blocks.resize(0); + + astc_ldr::pixel_stats_t& pixel_stats = enc_out.m_image_block_info(bx, by).m_pixel_stats; + + if (superpass_index == 0) + pixel_stats.init(total_block_pixels, block_pixels); + + const bool is_purely_solid_block = (pixel_stats.m_min == pixel_stats.m_max); + + // early out on totally solid blocks + if (is_purely_solid_block) + { + encode_block_output* pOut = out_blocks.enlarge(1); + pOut->clear(); + + astc_helpers::log_astc_block& log_blk = pOut->m_log_blk; + + log_blk.clear(); + log_blk.m_solid_color_flag_ldr = true; + + for (uint32_t c = 0; c < 4; c++) + log_blk.m_solid_color[c] = pixel_stats.m_min[c]; + + // Expand each component to 16-bits + for (uint32_t c = 0; c < 4; c++) + log_blk.m_solid_color[c] |= (uint16_t)(log_blk.m_solid_color[c]) << 8u; + + pOut->m_sse = eval_error(block_width, block_height, log_blk, pixel_stats, enc_cfg.m_cem_enc_params); + + ldr_astc_block_encode_image_output::block_info& block_info_out = enc_out.m_image_block_info(bx, by); + + block_info_out.m_low_freq_block_flag = true; + block_info_out.m_super_strong_edges = false; + block_info_out.m_very_strong_edges = false; + block_info_out.m_strong_edges = false; + block_info_out.m_packed_out_block_index = 0; + + // Create packed ASTC block + astc_helpers::astc_block& best_phys_block = packed_blocks(bx, by); + bool pack_success = astc_helpers::pack_astc_block(best_phys_block, log_blk); + if (!pack_success) + { + encoder_failed_flag.store(true); + return; + } + + output_block_devel_desc& out_devel_desc = output_block_devel_info(bx, by); + out_devel_desc.m_low_freq_block_flag = true; + out_devel_desc.m_super_strong_edges = false; + out_devel_desc.m_very_strong_edges = false; + out_devel_desc.m_strong_edges = false; + + { + std::lock_guard g(global_mutex); + + total_void_extent_blocks_skipped++; + + total_blocks_done++; + } + + return; + } + + float max_std_dev = 0.0f; + for (uint32_t i = 0; i < 4; i++) + max_std_dev = maximum(max_std_dev, pixel_stats.m_rgba_stats[i].m_std_dev); + + bool is_lum_only = true; + + for (uint32_t y = 0; y < block_height; y++) + { + for (uint32_t x = 0; x < block_width; x++) + { + const color_rgba& c = pixel_stats.m_pixels[x + y * block_width]; + bool is_lum_texel = (c.r == c.g) && (c.r == c.b); + if (!is_lum_texel) + { + is_lum_only = false; + break; + } + } + if (is_lum_only) + break; + } + + basisu::vector block_dct_energy(total_block_pixels); + + bool filter_horizontally_flag = false; + bool low_freq_block_flag = 0; + + { + basisu::vector block_floats(total_block_pixels); + basisu::vector block_dct(total_block_pixels); + basist::astc_ldr_t::fvec work; + + for (uint32_t c = 0; c < 4; c++) + { + for (uint32_t i = 0; i < total_block_pixels; i++) + block_floats[i] = pixel_stats.m_pixels_f[i][c]; + + dct.forward(block_floats.data(), block_dct.data(), work); + + for (uint32_t y = 0; y < block_height; y++) + for (uint32_t x = 0; x < block_width; x++) + block_dct_energy[x + y * block_width] += (float)enc_cfg.m_cem_enc_params.m_comp_weights[c] * squaref(block_dct[x + y * block_width]); + + } // c + + // Wipe DC + block_dct_energy[0] = 0.0f; + + float tot_energy = compute_preserved_dct_energy(block_width, block_height, block_dct_energy.get_ptr(), block_width, block_height); + + float h_energy_lost = compute_lost_dct_energy(block_width, block_height, block_dct_energy.get_ptr(), block_width / 2, block_height); + float v_energy_lost = compute_lost_dct_energy(block_width, block_height, block_dct_energy.get_ptr(), block_width, block_height / 2); + + filter_horizontally_flag = h_energy_lost < v_energy_lost; + + float hv2_lost_energy_fract = compute_lost_dct_energy(block_width, block_height, block_dct_energy.get_ptr(), 2, 2); + if (tot_energy) + hv2_lost_energy_fract /= tot_energy; + + if ((hv2_lost_energy_fract < .03f) || (max_std_dev < (1.0f / 255.0f))) + low_freq_block_flag = true; + } + + if (enc_cfg.m_debug_images) + vis_dct_low_freq_block.fill_box(bx * block_width, by * block_height, block_width, block_height, low_freq_block_flag ? color_rgba(255, 0, 0, 255) : g_black_color); + + bool active_chan_flags[4] = { }; + + // The number of channels with non-zero spans + uint32_t total_active_chans = 0; + // The indices of the channels with non-zero spans. + //uint32_t active_chan_list[4] = { 0 }; + + for (uint32_t i = 0; i < 4; i++) + { + if (pixel_stats.m_rgba_stats[i].m_range > 0.0f) + { + assert(pixel_stats.m_max[i] != pixel_stats.m_min[i]); + + active_chan_flags[i] = true; + + //active_chan_list[total_active_chans] = i; + total_active_chans++; + } + else + { + assert(pixel_stats.m_max[i] == pixel_stats.m_min[i]); + } + } + + basisu::comparative_stats cross_chan_stats[TOTAL_RGBA_CHAN_PAIRS]; + + // def=max correlation for each channel pair (or 1 if one of the channels is inactive) + float chan_pair_correlations[6] = { 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f }; + // 0=0, 1 + // 1=0, 2 + // 2=1, 2 + // 3=0, 3 + // 4=1, 3 + // 5=2, 3 + + float min_corr = 1.0f, max_corr = 0.0f; + + for (uint32_t pair_index = 0; pair_index < TOTAL_RGBA_CHAN_PAIRS; pair_index++) + { + const uint32_t chanA = g_rgba_chan_pairs[pair_index][0]; + const uint32_t chanB = g_rgba_chan_pairs[pair_index][1]; + + // If both channels were active, we've got usable correlation statistics. + if (active_chan_flags[chanA] && active_chan_flags[chanB]) + { + // TODO: This can be directly derived from the 3D/4D covariance matrix entries. + cross_chan_stats[pair_index].calc_pearson(total_block_pixels, + &pixel_stats.m_pixels_f[0][chanA], + &pixel_stats.m_pixels_f[0][chanB], + 4, 4, + &pixel_stats.m_rgba_stats[chanA], + &pixel_stats.m_rgba_stats[chanB]); + + chan_pair_correlations[pair_index] = fabsf(cross_chan_stats[pair_index].m_pearson); + + const float c = fabsf((float)cross_chan_stats[pair_index].m_pearson); + min_corr = minimum(min_corr, c); + max_corr = maximum(max_corr, c); + } + } + + // min_cor will be 1.0f if all channels inactive (solid) + + // Pixel the trial modes the encoder will use: RGB or RGBA (we don't currently support trying both) + + const bool used_alpha_encoder_modes = pixel_stats.m_has_alpha; + + float sobel_energy = 0.0f; + for (uint32_t y = 0; y < block_height; y++) + { + for (uint32_t x = 0; x < block_width; x++) + { + const color_rgba& s = orig_img_sobel_xy.get_clamped(bx * block_width + x, by * block_height + y); + sobel_energy += s[0] * s[0] + s[1] * s[1] + s[2] * s[2] + s[3] * s[3]; + } // x + } // y + + sobel_energy /= (float)total_block_pixels; + + // Configure low-level block encoder. + ldr_astc_lowlevel_block_encoder_params enc_blk_params; + + enc_blk_params.m_block_width = block_width; + enc_blk_params.m_block_height = block_height; + enc_blk_params.m_total_block_pixels = total_block_pixels; + enc_blk_params.m_bx = bx; + enc_blk_params.m_by = by; + + enc_blk_params.m_pOrig_img_sobel_xy_t = &orig_img_sobel_xy; + + enc_blk_params.m_num_trial_modes = encoder_trial_modes.size_u32(); + enc_blk_params.m_pTrial_modes = encoder_trial_modes.get_ptr(); + enc_blk_params.m_pGrouped_trial_modes = &grouped_encoder_trial_modes; + + enc_blk_params.m_pPart_data_p2 = pPart_data_p2; + enc_blk_params.m_pPart_data_p3 = pPart_data_p3; + enc_blk_params.m_pEnc_params = &enc_cfg.m_cem_enc_params; + + float ang_dot = saturate(pixel_stats.m_zero_rel_axis3.dot3(pixel_stats.m_mean_rel_axis3)); + const float pca_axis_angles = acosf(ang_dot) * (180.0f / (float)cPiD); + + enc_blk_params.m_use_alpha_or_opaque_modes = used_alpha_encoder_modes; + enc_blk_params.m_use_lum_direct_modes = is_lum_only; + + const bool filter_by_pca_angles_flag = (superpass_index == 1) ? enc_cfg.m_filter_by_pca_angles_flag_p2 : enc_cfg.m_filter_by_pca_angles_flag; + if (!filter_by_pca_angles_flag) + { + enc_blk_params.m_use_direct_modes = true; + enc_blk_params.m_use_base_scale_modes = true; + } + else + { + // TODO: Make selective based off edge blocks? + enc_blk_params.m_use_direct_modes = (!total_active_chans) || (pca_axis_angles > enc_cfg.m_use_direct_angle_thresh); + enc_blk_params.m_use_base_scale_modes = (pca_axis_angles <= enc_cfg.m_use_base_scale_angle_thresh); + } + + enc_blk_params.m_grid_hv_filtering = enc_cfg.m_grid_hv_filtering; + enc_blk_params.m_filter_horizontally_flag = filter_horizontally_flag; + + enc_blk_params.m_use_small_grids_only = low_freq_block_flag && enc_cfg.m_low_freq_block_filtering; + + enc_blk_params.m_subsets_enabled = enc_cfg.m_subsets_enabled && (!low_freq_block_flag || !enc_cfg.m_subsets_edge_filtering); + + enc_blk_params.m_subsets_edge_filtering = enc_cfg.m_subsets_edge_filtering; + + enc_blk_params.m_use_blue_contraction = enc_cfg.m_use_blue_contraction; + enc_blk_params.m_final_encode_try_base_ofs = enc_cfg.m_use_base_ofs; + + memcpy(enc_blk_params.m_superbucket_max_to_retain, enc_cfg.m_superbucket_max_to_retain, sizeof(enc_cfg.m_superbucket_max_to_retain)); + + memcpy(enc_blk_params.m_final_shortlist_fraction, enc_cfg.m_final_shortlist_fraction, sizeof(enc_blk_params.m_final_shortlist_fraction)); + memcpy(enc_blk_params.m_final_shortlist_min_size, enc_cfg.m_final_shortlist_min_size, sizeof(enc_cfg.m_final_shortlist_min_size)); + memcpy(enc_blk_params.m_final_shortlist_max_size, enc_cfg.m_final_shortlist_max_size, sizeof(enc_blk_params.m_final_shortlist_max_size)); + + enc_blk_params.m_part2_fraction_to_keep = enc_cfg.m_part2_fraction_to_keep; + enc_blk_params.m_part3_fraction_to_keep = enc_cfg.m_part3_fraction_to_keep; + enc_blk_params.m_base_parts2 = enc_cfg.m_base_parts2; + enc_blk_params.m_base_parts3 = enc_cfg.m_base_parts3; + enc_blk_params.m_gradient_descent_flag = enc_cfg.m_gradient_descent_flag; + enc_blk_params.m_polish_weights_flag = enc_cfg.m_polish_weights_flag; + enc_blk_params.m_qcd_enabled_flag = enc_cfg.m_qcd_enabled_flag; + enc_blk_params.m_bucket_pruning_passes = enc_cfg.m_bucket_pruning_passes; + + enc_blk_params.m_alpha_cems = used_alpha_encoder_modes; + + enc_blk_params.m_early_stop_wpsnr = enc_cfg.m_early_stop_wpsnr; + enc_blk_params.m_early_stop2_wpsnr = enc_cfg.m_early_stop2_wpsnr; + + enc_blk_params.m_final_encode_always_try_rgb_direct = enc_cfg.m_final_encode_always_try_rgb_direct; + + enc_blk_params.m_pDCT2F = &dct; + + // Determine DP usage + if (enc_cfg.m_force_all_dual_plane_chan_evals) + { + for (uint32_t i = 0; i < 4; i++) + enc_blk_params.m_dp_active_chans[i] = active_chan_flags[i]; + } + else + { + for (uint32_t i = 0; i < 3; i++) + enc_blk_params.m_dp_active_chans[i] = false; + + // Being very conservative with alpha here - always let the analytical evaluator consider it. + enc_blk_params.m_dp_active_chans[3] = pixel_stats.m_has_alpha; + + if (!enc_cfg.m_disable_rgb_dual_plane) + { + const float rg_corr = chan_pair_correlations[0]; + const float rb_corr = chan_pair_correlations[1]; + const float gb_corr = chan_pair_correlations[2]; + + int desired_dp_chan_rgb = -1; + + float min_p = minimum(rg_corr, rb_corr, gb_corr); + + if (min_p < enc_cfg.m_strong_dp_decorr_thresh_rgb) + { + const bool has_r = active_chan_flags[0], has_g = active_chan_flags[1]; + //const bool has_b = active_chan_flags[2]; + + uint32_t total_active_chans_rgb = 0; + for (uint32_t i = 0; i < 3; i++) + total_active_chans_rgb += active_chan_flags[i]; + + if (total_active_chans_rgb == 2) + { + if (!has_r) + desired_dp_chan_rgb = 1; + else if (!has_g) + desired_dp_chan_rgb = 0; + else + desired_dp_chan_rgb = 0; + } + else if (total_active_chans_rgb == 3) + { + // see if rg/rb is weakly correlated vs. gb + if ((rg_corr < gb_corr) && (rb_corr < gb_corr)) + desired_dp_chan_rgb = 0; + // see if gr/gb is weakly correlated vs. rb + else if ((rg_corr < rb_corr) && (gb_corr < rb_corr)) + desired_dp_chan_rgb = 1; + // assume b is weakest + else + desired_dp_chan_rgb = 2; + } + } + + if (desired_dp_chan_rgb != -1) + { + assert(active_chan_flags[desired_dp_chan_rgb]); + enc_blk_params.m_dp_active_chans[desired_dp_chan_rgb] = true; + } + } + } + + if (!enc_blk_params.m_dp_active_chans[0] && !enc_blk_params.m_dp_active_chans[1] && !enc_blk_params.m_dp_active_chans[2] && !enc_blk_params.m_dp_active_chans[3]) + { + enc_blk_params.m_use_dual_planes = false; + } + + astc_ldr::cem_encode_params temp_cem_enc_params; + if (superpass_index == 1) + { + enc_blk_params.m_base_parts2 = enc_cfg.m_base_parts2_p2; + enc_blk_params.m_base_parts3 = enc_cfg.m_base_parts3_p2; + enc_blk_params.m_part2_fraction_to_keep = 1; + enc_blk_params.m_part3_fraction_to_keep = 1; + + memcpy(enc_blk_params.m_superbucket_max_to_retain, enc_cfg.m_superbucket_max_to_retain_p2, sizeof(enc_cfg.m_superbucket_max_to_retain_p2)); + memcpy(enc_blk_params.m_final_shortlist_max_size, enc_cfg.m_final_shortlist_max_size_p2, sizeof(enc_cfg.m_final_shortlist_max_size_p2)); + + if (enc_cfg.m_second_pass_force_subsets_enabled) + enc_blk_params.m_subsets_enabled = true; + enc_blk_params.m_subsets_edge_filtering = false; + + if (enc_cfg.m_force_all_dp_chans_p2) + { + enc_blk_params.m_dp_active_chans[0] = active_chan_flags[0]; + enc_blk_params.m_dp_active_chans[1] = active_chan_flags[1]; + enc_blk_params.m_dp_active_chans[2] = active_chan_flags[2]; + enc_blk_params.m_dp_active_chans[3] = active_chan_flags[3]; + enc_blk_params.m_use_dual_planes = true; + + if (!enc_blk_params.m_dp_active_chans[0] && !enc_blk_params.m_dp_active_chans[1] && !enc_blk_params.m_dp_active_chans[2] && !enc_blk_params.m_dp_active_chans[3]) + { + enc_blk_params.m_use_dual_planes = false; + } + } + + enc_blk_params.m_gradient_descent_flag = true; + enc_blk_params.m_polish_weights_flag = true; + + enc_blk_params.m_use_direct_modes = true; + enc_blk_params.m_use_base_scale_modes = true; + + enc_blk_params.m_early_stop_wpsnr = enc_cfg.m_early_stop_wpsnr + 2.0f; + enc_blk_params.m_early_stop2_wpsnr = enc_cfg.m_early_stop2_wpsnr + 2.0f; + + if (enc_cfg.m_second_pass_total_weight_refine_passes) + { + temp_cem_enc_params = enc_cfg.m_cem_enc_params; + enc_blk_params.m_pEnc_params = &temp_cem_enc_params; + + temp_cem_enc_params.m_total_weight_refine_passes = enc_cfg.m_second_pass_total_weight_refine_passes; + temp_cem_enc_params.m_worst_weight_nudging_flag = true; + temp_cem_enc_params.m_endpoint_refinement_flag = true; + } + } + + scoped_ldr_astc_lowlevel_block_encoder scoped_block_encoder(encoder_pool); + if (scoped_block_encoder.get_ptr() == nullptr) + { + error_printf("Failed allocating thread local encode block temps\n"); + encoder_failed_flag.store(true); + return; + } + + // solid color + { + encode_block_output* pOut = out_blocks.enlarge(1); + pOut->clear(); + + astc_helpers::log_astc_block& log_blk = pOut->m_log_blk; + + log_blk.clear(); + log_blk.m_solid_color_flag_ldr = true; + + for (uint32_t c = 0; c < 4; c++) + log_blk.m_solid_color[c] = (uint16_t)clamp((int)std::round(pixel_stats.m_mean_f[c] * 255.0f), 0, 255); + + // Expand each component to 16-bits + for (uint32_t c = 0; c < 4; c++) + log_blk.m_solid_color[c] |= (uint16_t)(log_blk.m_solid_color[c]) << 8u; + + pOut->m_sse = eval_error(block_width, block_height, log_blk, pixel_stats, enc_cfg.m_cem_enc_params); + } + + encode_block_stats enc_block_stats; + + bool enc_status = scoped_block_encoder.get_ptr()->full_encode(enc_blk_params, pixel_stats, out_blocks, 0, enc_block_stats); + if (!enc_status) + { + encoder_failed_flag.store(true); + return; + } + +#if 1 + // --------------------- BLOCK BLURRING + // TODO - very slow, needs more configuration and tuning, experimental + const float BLUR_STD_DEV_THRESH = (15.0f / 255.0f); + const float BLUR_SOBEL_ENERGY_THRESH = 15000.0f; + + const bool use_blurs = (enc_cfg.m_blurring_enabled && (!selective_blurring || ((max_std_dev > BLUR_STD_DEV_THRESH) && (sobel_energy > BLUR_SOBEL_ENERGY_THRESH)))) || + (enc_cfg.m_blurring_enabled_p2 && (superpass_index == 1)); + + if (use_blurs) + { + { + assert(orig_img_blurred2.get_width()); + + color_rgba block_pixels_blurred2[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + orig_img_blurred2.extract_block_clamped(block_pixels_blurred2, bx * block_width, by * block_height, block_width, block_height); + + astc_ldr::pixel_stats_t pixel_stats_blurred2; + pixel_stats_blurred2.init(total_block_pixels, block_pixels_blurred2); + + enc_status = scoped_block_encoder.get_ptr()->full_encode(enc_blk_params, pixel_stats_blurred2, out_blocks, 1, enc_block_stats); + if (!enc_status) + { + encoder_failed_flag.store(true); + return; + } + } + + { + assert(orig_img_blurred3.get_width()); + + color_rgba block_pixels_blurred3[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + orig_img_blurred3.extract_block_clamped(block_pixels_blurred3, bx * block_width, by * block_height, block_width, block_height); + + astc_ldr::pixel_stats_t pixel_stats_blurred3; + pixel_stats_blurred3.init(total_block_pixels, block_pixels_blurred3); + + enc_status = scoped_block_encoder.get_ptr()->full_encode(enc_blk_params, pixel_stats_blurred3, out_blocks, 2, enc_block_stats); + if (!enc_status) + { + encoder_failed_flag.store(true); + return; + } + } + + { + assert(orig_img_blurred4.get_width()); + + color_rgba block_pixels_blurred4[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + orig_img_blurred4.extract_block_clamped(block_pixels_blurred4, bx * block_width, by * block_height, block_width, block_height); + + astc_ldr::pixel_stats_t pixel_stats_blurred4; + pixel_stats_blurred4.init(total_block_pixels, block_pixels_blurred4); + + enc_status = scoped_block_encoder.get_ptr()->full_encode(enc_blk_params, pixel_stats_blurred4, out_blocks, 3, enc_block_stats); + if (!enc_status) + { + encoder_failed_flag.store(true); + return; + } + } + + { + assert(orig_img_blurred5.get_width()); + + color_rgba block_pixels_blurred5[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + orig_img_blurred5.extract_block_clamped(block_pixels_blurred5, bx * block_width, by * block_height, block_width, block_height); + + astc_ldr::pixel_stats_t pixel_stats_blurred5; + pixel_stats_blurred5.init(total_block_pixels, block_pixels_blurred5); + + enc_status = scoped_block_encoder.get_ptr()->full_encode(enc_blk_params, pixel_stats_blurred5, out_blocks, 4, enc_block_stats); + if (!enc_status) + { + encoder_failed_flag.store(true); + return; + } + } + } +#endif + + // --------------------- WEIGHT GRID DCT CODING + if (enc_cfg.m_use_dct) + { + // apply DCT to weights + for (uint32_t out_block_iter = 0; out_block_iter < out_blocks.size_u32(); out_block_iter++) + { + if (out_blocks[out_block_iter].m_trial_mode_index < 0) + continue; + + astc_helpers::log_astc_block& log_astc_blk = out_blocks[out_block_iter].m_log_blk; + + const basist::astc_ldr_t::astc_block_grid_data* pGrid_data = basist::astc_ldr_t::find_astc_block_grid_data(block_width, block_height, log_astc_blk.m_grid_width, log_astc_blk.m_grid_height); + + const uint32_t num_planes = (log_astc_blk.m_dual_plane ? 2 : 1); + for (uint32_t plane_index = 0; plane_index < num_planes; plane_index++) + { + bitwise_coder c; + basist::astc_ldr_t::dct_syms syms; + code_block_weights(grid_coder, enc_cfg.m_base_q, plane_index, log_astc_blk, pGrid_data, c, syms); + + out_blocks[out_block_iter].m_packed_dct_plane_data[plane_index] = syms; + + c.flush(); + + basist::bitwise_decoder d; + d.init(c.get_bytes().data(), c.get_bytes().size_u32()); + + // ensure existing weights get blown away + for (uint32_t i = 0; i < (uint32_t)(log_astc_blk.m_grid_width * log_astc_blk.m_grid_height); i++) + log_astc_blk.m_weights[i * num_planes + plane_index] = 0; + + basist::astc_ldr_t::fvec dct_temp; + bool status = grid_coder.decode_block_weights(enc_cfg.m_base_q, plane_index, log_astc_blk, &d, pGrid_data, nullptr, dct_temp, nullptr); + + assert(status); + if (!status) + { + error_printf("grid_coder.decode_block_weights() failed!\n"); + + encoder_failed_flag.store(true); + return; + } + +#if 0 + { + astc_helpers::log_astc_block alt_log_astc_blk(log_astc_blk); + + for (uint32_t i = 0; i < (uint32_t)(log_astc_blk.m_grid_width * log_astc_blk.m_grid_height); i++) + alt_log_astc_blk.m_weights[i * num_planes + plane_index] = 0; + + status = grid_coder.decode_block_weights(q, plane_index, alt_log_astc_blk, nullptr, pGrid_data, &out_block_dct_stats[out_block_iter], &syms); + assert(status); + + for (uint32_t i = 0; i < (uint32_t)(log_astc_blk.m_grid_width * log_astc_blk.m_grid_height); i++) + { + assert(log_astc_blk.m_weights[i * num_planes + plane_index] == alt_log_astc_blk.m_weights[i * num_planes + plane_index]); + } + + } +#endif + // TODO: in theory, endpoints can be refined if they don't change the DCT span. + } + + out_blocks[out_block_iter].m_sse = eval_error(block_width, block_height, log_astc_blk, pixel_stats, enc_cfg.m_cem_enc_params); + + } // for + + } // use_dct + + // Find best output block + uint64_t best_out_blocks_err = UINT64_MAX; + uint32_t best_out_blocks_index = 0; + astc_helpers::log_astc_block best_out_blocks_log_astc_blk; + + for (uint32_t out_block_iter = 0; out_block_iter < out_blocks.size_u32(); out_block_iter++) + { + const astc_helpers::log_astc_block& log_astc_blk = out_blocks[out_block_iter].m_log_blk; + + color_rgba dec_pixels[astc_helpers::MAX_BLOCK_DIM * astc_helpers::MAX_BLOCK_DIM]; + bool dec_status = astc_helpers::decode_block(log_astc_blk, dec_pixels, block_width, block_height, enc_cfg.m_cem_enc_params.m_decode_mode_srgb ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8); + + assert(dec_status); + if (!dec_status) + { + encoder_failed_flag.store(true); + return; + } + + uint64_t total_err = 0; + for (uint32_t i = 0; i < total_block_pixels; i++) + total_err += weighted_color_error(block_pixels[i], dec_pixels[i], enc_cfg.m_cem_enc_params); + + // if not blurred + if (out_blocks[out_block_iter].m_blur_id == 0) + { + if (out_blocks[out_block_iter].m_sse != total_err) + { + assert(0); + fmt_error_printf("output block SSE invalid\n"); + encoder_failed_flag.store(true); + return; + } + } + + // Replace m_sse with the actual WSSE vs. the original source block (in case it was blurred) + out_blocks[out_block_iter].m_sse = total_err; + + if (total_err < best_out_blocks_err) + { + best_out_blocks_err = total_err; + best_out_blocks_log_astc_blk = log_astc_blk; + best_out_blocks_index = out_block_iter; + } + } // out_block_iter + +#if 0 + // TODO: Save memory, only minimally tested + if (enc_cfg.m_save_single_result) + { + basisu::vector new_out_blocks(1); + new_out_blocks[0] = out_blocks[best_out_blocks_index]; + + std::swap(out_blocks, new_out_blocks); + + best_out_blocks_index = 0; + } +#endif + + ldr_astc_block_encode_image_output::block_info& block_info_out = enc_out.m_image_block_info(bx, by); + + block_info_out.m_low_freq_block_flag = low_freq_block_flag; + block_info_out.m_super_strong_edges = scoped_block_encoder.get_ptr()->m_super_strong_edges; + block_info_out.m_very_strong_edges = scoped_block_encoder.get_ptr()->m_very_strong_edges; + block_info_out.m_strong_edges = scoped_block_encoder.get_ptr()->m_strong_edges; + block_info_out.m_packed_out_block_index = best_out_blocks_index; + + // Create packed ASTC block + astc_helpers::astc_block& best_phys_block = packed_blocks(bx, by); + bool pack_success = astc_helpers::pack_astc_block(best_phys_block, best_out_blocks_log_astc_blk); + if (!pack_success) + { + encoder_failed_flag.store(true); + return; + } + + output_block_devel_desc& out_devel_desc = output_block_devel_info(bx, by); + out_devel_desc.m_low_freq_block_flag = low_freq_block_flag; + out_devel_desc.m_super_strong_edges = scoped_block_encoder.get_ptr()->m_super_strong_edges; + out_devel_desc.m_very_strong_edges = scoped_block_encoder.get_ptr()->m_very_strong_edges; + out_devel_desc.m_strong_edges = scoped_block_encoder.get_ptr()->m_strong_edges; + + // Critical Section + { + std::lock_guard g(global_mutex); + + if (use_blurs) + total_blur_encodes++; + + if (out_blocks[best_out_blocks_index].m_blur_id) + total_blurred_blocks1++; + + if (superpass_index == 0) + { + // TODO: Add 2nd pass statistics + total_superbuckets_created += enc_block_stats.m_total_superbuckets_created; + total_buckets_created += enc_block_stats.m_total_buckets_created; + total_surrogate_encodes += enc_block_stats.m_total_surrogate_encodes; + total_full_encodes += enc_block_stats.m_total_full_encodes; + total_shortlist_candidates += enc_block_stats.m_total_shortlist_candidates; + } + else if (superpass_index == 1) + { + total_full_encodes_pass1 += enc_block_stats.m_total_full_encodes; + } + + total_blocks_done++; + if (enc_cfg.m_debug_output) + { + if (superpass_index == 1) + { + if ((total_blocks_done & 63) == 63) + { + float new_val = ((float)total_blocks_done * 100.0f) / (float)total_blocks_to_recompress; + if ((new_val - last_printed_progress_val) >= 5.0f) + { + last_printed_progress_val = new_val; + fmt_printf("{3.2}%\n", new_val); + } + } + } + else if ((total_blocks_done & 255) == 255) + { + float new_val = ((float)total_blocks_done * 100.0f) / (float)total_blocks; + if ((new_val - last_printed_progress_val) >= 5.0f) + { + last_printed_progress_val = new_val; + fmt_printf("{3.2}%\n", new_val); + } + } + } + + } // lock_guard (global_mutex) + + } // if (superpass_index == ...) + + }); + + if (encoder_failed_flag) + break; + + } // bx + + if (encoder_failed_flag) + break; + + } // by + + if (encoder_failed_flag) + { + fmt_error_printf("Main compressor block loop failed!\n"); + return false; + } + + job_pool.wait_for_all(); + + if (encoder_failed_flag) + { + fmt_error_printf("Main compressor block loop failed!\n"); + return false; + } + + if ((superpass_index == 0) && (enc_cfg.m_second_superpass_refinement) && (enc_cfg.m_second_superpass_fract_to_recompress > 0.0f)) + { + uint_vec block_wsse_indices(total_blocks); + + float_vec block_wsses(total_blocks); + for (uint32_t by = 0; by < num_blocks_y; by++) + { + for (uint32_t bx = 0; bx < num_blocks_x; bx++) + { + ldr_astc_block_encode_image_output::block_info& out_block_info = enc_out.m_image_block_info(bx, by); + + float wsse = (float)out_block_info.m_out_blocks[out_block_info.m_packed_out_block_index].m_sse; + + block_wsses[bx + by * num_blocks_x] = wsse; + } // bx + } // by + + indirect_sort(total_blocks, block_wsse_indices.data(), block_wsses.data()); + + if (block_wsses[block_wsse_indices[total_blocks - 1]] > 0.0f) + { + total_blocks_to_recompress = clamp((uint32_t)std::round((float)total_blocks * enc_cfg.m_second_superpass_fract_to_recompress), 0, total_blocks); + + image vis_recomp_img; + if (enc_cfg.m_debug_images) + vis_recomp_img.resize(width, height); + + for (uint32_t i = 0; i < total_blocks_to_recompress; i++) + { + const uint32_t block_index = block_wsse_indices[total_blocks - 1 - i]; + + const uint32_t block_x = block_index % num_blocks_x; + const uint32_t block_y = block_index / num_blocks_x; + + superpass2_recompress_block_flags(block_x, block_y) = true; + + if (enc_cfg.m_debug_images) + vis_recomp_img.fill_box(block_x * block_width, block_y * block_height, block_width, block_height, color_rgba(255, 255, 255, 255)); + } + + if (enc_cfg.m_debug_images) + save_png(enc_cfg.m_debug_file_prefix + "vis_recomp_img.png", vis_recomp_img); + } + } + + } // superpass_index + + if (enc_cfg.m_third_superpass_try_neighbors) + { + uint32_t total_superpass1_improved_blocks1 = 0; + uint32_t total_superpass1_improved_blocks2 = 0; + + // Merge pass 2's output into pass 0's/1's output, which can be done safely now. + for (uint32_t by = 0; by < num_blocks_y; by++) + { + for (uint32_t bx = 0; bx < num_blocks_x; bx++) + { + ldr_astc_block_encode_image_output::block_info& out_block_info = enc_out.m_image_block_info(bx, by); + + const ldr_astc_block_encode_image_output::block_info_superpass1& out_block_info_superpass1 = enc_out.m_image_block_info_superpass2(bx, by); + + for (uint32_t neighbor_index = 0; neighbor_index < basist::astc_ldr_t::cMaxConfigReuseNeighbors; neighbor_index++) + { + const int new_neighbor_index = out_block_info_superpass1.m_config_reuse_neighbor_out_block_indices[neighbor_index]; + + if (new_neighbor_index == cInvalidIndex) + { + // Can't reuse neighbor's best output block + continue; + } + + if (!out_block_info_superpass1.m_config_reuse_new_neighbor_out_block_flags[neighbor_index]) + { + // Reuses an existing, already encoded output block which matches the neighbor + assert((size_t)new_neighbor_index < out_block_info.m_out_blocks.size()); + continue; + } + + const uint32_t new_out_block_index = out_block_info.m_out_blocks.size_u32(); + + const encode_block_output& new_output_blk = out_block_info_superpass1.m_new_out_config_reuse_blocks[new_neighbor_index]; + + out_block_info.m_out_blocks.push_back(new_output_blk); + +#define BU_CHECK_NEIGHBOR_BEST (1) + +#if BU_CHECK_NEIGHBOR_BEST + // See if the solution has improved + if (new_output_blk.m_sse < out_block_info.m_out_blocks[out_block_info.m_packed_out_block_index].m_sse) + { + total_superpass1_improved_blocks1++; + + // Warning: This invalidate the neighbor indices + out_block_info.m_packed_out_block_index = new_out_block_index; + + //astc_helpers::astc_block& packed_block = enc_out.m_packed_phys_blocks(bx, by); + + bool pack_success = astc_helpers::pack_astc_block((astc_helpers::astc_block&)packed_blocks(bx, by), new_output_blk.m_log_blk); + if (!pack_success) + { + fmt_error_printf("astc_helpers::pack_astc_block failed\n"); + + return false; + } + } +#endif + + } // neighbor_index + + for (uint32_t j = 0; j < out_block_info_superpass1.m_new_out_config_endpoint_reuse_blocks.size(); j++) + { + const uint32_t new_out_block_index = out_block_info.m_out_blocks.size_u32(); + + const encode_block_output& new_output_blk = out_block_info_superpass1.m_new_out_config_endpoint_reuse_blocks[j]; + + out_block_info.m_out_blocks.push_back(new_output_blk); + +#define BU_CHECK_NEIGHBOR_BEST (1) + +#if BU_CHECK_NEIGHBOR_BEST + // See if the solution has improved + if (new_output_blk.m_sse < out_block_info.m_out_blocks[out_block_info.m_packed_out_block_index].m_sse) + { + total_superpass1_improved_blocks2++; + + // Warning: This invalidate the neighbor indices + out_block_info.m_packed_out_block_index = new_out_block_index; + + //astc_helpers::astc_block& packed_block = enc_out.m_packed_phys_blocks(bx, by); + + bool pack_success = astc_helpers::pack_astc_block((astc_helpers::astc_block&)packed_blocks(bx, by), new_output_blk.m_log_blk); + if (!pack_success) + { + fmt_error_printf("astc_helpers::pack_astc_block failed\n"); + + return false; + } + } +#endif + + } // j + + } // bx + } // by + + if (enc_cfg.m_debug_output) + { + fmt_debug_printf("Total superpass 1 improved blocks 1: {} {3.2}%\n", total_superpass1_improved_blocks1, ((float)total_superpass1_improved_blocks1 * 100.0f) / (float)(total_blocks)); + fmt_debug_printf("Total superpass 1 improved blocks 2: {} {3.2}%\n", total_superpass1_improved_blocks2, ((float)total_superpass1_improved_blocks2 * 100.0f) / (float)(total_blocks)); + } + } + + if (ASTC_LDR_CONSISTENCY_CHECKING) + { + if (enc_cfg.m_debug_output) + fmt_debug_printf("consistency checking\n"); + + // Consistency/sanity cross checking + //uint32_t total_blocks_using_neighbor_config = 0; + for (uint32_t by = 0; by < num_blocks_y; by++) + { + for (uint32_t bx = 0; bx < num_blocks_x; bx++) + { + const ldr_astc_block_encode_image_output::block_info& out_block_info = enc_out.m_image_block_info(bx, by); + +#if BU_CHECK_NEIGHBOR_BEST + uint64_t best_sse = UINT64_MAX; + uint32_t best_out_block_index = 0; + + for (uint32_t i = 0; i < out_block_info.m_out_blocks.size(); i++) + { + if (out_block_info.m_out_blocks[i].m_sse < best_sse) + { + best_sse = out_block_info.m_out_blocks[i].m_sse; + best_out_block_index = i; + } + } // i + + if (best_out_block_index != out_block_info.m_packed_out_block_index) + { + fmt_error_printf("consistency check failed\n"); + assert(0); + return false; + } +#endif + + if (out_block_info.m_out_blocks[out_block_info.m_packed_out_block_index].m_sse != + eval_error(block_width, block_height, out_block_info.m_out_blocks[out_block_info.m_packed_out_block_index].m_log_blk, out_block_info.m_pixel_stats, enc_cfg.m_cem_enc_params)) + { + fmt_error_printf("consistency check failed\n"); + assert(0); + return false; + } + + // Ensure packed output block matches the expected best WSSE block. + astc_helpers::astc_block packed_block; + bool pack_success = astc_helpers::pack_astc_block(packed_block, out_block_info.m_out_blocks[out_block_info.m_packed_out_block_index].m_log_blk); + if (!pack_success) + { + fmt_error_printf("astc_helpers::pack_astc_block failed\n"); + return false; + } + + if (memcmp(&packed_block, &enc_out.m_packed_phys_blocks(bx, by), sizeof(astc_helpers::astc_block)) != 0) + { + fmt_error_printf("consistency check failed\n"); + assert(0); + return false; + } + + // DCT check + if ((enc_cfg.m_use_dct) && (out_block_info.m_out_blocks[out_block_info.m_packed_out_block_index].m_trial_mode_index >= 0)) + { + const auto& best_log_blk = out_block_info.m_out_blocks[out_block_info.m_packed_out_block_index].m_log_blk; + if (best_log_blk.m_solid_color_flag_ldr) + { + fmt_error_printf("consistency check failed\n"); + assert(0); + return false; + } + + const basist::astc_ldr_t::astc_block_grid_data* pGrid_data = basist::astc_ldr_t::find_astc_block_grid_data(block_width, block_height, best_log_blk.m_grid_width, best_log_blk.m_grid_height); + const uint32_t total_planes = best_log_blk.m_num_partitions ? (best_log_blk.m_dual_plane ? 2 : 1) : 0; + + astc_helpers::log_astc_block verify_log_blk(best_log_blk); + + for (uint32_t plane_index = 0; plane_index < total_planes; plane_index++) + { + if (!out_block_info.m_out_blocks[out_block_info.m_packed_out_block_index].m_packed_dct_plane_data[plane_index].m_coeffs.size()) + { + fmt_error_printf("consistency check failed\n"); + assert(0); + return false; + } + + basist::astc_ldr_t::fvec dct_temp; + bool dec_status = grid_coder.decode_block_weights(enc_cfg.m_base_q, plane_index, verify_log_blk, nullptr, pGrid_data, nullptr, dct_temp, + &out_block_info.m_out_blocks[out_block_info.m_packed_out_block_index].m_packed_dct_plane_data[plane_index]); + + if (!dec_status) + { + fmt_error_printf("consistency check failed\n"); + assert(0); + return false; + } + + for (uint32_t i = 0; i < (uint32_t)(best_log_blk.m_grid_width * best_log_blk.m_grid_height); i++) + { + if (best_log_blk.m_weights[i * total_planes + plane_index] != verify_log_blk.m_weights[i * total_planes + plane_index]) + { + fmt_error_printf("consistency check failed\n"); + assert(0); + return false; + } + } + + } // plane_index + } + + } // bx + } // by + + if (enc_cfg.m_debug_output) + fmt_debug_printf("consistency checking PASSED\n"); + } + + //fmt_debug_printf("Total blocks using neighbor config: {} {3.2}%\n", total_blocks_using_neighbor_config, ((float)total_blocks_using_neighbor_config * 100.0f) / (float)(total_blocks)); + + // Debug output + uint_vec trial_mode_hist; + trial_mode_hist.resize(encoder_trial_modes.size()); + uint32_t total_alpha_blocks = 0; + + for (uint32_t by = 0; by < num_blocks_y; by++) + { + for (uint32_t bx = 0; bx < num_blocks_x; bx++) + { + const ldr_astc_block_encode_image_output::block_info& out_block_info = enc_out.m_image_block_info(bx, by); + const astc_ldr::pixel_stats_t& pixel_stats = out_block_info.m_pixel_stats; + + const encode_block_output& best_out_block = out_block_info.m_out_blocks[out_block_info.m_packed_out_block_index]; + const astc_helpers::log_astc_block& best_out_blocks_log_astc_blk = best_out_block.m_log_blk; + + if (pixel_stats.m_has_alpha) + total_alpha_blocks++; + + output_block_devel_desc& out_devel_desc = output_block_devel_info(bx, by); + out_devel_desc.m_had_alpha = pixel_stats.m_has_alpha; + out_devel_desc.m_trial_mode_index = best_out_block.m_trial_mode_index; + out_devel_desc.m_pTrial_modes = encoder_trial_modes.data(); + + if (out_devel_desc.m_trial_mode_index >= 0) + trial_mode_hist[out_devel_desc.m_trial_mode_index]++; + + //const float total_astc_weight_bits = log2f((float)astc_helpers::get_ise_levels(best_out_block.m_log_blk.m_weight_ise_range)) * + // best_out_block.m_log_blk.m_grid_width * best_out_block.m_log_blk.m_grid_height * (best_out_block.m_log_blk.m_dual_plane ? 2 : 1); + + //bool used_blue_contraction = astc_ldr::used_blue_contraction(best_out_blocks_log_astc_blk.m_color_endpoint_modes[0], best_out_blocks_log_astc_blk.m_endpoints, best_out_blocks_log_astc_blk.m_endpoint_ise_range); + + if (enc_cfg.m_debug_images) + { + color_rgba vis_col(g_black_color); + color_rgba vis2_col(g_black_color); + color_rgba dp_vis(g_black_color); + color_rgba base_ofs_vis(g_black_color); + //color_rgba dct_bits_abs_vis(g_black_color); + //color_rgba dct_bits_vs_astc_vis(g_black_color); + + const astc_ldr::partition_pattern_vec* pPat = nullptr; + + if (best_out_blocks_log_astc_blk.m_num_partitions == 2) + { + vis_col.set(0, 255, 0, 255); + + const astc_ldr::partitions_data* pPart_data = pPart_data_p2; + + const uint32_t part_seed_index = best_out_blocks_log_astc_blk.m_partition_id; + const uint32_t part_unique_index = pPart_data->m_part_seed_to_unique_index[part_seed_index]; + + pPat = &pPart_data->m_partition_pats[part_unique_index]; + } + else if (best_out_blocks_log_astc_blk.m_num_partitions == 3) + { + vis_col.set(0, 0, 255, 255); + + const astc_ldr::partitions_data* pPart_data = pPart_data_p3; + + const uint32_t part_seed_index = best_out_blocks_log_astc_blk.m_partition_id; + const uint32_t part_unique_index = pPart_data->m_part_seed_to_unique_index[part_seed_index]; + + pPat = &pPart_data->m_partition_pats[part_unique_index]; + } + + // vis_col.r = enc_blk_params.m_use_base_scale_modes ? 255 : 0; + // vis_col.g = enc_blk_params.m_use_direct_modes ? 255 : 0; + + if (!out_devel_desc.m_low_freq_block_flag) + { + if (out_devel_desc.m_super_strong_edges) + vis2_col.set(255, 0, 255, 255); + else if (out_devel_desc.m_very_strong_edges) + vis2_col.set(255, 0, 0, 255); + else if (out_devel_desc.m_strong_edges) + vis2_col.set(0, 255, 0, 255); + } + + if (pPat) + { + for (uint32_t y = 0; y < block_height; y++) + { + for (uint32_t x = 0; x < block_width; x++) + { + const uint32_t subset_idx = (*pPat)(x, y); + + color_rgba c(g_black_color); + + if (best_out_blocks_log_astc_blk.m_num_partitions == 2) + { + assert(subset_idx < 2); + c = subset_idx ? color_rgba(255, 0, 0, 255) : color_rgba(0, 255, 0, 255); + } + else + { + assert(best_out_blocks_log_astc_blk.m_num_partitions == 3); + assert(subset_idx < 3); + + if (subset_idx == 2) + c = color_rgba(0, 0, 255, 255); + else if (subset_idx == 1) + c = color_rgba(32, 0, 190, 255); + else + c = color_rgba(64, 0, 64, 255); + } + + vis_part_pat_img.set_clipped(bx * block_width + x, by * block_height + y, c); + } + } + } + + if (best_out_blocks_log_astc_blk.m_dual_plane) + dp_vis.g = 255; + + if ((best_out_blocks_log_astc_blk.m_color_endpoint_modes[0] == astc_helpers::CEM_LDR_RGB_BASE_PLUS_OFFSET) || + (best_out_blocks_log_astc_blk.m_color_endpoint_modes[0] == astc_helpers::CEM_LDR_RGBA_BASE_PLUS_OFFSET)) + { + base_ofs_vis.b = 255; + } + + vis_part_usage_img.fill_box(bx * block_width, by * block_height, block_width, block_height, vis_col); + vis_strong_edge.fill_box(bx * block_width, by * block_height, block_width, block_height, vis2_col); + vis_dp_img.fill_box(bx * block_width, by * block_height, block_width, block_height, dp_vis); + vis_base_ofs_img.fill_box(bx * block_width, by * block_height, block_width, block_height, base_ofs_vis); + } + + } // bx + + } // by + + const double total_enc_time = itm.get_elapsed_secs(); + + if (enc_cfg.m_debug_output) + fmt_debug_printf("ASTC packing complete\n"); + + image unpacked_img(width, height); + + // Unpack packed image, validate ASTC data with several decoders. + for (uint32_t by = 0; by < num_blocks_y; by++) + { + for (uint32_t bx = 0; bx < num_blocks_x; bx++) + { + const astc_helpers::astc_block* pPhys_block = &packed_blocks(bx, by); + + astc_helpers::log_astc_block log_blk; + bool status = astc_helpers::unpack_block(pPhys_block, log_blk, block_width, block_height); + if (!status) + { + fmt_error_printf("unpack_block() failed\n"); + return false; + } + + // Decode with our generic ASTC decoder. + color_rgba block_pixels[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + status = astc_helpers::decode_block(log_blk, block_pixels, block_width, block_height, enc_cfg.m_cem_enc_params.m_decode_mode_srgb ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8); + if (!status) + { + fmt_error_printf("decode_block() failed\n"); + return false; + } + + unpacked_img.set_block_clipped(block_pixels, bx * block_width, by * block_height, block_width, block_height); + + // Decode with the Android testing framework ASTC decoder + { + uint8_t dec_pixels_android[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS * 4]; + + bool android_success = basisu_astc::astc::decompress_ldr(dec_pixels_android, (const uint8_t*)pPhys_block, enc_cfg.m_cem_enc_params.m_decode_mode_srgb, block_width, block_height); + if (!android_success) + { + fmt_error_printf("Android ASTC decoder failed!\n"); + return false; + } + + if (memcmp(dec_pixels_android, block_pixels, total_block_pixels * 4) != 0) + { + fmt_error_printf("Android ASTC decoder mismatch!\n"); + return false; + } + } + + // Decode with our optimized XUASTC LDR decoder + { + color_rgba block_pixels_alt[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + status = astc_helpers::decode_block_xuastc_ldr(log_blk, block_pixels_alt, block_width, block_height, enc_cfg.m_cem_enc_params.m_decode_mode_srgb ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8); + if (!status) + { + fmt_error_printf("decode_block_xuastc_ldr() failed\n"); + return false; + } + + if (memcmp(block_pixels, block_pixels_alt, total_block_pixels * 4) != 0) + { + fmt_error_printf("XUASTC LDR ASTC decoder mismatch!\n"); + return false; + } + } + + } // bx + } // by + + if (enc_cfg.m_debug_images) + { + save_png(enc_cfg.m_debug_file_prefix + "dbg_astc_ldr_unpacked_img.png", unpacked_img); + + if (vis_part_usage_img.is_valid()) + save_png(enc_cfg.m_debug_file_prefix + "vis_part_usage.png", vis_part_usage_img); + + if (vis_part_pat_img.is_valid()) + save_png(enc_cfg.m_debug_file_prefix + "vis_part_pat_img.png", vis_part_pat_img); + + if (vis_strong_edge.is_valid()) + save_png(enc_cfg.m_debug_file_prefix + "vis_strong_edge.png", vis_strong_edge); + + if (vis_dct_low_freq_block.is_valid()) + save_png(enc_cfg.m_debug_file_prefix + "vis_dct_low_freq_block.png", vis_dct_low_freq_block); + + if (vis_dp_img.is_valid()) + save_png(enc_cfg.m_debug_file_prefix + "vis_dp.png", vis_dp_img); + + if (vis_base_ofs_img.is_valid()) + save_png(enc_cfg.m_debug_file_prefix + "vis_base_ofs.png", vis_base_ofs_img); + } + + if (enc_cfg.m_debug_output) + { + uint32_t cem_used_hist[16] = { 0 }; + uint32_t cem_used_bc[16] = { 0 }; + uint32_t cem_used_subsets[16] = { 0 }; + uint32_t cem_used_dp[16] = { 0 }; + uint32_t total_dp = 0, total_base_ofs = 0; + uint32_t subset_used_hist[4] = { 0 }; + uint32_t grid_usage_hist[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS * astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS + 1] = { 0 }; + + uint32_t total_header_bits = 0; + uint32_t total_weight_bits = 0; + uint32_t total_endpoint_bits = 0; + + uint32_t total_void_extent = 0; + + uint32_t used_endpoint_levels_hist[astc_helpers::LAST_VALID_ENDPOINT_ISE_RANGE - astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE + 1] = { 0 }; + uint32_t used_weight_levels_hist[astc_helpers::LAST_VALID_WEIGHT_ISE_RANGE - astc_helpers::FIRST_VALID_WEIGHT_ISE_RANGE + 1] = { 0 }; + + uint32_t total_blocks_using_subsets = 0; + + for (uint32_t by = 0; by < num_blocks_y; by++) + { + for (uint32_t bx = 0; bx < num_blocks_x; bx++) + { + const output_block_devel_desc& desc = output_block_devel_info(bx, by); + + const astc_helpers::astc_block* pPhys_block = &packed_blocks(bx, by); + + astc_helpers::log_astc_block log_blk; + bool status = astc_helpers::unpack_block(pPhys_block, log_blk, block_width, block_height); + if (!status) + { + fmt_error_printf("unpack_block() failed\n"); + return false; + } + + if (desc.m_trial_mode_index < 0) + { + total_void_extent++; + continue; + } + else + { + const basist::astc_ldr_t::trial_mode& tm = desc.m_pTrial_modes[desc.m_trial_mode_index]; + + const uint32_t actual_cem = log_blk.m_color_endpoint_modes[0]; + //assert(tm.m_cem == log_blk.m_color_endpoint_modes[0]); // may differ due to base+ofs usage + + assert((tm.m_ccs_index >= 0) == log_blk.m_dual_plane); + assert((!log_blk.m_dual_plane) || (tm.m_ccs_index == log_blk.m_color_component_selector)); + assert(tm.m_endpoint_ise_range == log_blk.m_endpoint_ise_range); + assert(tm.m_weight_ise_range == log_blk.m_weight_ise_range); + assert(tm.m_grid_width == log_blk.m_grid_width); + assert(tm.m_grid_height == log_blk.m_grid_height); + assert(tm.m_num_parts == log_blk.m_num_partitions); + + used_weight_levels_hist[open_range_check(tm.m_weight_ise_range - astc_helpers::FIRST_VALID_WEIGHT_ISE_RANGE, std::size(used_weight_levels_hist))]++; + used_endpoint_levels_hist[open_range_check(tm.m_endpoint_ise_range - astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE, std::size(used_endpoint_levels_hist))]++; + + cem_used_hist[actual_cem]++; + if (log_blk.m_dual_plane) + total_dp++; + + subset_used_hist[open_range_check(log_blk.m_num_partitions - 1, std::size(subset_used_hist))]++; + + bool used_bc = false; + for (uint32_t i = 0; i < tm.m_num_parts; i++) + { + if (astc_helpers::used_blue_contraction(actual_cem, log_blk.m_endpoints + i * astc_helpers::get_num_cem_values(actual_cem), log_blk.m_endpoint_ise_range)) + { + used_bc = true; + } + } + + if (used_bc) + cem_used_bc[actual_cem]++; + + if (tm.m_num_parts > 1) + cem_used_subsets[actual_cem]++; + + // TODO: add CCS index histogram per CEM + if (log_blk.m_dual_plane) + cem_used_dp[actual_cem]++; + + if ((actual_cem == astc_helpers::CEM_LDR_RGB_BASE_PLUS_OFFSET) || + (actual_cem == astc_helpers::CEM_LDR_RGBA_BASE_PLUS_OFFSET)) + { + total_base_ofs++; + } + + grid_usage_hist[open_range_check(log_blk.m_grid_width * log_blk.m_grid_height, std::size(grid_usage_hist))]++; + + if (tm.m_num_parts > 1) + total_blocks_using_subsets++; + } + + astc_helpers::pack_stats pack_stats; + pack_stats.clear(); + + astc_helpers::astc_block temp_phys_block; + int expected_endpoint_range = 0; + status = astc_helpers::pack_astc_block(temp_phys_block, log_blk, &expected_endpoint_range, &pack_stats); + assert(status); + + total_header_bits += pack_stats.m_header_bits; + total_weight_bits += pack_stats.m_weight_bits; + total_endpoint_bits += pack_stats.m_endpoint_bits; + + } // bx + } // by + + uint32_t total_used_modes = 0; + + fmt_debug_printf("--------------------- Trial Modes:\n"); + + for (uint32_t i = 0; i < trial_mode_hist.size(); i++) + { + if (!trial_mode_hist[i]) + continue; + + if (trial_mode_hist[i]) + total_used_modes++; + +#if 0 + const uint32_t total_mode_blocks = trial_mode_hist[i]; + + const uint32_t num_subsets = encoder_trial_modes[i].m_num_parts; + const uint32_t cem_index = encoder_trial_modes[i].m_cem; + + fmt_debug_printf("{}: {} {3.2}%: cem: {}, grid {}x{}, e: {} w: {}, ccs: {}, parts: {}, total base+ofs: {}, total direct: {}\n", i, total_mode_blocks, (float)total_mode_blocks * 100.0f / (float)total_blocks, + encoder_trial_modes[i].m_cem, + encoder_trial_modes[i].m_grid_width, encoder_trial_modes[i].m_grid_height, + astc_helpers::get_ise_levels(encoder_trial_modes[i].m_endpoint_ise_range), astc_helpers::get_ise_levels(encoder_trial_modes[i].m_weight_ise_range), + encoder_trial_modes[i].m_ccs_index, + encoder_trial_modes[i].m_num_parts, + used_base_offset_count[i], + used_rgb_direct_count[i]); +#endif + } + + fmt_debug_printf("\n"); + + fmt_debug_printf("Used endpoint ISE levels:\n"); + for (uint32_t i = 0; i < std::size(used_endpoint_levels_hist); i++) + fmt_debug_printf("{} levels: {}\n", astc_helpers::get_ise_levels(astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE + i), used_endpoint_levels_hist[i]); + + fmt_debug_printf("\nUsed weight ISE levels:\n"); + for (uint32_t i = 0; i < std::size(used_weight_levels_hist); i++) + fmt_debug_printf("{} levels: {}\n", astc_helpers::get_ise_levels(astc_helpers::FIRST_VALID_WEIGHT_ISE_RANGE + i), used_weight_levels_hist[i]); + + const uint32_t total_blocks_excluding_void_extent = total_blocks - total_void_extent; + + fmt_debug_printf("\nTotal blocks: {}, excluding void extent: {}\n", total_blocks, total_blocks_excluding_void_extent); + fmt_debug_printf("Total void extent blocks skipped by compressor: {}\n", total_void_extent_blocks_skipped); + fmt_debug_printf("Total final void extent blocks: {}\n", total_void_extent); + fmt_debug_printf("Total input blocks with alpha: {} {3.1}%\n", total_alpha_blocks, (float)total_alpha_blocks * 100.0f / (float)total_blocks); + + fmt_debug_printf("\nASTC phys avg block stats (including void extent):\n"); + fmt_debug_printf("Total header bits: {}, {} per block, {} per pixel\n", total_header_bits, (float)total_header_bits / (float)total_blocks, (float)total_header_bits / (float)(total_pixels)); + fmt_debug_printf("Total weight bits: {}, {} per block, {} per pixel\n", total_weight_bits, (float)total_weight_bits / (float)total_blocks, (float)total_weight_bits / (float)(total_pixels)); + fmt_debug_printf("Total endpoint bits: {}, {} per block, {} per pixel\n", total_endpoint_bits, (float)total_endpoint_bits / (float)total_blocks, (float)total_endpoint_bits / (float)(total_pixels)); + fmt_debug_printf("Total header+endpoint bits: {}, {} per block, {} per pixel\n", total_header_bits + total_endpoint_bits, + (float)(total_header_bits + total_endpoint_bits) / (float)total_blocks, (float)(total_header_bits + total_endpoint_bits) / (float)(total_pixels)); + fmt_debug_printf("Total header+endpoint+weight bits: {}, {} per block, {} per pixel\n", total_header_bits + total_endpoint_bits + total_weight_bits, + (float)(total_header_bits + total_endpoint_bits + total_weight_bits) / (float)total_blocks, (float)(total_header_bits + total_endpoint_bits + total_weight_bits) / (float)(total_pixels)); + + fmt_debug_printf("\nEncoder stats:\n"); + fmt_debug_printf("Total utilized encoder trial modes: {} {3.2}%\n", total_used_modes, (float)total_used_modes * 100.0f / (float)encoder_trial_modes.size()); + + const uint32_t total_blurred_blocks = total_blurred_blocks1 + total_blurred_blocks2 + total_blurred_blocks3 + total_blurred_blocks4; + + fmt_debug_printf("\nTotal blur encodes: {} ({3.2}%)\n", total_blur_encodes, (float)total_blur_encodes * 100.0f / (float)total_blocks); + fmt_debug_printf("Total blurred blocks: {} ({3.2}%)\n", total_blurred_blocks, (float)total_blurred_blocks * 100.0f / (float)total_blocks); + fmt_debug_printf("Total blurred1 blocks: {} ({3.2}%)\n", total_blurred_blocks1, (float)total_blurred_blocks1 * 100.0f / (float)total_blocks); + fmt_debug_printf("Total blurred2 blocks: {} ({3.2}%)\n", total_blurred_blocks2, (float)total_blurred_blocks2 * 100.0f / (float)total_blocks); + fmt_debug_printf("Total blurred3 blocks: {} ({3.2}%)\n", total_blurred_blocks3, (float)total_blurred_blocks3 * 100.0f / (float)total_blocks); + fmt_debug_printf("Total blurred4 blocks: {} ({3.2}%)\n", total_blurred_blocks4, (float)total_blurred_blocks4 * 100.0f / (float)total_blocks); + + fmt_debug_printf("\nTotal superbuckets created: {} ({4.1} per block)\n", total_superbuckets_created, (float)total_superbuckets_created / (float)total_blocks); + fmt_debug_printf("Total shortlist buckets created: {} ({4.1} per block)\n", total_buckets_created, (float)total_buckets_created / (float)total_blocks); + fmt_debug_printf("Total surrogate encodes: {} ({4.1} per block)\n", total_surrogate_encodes, (float)total_surrogate_encodes / (float)total_blocks); + fmt_debug_printf("Total shortlist candidates (before full encoding): {} ({4.1} per block)\n", total_shortlist_candidates, (float)total_shortlist_candidates / (float)total_blocks); + fmt_debug_printf("Total full encodes on superpass 0: {} ({4.1} per block)\n", total_full_encodes, (float)total_full_encodes / (float)total_blocks); + fmt_debug_printf("Total full encodes on superpass 1: {} ({4.1} per block)\n", total_full_encodes_pass1, (float)total_full_encodes_pass1 / (float)total_blocks); + fmt_debug_printf("Total full encodes on superpass 2: {} ({4.1} per block)\n", total_full_encodes_pass2, (float)total_full_encodes_pass2 / (float)total_blocks); + + debug_printf("\nTotal final encoded ASTC blocks using blue contraction: %u (%.2f%%)\n", total_used_bc, 100.0f * (float)total_used_bc / (float)total_blocks); + + fmt_debug_printf("Total final encoded ASTC blocks using dual planes: {} {3.2}%\n", total_dp, (float)total_dp * 100.0f / (float)total_blocks); + fmt_debug_printf("Total final encoded ASTC blocks using base+ofs: {} {3.2}%\n", total_dp, (float)total_base_ofs * 100.0f / (float)total_blocks); + fmt_debug_printf("Total final encoded ASTC blocks using subsets: {} {3.2}%\n", total_blocks_using_subsets, (float)total_blocks_using_subsets * 100.0f / (float)total_blocks); + + debug_printf("\nSubset usage histogram:\n"); + for (uint32_t i = 0; i < 4; i++) + fmt_debug_printf("{} subsets: {} {3.2}%\n", i + 1, subset_used_hist[i], (float)subset_used_hist[i] * 100.0f / (float)total_blocks); + debug_printf("\n"); + + debug_printf("CEM usage histogram:\n"); + for (uint32_t i = 0; i < 16; i++) + { + if (astc_helpers::is_cem_hdr(i)) + continue; + + std::string n(astc_helpers::get_cem_name(i)); + while (n.size() < 40) + n.push_back(' '); + + fmt_debug_printf("{}: {} {3.2}%, Used BC: {3.2}%, Used subsets: {3.2}%, Used DP: {3.2}%\n", + n, + cem_used_hist[i], + (float)cem_used_hist[i] * 100.0f / (float)total_blocks, + (float)cem_used_bc[i] * 100.0f / (float)total_blocks, + (float)cem_used_subsets[i] * 100.0f / (float)total_blocks, + (float)cem_used_dp[i] * 100.0f / (float)total_blocks); + } + debug_printf("\n"); + + debug_printf("Grid samples histogram:\n"); + for (uint32_t i = 1; i <= block_width * block_height; i++) + { + if (grid_usage_hist[i]) + fmt_debug_printf("{} samples: {} {3.2}%\n", i, grid_usage_hist[i], (float)grid_usage_hist[i] * 100.0f / (float)total_blocks); + } + debug_printf("\n"); + + fmt_debug_printf("orig vs. ASTC compressed:\n"); + print_image_metrics(orig_img, unpacked_img); + + fmt_debug_printf("Total encode time: {.3} secs, {.3} ms per block, {.1} blocks/sec\n", total_enc_time, total_enc_time * 1000.0f / total_blocks, total_blocks / total_enc_time); + + fmt_debug_printf("OK\n"); + } + + return true; +} + +//const uint32_t rice_zero_run_m = 3, rice_dct_coeff_m = 2; + +const uint_vec& separate_tm_index(uint32_t block_width, uint32_t block_height, const basist::astc_ldr_t::grouped_trial_modes& grouped_enc_trial_modes, const basist::astc_ldr_t::trial_mode& tm, + uint32_t& cem_index, uint32_t& subset_index, uint32_t& ccs_index, uint32_t& grid_size, uint32_t& grid_aniso) +{ + cem_index = tm.m_cem; + assert(cem_index < basist::astc_ldr_t::OTM_NUM_CEMS); + + subset_index = tm.m_num_parts - 1; + assert(subset_index < basist::astc_ldr_t::OTM_NUM_SUBSETS); + + ccs_index = tm.m_ccs_index + 1; + assert(ccs_index < basist::astc_ldr_t::OTM_NUM_CCS); + + grid_size = (tm.m_grid_width >= (block_width - 1)) && (tm.m_grid_height >= (block_height - 1)); + grid_aniso = basist::astc_ldr_t::calc_grid_aniso_val(tm.m_grid_width, tm.m_grid_height, block_width, block_height); + + const uint_vec& modes = grouped_enc_trial_modes.m_tm_groups[cem_index][subset_index][ccs_index][grid_size][grid_aniso]; + return modes; +} + +static bool compare_log_block_configs(const astc_helpers::log_astc_block& trial_log_blk, const astc_helpers::log_astc_block& neighbor_log_blk) +{ + assert(!trial_log_blk.m_solid_color_flag_ldr); + + if (neighbor_log_blk.m_solid_color_flag_ldr) + return false; + + if ((trial_log_blk.m_color_endpoint_modes[0] == neighbor_log_blk.m_color_endpoint_modes[0]) && + (trial_log_blk.m_dual_plane == neighbor_log_blk.m_dual_plane) && (trial_log_blk.m_color_component_selector == neighbor_log_blk.m_color_component_selector) && + (trial_log_blk.m_num_partitions == neighbor_log_blk.m_num_partitions) && (trial_log_blk.m_partition_id == neighbor_log_blk.m_partition_id) && + (trial_log_blk.m_grid_width == neighbor_log_blk.m_grid_width) && (trial_log_blk.m_grid_height == neighbor_log_blk.m_grid_height) && + (trial_log_blk.m_endpoint_ise_range == neighbor_log_blk.m_endpoint_ise_range) && (trial_log_blk.m_weight_ise_range == neighbor_log_blk.m_weight_ise_range)) + { + return true; + } + + return false; +} + +static bool compare_log_block_configs_and_endpoints(const astc_helpers::log_astc_block& trial_log_blk, const astc_helpers::log_astc_block& neighbor_log_blk) +{ + if (!compare_log_block_configs(trial_log_blk, neighbor_log_blk)) + return false; + + const uint32_t total_endpoint_vals = trial_log_blk.m_num_partitions * astc_helpers::get_num_cem_values(trial_log_blk.m_color_endpoint_modes[0]); + if (memcmp(trial_log_blk.m_endpoints, neighbor_log_blk.m_endpoints, total_endpoint_vals) == 0) + return true; + + return false; +} + +static bool compare_log_blocks_for_equality(const astc_helpers::log_astc_block& trial_log_blk, const astc_helpers::log_astc_block& neighbor_log_blk) +{ + if (trial_log_blk.m_solid_color_flag_ldr) + { + if (!neighbor_log_blk.m_solid_color_flag_ldr) + return false; + + for (uint32_t i = 0; i < 4; i++) + if (trial_log_blk.m_solid_color[i] != neighbor_log_blk.m_solid_color[i]) + return false; + + return true; + } + else if (neighbor_log_blk.m_solid_color_flag_ldr) + { + return false; + } + + assert(!trial_log_blk.m_solid_color_flag_ldr && !neighbor_log_blk.m_solid_color_flag_ldr); + + if ((trial_log_blk.m_color_endpoint_modes[0] == neighbor_log_blk.m_color_endpoint_modes[0]) && + (trial_log_blk.m_dual_plane == neighbor_log_blk.m_dual_plane) && (trial_log_blk.m_color_component_selector == neighbor_log_blk.m_color_component_selector) && + (trial_log_blk.m_num_partitions == neighbor_log_blk.m_num_partitions) && (trial_log_blk.m_partition_id == neighbor_log_blk.m_partition_id) && + (trial_log_blk.m_grid_width == neighbor_log_blk.m_grid_width) && (trial_log_blk.m_grid_height == neighbor_log_blk.m_grid_height) && + (trial_log_blk.m_endpoint_ise_range == neighbor_log_blk.m_endpoint_ise_range) && (trial_log_blk.m_weight_ise_range == neighbor_log_blk.m_weight_ise_range)) + { + const uint32_t total_endpoint_vals = trial_log_blk.m_num_partitions * astc_helpers::get_num_cem_values(trial_log_blk.m_color_endpoint_modes[0]); + if (memcmp(trial_log_blk.m_endpoints, neighbor_log_blk.m_endpoints, total_endpoint_vals) == 0) + { + const uint32_t total_weights = (trial_log_blk.m_dual_plane ? 2 : 1) * (trial_log_blk.m_grid_width * trial_log_blk.m_grid_height); + return memcmp(trial_log_blk.m_weights, neighbor_log_blk.m_weights, total_weights) == 0; + } + } + + return false; +} + +void configure_encoder_effort_level(int level, ldr_astc_block_encode_image_high_level_config& cfg) +{ + switch (level) + { + case 10: + { + cfg.m_second_superpass_refinement = true; + cfg.m_third_superpass_try_neighbors = true; + + cfg.m_subsets_enabled = true; + cfg.m_use_blue_contraction = true; + cfg.m_use_base_ofs = true; + + cfg.m_force_all_dual_plane_chan_evals = true; + cfg.m_filter_by_pca_angles_flag = false; + + cfg.m_superbucket_max_to_retain[0] = 256; + cfg.m_superbucket_max_to_retain[1] = 256; + cfg.m_superbucket_max_to_retain[2] = 256; + + cfg.m_base_parts2 = 128; + cfg.m_base_parts3 = 128; + cfg.m_part2_fraction_to_keep = 1; + cfg.m_part3_fraction_to_keep = 1; + + cfg.m_final_shortlist_fraction[0] = 1.0f; + cfg.m_final_shortlist_fraction[1] = 1.0f; + cfg.m_final_shortlist_fraction[2] = 1.0f; + + cfg.m_final_shortlist_max_size[0] = 128; + cfg.m_final_shortlist_max_size[1] = 128; + cfg.m_final_shortlist_max_size[2] = 128; + + // Second superpass + cfg.m_second_superpass_fract_to_recompress = .075f; + cfg.m_superbucket_max_to_retain_p2[0] = 1024; + cfg.m_superbucket_max_to_retain_p2[1] = 1024; + cfg.m_superbucket_max_to_retain_p2[2] = 1024; + cfg.m_final_shortlist_max_size_p2[0] = 256; + cfg.m_final_shortlist_max_size_p2[1] = 256; + cfg.m_final_shortlist_max_size_p2[2] = 256; + cfg.m_base_parts2_p2 = 128; + cfg.m_base_parts3_p2 = 128; + cfg.m_force_all_dp_chans_p2 = true; + cfg.m_filter_by_pca_angles_flag_p2 = false; + + cfg.m_final_encode_always_try_rgb_direct = true; + + cfg.m_early_stop_wpsnr = 90.0f; + cfg.m_early_stop2_wpsnr = 90.0f; + cfg.m_grid_hv_filtering = false; + cfg.m_low_freq_block_filtering = false; + + break; + } + case 9: + { + cfg.m_second_superpass_refinement = true; + cfg.m_third_superpass_try_neighbors = true; + + cfg.m_subsets_enabled = true; + cfg.m_use_blue_contraction = true; + cfg.m_use_base_ofs = true; + + cfg.m_force_all_dual_plane_chan_evals = false; + cfg.m_filter_by_pca_angles_flag = true; + + cfg.m_superbucket_max_to_retain[0] = 8; + cfg.m_superbucket_max_to_retain[1] = 16; + cfg.m_superbucket_max_to_retain[2] = 32; + + cfg.m_base_parts2 = 32; + cfg.m_base_parts3 = 32; + cfg.m_part2_fraction_to_keep = 2; + cfg.m_part3_fraction_to_keep = 2; + + cfg.m_final_shortlist_fraction[0] = 1.0f; + cfg.m_final_shortlist_fraction[1] = 1.0f; + cfg.m_final_shortlist_fraction[2] = 1.0f; + + cfg.m_final_shortlist_max_size[0] = 4; + cfg.m_final_shortlist_max_size[1] = 12; + cfg.m_final_shortlist_max_size[2] = 24; + + // Second superpass + cfg.m_second_superpass_fract_to_recompress = .075f; + cfg.m_superbucket_max_to_retain_p2[0] = 16; + cfg.m_superbucket_max_to_retain_p2[1] = 64; + cfg.m_superbucket_max_to_retain_p2[2] = 256; + cfg.m_final_shortlist_max_size_p2[0] = 8; + cfg.m_final_shortlist_max_size_p2[1] = 16; + cfg.m_final_shortlist_max_size_p2[2] = 32; + cfg.m_base_parts2_p2 = 64; + cfg.m_base_parts3_p2 = 64; + cfg.m_force_all_dp_chans_p2 = false; + cfg.m_filter_by_pca_angles_flag_p2 = false; + + cfg.m_final_encode_always_try_rgb_direct = false; + + cfg.m_early_stop_wpsnr = 75.0f; + cfg.m_early_stop2_wpsnr = 70.0f; + + break; + } + case 8: + { + cfg.m_second_superpass_refinement = true; + cfg.m_third_superpass_try_neighbors = true; + + cfg.m_subsets_enabled = true; + cfg.m_use_blue_contraction = true; + cfg.m_use_base_ofs = true; + + cfg.m_force_all_dual_plane_chan_evals = false; + cfg.m_filter_by_pca_angles_flag = true; + + cfg.m_superbucket_max_to_retain[0] = 4; + cfg.m_superbucket_max_to_retain[1] = 8; + cfg.m_superbucket_max_to_retain[2] = 16; + + cfg.m_base_parts2 = 16; + cfg.m_base_parts3 = 16; + cfg.m_part2_fraction_to_keep = 2; + cfg.m_part3_fraction_to_keep = 2; + + cfg.m_final_shortlist_fraction[0] = 1.0f; + cfg.m_final_shortlist_fraction[1] = 1.0f; + cfg.m_final_shortlist_fraction[2] = 1.0f; + + cfg.m_final_shortlist_max_size[0] = 3; + cfg.m_final_shortlist_max_size[1] = 8; + cfg.m_final_shortlist_max_size[2] = 12; + + // Second superpass + cfg.m_second_superpass_fract_to_recompress = .075f; + cfg.m_superbucket_max_to_retain_p2[0] = 16; + cfg.m_superbucket_max_to_retain_p2[1] = 64; + cfg.m_superbucket_max_to_retain_p2[2] = 256; + cfg.m_final_shortlist_max_size_p2[0] = 8; + cfg.m_final_shortlist_max_size_p2[1] = 16; + cfg.m_final_shortlist_max_size_p2[2] = 32; + cfg.m_base_parts2_p2 = 64; + cfg.m_base_parts3_p2 = 64; + cfg.m_force_all_dp_chans_p2 = false; + cfg.m_filter_by_pca_angles_flag_p2 = false; + + cfg.m_final_encode_always_try_rgb_direct = false; + + cfg.m_early_stop_wpsnr = 75.0f; + cfg.m_early_stop2_wpsnr = 70.0f; + break; + } + case 7: + { + cfg.m_second_superpass_refinement = true; + cfg.m_third_superpass_try_neighbors = true; + + cfg.m_subsets_enabled = true; + cfg.m_use_blue_contraction = true; + cfg.m_use_base_ofs = true; + + cfg.m_disable_rgb_dual_plane = false; + cfg.m_strong_dp_decorr_thresh_rgb = .9f; + + cfg.m_force_all_dual_plane_chan_evals = false; + cfg.m_filter_by_pca_angles_flag = true; + + cfg.m_superbucket_max_to_retain[0] = 3; + cfg.m_superbucket_max_to_retain[1] = 7; + cfg.m_superbucket_max_to_retain[2] = 12; + + cfg.m_base_parts2 = 12; + cfg.m_base_parts3 = 12; + cfg.m_part2_fraction_to_keep = 2; + cfg.m_part3_fraction_to_keep = 2; + + cfg.m_final_shortlist_fraction[0] = 1.0f; + cfg.m_final_shortlist_fraction[1] = 1.0f; + cfg.m_final_shortlist_fraction[2] = 1.0f; + + cfg.m_final_shortlist_max_size[0] = 2; + cfg.m_final_shortlist_max_size[1] = 4; + cfg.m_final_shortlist_max_size[2] = 8; + + cfg.m_gradient_descent_flag = true; + cfg.m_polish_weights_flag = true; + cfg.m_qcd_enabled_flag = true; + + cfg.m_bucket_pruning_passes = false; + cfg.m_cem_enc_params.m_max_ls_passes = 1; + + // Second superpass + cfg.m_second_superpass_fract_to_recompress = .075f; + cfg.m_superbucket_max_to_retain_p2[0] = 4; + cfg.m_superbucket_max_to_retain_p2[1] = 16; + cfg.m_superbucket_max_to_retain_p2[2] = 32; + cfg.m_final_shortlist_max_size_p2[0] = 4; + cfg.m_final_shortlist_max_size_p2[1] = 16; + cfg.m_final_shortlist_max_size_p2[2] = 32; + cfg.m_base_parts2_p2 = 32; + cfg.m_base_parts3_p2 = 8; + cfg.m_force_all_dp_chans_p2 = false; + cfg.m_filter_by_pca_angles_flag_p2 = true; + + cfg.m_early_stop_wpsnr = 65.0f; + cfg.m_early_stop2_wpsnr = 60.0f; + break; + } + case 6: + { + cfg.m_second_superpass_refinement = true; + cfg.m_third_superpass_try_neighbors = true; + + cfg.m_subsets_enabled = true; + cfg.m_use_blue_contraction = true; + cfg.m_use_base_ofs = true; + + cfg.m_disable_rgb_dual_plane = false; + cfg.m_strong_dp_decorr_thresh_rgb = .75f; + + cfg.m_force_all_dual_plane_chan_evals = false; + cfg.m_filter_by_pca_angles_flag = true; + + cfg.m_superbucket_max_to_retain[0] = 2; + cfg.m_superbucket_max_to_retain[1] = 5; + cfg.m_superbucket_max_to_retain[2] = 10; + + cfg.m_base_parts2 = 12; + cfg.m_base_parts3 = 10; + cfg.m_part2_fraction_to_keep = 2; + cfg.m_part3_fraction_to_keep = 2; + + cfg.m_final_shortlist_fraction[0] = 1.0f; + cfg.m_final_shortlist_fraction[1] = 1.0f; + cfg.m_final_shortlist_fraction[2] = 1.0f; + + cfg.m_final_shortlist_max_size[0] = 1; + cfg.m_final_shortlist_max_size[1] = 4; + cfg.m_final_shortlist_max_size[2] = 8; + + cfg.m_gradient_descent_flag = true; + cfg.m_polish_weights_flag = true; + cfg.m_qcd_enabled_flag = true; + + cfg.m_bucket_pruning_passes = false; + cfg.m_cem_enc_params.m_max_ls_passes = 1; + + // Second superpass + cfg.m_second_superpass_fract_to_recompress = .075f; + cfg.m_superbucket_max_to_retain_p2[0] = 2; + cfg.m_superbucket_max_to_retain_p2[1] = 8; + cfg.m_superbucket_max_to_retain_p2[2] = 16; + cfg.m_final_shortlist_max_size_p2[0] = 2; + cfg.m_final_shortlist_max_size_p2[1] = 8; + cfg.m_final_shortlist_max_size_p2[2] = 16; + cfg.m_base_parts2_p2 = 32; + cfg.m_base_parts3_p2 = 8; + cfg.m_force_all_dp_chans_p2 = false; + cfg.m_filter_by_pca_angles_flag_p2 = true; + + cfg.m_early_stop_wpsnr = 65.0f; + cfg.m_early_stop2_wpsnr = 60.0f; + break; + } + case 5: + { + cfg.m_second_superpass_refinement = true; + cfg.m_third_superpass_try_neighbors = true; + + cfg.m_subsets_enabled = true; + cfg.m_use_blue_contraction = true; + cfg.m_use_base_ofs = true; + + cfg.m_disable_rgb_dual_plane = false; + cfg.m_strong_dp_decorr_thresh_rgb = .75f; + + cfg.m_force_all_dual_plane_chan_evals = false; + cfg.m_filter_by_pca_angles_flag = true; + + cfg.m_superbucket_max_to_retain[0] = 1; + cfg.m_superbucket_max_to_retain[1] = 4; + cfg.m_superbucket_max_to_retain[2] = 8; + + cfg.m_base_parts2 = 12; + cfg.m_base_parts3 = 8; + cfg.m_part2_fraction_to_keep = 2; + cfg.m_part3_fraction_to_keep = 2; + + cfg.m_final_shortlist_fraction[0] = 1.0f; + cfg.m_final_shortlist_fraction[1] = 1.0f; + cfg.m_final_shortlist_fraction[2] = 1.0f; + + cfg.m_final_shortlist_max_size[0] = 1; + cfg.m_final_shortlist_max_size[1] = 4; + cfg.m_final_shortlist_max_size[2] = 8; + + cfg.m_gradient_descent_flag = true; + cfg.m_polish_weights_flag = true; + cfg.m_qcd_enabled_flag = false; + + cfg.m_bucket_pruning_passes = false; + cfg.m_cem_enc_params.m_max_ls_passes = 1; + + // Second superpass + cfg.m_second_superpass_fract_to_recompress = .075f; + cfg.m_superbucket_max_to_retain_p2[0] = 2; + cfg.m_superbucket_max_to_retain_p2[1] = 8; + cfg.m_superbucket_max_to_retain_p2[2] = 16; + cfg.m_final_shortlist_max_size_p2[0] = 2; + cfg.m_final_shortlist_max_size_p2[1] = 8; + cfg.m_final_shortlist_max_size_p2[2] = 16; + cfg.m_base_parts2_p2 = 32; + cfg.m_base_parts3_p2 = 8; + cfg.m_force_all_dp_chans_p2 = false; + cfg.m_filter_by_pca_angles_flag_p2 = true; + + cfg.m_early_stop_wpsnr = 65.0f; + cfg.m_early_stop2_wpsnr = 60.0f; + break; + } + case 4: + { + cfg.m_second_superpass_refinement = true; + cfg.m_third_superpass_try_neighbors = true; + + cfg.m_subsets_enabled = true; + cfg.m_use_blue_contraction = true; + cfg.m_use_base_ofs = true; + + cfg.m_disable_rgb_dual_plane = false; + cfg.m_strong_dp_decorr_thresh_rgb = .75f; + + cfg.m_force_all_dual_plane_chan_evals = false; + cfg.m_filter_by_pca_angles_flag = true; + + cfg.m_superbucket_max_to_retain[0] = 1; + cfg.m_superbucket_max_to_retain[1] = 4; + cfg.m_superbucket_max_to_retain[2] = 8; + + cfg.m_base_parts2 = 8; + cfg.m_base_parts3 = 4; + cfg.m_part2_fraction_to_keep = 2; + cfg.m_part3_fraction_to_keep = 2; + + cfg.m_final_shortlist_fraction[0] = 1.0f; + cfg.m_final_shortlist_fraction[1] = 1.0f; + cfg.m_final_shortlist_fraction[2] = 1.0f; + + cfg.m_final_shortlist_max_size[0] = 1; + cfg.m_final_shortlist_max_size[1] = 4; + cfg.m_final_shortlist_max_size[2] = 8; + + cfg.m_gradient_descent_flag = true; + cfg.m_polish_weights_flag = true; + cfg.m_qcd_enabled_flag = false; + + cfg.m_bucket_pruning_passes = false; + cfg.m_cem_enc_params.m_max_ls_passes = 1; + + // Second superpass + cfg.m_second_superpass_fract_to_recompress = .075f; + cfg.m_superbucket_max_to_retain_p2[0] = 2; + cfg.m_superbucket_max_to_retain_p2[1] = 8; + cfg.m_superbucket_max_to_retain_p2[2] = 16; + cfg.m_final_shortlist_max_size_p2[0] = 2; + cfg.m_final_shortlist_max_size_p2[1] = 8; + cfg.m_final_shortlist_max_size_p2[2] = 16; + cfg.m_base_parts2_p2 = 32; + cfg.m_base_parts3_p2 = 8; + cfg.m_force_all_dp_chans_p2 = false; + cfg.m_filter_by_pca_angles_flag_p2 = true; + + cfg.m_early_stop_wpsnr = 65.0f; + cfg.m_early_stop2_wpsnr = 60.0f; + break; + } + default: + case 3: + { + cfg.m_second_superpass_refinement = true; + cfg.m_third_superpass_try_neighbors = true; + + cfg.m_subsets_enabled = true; + cfg.m_use_blue_contraction = true; + cfg.m_use_base_ofs = false; + + cfg.m_disable_rgb_dual_plane = false; + cfg.m_strong_dp_decorr_thresh_rgb = .75f; + + cfg.m_force_all_dual_plane_chan_evals = false; + cfg.m_filter_by_pca_angles_flag = true; + + cfg.m_superbucket_max_to_retain[0] = 1; + cfg.m_superbucket_max_to_retain[1] = 4; + cfg.m_superbucket_max_to_retain[2] = 8; + + cfg.m_base_parts2 = 4; + cfg.m_base_parts3 = 2; + cfg.m_part2_fraction_to_keep = 2; + cfg.m_part3_fraction_to_keep = 2; + + cfg.m_final_shortlist_fraction[0] = 1.0f; + cfg.m_final_shortlist_fraction[1] = 1.0f; + cfg.m_final_shortlist_fraction[2] = 1.0f; + + cfg.m_final_shortlist_max_size[0] = 1; + cfg.m_final_shortlist_max_size[1] = 4; + cfg.m_final_shortlist_max_size[2] = 8; + + cfg.m_gradient_descent_flag = true; + cfg.m_polish_weights_flag = true; + cfg.m_qcd_enabled_flag = false; + + cfg.m_bucket_pruning_passes = false; + cfg.m_cem_enc_params.m_max_ls_passes = 1; + + // Second superpass + cfg.m_second_superpass_fract_to_recompress = .075f; + cfg.m_superbucket_max_to_retain_p2[0] = 2; + cfg.m_superbucket_max_to_retain_p2[1] = 8; + cfg.m_superbucket_max_to_retain_p2[2] = 16; + cfg.m_final_shortlist_max_size_p2[0] = 2; + cfg.m_final_shortlist_max_size_p2[1] = 8; + cfg.m_final_shortlist_max_size_p2[2] = 16; + cfg.m_base_parts2_p2 = 32; + cfg.m_base_parts3_p2 = 8; + cfg.m_force_all_dp_chans_p2 = false; + cfg.m_filter_by_pca_angles_flag_p2 = true; + + cfg.m_early_stop_wpsnr = 65.0f; + cfg.m_early_stop2_wpsnr = 60.0f; + break; + } + case 2: + { + // Level 2+ have subsets and RGB dual-plane enabled + cfg.m_second_superpass_refinement = false; + cfg.m_third_superpass_try_neighbors = true; + + cfg.m_subsets_enabled = true; + cfg.m_use_blue_contraction = true; + cfg.m_use_base_ofs = false; + cfg.m_disable_rgb_dual_plane = false; + + cfg.m_force_all_dual_plane_chan_evals = false; + cfg.m_filter_by_pca_angles_flag = true; + + cfg.m_superbucket_max_to_retain[0] = 1; + cfg.m_superbucket_max_to_retain[1] = 2; + cfg.m_superbucket_max_to_retain[2] = 3; + + cfg.m_base_parts2 = 1; + cfg.m_base_parts3 = 0; + cfg.m_part2_fraction_to_keep = 1; + cfg.m_part3_fraction_to_keep = 1; + + cfg.m_final_shortlist_fraction[0] = 1.0f; + cfg.m_final_shortlist_fraction[1] = 1.0f; + cfg.m_final_shortlist_fraction[2] = 1.0f; + + cfg.m_final_shortlist_max_size[0] = 1; + cfg.m_final_shortlist_max_size[1] = 2; + cfg.m_final_shortlist_max_size[2] = 3; + + cfg.m_gradient_descent_flag = false; + cfg.m_polish_weights_flag = true; + cfg.m_qcd_enabled_flag = false; + + cfg.m_bucket_pruning_passes = false; + cfg.m_cem_enc_params.m_max_ls_passes = 1; + + // Second superpass + cfg.m_second_superpass_fract_to_recompress = .04f; + cfg.m_second_pass_force_subsets_enabled = true; + cfg.m_superbucket_max_to_retain_p2[0] = 1; + cfg.m_superbucket_max_to_retain_p2[1] = 2; + cfg.m_superbucket_max_to_retain_p2[2] = 8; + cfg.m_final_shortlist_max_size_p2[0] = 1; + cfg.m_final_shortlist_max_size_p2[1] = 2; + cfg.m_final_shortlist_max_size_p2[2] = 8; + cfg.m_base_parts2_p2 = 16; + cfg.m_base_parts3_p2 = 0; + cfg.m_force_all_dp_chans_p2 = false; + cfg.m_filter_by_pca_angles_flag_p2 = true; + + cfg.m_early_stop_wpsnr = 45.0f; + cfg.m_early_stop2_wpsnr = 40.0f; + break; + } + case 1: + { + cfg.m_second_superpass_refinement = false; + cfg.m_third_superpass_try_neighbors = false; + + cfg.m_subsets_enabled = false; + cfg.m_use_blue_contraction = true; + cfg.m_use_base_ofs = false; + cfg.m_disable_rgb_dual_plane = true; + + cfg.m_force_all_dual_plane_chan_evals = false; + cfg.m_filter_by_pca_angles_flag = true; + + cfg.m_superbucket_max_to_retain[0] = 1; + cfg.m_superbucket_max_to_retain[1] = 1; + cfg.m_superbucket_max_to_retain[2] = 1; + + cfg.m_base_parts2 = 0; + cfg.m_base_parts3 = 0; + cfg.m_part2_fraction_to_keep = 1; + cfg.m_part3_fraction_to_keep = 1; + + cfg.m_final_shortlist_fraction[0] = 1.0f; + cfg.m_final_shortlist_fraction[1] = 1.0f; + cfg.m_final_shortlist_fraction[2] = 1.0f; + + cfg.m_final_shortlist_max_size[0] = 1; + cfg.m_final_shortlist_max_size[1] = 1; + cfg.m_final_shortlist_max_size[2] = 1; + + cfg.m_gradient_descent_flag = false; + cfg.m_polish_weights_flag = true; + cfg.m_qcd_enabled_flag = false; + + cfg.m_bucket_pruning_passes = false; + cfg.m_cem_enc_params.m_max_ls_passes = 1; + + cfg.m_early_stop_wpsnr = 45.0f; + cfg.m_early_stop2_wpsnr = 40.0f; + break; + } + case 0: + { + cfg.m_second_superpass_refinement = false; + cfg.m_third_superpass_try_neighbors = false; + + cfg.m_subsets_enabled = false; + cfg.m_use_blue_contraction = true; + cfg.m_use_base_ofs = false; + cfg.m_disable_rgb_dual_plane = true; + + cfg.m_force_all_dual_plane_chan_evals = false; + cfg.m_filter_by_pca_angles_flag = true; + + cfg.m_superbucket_max_to_retain[0] = 1; + cfg.m_superbucket_max_to_retain[1] = 1; + cfg.m_superbucket_max_to_retain[2] = 1; + + cfg.m_base_parts2 = 0; + cfg.m_base_parts3 = 0; + cfg.m_part2_fraction_to_keep = 1; + cfg.m_part3_fraction_to_keep = 1; + + cfg.m_final_shortlist_fraction[0] = 1.0f; + cfg.m_final_shortlist_fraction[1] = 1.0f; + cfg.m_final_shortlist_fraction[2] = 1.0f; + + cfg.m_final_shortlist_max_size[0] = 1; + cfg.m_final_shortlist_max_size[1] = 1; + cfg.m_final_shortlist_max_size[2] = 1; + + cfg.m_gradient_descent_flag = false; + cfg.m_polish_weights_flag = false; + cfg.m_qcd_enabled_flag = false; + + cfg.m_bucket_pruning_passes = false; + cfg.m_cem_enc_params.m_max_ls_passes = 1; + + cfg.m_early_stop_wpsnr = 45.0f; + cfg.m_early_stop2_wpsnr = 40.0f; + break; + } + } +} + +#if BASISD_SUPPORT_KTX2_ZSTD +static bool zstd_compress(const uint8_t* pData, size_t data_len, uint8_vec& comp_data, int zstd_level) +{ + if (!data_len) + { + comp_data.resize(0); + return true; + } + + assert(pData); + + comp_data.resize(ZSTD_compressBound(data_len)); + + size_t result = ZSTD_compress(comp_data.data(), comp_data.size(), pData, data_len, zstd_level); + + if (ZSTD_isError(result)) + { + comp_data.resize(0); + return false; + } + + if (result > UINT32_MAX) + { + comp_data.resize(0); + return false; + } + + comp_data.resize(result); + return true; +} + +static bool zstd_compress(const bitwise_coder& coder, uint8_vec& comp_data, int zstd_level) +{ + return zstd_compress(coder.get_bytes().data(), coder.get_bytes().size(), comp_data, zstd_level); +} + +static bool zstd_compress(const uint8_vec& vec, uint8_vec& comp_data, int zstd_level) +{ + return zstd_compress(vec.data(), vec.size(), comp_data, zstd_level); +} + +static uint32_t encode_values(bitwise_coder& coder, uint32_t total_values, const uint8_t* pVals, uint32_t endpoint_range) +{ + const uint32_t MAX_VALS = 64; + uint32_t bit_values[MAX_VALS], tq_values[(MAX_VALS + 2) / 3]; + uint32_t total_tq_values = 0, tq_accum = 0, tq_mul = 1; + + assert((total_values) && (total_values <= MAX_VALS)); + + const uint32_t ep_bits = astc_helpers::g_ise_range_table[endpoint_range][0]; + const uint32_t ep_trits = astc_helpers::g_ise_range_table[endpoint_range][1]; + const uint32_t ep_quints = astc_helpers::g_ise_range_table[endpoint_range][2]; + + for (uint32_t i = 0; i < total_values; i++) + { + uint32_t val = pVals[i]; + + uint32_t bits = val & ((1 << ep_bits) - 1); + uint32_t tq = val >> ep_bits; + + bit_values[i] = bits; + + if (ep_trits) + { + assert(tq < 3); + tq_accum += tq * tq_mul; + tq_mul *= 3; + if (tq_mul == 243) + { + assert(total_tq_values < BASISU_ARRAY_SIZE(tq_values)); + tq_values[total_tq_values++] = tq_accum; + tq_accum = 0; + tq_mul = 1; + } + } + else if (ep_quints) + { + assert(tq < 5); + tq_accum += tq * tq_mul; + tq_mul *= 5; + if (tq_mul == 125) + { + assert(total_tq_values < BASISU_ARRAY_SIZE(tq_values)); + tq_values[total_tq_values++] = tq_accum; + tq_accum = 0; + tq_mul = 1; + } + } + } + + uint32_t total_bits_output = 0; + + for (uint32_t i = 0; i < total_tq_values; i++) + { + const uint32_t num_bits = ep_trits ? 8 : 7; + coder.put_bits(tq_values[i], num_bits); + total_bits_output += num_bits; + } + + if (tq_mul > 1) + { + uint32_t num_bits; + if (ep_trits) + { + if (tq_mul == 3) + num_bits = 2; + else if (tq_mul == 9) + num_bits = 4; + else if (tq_mul == 27) + num_bits = 5; + else //if (tq_mul == 81) + num_bits = 7; + } + else + { + if (tq_mul == 5) + num_bits = 3; + else //if (tq_mul == 25) + num_bits = 5; + } + coder.put_bits(tq_accum, num_bits); + total_bits_output += num_bits; + } + + for (uint32_t i = 0; i < total_values; i++) + { + coder.put_bits(bit_values[i], ep_bits); + total_bits_output += ep_bits; + } + + return total_bits_output; +} + +static bool compress_image_full_zstd( + const image& orig_img, uint8_vec& comp_data, vector2D& coded_blocks, + const astc_ldr_encode_config& global_cfg, + job_pool& job_pool, + ldr_astc_block_encode_image_high_level_config& enc_cfg, const ldr_astc_block_encode_image_output& enc_out) +{ + BASISU_NOTE_UNUSED(job_pool); + + const uint32_t width = orig_img.get_width(), height = orig_img.get_height(); + + const uint32_t block_width = global_cfg.m_astc_block_width; + const uint32_t block_height = global_cfg.m_astc_block_height; + const uint32_t total_block_pixels = block_width * block_height; + + const uint32_t total_pixels = width * height; + const uint32_t num_blocks_x = (width + block_width - 1) / block_width; + const uint32_t num_blocks_y = (height + block_height - 1) / block_height; + const uint32_t total_blocks = num_blocks_x * num_blocks_y; + const bool has_alpha = orig_img.has_alpha(); + + // Mode + uint8_vec mode_bytes; + mode_bytes.reserve(8192); + + bitwise_coder raw_bits; + raw_bits.init(8192); + + uint8_vec solid_dpcm_bytes; + solid_dpcm_bytes.reserve(8192); + + // Endpoints + uint8_vec endpoint_dpcm_reuse_indices; + endpoint_dpcm_reuse_indices.reserve(8192); + + bitwise_coder use_bc_bits; + use_bc_bits.init(1024); + + bitwise_coder endpoint_dpcm_3bit; + endpoint_dpcm_3bit.init(1024); + + bitwise_coder endpoint_dpcm_4bit; + endpoint_dpcm_4bit.init(1024); + + uint8_vec endpoint_dpcm_5bit; + endpoint_dpcm_5bit.reserve(8192); + + uint8_vec endpoint_dpcm_6bit; + endpoint_dpcm_6bit.reserve(8192); + + uint8_vec endpoint_dpcm_7bit; + endpoint_dpcm_7bit.reserve(8192); + + uint8_vec endpoint_dpcm_8bit; + endpoint_dpcm_8bit.reserve(8192); + + // Weights + bitwise_coder mean0_bits; + uint8_vec mean1_bytes; + uint8_vec run_bytes; + uint8_vec coeff_bytes; + bitwise_coder sign_bits; + bitwise_coder weight2_bits; + bitwise_coder weight3_bits; + bitwise_coder weight4_bits; + uint8_vec weight8_bits; + + mean0_bits.init(1024); + mean1_bytes.reserve(1024); + run_bytes.reserve(8192); + coeff_bytes.reserve(8192); + sign_bits.init(1024); + weight2_bits.init(1024); + weight3_bits.init(1024); + weight4_bits.init(1024); + weight8_bits.reserve(8192); + + const float replacement_min_psnr = has_alpha ? global_cfg.m_replacement_min_psnr_alpha : global_cfg.m_replacement_min_psnr; + const float psnr_trial_diff_thresh = has_alpha ? global_cfg.m_psnr_trial_diff_thresh_alpha : global_cfg.m_psnr_trial_diff_thresh; + const float psnr_trial_diff_thresh_edge = has_alpha ? global_cfg.m_psnr_trial_diff_thresh_edge_alpha : global_cfg.m_psnr_trial_diff_thresh_edge; + const float total_comp_weights = enc_cfg.m_cem_enc_params.get_total_comp_weights(); + + basist::astc_ldr_t::grid_weight_dct grid_dct; + grid_dct.init(block_width, block_height); + + coded_blocks.resize(num_blocks_x, num_blocks_y); + for (uint32_t y = 0; y < num_blocks_y; y++) + for (uint32_t x = 0; x < num_blocks_x; x++) + coded_blocks(x, y).clear(); + + vector2D prev_block_states(num_blocks_x, num_blocks_y); + + int part2_hash[basist::astc_ldr_t::PART_HASH_SIZE]; + std::fill(part2_hash, part2_hash + basist::astc_ldr_t::PART_HASH_SIZE, -1); + + int part3_hash[basist::astc_ldr_t::PART_HASH_SIZE]; + std::fill(part3_hash, part3_hash + basist::astc_ldr_t::PART_HASH_SIZE, -1); + + int tm_hash[basist::astc_ldr_t::TM_HASH_SIZE]; + std::fill(tm_hash, tm_hash + basist::astc_ldr_t::TM_HASH_SIZE, -1); + + const bool use_run_commands_global_enable = true; + const bool endpoint_dpcm_global_enable = true; + + uint32_t cur_run_len = 0; + + uint32_t total_runs = 0, total_run_blocks = 0, total_nonrun_blocks = 0; + uint32_t total_lossy_replacements = 0; + uint32_t total_solid_blocks = 0; + uint32_t total_full_reuse_commands = 0; + uint32_t total_raw_commands = 0; + uint32_t total_reuse_full_cfg_emitted = 0; + uint32_t total_full_cfg_emitted = 0; + uint32_t num_part_hash_probes = 0; + uint32_t num_part_hash_hits = 0; + uint32_t total_used_endpoint_dpcm = 0; + uint32_t total_used_endpoint_raw = 0; + uint32_t total_used_dct = 0; + uint32_t total_used_weight_dpcm = 0; + uint32_t num_tm_hash_hits = 0, num_tm_hash_probes = 0; + + raw_bits.put_bits(basist::astc_ldr_t::FULL_ZSTD_HEADER_MARKER, basist::astc_ldr_t::FULL_ZSTD_HEADER_MARKER_BITS); + + const int block_dim_index = astc_helpers::find_astc_block_size_index(block_width, block_height); + assert((block_dim_index >= 0) && (block_dim_index < (int)astc_helpers::NUM_ASTC_BLOCK_SIZES)); + + raw_bits.put_bits(block_dim_index, 4); + + raw_bits.put_bits(enc_cfg.m_cem_enc_params.m_decode_mode_srgb, 1); + + raw_bits.put_bits(width, 16); + raw_bits.put_bits(height, 16); + + raw_bits.put_bits(has_alpha, 1); + + raw_bits.put_bits(enc_cfg.m_use_dct, 1); + if (enc_cfg.m_use_dct) + { + const int int_q = clamp((int)std::round(global_cfg.m_dct_quality * 2.0f), 0, 200); + raw_bits.put_bits(int_q, 8); + } + + const uint32_t FULL_ZSTD_MAX_RUN_LEN = 64; + + for (uint32_t by = 0; by < num_blocks_y; by++) + { + //const uint32_t base_y = by * block_height; + + for (uint32_t bx = 0; bx < num_blocks_x; bx++) + { + //const uint32_t base_x = bx * block_width; + //raw_bits.put_bits(0xA1, 8); + + basist::astc_ldr_t::prev_block_state_full_zstd& prev_state = prev_block_states(bx, by); + + const basist::astc_ldr_t::prev_block_state_full_zstd* pLeft_state = bx ? &prev_block_states(bx - 1, by) : nullptr; + const basist::astc_ldr_t::prev_block_state_full_zstd* pUpper_state = by ? &prev_block_states(bx, by - 1) : nullptr; + const basist::astc_ldr_t::prev_block_state_full_zstd* pDiag_state = (bx && by) ? &prev_block_states(bx - 1, by - 1) : nullptr; + + const ldr_astc_block_encode_image_output::block_info& blk_info = enc_out.m_image_block_info(bx, by); + + uint32_t best_packed_out_block_index = blk_info.m_packed_out_block_index; + + // check for run + if ((use_run_commands_global_enable) && (bx || by)) + { + const encode_block_output& blk_out = blk_info.m_out_blocks[best_packed_out_block_index]; + const astc_helpers::log_astc_block& cur_log_blk = blk_out.m_log_blk; + + const astc_helpers::log_astc_block& prev_log_blk = bx ? coded_blocks(bx - 1, by) : coded_blocks(0, by - 1); + const basist::astc_ldr_t::prev_block_state_full_zstd* pPrev_block_state = bx ? pLeft_state : pUpper_state; + + assert(pPrev_block_state); + + if (compare_log_blocks_for_equality(cur_log_blk, prev_log_blk)) + { + // Left or upper is exactly the same logical block, so expand the run. + cur_run_len++; + + // Accept the previous block (left or upper) as if it's been coded normally. + + coded_blocks(bx, by) = prev_log_blk; + + //prev_state.m_was_solid_color = pPrev_block_state->m_was_solid_color; + prev_state.m_tm_index = pPrev_block_state->m_tm_index; + //prev_state.m_base_cem_index = pPrev_block_state->m_base_cem_index; + + if (cur_run_len == FULL_ZSTD_MAX_RUN_LEN) + { + total_runs++; + total_run_blocks += cur_run_len; + mode_bytes.push_back((uint8_t)((uint32_t)basist::astc_ldr_t::xuastc_zstd_mode::cMODE_RUN | ((cur_run_len - 1) << 2))); + cur_run_len = 0; + } + + continue; + } + } + + if (cur_run_len) + { + assert(cur_run_len <= FULL_ZSTD_MAX_RUN_LEN); + + total_runs++; + total_run_blocks += cur_run_len; + mode_bytes.push_back((uint8_t)((uint32_t)basist::astc_ldr_t::xuastc_zstd_mode::cMODE_RUN | ((cur_run_len - 1) << 2))); + cur_run_len = 0; + } + + total_nonrun_blocks++; + + // TODO: Move this to a prepass that's shared between arith/zstd + const float ref_wmse = (float)blk_info.m_out_blocks[best_packed_out_block_index].m_sse / (total_comp_weights * (float)total_block_pixels); + const float ref_wpsnr = (ref_wmse > 1e-5f) ? 20.0f * log10f(255.0f / sqrtf(ref_wmse)) : 10000.0f; + + if ((global_cfg.m_lossy_supercompression) && (ref_wpsnr >= replacement_min_psnr) && + (!blk_info.m_out_blocks[blk_info.m_packed_out_block_index].m_log_blk.m_solid_color_flag_ldr)) + { + const float psnr_thresh = blk_info.m_strong_edges ? psnr_trial_diff_thresh_edge : psnr_trial_diff_thresh; + + float best_alt_wpsnr = 0.0f; + bool found_alternative = false; + + // Pass: 0 consider full config+part ID endpoint reuse + // Pass: 1 fall back to just full config+part ID reuse (no endpoints) + for (uint32_t pass = 0; pass < 2; pass++) + { + // Iterate through all available alternative candidates + for (uint32_t out_block_iter = 0; out_block_iter < blk_info.m_out_blocks.size(); out_block_iter++) + { + if (out_block_iter == blk_info.m_packed_out_block_index) + continue; + + const float trial_wmse = (float)blk_info.m_out_blocks[out_block_iter].m_sse / (total_comp_weights * (float)total_block_pixels); + const float trial_wpsnr = (trial_wmse > 1e-5f) ? 20.0f * log10f(255.0f / sqrtf(trial_wmse)) : 10000.0f; + + // Reject if PSNR too low + if (trial_wpsnr < (ref_wpsnr - psnr_thresh)) + continue; + + // Reject if inferior than best found so far + if (trial_wpsnr < best_alt_wpsnr) + continue; + + const astc_helpers::log_astc_block& trial_log_blk = blk_info.m_out_blocks[out_block_iter].m_log_blk; + + if (trial_log_blk.m_solid_color_flag_ldr) + continue; + + // Examine nearby neighbors + for (uint32_t i = 0; i < basist::astc_ldr_t::cMaxConfigReuseNeighbors; i++) + { + int dx = 0, dy = 0; + switch (i) + { + case 0: dx = -1; break; + case 1: dy = -1; break; + case 2: dx = -1; dy = -1; break; + default: assert(0); break; + } + + const int n_bx = bx + dx, n_by = by + dy; + if ((n_bx < 0) || (n_by < 0)) + continue; + + astc_helpers::log_astc_block& neighbor_log_blk = coded_blocks(n_bx, n_by); + + if (neighbor_log_blk.m_solid_color_flag_ldr) + continue; + + bool accept_flag = false; + if (pass == 0) + { + // prefer full config+endpoint equality first + accept_flag = compare_log_block_configs_and_endpoints(trial_log_blk, neighbor_log_blk); + } + else + { + // next check for just config equality + accept_flag = compare_log_block_configs(trial_log_blk, neighbor_log_blk); + } + + if (accept_flag) + { + best_alt_wpsnr = trial_wpsnr; + best_packed_out_block_index = out_block_iter; + found_alternative = true; + break; + } + + } // i + + } // out_block_iter + + if (found_alternative) + break; + + } // pass + + if (best_packed_out_block_index != blk_info.m_packed_out_block_index) + total_lossy_replacements++; + + } // global_cfg.m_lossy_supercompression + + const encode_block_output& blk_out = blk_info.m_out_blocks[best_packed_out_block_index]; + + astc_helpers::log_astc_block& cur_log_blk = coded_blocks(bx, by); + + cur_log_blk = blk_out.m_log_blk; + + // Solid color/void extent + if (blk_out.m_trial_mode_index < 0) + { + assert(cur_log_blk.m_solid_color_flag_ldr); + + total_solid_blocks++; + + mode_bytes.push_back((uint8_t)basist::astc_ldr_t::xuastc_zstd_mode::cMODE_SOLID); + + uint32_t cur_solid_color[4]; + for (uint32_t i = 0; i < 4; i++) + cur_solid_color[i] = blk_out.m_log_blk.m_solid_color[i] >> 8; + + uint32_t prev_solid_color[4] = { 0 }; + + const uint32_t num_comps = has_alpha ? 4 : 3; + + astc_helpers::log_astc_block* pPrev_log_blk = bx ? &coded_blocks(bx - 1, by) : (by ? &coded_blocks(bx, by - 1) : nullptr); + if (pPrev_log_blk) + { + if (pPrev_log_blk->m_solid_color_flag_ldr) + { + prev_solid_color[0] = pPrev_log_blk->m_solid_color[0] >> 8; + prev_solid_color[1] = pPrev_log_blk->m_solid_color[1] >> 8; + prev_solid_color[2] = pPrev_log_blk->m_solid_color[2] >> 8; + prev_solid_color[3] = pPrev_log_blk->m_solid_color[3] >> 8; + } + else + { + // Decode previous block's first CEM, use the halfway point as the predictor. + color_rgba prev_l, prev_h; + decode_endpoints(pPrev_log_blk->m_color_endpoint_modes[0], pPrev_log_blk->m_endpoints, pPrev_log_blk->m_endpoint_ise_range, prev_l, prev_h); + + prev_solid_color[0] = (prev_l[0] + prev_h[0] + 1) >> 1; + prev_solid_color[1] = (prev_l[1] + prev_h[1] + 1) >> 1; + prev_solid_color[2] = (prev_l[2] + prev_h[2] + 1) >> 1; + prev_solid_color[3] = (prev_l[3] + prev_h[3] + 1) >> 1; + } + } + + for (uint32_t i = 0; i < num_comps; i++) + { + const uint32_t delta = (cur_solid_color[i] - prev_solid_color[i]) & 0xFF; + solid_dpcm_bytes.push_back((uint8_t)delta); + } + + //prev_state.m_was_solid_color = true; + prev_state.m_tm_index = -1; + //prev_state.m_base_cem_index = astc_helpers::CEM_LDR_RGB_DIRECT; + + continue; + } + + assert(!cur_log_blk.m_solid_color_flag_ldr); + + int full_cfg_endpoint_reuse_index = -1; + + for (uint32_t i = 0; i < basist::astc_ldr_t::cMaxConfigReuseNeighbors; i++) + { + int dx = 0, dy = 0; + switch (i) + { + case 0: dx = -1; break; + case 1: dy = -1; break; + case 2: dx = -1; dy = -1; break; + default: assert(0); break; + } + + const int n_bx = bx + dx, n_by = by + dy; + if ((n_bx < 0) || (n_by < 0)) + continue; + + astc_helpers::log_astc_block& neighbor_log_blk = coded_blocks(n_bx, n_by); + + if (neighbor_log_blk.m_solid_color_flag_ldr) + continue; + + if (compare_log_block_configs_and_endpoints(cur_log_blk, neighbor_log_blk)) + { + full_cfg_endpoint_reuse_index = i; + break; + } + } // i + + if (full_cfg_endpoint_reuse_index >= 0) + { + // Reused full config, part ID and endpoint values from an immediate neighbor + mode_bytes.push_back((uint8_t)((uint32_t)basist::astc_ldr_t::xuastc_zstd_mode::cMODE_REUSE_CFG_ENDPOINTS_LEFT + (full_cfg_endpoint_reuse_index << 2))); + + total_full_reuse_commands++; + + const basist::astc_ldr_t::prev_block_state_full_zstd* pReused_cfg_state = nullptr; + + switch (full_cfg_endpoint_reuse_index) + { + case 0: pReused_cfg_state = pLeft_state; break; + case 1: pReused_cfg_state = pUpper_state; break; + case 2: pReused_cfg_state = pDiag_state; break; + default: assert(0); break; + } + + if (!pReused_cfg_state) + { + assert(0); + fmt_error_printf("encoding internal failure\n"); + return false; + } + + assert(pReused_cfg_state->m_tm_index == blk_out.m_trial_mode_index); + + prev_state.m_tm_index = blk_out.m_trial_mode_index; + } + else + { + // No nearby full config+part ID+endpoint reuse, so send raw command + // Must send endpoints too. + total_raw_commands++; + + // Format of mode byte (UD bit used in modes other than raw) + // 7 6 5 4 3 2 1 0 + // UD C ED HH BO I I M + + // MMM=mode + // II=neighbor reuse index [0,3], 3=no reuse + // BO=base offset flag + // HH=partition hash hit flag + // ED=endpoint DPCM flag + // C=config hash table hit + // UD=use DCT flag + + mode_bytes.push_back((uint8_t)basist::astc_ldr_t::xuastc_zstd_mode::cMODE_RAW); + + const uint32_t cur_actual_cem = cur_log_blk.m_color_endpoint_modes[0]; + const uint32_t total_endpoint_vals = astc_helpers::get_num_cem_values(cur_actual_cem); + + // DO NOT use tm.m_cem because the encoder may have selected a base+ofs variant instead. Use cur_actual_cem. + const basist::astc_ldr_t::trial_mode& tm = enc_out.m_encoder_trial_modes[blk_out.m_trial_mode_index]; + + // Check for config+part ID neighbor reuse (partial refuse) + int neighbor_cfg_match_index = -1; + for (uint32_t i = 0; i < basist::astc_ldr_t::cMaxConfigReuseNeighbors; i++) + { + const basist::astc_ldr_t::prev_block_state_full_zstd* pNeighbor_state = nullptr; + + int dx = 0, dy = 0; + switch (i) + { + case 0: dx = -1; pNeighbor_state = pLeft_state; break; + case 1: dy = -1; pNeighbor_state = pUpper_state; break; + case 2: dx = -1; dy = -1; pNeighbor_state = pDiag_state; break; + default: assert(0); break; + } + + if (!pNeighbor_state) + continue; + + const int n_bx = bx + dx, n_by = by + dy; + assert((n_bx >= 0) && (n_by >= 0)); + + astc_helpers::log_astc_block& neighbor_log_blk = coded_blocks(n_bx, n_by); + + if (pNeighbor_state->m_tm_index != blk_out.m_trial_mode_index) + continue; + + if (neighbor_log_blk.m_color_endpoint_modes[0] != cur_log_blk.m_color_endpoint_modes[0]) + continue; + + if (neighbor_log_blk.m_partition_id != cur_log_blk.m_partition_id) + continue; + + assert(neighbor_log_blk.m_dual_plane == cur_log_blk.m_dual_plane); + assert(neighbor_log_blk.m_color_component_selector == cur_log_blk.m_color_component_selector); + assert(neighbor_log_blk.m_num_partitions == cur_log_blk.m_num_partitions); + assert(neighbor_log_blk.m_grid_width == cur_log_blk.m_grid_width); + assert(neighbor_log_blk.m_grid_height == cur_log_blk.m_grid_height); + assert(neighbor_log_blk.m_endpoint_ise_range == cur_log_blk.m_endpoint_ise_range); + assert(neighbor_log_blk.m_weight_ise_range == cur_log_blk.m_weight_ise_range); + + neighbor_cfg_match_index = i; + break; + } + + if (neighbor_cfg_match_index >= 0) + { + // Partial reuse (config+partition ID, but not endpoints). + // OR 2-bits into the mode byte + mode_bytes.back() |= (uint8_t)(neighbor_cfg_match_index << 1); + + const basist::astc_ldr_t::prev_block_state_full_zstd* pReused_cfg_state = nullptr; + + switch (neighbor_cfg_match_index) + { + case 0: pReused_cfg_state = pLeft_state; break; + case 1: pReused_cfg_state = pUpper_state; break; + case 2: pReused_cfg_state = pDiag_state; break; + default: assert(0); break; + } + + if (!pReused_cfg_state) + { + assert(0); + fmt_error_printf("encoding internal failure\n"); + return false; + } + + assert(pReused_cfg_state->m_tm_index == blk_out.m_trial_mode_index); + + prev_state.m_tm_index = blk_out.m_trial_mode_index; + + total_reuse_full_cfg_emitted++; + } + else + { + // No reuse - must send config, so pack it. Then send endpoints. + total_full_cfg_emitted++; + + // OR 2-bits into the mode byte (so now 5 bits total) + mode_bytes.back() |= (uint8_t)(((uint32_t)basist::astc_ldr_t::cMaxConfigReuseNeighbors) << 1); + + // Pack tm index (ASTC base config) + { + num_tm_hash_probes++; + + uint32_t tm_h = basist::astc_ldr_t::tm_hash_index(blk_out.m_trial_mode_index); + + if (tm_hash[tm_h] == blk_out.m_trial_mode_index) + { + num_tm_hash_hits++; + + mode_bytes.back() |= (uint8_t)basist::astc_ldr_t::XUASTC_LDR_MODE_BYTE_TM_HASH_HIT_FLAG; // tm hash hit flag + + raw_bits.put_bits(tm_h, basist::astc_ldr_t::TM_HASH_BITS); + } + else + { + raw_bits.put_truncated_binary(blk_out.m_trial_mode_index, (uint32_t)enc_out.m_encoder_trial_modes.size()); + + tm_hash[tm_h] = blk_out.m_trial_mode_index; + } + } + + prev_state.m_tm_index = blk_out.m_trial_mode_index; + + // Send base_ofs bit if the tm is direct + if ((tm.m_cem == astc_helpers::CEM_LDR_RGB_DIRECT) || (tm.m_cem == astc_helpers::CEM_LDR_RGBA_DIRECT)) + { + const bool is_base_ofs = (cur_log_blk.m_color_endpoint_modes[0] == astc_helpers::CEM_LDR_RGB_BASE_PLUS_OFFSET) || + (cur_log_blk.m_color_endpoint_modes[0] == astc_helpers::CEM_LDR_RGBA_BASE_PLUS_OFFSET); + + if (is_base_ofs) + mode_bytes.back() |= basist::astc_ldr_t::XUASTC_LDR_MODE_BYTE_IS_BASE_OFS_FLAG; // base_ofs bit + } + + if (tm.m_num_parts > 1) + { + // Send unique part pattern ID + const astc_ldr::partitions_data* pPart_data = (tm.m_num_parts == 2) ? &enc_out.m_part_data_p2 : &enc_out.m_part_data_p3; + + const uint32_t astc_pat_index = cur_log_blk.m_partition_id; + const uint32_t unique_pat_index = pPart_data->m_part_seed_to_unique_index[astc_pat_index]; + const uint32_t total_unique_indices = pPart_data->m_total_unique_patterns; + assert(unique_pat_index < total_unique_indices); + + num_part_hash_probes++; + + int* pPart_hash = (tm.m_num_parts == 2) ? part2_hash : part3_hash; + + const uint32_t h = basist::astc_ldr_t::part_hash_index(unique_pat_index); + + if (pPart_hash[h] != (int)unique_pat_index) + { +#if defined(_DEBUG) || defined(DEBUG) + // sanity + for (uint32_t i = 0; i < basist::astc_ldr_t::PART_HASH_SIZE; i++) + { + assert(pPart_hash[i] != (int)unique_pat_index); + } +#endif + + raw_bits.put_truncated_binary(unique_pat_index, total_unique_indices); + } + else + { + num_part_hash_hits++; + + mode_bytes.back() |= basist::astc_ldr_t::XUASTC_LDR_MODE_BYTE_PART_HASH_HIT; // hash pat_index hit bit + raw_bits.put_bits(h, basist::astc_ldr_t::PART_HASH_BITS); + } + + pPart_hash[basist::astc_ldr_t::part_hash_index(unique_pat_index)] = unique_pat_index; + } + } + + // Send endpoints + const int num_endpoint_levels = astc_helpers::get_ise_levels(cur_log_blk.m_endpoint_ise_range); + const auto& endpoint_ise_to_rank = astc_helpers::g_dequant_tables.get_endpoint_tab(cur_log_blk.m_endpoint_ise_range).m_ISE_to_rank; + + bool endpoints_use_bc[astc_helpers::MAX_PARTITIONS] = { false }; + + if (astc_helpers::cem_supports_bc(cur_actual_cem)) + { + for (uint32_t part_iter = 0; part_iter < tm.m_num_parts; part_iter++) + { + const bool cur_uses_bc = astc_helpers::used_blue_contraction(cur_actual_cem, cur_log_blk.m_endpoints + part_iter * total_endpoint_vals, cur_log_blk.m_endpoint_ise_range); + + endpoints_use_bc[part_iter] = cur_uses_bc; + + } // part_iter + } + + int best_reuse_bx = -1, best_reuse_by = -1; + uint32_t best_reuse_index = 0; + const astc_helpers::log_astc_block* pEndpoint_pred_log_blk = nullptr; + + if (endpoint_dpcm_global_enable) + { + int64_t best_trial_delta2 = INT64_MAX; + float best_trial_bits = BIG_FLOAT_VAL; + + // TODO: Decide if DPCM is even worth it. + const float N = (float)(total_endpoint_vals * tm.m_num_parts); + + for (uint32_t reuse_index = 0; reuse_index < basist::astc_6x6_hdr::NUM_REUSE_XY_DELTAS; reuse_index++) + { + const int rx = (int)bx + basist::astc_6x6_hdr::g_reuse_xy_deltas[reuse_index].m_x; + const int ry = (int)by + basist::astc_6x6_hdr::g_reuse_xy_deltas[reuse_index].m_y; + if ((rx < 0) || (ry < 0) || (rx >= (int)num_blocks_x) || (ry >= (int)num_blocks_y)) + continue; + + const astc_helpers::log_astc_block* pTrial_log_blk = &coded_blocks(rx, ry); + if (pTrial_log_blk->m_solid_color_flag_ldr) + continue; + + uint8_t trial_predicted_endpoints[astc_helpers::MAX_PARTITIONS][astc_helpers::MAX_CEM_ENDPOINT_VALS] = { }; + + uint32_t part_iter; + for (part_iter = 0; part_iter < tm.m_num_parts; part_iter++) + { + const bool always_repack_flag = false; + bool blue_contraction_clamped_flag = false, base_ofs_clamped_flag = false; + + bool conv_status = basist::astc_ldr_t::convert_endpoints_across_cems( + pTrial_log_blk->m_color_endpoint_modes[0], pTrial_log_blk->m_endpoint_ise_range, pTrial_log_blk->m_endpoints, + cur_actual_cem, cur_log_blk.m_endpoint_ise_range, trial_predicted_endpoints[part_iter], + always_repack_flag, + endpoints_use_bc[part_iter], false, + blue_contraction_clamped_flag, base_ofs_clamped_flag); + + if (!conv_status) + break; + } // part_iter + + if (part_iter < tm.m_num_parts) + continue; // failed + + int64_t trial_endpoint_delta2 = 0; + for (part_iter = 0; part_iter < tm.m_num_parts; part_iter++) + { + for (uint32_t val_iter = 0; val_iter < total_endpoint_vals; val_iter++) + { + int cur_e_rank = endpoint_ise_to_rank[cur_log_blk.m_endpoints[part_iter * total_endpoint_vals + val_iter]]; + int prev_e_rank = endpoint_ise_to_rank[trial_predicted_endpoints[part_iter][val_iter]]; + + int e_delta = cur_e_rank - prev_e_rank; + + trial_endpoint_delta2 += e_delta * e_delta; + + } // val_iter + + } // part_iter + + const float mse = (float)trial_endpoint_delta2 / N; + + // Gaussian entropy estimate - precomputed 0.5 * log2(2*pi*e) = ~2.0470956f + const float k_const = 2.0470956f; + + float bits_per_sym = 0.5f * log2f(basisu::maximum(mse, 1e-9f)) + k_const; + + bits_per_sym = clamp(bits_per_sym, 0.05f, 8.0f); + + // total est bits for this block’s endpoints + float total_est_bits = bits_per_sym * N; + + if (total_est_bits < best_trial_bits) + { + best_trial_delta2 = trial_endpoint_delta2; + best_trial_bits = total_est_bits; + + best_reuse_bx = rx; + best_reuse_by = ry; + best_reuse_index = reuse_index; + + if (!best_trial_delta2) + break; + } + + } // reuse_index + + if (best_reuse_bx >= 0) + { + pEndpoint_pred_log_blk = &coded_blocks(best_reuse_bx, best_reuse_by); + + assert(!pEndpoint_pred_log_blk->m_solid_color_flag_ldr); + } + + } // if (endpoint_dpcm_global_enable) + + uint8_t predicted_endpoints[astc_helpers::MAX_PARTITIONS][astc_helpers::MAX_CEM_ENDPOINT_VALS] = { }; + + bool use_dpcm_endpoints = false; + + if (pEndpoint_pred_log_blk) + { + use_dpcm_endpoints = true; + + assert(cur_log_blk.m_num_partitions == tm.m_num_parts); + + for (uint32_t part_iter = 0; part_iter < tm.m_num_parts; part_iter++) + { + const bool always_repack_flag = false; + bool blue_contraction_clamped_flag = false, base_ofs_clamped_flag = false; + + bool conv_status = basist::astc_ldr_t::convert_endpoints_across_cems( + pEndpoint_pred_log_blk->m_color_endpoint_modes[0], pEndpoint_pred_log_blk->m_endpoint_ise_range, pEndpoint_pred_log_blk->m_endpoints, + cur_actual_cem, cur_log_blk.m_endpoint_ise_range, predicted_endpoints[part_iter], + always_repack_flag, + endpoints_use_bc[part_iter], false, + blue_contraction_clamped_flag, base_ofs_clamped_flag); + + if (!conv_status) + { + // In practice, should never happen + use_dpcm_endpoints = false; + break; + } + } + } + + // TODO: Decide what is cheaper, endpoint DPCM vs. raw + + if (use_dpcm_endpoints) + { + // DPCM flag bit + mode_bytes.back() |= basist::astc_ldr_t::XUASTC_LDR_MODE_BYTE_DPCM_ENDPOINTS_FLAG; + + endpoint_dpcm_reuse_indices.push_back((uint8_t)best_reuse_index); + + if (astc_helpers::cem_supports_bc(cur_actual_cem)) + { + for (uint32_t part_iter = 0; part_iter < tm.m_num_parts; part_iter++) + { + use_bc_bits.put_bits(endpoints_use_bc[part_iter], 1); + + } // part_iter + } + + for (uint32_t part_iter = 0; part_iter < tm.m_num_parts; part_iter++) + { + for (uint32_t val_iter = 0; val_iter < total_endpoint_vals; val_iter++) + { + int cur_e_rank = endpoint_ise_to_rank[cur_log_blk.m_endpoints[part_iter * total_endpoint_vals + val_iter]]; + int prev_e_rank = endpoint_ise_to_rank[predicted_endpoints[part_iter][val_iter]]; + + int e_val = imod(cur_e_rank - prev_e_rank, num_endpoint_levels); + + if (num_endpoint_levels <= 8) + endpoint_dpcm_3bit.put_bits(e_val, 4); + else if (num_endpoint_levels <= 16) + endpoint_dpcm_4bit.put_bits(e_val, 4); + else if (num_endpoint_levels <= 32) + endpoint_dpcm_5bit.push_back((uint8_t)e_val); + else if (num_endpoint_levels <= 64) + endpoint_dpcm_6bit.push_back((uint8_t)e_val); + else if (num_endpoint_levels <= 128) + endpoint_dpcm_7bit.push_back((uint8_t)e_val); + else if (num_endpoint_levels <= 256) + endpoint_dpcm_8bit.push_back((uint8_t)e_val); + + } // val_iter + + } // part_iter + + total_used_endpoint_dpcm++; + } + else + { + encode_values(raw_bits, tm.m_num_parts * total_endpoint_vals, cur_log_blk.m_endpoints, cur_log_blk.m_endpoint_ise_range); + + total_used_endpoint_raw++; + } // if (use_dpcm_endpoints) + + } // if (full_cfg_endpoint_reuse_index >= 0) + + // ------------------------------------ Send weights + + const uint32_t total_planes = cur_log_blk.m_dual_plane ? 2 : 1; + const uint32_t total_weights = cur_log_blk.m_grid_width * cur_log_blk.m_grid_height; + + const int num_weight_levels = astc_helpers::get_ise_levels(cur_log_blk.m_weight_ise_range); + const auto& weight_ise_to_rank = astc_helpers::g_dequant_tables.get_weight_tab(cur_log_blk.m_weight_ise_range).m_ISE_to_rank; + + bool use_dct = enc_cfg.m_use_dct; + + // TODO - tune this threshold + const uint32_t SWITCH_TO_DPCM_NUM_COEFF_THRESH = (cur_log_blk.m_grid_width * cur_log_blk.m_grid_height * 45 + 64) >> 7; + + if (use_dct) + { + for (uint32_t plane_iter = 0; plane_iter < total_planes; plane_iter++) + { + const basist::astc_ldr_t::dct_syms& syms = blk_out.m_packed_dct_plane_data[plane_iter]; + if (syms.m_max_coeff_mag > basist::astc_ldr_t::DCT_MAX_ARITH_COEFF_MAG) + { + use_dct = false; + break; + } + + if (syms.m_coeffs.size() > SWITCH_TO_DPCM_NUM_COEFF_THRESH) + { + use_dct = false; + break; + } + } + } + + // MSB of mode byte=use DCT + if (enc_cfg.m_use_dct) + { + assert((mode_bytes.back() & basist::astc_ldr_t::XUASTC_LDR_MODE_BYTE_USE_DCT) == 0); + + if (use_dct) + mode_bytes.back() |= basist::astc_ldr_t::XUASTC_LDR_MODE_BYTE_USE_DCT; + } + + if (use_dct) + { + total_used_dct++; + + if (total_planes > 1) + { + assert(blk_out.m_packed_dct_plane_data[0].m_num_dc_levels == blk_out.m_packed_dct_plane_data[1].m_num_dc_levels); + } + + for (uint32_t plane_iter = 0; plane_iter < total_planes; plane_iter++) + { + const basist::astc_ldr_t::dct_syms& syms = blk_out.m_packed_dct_plane_data[plane_iter]; + + if (syms.m_num_dc_levels == basist::astc_ldr_t::DCT_MEAN_LEVELS1) + mean1_bytes.push_back((uint8_t)syms.m_dc_sym); + else + { + assert(syms.m_num_dc_levels == basist::astc_ldr_t::DCT_MEAN_LEVELS0); + mean0_bits.put_bits(syms.m_dc_sym, 4); + } + + for (uint32_t i = 0; i < syms.m_coeffs.size(); i++) + { + if (syms.m_coeffs[i].m_coeff == INT16_MAX) + { + run_bytes.push_back(basist::astc_ldr_t::DCT_RUN_LEN_EOB_SYM_INDEX); + } + else + { + run_bytes.push_back((uint8_t)syms.m_coeffs[i].m_num_zeros); + + sign_bits.put_bits(syms.m_coeffs[i].m_coeff < 0, 1); + + assert((syms.m_coeffs[i].m_coeff != 0) && (iabs(syms.m_coeffs[i].m_coeff) <= 255)); + + coeff_bytes.push_back((uint8_t)(iabs(syms.m_coeffs[i].m_coeff) - 1)); + } + } + + } // plane_iter + } + else + { + total_used_weight_dpcm++; + + for (uint32_t plane_iter = 0; plane_iter < total_planes; plane_iter++) + { + int prev_w = num_weight_levels / 2; + + for (uint32_t weight_iter = 0; weight_iter < total_weights; weight_iter++) + { + int ise_w = cur_log_blk.m_weights[plane_iter + weight_iter * total_planes]; + int w = weight_ise_to_rank[ise_w]; + + int w_to_code = w; + w_to_code = imod(w - prev_w, num_weight_levels); + + prev_w = w; + + if (num_weight_levels <= 4) + weight2_bits.put_bits((uint8_t)w_to_code, 2); + else if (num_weight_levels <= 8) + weight3_bits.put_bits((uint8_t)w_to_code, 4); + else if (num_weight_levels <= 16) + weight4_bits.put_bits((uint8_t)w_to_code, 4); + else + weight8_bits.push_back((uint8_t)w_to_code); + + } // weight_iter + + } // plane_iter + } + + } // bx + + if (cur_run_len) + { + assert(cur_run_len <= FULL_ZSTD_MAX_RUN_LEN); + + total_runs++; + total_run_blocks += cur_run_len; + mode_bytes.push_back((uint8_t)((uint32_t)basist::astc_ldr_t::xuastc_zstd_mode::cMODE_RUN | ((cur_run_len - 1) << 2))); + cur_run_len = 0; + } + + } // by + + raw_bits.put_bits(basist::astc_ldr_t::FINAL_SYNC_MARKER, basist::astc_ldr_t::FINAL_SYNC_MARKER_BITS); + + raw_bits.flush(); + endpoint_dpcm_3bit.flush(); + endpoint_dpcm_4bit.flush(); + use_bc_bits.flush(); + + mean0_bits.flush(); + sign_bits.flush(); + weight2_bits.flush(); + weight3_bits.flush(); + weight4_bits.flush(); + + const uint32_t zstd_level = 9; + + uint8_vec comp_mode, comp_solid_dpcm, comp_endpoint_dpcm_reuse_indices; + uint8_vec comp_use_bc_bits, comp_endpoint_dpcm_3bit, comp_endpoint_dpcm_4bit, comp_endpoint_dpcm_5bit, comp_endpoint_dpcm_6bit, comp_endpoint_dpcm_7bit, comp_endpoint_dpcm_8bit; + + // Mode + if (!zstd_compress(mode_bytes, comp_mode, zstd_level)) return false; + if (!zstd_compress(solid_dpcm_bytes, comp_solid_dpcm, zstd_level)) return false; + + // Endpoints + if (!zstd_compress(endpoint_dpcm_reuse_indices, comp_endpoint_dpcm_reuse_indices, zstd_level)) return false; + if (!zstd_compress(use_bc_bits, comp_use_bc_bits, zstd_level)) return false; + if (!zstd_compress(endpoint_dpcm_3bit, comp_endpoint_dpcm_3bit, zstd_level)) return false; + if (!zstd_compress(endpoint_dpcm_4bit, comp_endpoint_dpcm_4bit, zstd_level)) return false; + if (!zstd_compress(endpoint_dpcm_5bit, comp_endpoint_dpcm_5bit, zstd_level)) return false; + if (!zstd_compress(endpoint_dpcm_6bit, comp_endpoint_dpcm_6bit, zstd_level)) return false; + if (!zstd_compress(endpoint_dpcm_7bit, comp_endpoint_dpcm_7bit, zstd_level)) return false; + if (!zstd_compress(endpoint_dpcm_8bit, comp_endpoint_dpcm_8bit, zstd_level)) return false; + + // Weights + uint8_vec comp_mean0, comp_mean1, comp_run, comp_coeff, comp_weight2, comp_weight3, comp_weight4, comp_weight8; + + if (!zstd_compress(mean0_bits, comp_mean0, zstd_level)) return false; + if (!zstd_compress(mean1_bytes, comp_mean1, zstd_level)) return false; + if (!zstd_compress(run_bytes, comp_run, zstd_level)) return false; + if (!zstd_compress(coeff_bytes, comp_coeff, zstd_level)) return false; + if (!zstd_compress(weight2_bits, comp_weight2, zstd_level)) return false; + if (!zstd_compress(weight3_bits, comp_weight3, zstd_level)) return false; + if (!zstd_compress(weight4_bits, comp_weight4, zstd_level)) return false; + if (!zstd_compress(weight8_bits, comp_weight8, zstd_level)) return false; + + basist::astc_ldr_t::xuastc_ldr_full_zstd_header hdr; + clear_obj(hdr); + + hdr.m_flags = (uint8_t)basist::astc_ldr_t::xuastc_ldr_syntax::cFullZStd; + + hdr.m_raw_bits_len = (uint32_t)raw_bits.get_bytes().size(); + hdr.m_mode_bytes_len = (uint32_t)comp_mode.size(); + hdr.m_solid_dpcm_bytes_len = (uint32_t)comp_solid_dpcm.size(); + + hdr.m_endpoint_dpcm_reuse_indices_len = (uint32_t)comp_endpoint_dpcm_reuse_indices.size(); + hdr.m_use_bc_bits_len = (uint32_t)comp_use_bc_bits.size(); + hdr.m_endpoint_dpcm_3bit_len = (uint32_t)comp_endpoint_dpcm_3bit.size(); + hdr.m_endpoint_dpcm_4bit_len = (uint32_t)comp_endpoint_dpcm_4bit.size(); + hdr.m_endpoint_dpcm_5bit_len = (uint32_t)comp_endpoint_dpcm_5bit.size(); + hdr.m_endpoint_dpcm_6bit_len = (uint32_t)comp_endpoint_dpcm_6bit.size(); + hdr.m_endpoint_dpcm_7bit_len = (uint32_t)comp_endpoint_dpcm_7bit.size(); + hdr.m_endpoint_dpcm_8bit_len = (uint32_t)comp_endpoint_dpcm_8bit.size(); + + hdr.m_mean0_bits_len = (uint32_t)comp_mean0.size(); + hdr.m_mean1_bytes_len = (uint32_t)comp_mean1.size(); + hdr.m_run_bytes_len = (uint32_t)comp_run.size(); + hdr.m_coeff_bytes_len = (uint32_t)comp_coeff.size(); + hdr.m_sign_bits_len = (uint32_t)sign_bits.get_bytes().size(); + hdr.m_weight2_bits_len = (uint32_t)comp_weight2.size(); + hdr.m_weight3_bits_len = (uint32_t)comp_weight3.size(); + hdr.m_weight4_bits_len = (uint32_t)comp_weight4.size(); + hdr.m_weight8_bytes_len = (uint32_t)comp_weight8.size(); + + comp_data.reserve(8192); + + comp_data.resize(sizeof(hdr)); + memcpy(comp_data.data(), &hdr, sizeof(hdr)); + + comp_data.append(raw_bits.get_bytes()); + comp_data.append(comp_mode); + comp_data.append(comp_solid_dpcm); + + comp_data.append(comp_endpoint_dpcm_reuse_indices); + comp_data.append(comp_use_bc_bits); + comp_data.append(comp_endpoint_dpcm_3bit); + comp_data.append(comp_endpoint_dpcm_4bit); + comp_data.append(comp_endpoint_dpcm_5bit); + comp_data.append(comp_endpoint_dpcm_6bit); + comp_data.append(comp_endpoint_dpcm_7bit); + comp_data.append(comp_endpoint_dpcm_8bit); + + comp_data.append(comp_mean0); + comp_data.append(comp_mean1); + comp_data.append(comp_run); + comp_data.append(comp_coeff); + comp_data.append(sign_bits.get_bytes()); + comp_data.append(comp_weight2); + comp_data.append(comp_weight3); + comp_data.append(comp_weight4); + comp_data.append(comp_weight8); + + if (comp_data.size() > UINT32_MAX) + return false; + + if ((global_cfg.m_debug_images) || (global_cfg.m_debug_output)) + { + image coded_img(width, height); + + vector2D phys_blocks(num_blocks_x, num_blocks_y); + + for (uint32_t by = 0; by < num_blocks_y; by++) + { + for (uint32_t bx = 0; bx < num_blocks_x; bx++) + { + const astc_helpers::log_astc_block& log_blk = coded_blocks(bx, by); + + color_rgba block_pixels[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + + bool status = astc_helpers::decode_block(log_blk, block_pixels, block_width, block_height, enc_cfg.m_cem_enc_params.m_decode_mode_srgb ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8); + if (!status) + { + fmt_error_printf("astc_helpers::decode_block() failed\n"); + return false; + } + + // Be positive the logical block can be unpacked correctly as XUASTC LDR. + color_rgba block_pixels_alt[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + bool status_alt = astc_helpers::decode_block_xuastc_ldr(log_blk, block_pixels_alt, block_width, block_height, enc_cfg.m_cem_enc_params.m_decode_mode_srgb ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8); + if (!status_alt) + { + fmt_error_printf("astc_helpers::decode_block_xuastc_ldr() failed\n"); + return false; + } + + if (memcmp(block_pixels, block_pixels_alt, sizeof(color_rgba) * block_width * block_height) != 0) + { + fmt_error_printf("astc_helpers::decode_block_xuastc_ldr() decode pixel mismatch\n"); + return false; + } + + coded_img.set_block_clipped(block_pixels, bx * block_width, by * block_height, block_width, block_height); + + } // bx + + } //by + + if (global_cfg.m_debug_images) + save_png(global_cfg.m_debug_file_prefix + "coded_img.png", coded_img); + + if (global_cfg.m_debug_output) + { + debug_printf("Orig image vs. coded img:\n"); + print_image_metrics(orig_img, coded_img); + } + } + + if (global_cfg.m_debug_output) + { + fmt_debug_printf("Zstd compressed sizes:\n"); + + fmt_debug_printf(" Raw bytes: {}\n", (uint64_t)raw_bits.get_bytes().size()); + fmt_debug_printf(" Mode bytes: {}, comp size: {}\n", (uint64_t)mode_bytes.size(), (uint64_t)comp_mode.size()); + fmt_debug_printf(" Solid DPCM bytes: {}, comp size: {}\n", (uint64_t)solid_dpcm_bytes.size(), (uint64_t)comp_solid_dpcm.size()); + + fmt_debug_printf(" \n Endpoint DPCM Reuse Bytes: {}, comp size: {}\n", (uint64_t)endpoint_dpcm_reuse_indices.size(), (uint64_t)comp_endpoint_dpcm_reuse_indices.size()); + fmt_debug_printf(" Use BC bits bytes: {}, comp_size: {}\n", (uint64_t)use_bc_bits.get_bytes().size(), (uint64_t)comp_use_bc_bits.size()); + fmt_debug_printf(" Endpoint DPCM 3 bits: {}, comp size: {}\n", (uint64_t)endpoint_dpcm_3bit.get_bytes().size(), (uint64_t)comp_endpoint_dpcm_3bit.size()); + fmt_debug_printf(" Endpoint DPCM 4 bits: {}, comp size: {}\n", (uint64_t)endpoint_dpcm_4bit.get_bytes().size(), (uint64_t)comp_endpoint_dpcm_4bit.size()); + fmt_debug_printf(" Endpoint DPCM 5 bits: {}, comp size: {}\n", (uint64_t)endpoint_dpcm_5bit.size(), (uint64_t)comp_endpoint_dpcm_5bit.size()); + fmt_debug_printf(" Endpoint DPCM 6 bits: {}, comp size: {}\n", (uint64_t)endpoint_dpcm_6bit.size(), (uint64_t)comp_endpoint_dpcm_6bit.size()); + fmt_debug_printf(" Endpoint DPCM 7 bits: {}, comp size: {}\n", (uint64_t)endpoint_dpcm_7bit.size(), (uint64_t)comp_endpoint_dpcm_7bit.size()); + fmt_debug_printf(" Endpoint DPCM 8 bits: {}, comp size: {}\n", (uint64_t)endpoint_dpcm_8bit.size(), (uint64_t)comp_endpoint_dpcm_8bit.size()); + + fmt_debug_printf(" \n Mean0 bytes: {} comp size: {}\n", (uint64_t)mean0_bits.get_bytes().size(), (uint64_t)comp_mean0.size()); + fmt_debug_printf(" Mean1 bytes: {} comp size: {}\n", (uint64_t)mean1_bytes.size(), (uint64_t)comp_mean1.size()); + fmt_debug_printf(" Run bytes: {} comp size: {}\n", (uint64_t)run_bytes.size(), (uint64_t)comp_run.size()); + fmt_debug_printf(" Coeff bytes: {} comp size: {}\n", (uint64_t)coeff_bytes.size(), (uint64_t)comp_coeff.size()); + fmt_debug_printf(" Sign bytes: {}\n", (uint64_t)sign_bits.get_bytes().size()); + fmt_debug_printf(" Weight2 bytes: {} comp size: {}\n", (uint64_t)weight2_bits.get_bytes().size(), (uint64_t)comp_weight2.size()); + fmt_debug_printf(" Weight3 bytes: {} comp size: {}\n", (uint64_t)weight3_bits.get_bytes().size(), (uint64_t)comp_weight3.size()); + fmt_debug_printf(" Weight4 bytes: {} comp size: {}\n", (uint64_t)weight4_bits.get_bytes().size(), (uint64_t)comp_weight4.size()); + fmt_debug_printf(" Weight8 bytes: {} comp size: {}\n", (uint64_t)weight8_bits.size(), (uint64_t)comp_weight8.size()); + + fmt_debug_printf("\nTotal blocks: {}\n", total_blocks); + fmt_debug_printf("Total runs: {}, run blocks: {}, non-run blocks: {}\n", total_runs, total_run_blocks, total_nonrun_blocks); + fmt_debug_printf("Total lossy replacements: {}\n", total_lossy_replacements); + fmt_debug_printf("Total solid blocks: {}\n", total_solid_blocks); + fmt_debug_printf("Total full reuse commands: {}\n", total_full_reuse_commands); + fmt_debug_printf("Total raw commands: {}\n", total_raw_commands); + fmt_debug_printf("Total reuse full cfg emitted: {}\n", total_reuse_full_cfg_emitted); + fmt_debug_printf("Total full cfg emitted: {}\n", total_full_cfg_emitted); + fmt_debug_printf("Num part hash probes: {}, num part hash hits: {}\n", num_part_hash_probes, num_part_hash_hits); + fmt_debug_printf("Total used endpoint dpcm: {}, total used endpoint raw: {}\n", total_used_endpoint_dpcm, total_used_endpoint_raw); + fmt_debug_printf("Total used weight DCT: {}, total used weight DPCM: {}\n", total_used_dct, total_used_weight_dpcm); + fmt_debug_printf("Total tm hash probes: {}, total tm hash_hits: {}\n", num_tm_hash_probes, num_tm_hash_hits); + + fmt_debug_printf("\nCompressed to {} bytes, {3.3}bpp\n\n", comp_data.size_u32(), ((float)comp_data.size() * 8.0f) / (float)total_pixels); + } + + return true; +} +#endif + +bool compress_image( + const image& orig_img, uint8_vec& comp_data, vector2D& coded_blocks, + const astc_ldr_encode_config& global_cfg, + job_pool& job_pool) +{ + assert(g_initialized); + + if (global_cfg.m_debug_output) + { + fmt_debug_printf("\n------------------- astc_ldr::compress_image\n"); + + fmt_debug_printf("\nglobal_cfg:\n"); + global_cfg.debug_print(); + fmt_debug_printf("\n"); + } + + comp_data.resize(0); + + if (!g_initialized) + return false; + + const uint32_t width = orig_img.get_width(), height = orig_img.get_height(); + + if (!is_in_range(width, 1, (int)MAX_WIDTH) || !is_in_range(height, 1, (int)MAX_HEIGHT)) + return false; + + if (!astc_helpers::is_valid_block_size(global_cfg.m_astc_block_width, global_cfg.m_astc_block_height)) + return false; + + const uint32_t block_width = global_cfg.m_astc_block_width; + const uint32_t block_height = global_cfg.m_astc_block_height; + const uint32_t total_block_pixels = block_width * block_height; + + const uint32_t total_pixels = width * height; + const uint32_t num_blocks_x = (width + block_width - 1) / block_width; + const uint32_t num_blocks_y = (height + block_height - 1) / block_height; + const uint32_t total_blocks = num_blocks_x * num_blocks_y; + const bool has_alpha = orig_img.has_alpha(); + + if (global_cfg.m_debug_output) + fmt_debug_printf("Encoding image dimensions {}x{}, has alpha: {}\n", orig_img.get_width(), orig_img.get_height(), has_alpha); + + ldr_astc_block_encode_image_high_level_config enc_cfg; + + enc_cfg.m_block_width = block_width; + enc_cfg.m_block_height = block_height; + enc_cfg.m_pJob_pool = &job_pool; + + enc_cfg.m_use_dct = global_cfg.m_use_dct; + + if (!is_in_range(global_cfg.m_dct_quality, 1.0f, 100.0f)) + return false; + + const int int_q = clamp((int)std::round(global_cfg.m_dct_quality * 2.0f), 0, 200); + enc_cfg.m_base_q = (float)int_q / 2.0f; + + if (global_cfg.m_debug_output) + fmt_debug_printf("Use DCT: {}, base q: {}, lossy supercompression: {}\n", enc_cfg.m_use_dct, enc_cfg.m_base_q, global_cfg.m_lossy_supercompression); + + const float replacement_min_psnr = has_alpha ? global_cfg.m_replacement_min_psnr_alpha : global_cfg.m_replacement_min_psnr; + const float psnr_trial_diff_thresh = has_alpha ? global_cfg.m_psnr_trial_diff_thresh_alpha : global_cfg.m_psnr_trial_diff_thresh; + const float psnr_trial_diff_thresh_edge = has_alpha ? global_cfg.m_psnr_trial_diff_thresh_edge_alpha : global_cfg.m_psnr_trial_diff_thresh_edge; + + enc_cfg.m_blurring_enabled = global_cfg.m_block_blurring_p1; + enc_cfg.m_blurring_enabled_p2 = global_cfg.m_block_blurring_p2; + + for (uint32_t i = 0; i < 4; i++) + { + enc_cfg.m_cem_enc_params.m_comp_weights[i] = global_cfg.m_comp_weights[i]; + + if (!is_in_range(global_cfg.m_comp_weights[i], 1, 256)) + return false; + } + + int cfg_effort_level = global_cfg.m_effort_level; + if (global_cfg.m_debug_output) + fmt_debug_printf("Using cfg effort level: {}\n", cfg_effort_level); + + configure_encoder_effort_level(cfg_effort_level, enc_cfg); + + if (global_cfg.m_force_disable_subsets) + { + enc_cfg.m_subsets_enabled = false; + enc_cfg.m_second_pass_force_subsets_enabled = false; + } + + if (global_cfg.m_force_disable_rgb_dual_plane) + { + enc_cfg.m_disable_rgb_dual_plane = true; + enc_cfg.m_force_all_dp_chans_p2 = false; + } + + enc_cfg.m_cem_enc_params.m_decode_mode_srgb = global_cfg.m_astc_decode_mode_srgb; + + enc_cfg.m_debug_output = global_cfg.m_debug_output; + enc_cfg.m_debug_images = global_cfg.m_debug_images; + enc_cfg.m_debug_file_prefix = global_cfg.m_debug_file_prefix; + + ldr_astc_block_encode_image_output enc_out; + + const bool enc_status = ldr_astc_block_encode_image(orig_img, enc_cfg, enc_out); + + if (global_cfg.m_debug_output) + fmt_debug_printf("ldr_astc_block_encode_image: {}\n", enc_status); + + if (!enc_status) + return false; + + basist::astc_ldr_t::xuastc_ldr_syntax syntax = global_cfg.m_compressed_syntax; + + if (syntax >= basist::astc_ldr_t::xuastc_ldr_syntax::cTotal) + { + assert(0); + return false; + } + + // Switch to full adaptive arithmetic coding on the smallest mipmaps to avoid ZStd overhead. + const uint32_t DISABLE_FASTER_FORMAT_TOTAL_BLOCKS_THRESH = 64; + if (total_blocks <= DISABLE_FASTER_FORMAT_TOTAL_BLOCKS_THRESH) + syntax = basist::astc_ldr_t::xuastc_ldr_syntax::cFullArith; + + if (syntax == basist::astc_ldr_t::xuastc_ldr_syntax::cFullZStd) + { +#if BASISD_SUPPORT_KTX2_ZSTD + // Full ZStd syntax is so different we'll move that to another function. + return compress_image_full_zstd( + orig_img, comp_data, coded_blocks, + global_cfg, + job_pool, + enc_cfg, enc_out); +#else + fmt_error_printf("Full ZStd syntax not supported in this build (set BASISD_SUPPORT_KTX2_ZSTD to 1)\n"); + return false; +#endif + } + + const bool use_faster_format = (syntax == basist::astc_ldr_t::xuastc_ldr_syntax::cHybridArithZStd); + +#if !BASISD_SUPPORT_KTX2_ZSTD + if (use_faster_format) + { + fmt_error_printf("Full ZStd syntax not supported in this build (set BASISD_SUPPORT_KTX2_ZSTD to 1)\n"); + return false; + } +#endif + + // Either full arithmetic, or hybrid arithmetic+ZStd for weight symbols. + basist::astc_ldr_t::xuastc_ldr_arith_header hdr; + clear_obj(hdr); + + bitwise_coder mean0_bits; + uint8_vec mean1_bytes; + uint8_vec run_bytes; + uint8_vec coeff_bytes; + bitwise_coder sign_bits; + bitwise_coder weight2_bits; + bitwise_coder weight3_bits; + bitwise_coder weight4_bits; + uint8_vec weight8_bits; + + if (use_faster_format) + { + mean0_bits.init(1024); + mean1_bytes.reserve(1024); + run_bytes.reserve(8192); + coeff_bytes.reserve(8192); + sign_bits.init(1024); + weight2_bits.init(1024); + weight3_bits.init(1024); + weight4_bits.init(1024); + weight8_bits.reserve(8192); + } + + interval_timer itm; + itm.start(); + + basist::arith::arith_enc enc; + enc.init(1024 * 1024); + + enc.put_bits(basist::astc_ldr_t::ARITH_HEADER_MARKER, basist::astc_ldr_t::ARITH_HEADER_MARKER_BITS); + + const int block_dim_index = astc_helpers::find_astc_block_size_index(block_width, block_height); + assert((block_dim_index >= 0) && (block_dim_index < (int)astc_helpers::NUM_ASTC_BLOCK_SIZES)); + + enc.put_bits(block_dim_index, 4); + + enc.put_bit(enc_cfg.m_cem_enc_params.m_decode_mode_srgb); + + enc.put_bits(width, 16); + enc.put_bits(height, 16); + + enc.put_bit(has_alpha); + + enc.put_bits(enc_cfg.m_use_dct, 1); + if (enc_cfg.m_use_dct) + enc.put_bits(int_q, 8); + + basist::arith::arith_data_model mode_model((uint32_t)basist::astc_ldr_t::xuastc_mode::cMODE_TOTAL); + + basist::arith::arith_data_model solid_color_dpcm_model[4]; + for (uint32_t i = 0; i < 4; i++) + solid_color_dpcm_model[i].init(256, true); + + basist::arith::arith_data_model raw_endpoint_models[astc_helpers::TOTAL_ENDPOINT_ISE_RANGES]; + for (uint32_t i = 0; i < astc_helpers::TOTAL_ENDPOINT_ISE_RANGES; i++) + raw_endpoint_models[i].init(astc_helpers::get_ise_levels(astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE + i)); + + basist::arith::arith_data_model dpcm_endpoint_models[astc_helpers::TOTAL_ENDPOINT_ISE_RANGES]; + for (uint32_t i = 0; i < astc_helpers::TOTAL_ENDPOINT_ISE_RANGES; i++) + dpcm_endpoint_models[i].init(astc_helpers::get_ise_levels(astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE + i)); + + basist::arith::arith_data_model raw_weight_models[astc_helpers::TOTAL_WEIGHT_ISE_RANGES]; + for (uint32_t i = 0; i < astc_helpers::TOTAL_WEIGHT_ISE_RANGES; i++) + raw_weight_models[i].init(astc_helpers::get_ise_levels(astc_helpers::FIRST_VALID_WEIGHT_ISE_RANGE + i)); + + basist::arith::arith_bit_model is_base_ofs_model; + basist::arith::arith_bit_model use_dct_model[4]; + basist::arith::arith_bit_model use_dpcm_endpoints_model; + + basist::arith::arith_data_model cem_index_model[8]; + for (uint32_t i = 0; i < 8; i++) + cem_index_model[i].init(basist::astc_ldr_t::OTM_NUM_CEMS); + + basist::arith::arith_data_model subset_index_model[basist::astc_ldr_t::OTM_NUM_SUBSETS]; + for (uint32_t i = 0; i < basist::astc_ldr_t::OTM_NUM_SUBSETS; i++) + subset_index_model[i].init(basist::astc_ldr_t::OTM_NUM_SUBSETS); + + basist::arith::arith_data_model ccs_index_model[basist::astc_ldr_t::OTM_NUM_CCS]; + for (uint32_t i = 0; i < basist::astc_ldr_t::OTM_NUM_CCS; i++) + ccs_index_model[i].init(basist::astc_ldr_t::OTM_NUM_CCS); + + basist::arith::arith_data_model grid_size_model[basist::astc_ldr_t::OTM_NUM_GRID_SIZES]; + for (uint32_t i = 0; i < basist::astc_ldr_t::OTM_NUM_GRID_SIZES; i++) + grid_size_model[i].init(basist::astc_ldr_t::OTM_NUM_GRID_SIZES); + + basist::arith::arith_data_model grid_aniso_model[basist::astc_ldr_t::OTM_NUM_GRID_ANISOS]; + for (uint32_t i = 0; i < basist::astc_ldr_t::OTM_NUM_GRID_ANISOS; i++) + grid_aniso_model[i].init(basist::astc_ldr_t::OTM_NUM_GRID_ANISOS); + + basist::arith::arith_data_model dct_run_len_model(65); // [0,63] or 64=EOB + basist::arith::arith_data_model dct_coeff_mag(255); // [1,255] (blocks with larger mags go DPCM) + + double total_header_bits = 0.0f, total_weight_bits = 0.0f, total_endpoint_bits = 0.0f; + + uint32_t total_solid_blocks = 0, total_used_dct = 0, total_used_weight_dpcm = 0; + + basist::astc_ldr_t::grid_weight_dct grid_dct; + grid_dct.init(block_width, block_height); + + vector2D prev_block_states(num_blocks_x, num_blocks_y); + + coded_blocks.resize(num_blocks_x, num_blocks_y); + for (uint32_t y = 0; y < num_blocks_y; y++) + for (uint32_t x = 0; x < num_blocks_x; x++) + coded_blocks(x, y).clear(); + + const bool endpoint_dpcm_global_enable = true; + uint32_t total_used_endpoint_dpcm = 0, total_used_endpoint_raw = 0; + + basist::arith::arith_data_model submode_models[basist::astc_ldr_t::OTM_NUM_CEMS][basist::astc_ldr_t::OTM_NUM_SUBSETS][basist::astc_ldr_t::OTM_NUM_CCS][basist::astc_ldr_t::OTM_NUM_GRID_SIZES][basist::astc_ldr_t::OTM_NUM_GRID_ANISOS]; + + basist::arith::arith_bit_model endpoints_use_bc_models[4]; + + basist::arith::arith_data_model endpoint_reuse_delta_model(basist::astc_6x6_hdr::NUM_REUSE_XY_DELTAS); + + basist::arith::arith_data_model weight_mean_models[2]; + weight_mean_models[0].init(basist::astc_ldr_t::DCT_MEAN_LEVELS0); + weight_mean_models[1].init(basist::astc_ldr_t::DCT_MEAN_LEVELS1); + + basist::arith::arith_data_model config_reuse_model[4]; + for (uint32_t i = 0; i < 4; i++) + config_reuse_model[i].init(basist::astc_ldr_t::cMaxConfigReuseNeighbors + 1); + + uint32_t total_reuse_full_cfg_emitted = 0, total_full_cfg_emitted = 0; + + // TODO: check weights for >= 0 + const float total_comp_weights = enc_cfg.m_cem_enc_params.get_total_comp_weights(); + + uint32_t total_lossy_replacements = 0; + uint32_t total_full_reuse_commands = 0; + uint32_t total_raw_commands = 0; + + if (global_cfg.m_debug_output) + fmt_debug_printf("Supercompressor init time: {} secs\n", itm.get_elapsed_secs()); + + uint32_t total_runs = 0, total_run_blocks = 0; + uint32_t cur_run_len = 0; + const bool use_run_commands = true; + uint32_t total_nonrun_blocks = 0; + + int part2_hash[basist::astc_ldr_t::PART_HASH_SIZE]; + std::fill(part2_hash, part2_hash + basist::astc_ldr_t::PART_HASH_SIZE, -1); + + int part3_hash[basist::astc_ldr_t::PART_HASH_SIZE]; + std::fill(part3_hash, part3_hash + basist::astc_ldr_t::PART_HASH_SIZE, -1); + + basist::arith::arith_bit_model use_part_hash_model[4]; + basist::arith::arith_data_model part2_hash_index_model(basist::astc_ldr_t::PART_HASH_SIZE, true); + basist::arith::arith_data_model part3_hash_index_model(basist::astc_ldr_t::PART_HASH_SIZE, true); + + uint32_t num_part_hash_probes = 0, num_part_hash_hits = 0; + uint32_t total_dct_syms = 0, total_dpcm_syms = 0; + + basist::arith::arith_gamma_contexts m_run_len_contexts; + + image vis_img; + if (global_cfg.m_debug_images) + { + vis_img.resize(width, height); + } + + itm.start(); + + for (uint32_t by = 0; by < num_blocks_y; by++) + { + const uint32_t base_y = by * block_height; + + for (uint32_t bx = 0; bx < num_blocks_x; bx++) + { + const uint32_t base_x = bx * block_width; + + basist::astc_ldr_t::prev_block_state& prev_state = prev_block_states(bx, by); + const basist::astc_ldr_t::prev_block_state* pLeft_state = bx ? &prev_block_states(bx - 1, by) : nullptr; + const basist::astc_ldr_t::prev_block_state* pUpper_state = by ? &prev_block_states(bx, by - 1) : nullptr; + const basist::astc_ldr_t::prev_block_state* pDiag_state = (bx && by) ? &prev_block_states(bx - 1, by - 1) : nullptr; + const basist::astc_ldr_t::prev_block_state* pPred_state = pLeft_state ? pLeft_state : pUpper_state; // left or upper, or nullptr on first block + + const ldr_astc_block_encode_image_output::block_info& blk_info = enc_out.m_image_block_info(bx, by); + + uint32_t best_packed_out_block_index = blk_info.m_packed_out_block_index; + + // check for run + if ((use_run_commands) && (bx || by)) + { + const encode_block_output& blk_out = blk_info.m_out_blocks[best_packed_out_block_index]; + const astc_helpers::log_astc_block& cur_log_blk = blk_out.m_log_blk; + + const astc_helpers::log_astc_block& prev_log_blk = bx ? coded_blocks(bx - 1, by) : coded_blocks(0, by - 1); + const basist::astc_ldr_t::prev_block_state* pPrev_block_state = bx ? pLeft_state : pUpper_state; + + assert(pPrev_block_state); + + if (compare_log_blocks_for_equality(cur_log_blk, prev_log_blk)) + { + // Left or upper is exactly the same logical block, so expand the run. + cur_run_len++; + + // Accept the previous block (left or upper) as if it's been coded normally. + + coded_blocks(bx, by) = prev_log_blk; + + prev_state.m_was_solid_color = pPrev_block_state->m_was_solid_color; + prev_state.m_used_weight_dct = pPrev_block_state->m_used_weight_dct; + prev_state.m_first_endpoint_uses_bc = pPrev_block_state->m_first_endpoint_uses_bc; + prev_state.m_reused_full_cfg = true; + prev_state.m_used_part_hash = pPrev_block_state->m_used_part_hash; + prev_state.m_tm_index = pPrev_block_state->m_tm_index; + prev_state.m_base_cem_index = pPrev_block_state->m_base_cem_index; + prev_state.m_subset_index = pPrev_block_state->m_subset_index; + prev_state.m_ccs_index = pPrev_block_state->m_ccs_index; + prev_state.m_grid_size = pPrev_block_state->m_grid_size; + prev_state.m_grid_aniso = pPrev_block_state->m_grid_aniso; + + continue; + } + } + + if (cur_run_len) + { + total_runs++; + total_run_blocks += cur_run_len; + + total_header_bits += enc.encode_and_return_price((uint32_t)basist::astc_ldr_t::xuastc_mode::cMODE_RUN, mode_model); + total_header_bits += enc.put_gamma_and_return_price(cur_run_len, m_run_len_contexts); + cur_run_len = 0; + } + + total_nonrun_blocks++; + + const float ref_wmse = (float)blk_info.m_out_blocks[best_packed_out_block_index].m_sse / (total_comp_weights * (float)total_block_pixels); + const float ref_wpsnr = (ref_wmse > 1e-5f) ? 20.0f * log10f(255.0f / sqrtf(ref_wmse)) : 10000.0f; + + if ((global_cfg.m_lossy_supercompression) && (ref_wpsnr >= replacement_min_psnr) && + (!blk_info.m_out_blocks[blk_info.m_packed_out_block_index].m_log_blk.m_solid_color_flag_ldr)) + { + const float psnr_thresh = blk_info.m_strong_edges ? psnr_trial_diff_thresh_edge : psnr_trial_diff_thresh; + + float best_alt_wpsnr = 0.0f; + bool found_alternative = false; + + // Pass: 0 consider full config+part ID endpoint reuse + // Pass: 1 fall back to just full config+part ID reuse (no endpoints) + for (uint32_t pass = 0; pass < 2; pass++) + { + // Iterate through all available alternative candidates + for (uint32_t out_block_iter = 0; out_block_iter < blk_info.m_out_blocks.size(); out_block_iter++) + { + if (out_block_iter == blk_info.m_packed_out_block_index) + continue; + + const float trial_wmse = (float)blk_info.m_out_blocks[out_block_iter].m_sse / (total_comp_weights * (float)total_block_pixels); + const float trial_wpsnr = (trial_wmse > 1e-5f) ? 20.0f * log10f(255.0f / sqrtf(trial_wmse)) : 10000.0f; + + // Reject if PSNR too low + if (trial_wpsnr < (ref_wpsnr - psnr_thresh)) + continue; + + // Reject if inferior than best found so far + if (trial_wpsnr < best_alt_wpsnr) + continue; + + const astc_helpers::log_astc_block& trial_log_blk = blk_info.m_out_blocks[out_block_iter].m_log_blk; + + if (trial_log_blk.m_solid_color_flag_ldr) + continue; + + // Examine nearby neighbors + for (uint32_t i = 0; i < basist::astc_ldr_t::cMaxConfigReuseNeighbors; i++) + { + int dx = 0, dy = 0; + switch (i) + { + case 0: dx = -1; break; + case 1: dy = -1; break; + case 2: dx = -1; dy = -1; break; + default: assert(0); break; + } + + const int n_bx = bx + dx, n_by = by + dy; + if ((n_bx < 0) || (n_by < 0)) + continue; + + astc_helpers::log_astc_block& neighbor_log_blk = coded_blocks(n_bx, n_by); + + if (neighbor_log_blk.m_solid_color_flag_ldr) + continue; + + bool accept_flag = false; + if (pass == 0) + { + // prefer full config+endpoint equality first + accept_flag = compare_log_block_configs_and_endpoints(trial_log_blk, neighbor_log_blk); + } + else + { + // next check for just config equality + accept_flag = compare_log_block_configs(trial_log_blk, neighbor_log_blk); + } + + if (accept_flag) + { + best_alt_wpsnr = trial_wpsnr; + best_packed_out_block_index = out_block_iter; + found_alternative = true; + break; + } + + } // i + + } // out_block_iter + + if (found_alternative) + break; + + } // pass + + if (best_packed_out_block_index != blk_info.m_packed_out_block_index) + total_lossy_replacements++; + + } // global_cfg.m_lossy_supercompression + + const encode_block_output& blk_out = blk_info.m_out_blocks[best_packed_out_block_index]; + + astc_helpers::log_astc_block& cur_log_blk = coded_blocks(bx, by); + + cur_log_blk = blk_out.m_log_blk; + + // TODO: Add mode model context + + if (blk_out.m_trial_mode_index < 0) + { + assert(cur_log_blk.m_solid_color_flag_ldr); + + total_solid_blocks++; + + //total_header_bits += mode_model.get_price(cMODE_SOLID) + (float)(8 * (has_alpha ? 4 : 3)); + total_header_bits += mode_model.get_price((uint32_t)basist::astc_ldr_t::xuastc_mode::cMODE_SOLID); + enc.encode((uint32_t)basist::astc_ldr_t::xuastc_mode::cMODE_SOLID, mode_model); + + uint32_t cur_solid_color[4]; + for (uint32_t i = 0; i < 4; i++) + cur_solid_color[i] = blk_out.m_log_blk.m_solid_color[i] >> 8; + + uint32_t prev_solid_color[4] = { 0 }; + + const uint32_t num_comps = has_alpha ? 4 : 3; + + astc_helpers::log_astc_block* pPrev_log_blk = bx ? &coded_blocks(bx - 1, by) : (by ? &coded_blocks(bx, by - 1) : nullptr); + if (pPrev_log_blk) + { + if (pPrev_log_blk->m_solid_color_flag_ldr) + { + prev_solid_color[0] = pPrev_log_blk->m_solid_color[0] >> 8; + prev_solid_color[1] = pPrev_log_blk->m_solid_color[1] >> 8; + prev_solid_color[2] = pPrev_log_blk->m_solid_color[2] >> 8; + prev_solid_color[3] = pPrev_log_blk->m_solid_color[3] >> 8; + } + else + { +#if 0 + color_rgba prev_block_pixels[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + bool dec_status = astc_helpers::decode_block(*pPrev_log_blk, prev_block_pixels, block_width, block_height, enc_cfg.m_cem_enc_params.m_decode_mode_srgb ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8); + if (!dec_status) + { + fmt_error_printf("decode_block() failed\n"); + return false; + } + + for (uint32_t i = 0; i < total_block_pixels; i++) + { + for (uint32_t j = 0; j < num_comps; j++) + prev_solid_color[j] += prev_block_pixels[i][j]; + } + + for (uint32_t j = 0; j < num_comps; j++) + prev_solid_color[j] = (prev_solid_color[j] + (total_block_pixels / 2)) / total_block_pixels; +#endif + // Decode previous block's first CEM, use the halfway point as the predictor. + color_rgba prev_l, prev_h; + decode_endpoints(pPrev_log_blk->m_color_endpoint_modes[0], pPrev_log_blk->m_endpoints, pPrev_log_blk->m_endpoint_ise_range, prev_l, prev_h); + + prev_solid_color[0] = (prev_l[0] + prev_h[0] + 1) >> 1; + prev_solid_color[1] = (prev_l[1] + prev_h[1] + 1) >> 1; + prev_solid_color[2] = (prev_l[2] + prev_h[2] + 1) >> 1; + prev_solid_color[3] = (prev_l[3] + prev_h[3] + 1) >> 1; + } + } + + for (uint32_t i = 0; i < num_comps; i++) + { + const uint32_t delta = (cur_solid_color[i] - prev_solid_color[i]) & 0xFF; + + total_header_bits += enc.encode_and_return_price(delta, solid_color_dpcm_model[i]); + } + + // Bias the statistics towards using DCT (most common case). + prev_state.m_was_solid_color = true; + prev_state.m_used_weight_dct = enc_cfg.m_use_dct; + prev_state.m_first_endpoint_uses_bc = true; + prev_state.m_tm_index = -1; + prev_state.m_base_cem_index = astc_helpers::CEM_LDR_RGB_DIRECT; + prev_state.m_subset_index = 0; + prev_state.m_ccs_index = 0; + prev_state.m_grid_size = 0; + prev_state.m_grid_aniso = 0; + prev_state.m_reused_full_cfg = false; + prev_state.m_used_part_hash = true; // bias to true + + continue; + } + + //-------------------------------------------- + // for (uint32_t out_block_iter = 0; out_block_iter < blk_info.m_out_blocks.size(); out_block_iter++) + int full_cfg_endpoint_reuse_index = -1; + + for (uint32_t i = 0; i < basist::astc_ldr_t::cMaxConfigReuseNeighbors; i++) + { + int dx = 0, dy = 0; + switch (i) + { + case 0: dx = -1; break; + case 1: dy = -1; break; + case 2: dx = -1; dy = -1; break; + default: assert(0); break; + } + + const int n_bx = bx + dx, n_by = by + dy; + if ((n_bx < 0) || (n_by < 0)) + continue; + + astc_helpers::log_astc_block& neighbor_log_blk = coded_blocks(n_bx, n_by); + + if (neighbor_log_blk.m_solid_color_flag_ldr) + continue; + + if (compare_log_block_configs_and_endpoints(cur_log_blk, neighbor_log_blk)) + { + full_cfg_endpoint_reuse_index = i; + break; + } + } // i + //-------------------------------------------- + + if (full_cfg_endpoint_reuse_index >= 0) + { + // Reused full config, part ID and endpoint values from an immediate neighbor + total_header_bits += enc.encode_and_return_price((uint32_t)basist::astc_ldr_t::xuastc_mode::cMODE_REUSE_CFG_ENDPOINTS_LEFT + full_cfg_endpoint_reuse_index, mode_model); + + total_full_reuse_commands++; + + const basist::astc_ldr_t::prev_block_state* pReused_cfg_state = nullptr; + + switch (full_cfg_endpoint_reuse_index) + { + case 0: pReused_cfg_state = pLeft_state; break; + case 1: pReused_cfg_state = pUpper_state; break; + case 2: pReused_cfg_state = pDiag_state; break; + default: assert(0); break; + } + + if (!pReused_cfg_state) + { + assert(0); + fmt_error_printf("encoding internal failure\n"); + return false; + } + + assert(pReused_cfg_state->m_tm_index == blk_out.m_trial_mode_index); + + prev_state.m_tm_index = blk_out.m_trial_mode_index; + prev_state.m_base_cem_index = pReused_cfg_state->m_base_cem_index; + prev_state.m_subset_index = pReused_cfg_state->m_subset_index; + prev_state.m_ccs_index = pReused_cfg_state->m_ccs_index; + prev_state.m_grid_size = pReused_cfg_state->m_grid_size; + prev_state.m_grid_aniso = pReused_cfg_state->m_grid_aniso; + prev_state.m_used_part_hash = pReused_cfg_state->m_used_part_hash; + prev_state.m_reused_full_cfg = true; + + const uint32_t cur_actual_cem = cur_log_blk.m_color_endpoint_modes[0]; + + if (astc_helpers::cem_supports_bc(cur_actual_cem)) + { + prev_state.m_first_endpoint_uses_bc = astc_helpers::used_blue_contraction(cur_actual_cem, cur_log_blk.m_endpoints, cur_log_blk.m_endpoint_ise_range); + assert(prev_state.m_first_endpoint_uses_bc == pReused_cfg_state->m_first_endpoint_uses_bc); + } + } + else + { + total_raw_commands++; + + // Send mode + total_header_bits += mode_model.get_price((uint32_t)basist::astc_ldr_t::xuastc_mode::cMODE_RAW); + enc.encode((uint32_t)basist::astc_ldr_t::xuastc_mode::cMODE_RAW, mode_model); + + const uint32_t cur_actual_cem = cur_log_blk.m_color_endpoint_modes[0]; + //const bool actual_cem_supports_bc = astc_helpers::cem_supports_bc(cur_actual_cem); + const uint32_t total_endpoint_vals = astc_helpers::get_num_cem_values(cur_actual_cem); + + // DO NOT use tm.m_cem because the encoder may have selected a base+ofs variant instead. Use cur_actual_cem. + const basist::astc_ldr_t::trial_mode& tm = enc_out.m_encoder_trial_modes[blk_out.m_trial_mode_index]; + + // Check for config+part ID neighbor reuse + int neighbor_cfg_match_index = -1; + for (uint32_t i = 0; i < basist::astc_ldr_t::cMaxConfigReuseNeighbors; i++) + { + const basist::astc_ldr_t::prev_block_state* pNeighbor_state = nullptr; + + int dx = 0, dy = 0; + switch (i) + { + case 0: dx = -1; pNeighbor_state = pLeft_state; break; + case 1: dy = -1; pNeighbor_state = pUpper_state; break; + case 2: dx = -1; dy = -1; pNeighbor_state = pDiag_state; break; + default: assert(0); break; + } + + if (!pNeighbor_state) + continue; + + const int n_bx = bx + dx, n_by = by + dy; + assert((n_bx >= 0) && (n_by >= 0)); + + astc_helpers::log_astc_block& neighbor_log_blk = coded_blocks(n_bx, n_by); + + if (pNeighbor_state->m_tm_index != blk_out.m_trial_mode_index) + continue; + + if (neighbor_log_blk.m_color_endpoint_modes[0] != cur_log_blk.m_color_endpoint_modes[0]) + continue; + + if (neighbor_log_blk.m_partition_id != cur_log_blk.m_partition_id) + continue; + + assert(neighbor_log_blk.m_dual_plane == cur_log_blk.m_dual_plane); + assert(neighbor_log_blk.m_color_component_selector == cur_log_blk.m_color_component_selector); + assert(neighbor_log_blk.m_num_partitions == cur_log_blk.m_num_partitions); + assert(neighbor_log_blk.m_grid_width == cur_log_blk.m_grid_width); + assert(neighbor_log_blk.m_grid_height == cur_log_blk.m_grid_height); + assert(neighbor_log_blk.m_endpoint_ise_range == cur_log_blk.m_endpoint_ise_range); + assert(neighbor_log_blk.m_weight_ise_range == cur_log_blk.m_weight_ise_range); + + neighbor_cfg_match_index = i; + break; + } + + uint32_t reuse_full_cfg_model_index = 0; + if (pLeft_state) + reuse_full_cfg_model_index = pLeft_state->m_reused_full_cfg; + else + reuse_full_cfg_model_index = 1; + + if (pUpper_state) + reuse_full_cfg_model_index |= pUpper_state->m_reused_full_cfg ? 2 : 0; + else + reuse_full_cfg_model_index |= 2; + + if (neighbor_cfg_match_index >= 0) + { + total_header_bits += enc.encode_and_return_price(neighbor_cfg_match_index, config_reuse_model[reuse_full_cfg_model_index]); + + const basist::astc_ldr_t::prev_block_state* pReused_cfg_state = nullptr; + + switch (neighbor_cfg_match_index) + { + case 0: pReused_cfg_state = pLeft_state; break; + case 1: pReused_cfg_state = pUpper_state; break; + case 2: pReused_cfg_state = pDiag_state; break; + default: assert(0); break; + } + + if (!pReused_cfg_state) + { + assert(0); + fmt_error_printf("encoding internal failure\n"); + return false; + } + + assert(pReused_cfg_state->m_tm_index == blk_out.m_trial_mode_index); + + prev_state.m_tm_index = blk_out.m_trial_mode_index; + prev_state.m_base_cem_index = pReused_cfg_state->m_base_cem_index; + prev_state.m_subset_index = pReused_cfg_state->m_subset_index; + prev_state.m_ccs_index = pReused_cfg_state->m_ccs_index; + prev_state.m_grid_size = pReused_cfg_state->m_grid_size; + prev_state.m_grid_aniso = pReused_cfg_state->m_grid_aniso; + prev_state.m_used_part_hash = pReused_cfg_state->m_used_part_hash; + prev_state.m_reused_full_cfg = true; + + total_reuse_full_cfg_emitted++; + } + else + { + total_full_cfg_emitted++; + + total_header_bits += enc.encode_and_return_price(basist::astc_ldr_t::cMaxConfigReuseNeighbors, config_reuse_model[reuse_full_cfg_model_index]); + + // ------------------------------------------- Set TM index + { + uint32_t cem_index, subset_index, ccs_index, grid_size, grid_aniso; + + const uint_vec& submodes = separate_tm_index(block_width, block_height, enc_out.m_grouped_encoder_trial_modes, tm, + cem_index, subset_index, ccs_index, grid_size, grid_aniso); + + // TODO: sort this + uint32_t submode_index; + for (submode_index = 0; submode_index < submodes.size(); submode_index++) + if (submodes[submode_index] == (uint32_t)blk_out.m_trial_mode_index) + break; + + if (submode_index == submodes.size_u32()) + { + assert(0); + fmt_error_printf("Failed finding mode\n"); + return false; + } + + uint32_t prev_cem_index = astc_helpers::CEM_LDR_RGB_DIRECT; + uint32_t prev_subset_index = 0; + uint32_t prev_ccs_index = 0; + uint32_t prev_grid_size = 0; + uint32_t prev_grid_aniso = 0; + + if (pPred_state) + { + prev_cem_index = pPred_state->m_base_cem_index; + prev_subset_index = pPred_state->m_subset_index; + prev_ccs_index = pPred_state->m_ccs_index; + prev_grid_size = pPred_state->m_grid_size; + prev_grid_aniso = pPred_state->m_grid_aniso; + } + + const uint32_t ldrcem_index = basist::astc_ldr_t::cem_to_ldrcem_index(prev_cem_index); + + total_header_bits += cem_index_model[ldrcem_index].get_price(cem_index); + enc.encode(cem_index, cem_index_model[ldrcem_index]); + + total_header_bits += subset_index_model[prev_subset_index].get_price(subset_index); + enc.encode(subset_index, subset_index_model[prev_subset_index]); + + total_header_bits += ccs_index_model[prev_ccs_index].get_price(ccs_index); + enc.encode(ccs_index, ccs_index_model[prev_ccs_index]); + + total_header_bits += grid_size_model[prev_grid_size].get_price(grid_size); + enc.encode(grid_size, grid_size_model[prev_grid_size]); + + total_header_bits += grid_aniso_model[prev_grid_aniso].get_price(grid_aniso); + enc.encode(grid_aniso, grid_aniso_model[prev_grid_aniso]); + + if (submodes.size() > 1) + { + basist::arith::arith_data_model& submode_model = submode_models[cem_index][subset_index][ccs_index][grid_size][grid_aniso]; + if (!submode_model.get_num_data_syms()) + submode_model.init(submodes.size_u32(), true); + + total_header_bits += submode_model.get_price(submode_index); + enc.encode(submode_index, submode_model); + } + + prev_state.m_tm_index = blk_out.m_trial_mode_index; + prev_state.m_base_cem_index = cem_index; + prev_state.m_subset_index = subset_index; + prev_state.m_ccs_index = ccs_index; + prev_state.m_grid_size = grid_size; + prev_state.m_grid_aniso = grid_aniso; + prev_state.m_reused_full_cfg = false; + } + + // Send base_ofs bit if the tm is direct + if ((tm.m_cem == astc_helpers::CEM_LDR_RGB_DIRECT) || (tm.m_cem == astc_helpers::CEM_LDR_RGBA_DIRECT)) + { + const bool is_base_ofs = (cur_log_blk.m_color_endpoint_modes[0] == astc_helpers::CEM_LDR_RGB_BASE_PLUS_OFFSET) || + (cur_log_blk.m_color_endpoint_modes[0] == astc_helpers::CEM_LDR_RGBA_BASE_PLUS_OFFSET); + + total_header_bits += is_base_ofs_model.get_price(is_base_ofs); + enc.encode(is_base_ofs, is_base_ofs_model); + } + + if (tm.m_num_parts > 1) + { + // Send unique part pattern ID + astc_ldr::partitions_data* pPart_data = (tm.m_num_parts == 2) ? &enc_out.m_part_data_p2 : &enc_out.m_part_data_p3; + + const uint32_t astc_pat_index = cur_log_blk.m_partition_id; + const uint32_t unique_pat_index = pPart_data->m_part_seed_to_unique_index[astc_pat_index]; + const uint32_t total_unique_indices = pPart_data->m_total_unique_patterns; + assert(unique_pat_index < total_unique_indices); + + num_part_hash_probes++; + + uint32_t use_part_model_index = 0; + if (pLeft_state) + use_part_model_index = pLeft_state->m_used_part_hash; + else + use_part_model_index = 1; + if (pUpper_state) + use_part_model_index |= pUpper_state->m_used_part_hash ? 2 : 0; + else + use_part_model_index |= 2; + + int* pPart_hash = (tm.m_num_parts == 2) ? part2_hash : part3_hash; + + const uint32_t h = basist::astc_ldr_t::part_hash_index(unique_pat_index); + + if (pPart_hash[h] != (int)unique_pat_index) + { +#if defined(_DEBUG) || defined(DEBUG) + // sanity + for (uint32_t i = 0; i < basist::astc_ldr_t::PART_HASH_SIZE; i++) + { + assert(pPart_hash[i] != (int)unique_pat_index); + } +#endif + + total_header_bits += enc.encode_and_return_price(0, use_part_hash_model[use_part_model_index]); + total_header_bits += enc.put_truncated_binary(unique_pat_index, total_unique_indices); + + if (global_cfg.m_debug_images) + { + vis_img.fill_box(base_x, base_y, block_width, block_height, color_rgba(0, 0, 255, 255)); + } + + prev_state.m_used_part_hash = false; + } + else + { + num_part_hash_hits++; + + if (global_cfg.m_debug_images) + { + vis_img.fill_box(base_x, base_y, block_width, block_height, color_rgba(255, 0, 0, 255)); + } + + total_header_bits += enc.encode_and_return_price(1, use_part_hash_model[use_part_model_index]); + total_header_bits += enc.encode_and_return_price(h, (tm.m_num_parts == 2) ? part2_hash_index_model : part3_hash_index_model); + + prev_state.m_used_part_hash = true; + } + + pPart_hash[basist::astc_ldr_t::part_hash_index(unique_pat_index)] = unique_pat_index; + } + else + { + prev_state.m_used_part_hash = true; // bias to true + } + + } // if (neighbor_cfg_match_index >= 0) + + // ----------------------------------------- Send endpoints + const int num_endpoint_levels = astc_helpers::get_ise_levels(cur_log_blk.m_endpoint_ise_range); + const auto& endpoint_ise_to_rank = astc_helpers::g_dequant_tables.get_endpoint_tab(cur_log_blk.m_endpoint_ise_range).m_ISE_to_rank; + + uint32_t bc_model_index = 0; + if (pLeft_state) + bc_model_index = pLeft_state->m_first_endpoint_uses_bc; + else + bc_model_index = 1; + + if (pUpper_state) + bc_model_index |= pUpper_state->m_first_endpoint_uses_bc ? 2 : 0; + else + bc_model_index |= 2; + + bool endpoints_use_bc[astc_helpers::MAX_PARTITIONS] = { false }; + + if (astc_helpers::cem_supports_bc(cur_actual_cem)) + { + for (uint32_t part_iter = 0; part_iter < tm.m_num_parts; part_iter++) + { + const bool cur_uses_bc = astc_helpers::used_blue_contraction(cur_actual_cem, cur_log_blk.m_endpoints + part_iter * total_endpoint_vals, cur_log_blk.m_endpoint_ise_range); + + endpoints_use_bc[part_iter] = cur_uses_bc; + + } // part_iter + + prev_state.m_first_endpoint_uses_bc = endpoints_use_bc[0]; + } + + int best_reuse_bx = -1, best_reuse_by = -1; + uint32_t best_reuse_index = 0; + const astc_helpers::log_astc_block* pEndpoint_pred_log_blk = nullptr; + + if (endpoint_dpcm_global_enable) + { + int64_t best_trial_delta2 = INT64_MAX; + float best_trial_bits = BIG_FLOAT_VAL; + + //auto& trial_dpcm_model = dpcm_endpoint_models[cur_log_blk.m_endpoint_ise_range - astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE]; + + for (uint32_t reuse_index = 0; reuse_index < basist::astc_6x6_hdr::NUM_REUSE_XY_DELTAS; reuse_index++) + { + const int rx = (int)bx + basist::astc_6x6_hdr::g_reuse_xy_deltas[reuse_index].m_x; + const int ry = (int)by + basist::astc_6x6_hdr::g_reuse_xy_deltas[reuse_index].m_y; + if ((rx < 0) || (ry < 0) || (rx >= (int)num_blocks_x) || (ry >= (int)num_blocks_y)) + continue; + + const astc_helpers::log_astc_block* pTrial_log_blk = &coded_blocks(rx, ry); + if (pTrial_log_blk->m_solid_color_flag_ldr) + continue; + + uint8_t trial_predicted_endpoints[astc_helpers::MAX_PARTITIONS][astc_helpers::MAX_CEM_ENDPOINT_VALS] = { }; + + uint32_t part_iter; + for (part_iter = 0; part_iter < tm.m_num_parts; part_iter++) + { + const bool always_repack_flag = false; + bool blue_contraction_clamped_flag = false, base_ofs_clamped_flag = false; + + bool conv_status = basist::astc_ldr_t::convert_endpoints_across_cems( + pTrial_log_blk->m_color_endpoint_modes[0], pTrial_log_blk->m_endpoint_ise_range, pTrial_log_blk->m_endpoints, + cur_actual_cem, cur_log_blk.m_endpoint_ise_range, trial_predicted_endpoints[part_iter], + always_repack_flag, + endpoints_use_bc[part_iter], false, + blue_contraction_clamped_flag, base_ofs_clamped_flag); + + if (!conv_status) + break; + } // part_iter + + if (part_iter < tm.m_num_parts) + continue; // failed + + int64_t trial_endpoint_delta2 = 0; + for (part_iter = 0; part_iter < tm.m_num_parts; part_iter++) + { + for (uint32_t val_iter = 0; val_iter < total_endpoint_vals; val_iter++) + { + int cur_e_rank = endpoint_ise_to_rank[cur_log_blk.m_endpoints[part_iter * total_endpoint_vals + val_iter]]; + int prev_e_rank = endpoint_ise_to_rank[trial_predicted_endpoints[part_iter][val_iter]]; + + int e_delta = cur_e_rank - prev_e_rank; + + trial_endpoint_delta2 += e_delta * e_delta; + + } // val_iter + + } // part_iter + + const float N = (float)(total_endpoint_vals * tm.m_num_parts); + const float mse = (float)trial_endpoint_delta2 / N; + + // Gaussian entropy estimate - precomputed 0.5 * log2(2*pi*e) = ~2.0470956f + const float k_const = 2.0470956f; + + float bits_per_sym = 0.5f * log2f(basisu::maximum(mse, 1e-9f)) + k_const; + + bits_per_sym = clamp(bits_per_sym, 0.05f, 8.0f); + + // total est bits for this block’s endpoints + float total_est_bits = bits_per_sym * N; + + total_est_bits += endpoint_reuse_delta_model.get_price(reuse_index); + + if (total_est_bits < best_trial_bits) + { + best_trial_delta2 = trial_endpoint_delta2; + best_trial_bits = total_est_bits; + + best_reuse_bx = rx; + best_reuse_by = ry; + best_reuse_index = reuse_index; + + if (!best_trial_delta2) + break; + } + + } // reuse_index + + if (best_reuse_bx >= 0) + { + pEndpoint_pred_log_blk = &coded_blocks(best_reuse_bx, best_reuse_by); + + assert(!pEndpoint_pred_log_blk->m_solid_color_flag_ldr); + } + + } // if (endpoint_dpcm_global_enable) + + uint8_t predicted_endpoints[astc_helpers::MAX_PARTITIONS][astc_helpers::MAX_CEM_ENDPOINT_VALS] = { }; + + bool use_dpcm_endpoints = false; + + if (pEndpoint_pred_log_blk) + { + use_dpcm_endpoints = true; + + assert(cur_log_blk.m_num_partitions == tm.m_num_parts); + + for (uint32_t part_iter = 0; part_iter < tm.m_num_parts; part_iter++) + { + const bool always_repack_flag = false; + bool blue_contraction_clamped_flag = false, base_ofs_clamped_flag = false; + + bool conv_status = basist::astc_ldr_t::convert_endpoints_across_cems( + pEndpoint_pred_log_blk->m_color_endpoint_modes[0], pEndpoint_pred_log_blk->m_endpoint_ise_range, pEndpoint_pred_log_blk->m_endpoints, + cur_actual_cem, cur_log_blk.m_endpoint_ise_range, predicted_endpoints[part_iter], + always_repack_flag, + endpoints_use_bc[part_iter], false, + blue_contraction_clamped_flag, base_ofs_clamped_flag); + + if (!conv_status) + { + // In practice, should never happen + use_dpcm_endpoints = false; + break; + } + } + } + + // TODO: Decide what is cheaper, endpoint DPCM vs. raw + + if (use_dpcm_endpoints) + { + total_endpoint_bits += enc.encode_and_return_price(1, use_dpcm_endpoints_model); + + total_endpoint_bits += enc.encode_and_return_price(best_reuse_index, endpoint_reuse_delta_model); + + if (astc_helpers::cem_supports_bc(cur_actual_cem)) + { + for (uint32_t part_iter = 0; part_iter < tm.m_num_parts; part_iter++) + { + total_endpoint_bits += enc.encode_and_return_price(endpoints_use_bc[part_iter], endpoints_use_bc_models[bc_model_index]); + + } // part_iter + } + + // TODO: Perhaps separate DPCM models by CEM, entry index + auto& dpcm_model = dpcm_endpoint_models[cur_log_blk.m_endpoint_ise_range - astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE]; + + for (uint32_t part_iter = 0; part_iter < tm.m_num_parts; part_iter++) + { + for (uint32_t val_iter = 0; val_iter < total_endpoint_vals; val_iter++) + { + int cur_e_rank = endpoint_ise_to_rank[cur_log_blk.m_endpoints[part_iter * total_endpoint_vals + val_iter]]; + int prev_e_rank = endpoint_ise_to_rank[predicted_endpoints[part_iter][val_iter]]; + + int e_val = imod(cur_e_rank - prev_e_rank, num_endpoint_levels); + + total_endpoint_bits += dpcm_model.get_price(e_val); + enc.encode(e_val, dpcm_model); + + } // val_iter + + } // part_iter + + total_used_endpoint_dpcm++; + } + else + { + total_endpoint_bits += enc.encode_and_return_price(0, use_dpcm_endpoints_model); + + for (uint32_t part_iter = 0; part_iter < tm.m_num_parts; part_iter++) + { + for (uint32_t val_iter = 0; val_iter < total_endpoint_vals; val_iter++) + { + auto& model = raw_endpoint_models[cur_log_blk.m_endpoint_ise_range - astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE]; + uint32_t e_val = cur_log_blk.m_endpoints[part_iter * total_endpoint_vals + val_iter]; + + total_endpoint_bits += model.get_price(e_val); + enc.encode(e_val, model); + + } // val_iter + + } // part_iter + + total_used_endpoint_raw++; + } + + } // if (full_cfg_endpoint_reuse_index >= 0) + + // ------------------------------------ Send weights + const uint32_t total_planes = cur_log_blk.m_dual_plane ? 2 : 1; + const uint32_t total_weights = cur_log_blk.m_grid_width * cur_log_blk.m_grid_height; + + const int num_weight_levels = astc_helpers::get_ise_levels(cur_log_blk.m_weight_ise_range); + const auto& weight_ise_to_rank = astc_helpers::g_dequant_tables.get_weight_tab(cur_log_blk.m_weight_ise_range).m_ISE_to_rank; + + uint32_t use_dct_model_index = 0; + + if (enc_cfg.m_use_dct) + { + if (pLeft_state) + use_dct_model_index = pLeft_state->m_used_weight_dct; + else + use_dct_model_index = 1; + + if (pUpper_state) + use_dct_model_index |= pUpper_state->m_used_weight_dct ? 2 : 0; + else + use_dct_model_index |= 2; + } + + if (use_faster_format) + { + bool use_dct = enc_cfg.m_use_dct; + + // TODO - tune this threshold + //const uint32_t SWITCH_TO_DPCM_NUM_COEFF_THRESH = (cur_log_blk.m_grid_width * cur_log_blk.m_grid_height * 102 + 64) >> 7; + const uint32_t SWITCH_TO_DPCM_NUM_COEFF_THRESH = (cur_log_blk.m_grid_width * cur_log_blk.m_grid_height * 45 + 64) >> 7; + + if (use_dct) + { + for (uint32_t plane_iter = 0; plane_iter < total_planes; plane_iter++) + { + const basist::astc_ldr_t::dct_syms& syms = blk_out.m_packed_dct_plane_data[plane_iter]; + if (syms.m_max_coeff_mag > basist::astc_ldr_t::DCT_MAX_ARITH_COEFF_MAG) + { + use_dct = false; + break; + } + + if (syms.m_coeffs.size() > SWITCH_TO_DPCM_NUM_COEFF_THRESH) + { + use_dct = false; + break; + } + } + } + + if (enc_cfg.m_use_dct) + { + total_weight_bits += use_dct_model[use_dct_model_index].get_price(use_dct); + enc.encode(use_dct, use_dct_model[use_dct_model_index]); + } + + if (use_dct) + { + prev_state.m_used_weight_dct = true; + + total_used_dct++; + + if (total_planes > 1) + { + assert(blk_out.m_packed_dct_plane_data[0].m_num_dc_levels == blk_out.m_packed_dct_plane_data[1].m_num_dc_levels); + } + + for (uint32_t plane_iter = 0; plane_iter < total_planes; plane_iter++) + { + const basist::astc_ldr_t::dct_syms& syms = blk_out.m_packed_dct_plane_data[plane_iter]; + + if (syms.m_num_dc_levels == basist::astc_ldr_t::DCT_MEAN_LEVELS1) + mean1_bytes.push_back((uint8_t)syms.m_dc_sym); + else + { + assert(syms.m_num_dc_levels == basist::astc_ldr_t::DCT_MEAN_LEVELS0); + mean0_bits.put_bits(syms.m_dc_sym, 4); + } + + for (uint32_t i = 0; i < syms.m_coeffs.size(); i++) + { + if (syms.m_coeffs[i].m_coeff == INT16_MAX) + { + run_bytes.push_back(basist::astc_ldr_t::DCT_RUN_LEN_EOB_SYM_INDEX); + } + else + { + run_bytes.push_back((uint8_t)syms.m_coeffs[i].m_num_zeros); + + sign_bits.put_bits(syms.m_coeffs[i].m_coeff < 0, 1); + + assert((syms.m_coeffs[i].m_coeff != 0) && (iabs(syms.m_coeffs[i].m_coeff) <= 255)); + + coeff_bytes.push_back((uint8_t)(iabs(syms.m_coeffs[i].m_coeff) - 1)); + } + } + + } // plane_iter + } + else + { + total_used_weight_dpcm++; + + for (uint32_t plane_iter = 0; plane_iter < total_planes; plane_iter++) + { + int prev_w = num_weight_levels / 2; + + for (uint32_t weight_iter = 0; weight_iter < total_weights; weight_iter++) + { + int ise_w = cur_log_blk.m_weights[plane_iter + weight_iter * total_planes]; + int w = weight_ise_to_rank[ise_w]; + + int w_to_code = w; + w_to_code = imod(w - prev_w, num_weight_levels); + + prev_w = w; + + if (num_weight_levels <= 4) + weight2_bits.put_bits((uint8_t)w_to_code, 2); + else if (num_weight_levels <= 8) + weight3_bits.put_bits((uint8_t)w_to_code, 4); + else if (num_weight_levels <= 16) + weight4_bits.put_bits((uint8_t)w_to_code, 4); + else + weight8_bits.push_back((uint8_t)w_to_code); + + } // weight_iter + + } // plane_iter + } + } + else + { + float total_dpcm_bits = 0.0f, total_dct_bits = 0.0f; + const float FORBID_DCT_BITS = 1e+8f; + + for (uint32_t plane_iter = 0; plane_iter < total_planes; plane_iter++) + { + int prev_w = num_weight_levels / 2; + + for (uint32_t weight_iter = 0; weight_iter < total_weights; weight_iter++) + { + const auto& model = raw_weight_models[cur_log_blk.m_weight_ise_range - astc_helpers::FIRST_VALID_WEIGHT_ISE_RANGE]; + + int ise_w = cur_log_blk.m_weights[plane_iter + weight_iter * total_planes]; + int w = weight_ise_to_rank[ise_w]; + + int w_to_code = w; + w_to_code = imod(w - prev_w, num_weight_levels); + + prev_w = w; + + total_dpcm_bits += model.get_price(w_to_code); + + } // weight_iter + + } // plane_iter + + if (enc_cfg.m_use_dct) + { + for (uint32_t plane_iter = 0; plane_iter < total_planes; plane_iter++) + { + const basist::astc_ldr_t::dct_syms& syms = blk_out.m_packed_dct_plane_data[plane_iter]; + if (syms.m_max_coeff_mag > basist::astc_ldr_t::DCT_MAX_ARITH_COEFF_MAG) + { + total_dct_bits = FORBID_DCT_BITS; + break; + } + } + + if (total_dct_bits < FORBID_DCT_BITS) + { + for (uint32_t plane_iter = 0; plane_iter < total_planes; plane_iter++) + { + const basist::astc_ldr_t::dct_syms& syms = blk_out.m_packed_dct_plane_data[plane_iter]; + + assert((syms.m_num_dc_levels == basist::astc_ldr_t::DCT_MEAN_LEVELS0) || (syms.m_num_dc_levels == basist::astc_ldr_t::DCT_MEAN_LEVELS1)); + + total_dct_bits += weight_mean_models[(syms.m_num_dc_levels == basist::astc_ldr_t::DCT_MEAN_LEVELS1) ? 1 : 0].get_price(syms.m_dc_sym); + + for (uint32_t i = 0; i < syms.m_coeffs.size(); i++) + { + if (syms.m_coeffs[i].m_coeff == INT16_MAX) + { + total_dct_bits += dct_run_len_model.get_price(basist::astc_ldr_t::DCT_RUN_LEN_EOB_SYM_INDEX); + } + else + { + assert(syms.m_coeffs[i].m_num_zeros < basist::astc_ldr_t::DCT_RUN_LEN_EOB_SYM_INDEX); + + total_dct_bits += dct_run_len_model.get_price(syms.m_coeffs[i].m_num_zeros); + + total_dct_bits += 1.0f; // sign bit + assert((syms.m_coeffs[i].m_coeff != 0) && (iabs(syms.m_coeffs[i].m_coeff) <= 255)); + total_dct_bits += dct_coeff_mag.get_price(iabs(syms.m_coeffs[i].m_coeff) - 1); + } + } // i + } // plane_iter + } + } + + // TODO: Check if any DCT coeff overflows 8-bit mags, switch to DPCM. (In practice, not needed.) + bool use_dct = false; + if ((enc_cfg.m_use_dct) && + (total_dct_bits < FORBID_DCT_BITS) && + ((total_dct_bits + use_dct_model[use_dct_model_index].get_price(1)) <= (total_dpcm_bits + use_dct_model[use_dct_model_index].get_price(0)))) + { + use_dct = true; + } + + if (enc_cfg.m_use_dct) + { + total_weight_bits += use_dct_model[use_dct_model_index].get_price(use_dct); + enc.encode(use_dct, use_dct_model[use_dct_model_index]); + } + + if (use_dct) + { + prev_state.m_used_weight_dct = true; + + total_used_dct++; + + if (total_planes > 1) + { + assert(blk_out.m_packed_dct_plane_data[0].m_num_dc_levels == blk_out.m_packed_dct_plane_data[1].m_num_dc_levels); + } + + for (uint32_t plane_iter = 0; plane_iter < total_planes; plane_iter++) + { + const basist::astc_ldr_t::dct_syms& syms = blk_out.m_packed_dct_plane_data[plane_iter]; + + total_weight_bits += enc.encode_and_return_price(syms.m_dc_sym, weight_mean_models[(syms.m_num_dc_levels == basist::astc_ldr_t::DCT_MEAN_LEVELS1) ? 1 : 0]); + + for (uint32_t i = 0; i < syms.m_coeffs.size(); i++) + { + if (syms.m_coeffs[i].m_coeff == INT16_MAX) + { + total_weight_bits += enc.encode_and_return_price(basist::astc_ldr_t::DCT_RUN_LEN_EOB_SYM_INDEX, dct_run_len_model); + + total_dct_syms++; + } + else + { + total_weight_bits += enc.encode_and_return_price(syms.m_coeffs[i].m_num_zeros, dct_run_len_model); + + total_dct_syms++; + + enc.put_bit(syms.m_coeffs[i].m_coeff < 0); + total_weight_bits += 1.0f; + + assert((syms.m_coeffs[i].m_coeff != 0) && (iabs(syms.m_coeffs[i].m_coeff) <= 255)); + total_weight_bits += enc.encode_and_return_price(iabs(syms.m_coeffs[i].m_coeff) - 1, dct_coeff_mag); + + total_dct_syms++; + } + } + + } // plane_iter + } + else + { + total_used_weight_dpcm++; + auto& model = raw_weight_models[cur_log_blk.m_weight_ise_range - astc_helpers::FIRST_VALID_WEIGHT_ISE_RANGE]; + + for (uint32_t plane_iter = 0; plane_iter < total_planes; plane_iter++) + { + int prev_w = num_weight_levels / 2; + + for (uint32_t weight_iter = 0; weight_iter < total_weights; weight_iter++) + { + int ise_w = cur_log_blk.m_weights[plane_iter + weight_iter * total_planes]; + int w = weight_ise_to_rank[ise_w]; + + int w_to_code = w; + w_to_code = imod(w - prev_w, num_weight_levels); + + prev_w = w; + + total_weight_bits += model.get_price(w_to_code); + enc.encode(w_to_code, model); + + total_dpcm_syms++; + + } // weight_iter + + } // plane_iter + } + + } // use_faster_format + + } // bx + + if (cur_run_len) + { + total_runs++; + total_run_blocks += cur_run_len; + + total_header_bits += enc.encode_and_return_price((uint32_t)basist::astc_ldr_t::xuastc_mode::cMODE_RUN, mode_model); + total_header_bits += enc.put_gamma_and_return_price(cur_run_len, m_run_len_contexts); + cur_run_len = 0; + } + + } // by + + enc.put_bits(basist::astc_ldr_t::FINAL_SYNC_MARKER, basist::astc_ldr_t::FINAL_SYNC_MARKER_BITS); + + enc.flush(); + + if (global_cfg.m_debug_output) + { + fmt_debug_printf("Encoding time: {} secs\n", itm.get_elapsed_secs()); + } + + if (global_cfg.m_debug_images) + { + save_png(global_cfg.m_debug_file_prefix + "vis_img.png", vis_img); + } + + if ((global_cfg.m_debug_images) || (global_cfg.m_debug_output)) + { + image coded_img(width, height); + + vector2D phys_blocks(num_blocks_x, num_blocks_y); + + for (uint32_t by = 0; by < num_blocks_y; by++) + { + for (uint32_t bx = 0; bx < num_blocks_x; bx++) + { + const astc_helpers::log_astc_block& log_blk = coded_blocks(bx, by); + + color_rgba block_pixels[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + + bool status = astc_helpers::decode_block(log_blk, block_pixels, block_width, block_height, enc_cfg.m_cem_enc_params.m_decode_mode_srgb ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8); + if (!status) + { + fmt_error_printf("astc_helpers::decode_block() failed\n"); + return false; + } + + // Be positive the logical block can be unpacked correctly as XUASTC LDR. + color_rgba block_pixels_alt[astc_ldr::ASTC_LDR_MAX_BLOCK_PIXELS]; + bool status_alt = astc_helpers::decode_block_xuastc_ldr(log_blk, block_pixels_alt, block_width, block_height, enc_cfg.m_cem_enc_params.m_decode_mode_srgb ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8); + if (!status_alt) + { + fmt_error_printf("astc_helpers::decode_block_xuastc_ldr() failed\n"); + return false; + } + + if (memcmp(block_pixels, block_pixels_alt, sizeof(color_rgba) * block_width * block_height) != 0) + { + fmt_error_printf("astc_helpers::decode_block_xuastc_ldr() decode pixel mismatch\n"); + return false; + } + + coded_img.set_block_clipped(block_pixels, bx * block_width, by * block_height, block_width, block_height); + + } // bx + + } //by + + if (global_cfg.m_debug_images) + save_png(global_cfg.m_debug_file_prefix + "coded_img.png", coded_img); + + if (global_cfg.m_debug_output) + { + debug_printf("Orig image vs. coded img:\n"); + print_image_metrics(orig_img, coded_img); + } + } + + const uint64_t comp_data_size = enc.get_data_buf().size(); + if (comp_data_size > UINT32_MAX) + return false; + + uint8_vec suffix_bytes; + + if (use_faster_format) + { +#if !BASISD_SUPPORT_KTX2_ZSTD + fmt_error_printf("Full ZStd syntax not supported in this build (set BASISD_SUPPORT_KTX2_ZSTD to 1)\n"); + return false; +#else + suffix_bytes.reserve(8192); + + mean0_bits.flush(); + sign_bits.flush(); + weight2_bits.flush(); + weight3_bits.flush(); + weight4_bits.flush(); + + const uint32_t zstd_level = 9; + + uint8_vec comp_mean0, comp_mean1, comp_run, comp_coeff, comp_weight2, comp_weight3, comp_weight4, comp_weight8; + + if (!zstd_compress(mean0_bits.get_bytes().data(), mean0_bits.get_bytes().size(), comp_mean0, zstd_level)) + return false; + if (!zstd_compress(mean1_bytes.data(), mean1_bytes.size(), comp_mean1, zstd_level)) + return false; + if (!zstd_compress(run_bytes.data(), run_bytes.size(), comp_run, zstd_level)) + return false; + if (!zstd_compress(coeff_bytes.data(), coeff_bytes.size(), comp_coeff, zstd_level)) + return false; + if (!zstd_compress(weight2_bits.get_bytes().data(), weight2_bits.get_bytes().size(), comp_weight2, zstd_level)) + return false; + if (!zstd_compress(weight3_bits.get_bytes().data(), weight3_bits.get_bytes().size(), comp_weight3, zstd_level)) + return false; + if (!zstd_compress(weight4_bits.get_bytes().data(), weight4_bits.get_bytes().size(), comp_weight4, zstd_level)) + return false; + if (!zstd_compress(weight8_bits.data(), weight8_bits.size(), comp_weight8, zstd_level)) + return false; + + hdr.m_flags = (uint8_t)basist::astc_ldr_t::xuastc_ldr_syntax::cHybridArithZStd; + + hdr.m_arith_bytes_len = (uint32_t)comp_data_size; + hdr.m_mean0_bits_len = (uint32_t)comp_mean0.size(); + hdr.m_mean1_bytes_len = (uint32_t)comp_mean1.size(); + hdr.m_run_bytes_len = (uint32_t)comp_run.size(); + hdr.m_coeff_bytes_len = (uint32_t)comp_coeff.size(); + hdr.m_sign_bits_len = (uint32_t)sign_bits.get_bytes().size(); + hdr.m_weight2_bits_len = (uint32_t)comp_weight2.size(); + hdr.m_weight3_bits_len = (uint32_t)comp_weight3.size(); + hdr.m_weight4_bits_len = (uint32_t)comp_weight4.size(); + hdr.m_weight8_bytes_len = (uint32_t)comp_weight8.size(); + + suffix_bytes.append(comp_mean0); + suffix_bytes.append(comp_mean1); + suffix_bytes.append(comp_run); + suffix_bytes.append(comp_coeff); + suffix_bytes.append(sign_bits.get_bytes()); + suffix_bytes.append(comp_weight2); + suffix_bytes.append(comp_weight3); + suffix_bytes.append(comp_weight4); + suffix_bytes.append(comp_weight8); + + if (global_cfg.m_debug_output) + { + fmt_debug_printf("Zstd compressed sizes:\n"); + fmt_debug_printf(" Mean0 bytes: {} comp size: {}\n", (uint64_t)mean0_bits.get_bytes().size(), (uint64_t)comp_mean0.size()); + fmt_debug_printf(" Mean1 bytes: {} comp size: {}\n", (uint64_t)mean1_bytes.size(), (uint64_t)comp_mean1.size()); + fmt_debug_printf(" Run bytes: {} comp size: {}\n", (uint64_t)run_bytes.size(), (uint64_t)comp_run.size()); + fmt_debug_printf(" Coeff bytes: {} comp size: {}\n", (uint64_t)coeff_bytes.size(), (uint64_t)comp_coeff.size()); + fmt_debug_printf(" Sign bytes: {}\n", (uint64_t)sign_bits.get_bytes().size()); + fmt_debug_printf(" Weight2 bytes: {} comp size: {}\n", (uint64_t)weight2_bits.get_bytes().size(), (uint64_t)comp_weight2.size()); + fmt_debug_printf(" Weight3 bytes: {} comp size: {}\n", (uint64_t)weight3_bits.get_bytes().size(), (uint64_t)comp_weight3.size()); + fmt_debug_printf(" Weight4 bytes: {} comp size: {}\n", (uint64_t)weight4_bits.get_bytes().size(), (uint64_t)comp_weight4.size()); + fmt_debug_printf(" Weight8 bytes: {} comp size: {}\n", (uint64_t)weight8_bits.size(), (uint64_t)comp_weight8.size()); + } +#endif + } + + assert(comp_data.size() == 0); + if (use_faster_format) + { + comp_data.resize(sizeof(hdr)); + memcpy(comp_data.data(), &hdr, sizeof(hdr)); + } + else + { + comp_data.push_back((uint8_t)basist::astc_ldr_t::xuastc_ldr_syntax::cFullArith); + } + + comp_data.append(enc.get_data_buf()); + + comp_data.append(suffix_bytes); + + if (comp_data.size() > UINT32_MAX) + return false; + + if (global_cfg.m_debug_output) + { + fmt_debug_printf("Total blocks: {}\n", total_blocks); + fmt_debug_printf("Total lossy replacements made by supercompression layer: {} {3.2}%\n", total_lossy_replacements, (float)total_lossy_replacements * 100.0f / (float)total_blocks); + fmt_debug_printf("Total runs: {}, total run blocks: {} {3.2}%\n", total_runs, total_run_blocks, (float)total_run_blocks * 100.0f / (float)total_blocks); + fmt_debug_printf("Total blocks coded (not inside runs): {} {3.2}%\n", total_nonrun_blocks, (float)total_nonrun_blocks * 100.0f / (float)total_blocks); + fmt_debug_printf("num_part_hash_probes: {}, num_part_hash_hits: {} {3.2}%\n", num_part_hash_probes, num_part_hash_hits, num_part_hash_probes ? ((float)num_part_hash_hits * 100.0f / (float)num_part_hash_probes) : 0); + fmt_debug_printf("Total DCT syms: {}, DPCM syms: {}\n", total_dct_syms, total_dpcm_syms); + + const uint32_t total_non_void_extent_blocks = total_blocks - total_solid_blocks; + + fmt_debug_printf("Total blocks using void extent: {} {3.2}%\n", + total_solid_blocks, (float)total_solid_blocks * 100.0f / (float)total_blocks); + + fmt_debug_printf("Total non void-extent blocks: {} {3.2}%\n", + total_non_void_extent_blocks, (float)total_non_void_extent_blocks * 100.0f / (float)total_blocks); + + fmt_debug_printf("Total full cfg+part ID+endpoint reuse commands: {} {3.2}%\n", + total_full_reuse_commands, (float)total_full_reuse_commands * 100.0f / (float)total_blocks); + + fmt_debug_printf("Total raw commands: {} {3.2}%\n", + total_raw_commands, (float)total_raw_commands * 100.0f / (float)total_blocks); + + fmt_debug_printf("Total reuse cfg+part ID emitted: {} {3.2}%, Total full cfg emitted: {} {3.2}%\n", + total_reuse_full_cfg_emitted, (float)total_reuse_full_cfg_emitted * 100.0f / (float)total_blocks, + total_full_cfg_emitted, (float)total_full_cfg_emitted * 100.0f / (float)total_blocks); + + fmt_debug_printf("Total coded endpoints using DPCM: {} {3.2}%\n", + total_used_endpoint_dpcm, (float)total_used_endpoint_dpcm * 100.0f / (float)total_non_void_extent_blocks); + + fmt_debug_printf("Total coded endpoints using RAW: {} {3.2}%\n", + total_used_endpoint_raw, (float)total_used_endpoint_raw * 100.0f / (float)total_non_void_extent_blocks); + + fmt_debug_printf("Total coded blocks using weight DCT: {} {3.2}%, total blocks using weight DPCM: {} {3.2}%\n", + total_used_dct, (float)total_used_dct * 100.0f / total_non_void_extent_blocks, + total_used_weight_dpcm, (float)total_used_weight_dpcm * 100.0f / (float)total_non_void_extent_blocks); + + fmt_debug_printf("Total header bits: {} bytes: {}, bpp: {}, bits per non-void extent block: {}\nTotal endpoint bits: {}, bytes: {}, bpp: {}, bits per non-void extent block: {}\nTotal weight bits: {}, bytes: {}, bpp: {}, bits per non-void extent block: {}\nTotal_bits: {} bytes: {}, bpp {}, bits per non-void extent block: {}\n", + total_header_bits, total_header_bits / 8.0f, total_header_bits / (double)total_pixels, total_header_bits / (double)total_non_void_extent_blocks, + total_endpoint_bits, total_endpoint_bits / 8.0f, total_endpoint_bits / (double)total_pixels, total_endpoint_bits / (double)total_non_void_extent_blocks, + total_weight_bits, total_weight_bits / 8.0f, total_weight_bits / (double)total_pixels, total_weight_bits / (double)total_non_void_extent_blocks, + total_header_bits + total_endpoint_bits + total_weight_bits, + (total_header_bits + total_endpoint_bits + total_weight_bits) / 8.0f, + (total_header_bits + total_endpoint_bits + total_weight_bits) / (double)total_pixels, + (total_header_bits + total_endpoint_bits + total_weight_bits) / (double)total_non_void_extent_blocks); + + fmt_debug_printf("Compressed to {} bytes, {3.3}bpp\n\n", comp_data.size_u32(), ((float)comp_data.size() * 8.0f) / (float)total_pixels); + +#if 0 + for (uint32_t i = 0; i < 4; i++) + { + solid_color_dpcm_model[i].print_prices(fmt_string("solid_color_dpcm_model[{}]:\n\n", i).c_str()); + } +#endif + } + + return true; +} + +void encoder_init() +{ + if (g_initialized) + return; + + g_initialized = true; +} + +void deblock_filter(uint32_t filter_block_width, uint32_t filter_block_height, const image& src_img, image& dst_img, bool stronger_filtering, int SKIP_THRESH) +{ + image temp_img(src_img); + + for (int y = 0; y < (int)src_img.get_height(); y++) + { + for (int x = filter_block_width; x < (int)src_img.get_width(); x += filter_block_width) + { + color_rgba ll(src_img.get_clamped(x - 2, y)); + color_rgba l(src_img.get_clamped(x - 1, y)); + color_rgba r(src_img.get_clamped(x, y)); + color_rgba rr(src_img.get_clamped(x + 1, y)); + + if (SKIP_THRESH < 256) + { + bool skip_flag = false; + for (uint32_t c = 0; c < 4; c++) + { + int delta = iabs((int)l[c] - (int)r[c]); + if (delta > SKIP_THRESH) + { + skip_flag = true; + break; + } + } + + if (skip_flag) + continue; + } + + color_rgba ml, mr; + for (uint32_t c = 0; c < 4; c++) + { + if (stronger_filtering) + { + ml[c] = (3 * l[c] + 2 * r[c] + ll[c] + 3) / 6; + mr[c] = (3 * r[c] + 2 * l[c] + rr[c] + 3) / 6; + } + else + { + ml[c] = (5 * l[c] + 2 * r[c] + ll[c] + 4) / 8; + mr[c] = (5 * r[c] + 2 * l[c] + rr[c] + 4) / 8; + } + } + + temp_img.set_clipped(x - 1, y, ml); + temp_img.set_clipped(x, y, mr); + + } // x + + } // y + + dst_img = temp_img; + + for (int x = 0; x < (int)temp_img.get_width(); x++) + { + for (int y = filter_block_height; y < (int)temp_img.get_height(); y += filter_block_height) + { + color_rgba uu(temp_img.get_clamped(x, y - 2)); + color_rgba u(temp_img.get_clamped(x, y - 1)); + color_rgba d(temp_img.get_clamped(x, y)); + color_rgba dd(temp_img.get_clamped(x, y + 1)); + + if (SKIP_THRESH < 256) + { + bool skip_flag = false; + for (uint32_t c = 0; c < 4; c++) + { + int delta = iabs((int)u[c] - (int)d[c]); + if (delta > SKIP_THRESH) + { + skip_flag = true; + break; + } + } + + if (skip_flag) + continue; + } + + color_rgba mu, md; + for (uint32_t c = 0; c < 4; c++) + { + if (stronger_filtering) + { + mu[c] = (3 * u[c] + 2 * d[c] + uu[c] + 3) / 6; + md[c] = (3 * d[c] + 2 * u[c] + dd[c] + 3) / 6; + } + else + { + mu[c] = (5 * u[c] + 2 * d[c] + uu[c] + 4) / 8; + md[c] = (5 * d[c] + 2 * u[c] + dd[c] + 4) / 8; + } + } + + dst_img.set_clipped(x, y - 1, mu); + dst_img.set_clipped(x, y, md); + + } // x + + } // y +} + +} // namespace astc_ldr +} // namespace basisu diff --git a/external/basis_universal/encoder/basisu_astc_ldr_encode.h b/external/basis_universal/encoder/basisu_astc_ldr_encode.h new file mode 100644 index 0000000000..c9e0e836d0 --- /dev/null +++ b/external/basis_universal/encoder/basisu_astc_ldr_encode.h @@ -0,0 +1,118 @@ +// File: basisu_astc_ldr_encode.cpp +#pragma once +#include "basisu_enc.h" +#include "../transcoder/basisu_astc_helpers.h" + +namespace basisu { +namespace astc_ldr { + + void encoder_init(); + + const int EFFORT_LEVEL_MIN = 0, EFFORT_LEVEL_MAX = 10, EFFORT_LEVEL_DEF = 3; + const int DCT_QUALITY_MIN = 1, DCT_QUALITY_MAX = 100; + + struct astc_ldr_encode_config + { + astc_ldr_encode_config() + { + } + + void clear() + { + *this = astc_ldr_encode_config(); + } + + // ASTC LDR block dimensions. Must be a valid ASTC block dimension. Any supported from 4x4-12x12, including unequal dimensions. + uint32_t m_astc_block_width = 6; + uint32_t m_astc_block_height = 6; + + // If true, the encoder assumes all ASTC blocks will be decompressed using sRGB vs. LDR8 mode. This corresponds to astcenc's -cs vs. cl color profiles. + // This should match how the texture is later decoded by the GPU for maximum quality. This bit is stored into the output file. + bool m_astc_decode_mode_srgb = true; + + // If true, trade off some compression (3-10%) for faster decompression. + // If false, favor highest compression, but slower decompression. + //bool m_use_faster_format = true; + + basist::astc_ldr_t::xuastc_ldr_syntax m_compressed_syntax = basist::astc_ldr_t::xuastc_ldr_syntax::cFullArith; + + // Encoder CPU effort vs. quality. [0,10], higher=better. + // 0=extremely fast but very brittle (no subsets) + // 1=first 2 subset effort level + // 10=extremely high CPU requirements. + uint32_t m_effort_level = 3; + + // Weight grid DCT quality [1,100] - higher=better quality (JPEG-style). + float m_dct_quality = 85; + + // true=use weight grid DCT, false=always use DPCM + bool m_use_dct = false; + + // true=use lossy supercompression, false=supercompression stage is always lossless. + bool m_lossy_supercompression = false; + + // Channel weights used to compute RGBA colorspace L2 errors. Must be >= 1. + uint32_t m_comp_weights[4] = { 1, 1, 1, 1 }; + + // Lossy supercompression stage parameters for RGB vs. RGBA image inputs. + // (Bounded RDO - explictly not Lagrangian.) + float m_replacement_min_psnr = 35.0f; // if the block's base PSNR is less than this, it cannot be changed + float m_psnr_trial_diff_thresh = 1.5f; // reject candidates if their PSNR is lower than m_replacement_min_psnr-m_psnr_trial_diff_thresh + float m_psnr_trial_diff_thresh_edge = 1.0f; // edge variant + + // Lossy supercompression settings - alpha texture variants + float m_replacement_min_psnr_alpha = 38.0f; + float m_psnr_trial_diff_thresh_alpha = .75f; + float m_psnr_trial_diff_thresh_edge_alpha = .5f; + + // If true, try encoding blurred blocks, in addition to unblurred, for superpass 1 and 2. + // Higher quality, but massively slower and not yet tuned/refined. + bool m_block_blurring_p1 = false, m_block_blurring_p2 = false; + + // If true, no matter what effort level subset usage will be disabled. + bool m_force_disable_subsets = false; + + // If true, no matter what effort level RGB dual plane usage will be disabled. + bool m_force_disable_rgb_dual_plane = false; + + bool m_debug_images = false; + bool m_debug_output = false; + + std::string m_debug_file_prefix; + + void debug_print() const + { + fmt_debug_printf("ASTC block dimensions: {}x{}\n", m_astc_block_width, m_astc_block_height); + fmt_debug_printf("ASTC decode profile mode sRGB: {}\n", m_astc_decode_mode_srgb); + fmt_debug_printf("Syntax: {}\n", (uint32_t)m_compressed_syntax); + fmt_debug_printf("Effort level: {}\n", m_effort_level); + fmt_debug_printf("Use DCT: {}\n", m_use_dct); + fmt_debug_printf("DCT quality level (1-100): {}\n", m_dct_quality); + fmt_debug_printf("Comp weights: {} {} {} {}\n", m_comp_weights[0], m_comp_weights[1], m_comp_weights[2], m_comp_weights[3]); + fmt_debug_printf("Block blurring: {} {}\n", m_block_blurring_p1, m_block_blurring_p2); + fmt_debug_printf("Force disable subsets: {}\n", m_force_disable_subsets); + fmt_debug_printf("Force disable RGB dual plane: {}\n", m_force_disable_rgb_dual_plane); + + fmt_debug_printf("\nLossy supercompression: {}\n", m_lossy_supercompression); + fmt_debug_printf("m_replacement_min_psnr: {}\n", m_replacement_min_psnr); + fmt_debug_printf("m_psnr_trial_diff_thresh: {}\n", m_psnr_trial_diff_thresh); + fmt_debug_printf("m_psnr_trial_diff_thresh_edge: {}\n", m_psnr_trial_diff_thresh_edge); + fmt_debug_printf("m_replacement_min_psnr_alpha: {}\n", m_replacement_min_psnr_alpha); + fmt_debug_printf("m_psnr_trial_diff_thresh_alpha: {}\n", m_psnr_trial_diff_thresh_alpha); + fmt_debug_printf("m_psnr_trial_diff_thresh_edge_alpha: {}\n", m_psnr_trial_diff_thresh_edge_alpha); + + fmt_debug_printf("m_debug_images: {}\n", m_debug_images); + } + }; + + bool compress_image( + const image& orig_img, uint8_vec &comp_data, vector2D& coded_blocks, + const astc_ldr_encode_config& global_cfg, + job_pool& job_pool); + + void deblock_filter(uint32_t filter_block_width, uint32_t filter_block_height, const image& src_img, image& dst_img, bool stronger_filtering = false, int SKIP_THRESH = 24); + +} // namespace astc_ldr +} // namespace basisu + + diff --git a/external/basis_universal/encoder/basisu_backend.cpp b/external/basis_universal/encoder/basisu_backend.cpp index 8096e25834..adc791eff7 100644 --- a/external/basis_universal/encoder/basisu_backend.cpp +++ b/external/basis_universal/encoder/basisu_backend.cpp @@ -1,5 +1,5 @@ // basisu_backend.cpp -// Copyright (C) 2019-2024 Binomial LLC. All Rights Reserved. +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -54,7 +54,7 @@ namespace basisu m_pFront_end = pFront_end; m_params = params; m_slices = slice_descs; - + debug_printf("basisu_backend::Init: Slices: %u, ETC1S: %u, EndpointRDOQualityThresh: %f, SelectorRDOQualityThresh: %f\n", m_slices.size(), params.m_etc1s, @@ -196,7 +196,7 @@ namespace basisu m_endpoint_remap_table_old_to_new = reorderer.get_remap_table(); } - // For endpoints, old_to_new[] may not be bijective! + // For endpoints, old_to_new[] may not be bijective! // Some "old" entries may be unused and don't get remapped into the "new" array. m_old_endpoint_was_used.clear(); @@ -220,13 +220,13 @@ namespace basisu } // slice_index debug_printf("basisu_backend::reoptimize_and_sort_endpoints_codebook: First old entry index: %u\n", first_old_entry_index); - + m_new_endpoint_was_used.clear(); m_new_endpoint_was_used.resize(r.get_total_endpoint_clusters()); m_endpoint_remap_table_new_to_old.clear(); m_endpoint_remap_table_new_to_old.resize(r.get_total_endpoint_clusters()); - + // Set unused entries in the new array to point to the first used entry in the old array. m_endpoint_remap_table_new_to_old.set_all(first_old_entry_index); @@ -235,7 +235,7 @@ namespace basisu if (m_old_endpoint_was_used[old_index]) { const uint32_t new_index = m_endpoint_remap_table_old_to_new[old_index]; - + m_new_endpoint_was_used[new_index] = true; m_endpoint_remap_table_new_to_old[new_index] = old_index; @@ -612,7 +612,7 @@ namespace basisu sort_selector_codebook(); check_for_valid_cr_blocks(); - + debug_printf("Elapsed time: %3.3f secs\n", tm.get_elapsed_secs()); } @@ -666,14 +666,14 @@ namespace basisu if (m_params.m_debug_images) { image gi_unpacked; - gi.unpack(gi_unpacked); + gi.unpack(gi_unpacked, false); char buf[256]; -#ifdef _WIN32 +#ifdef _WIN32 sprintf_s(buf, sizeof(buf), "basisu_backend_slice_%u.png", slice_index); #else snprintf(buf, sizeof(buf), "basisu_backend_slice_%u.png", slice_index); -#endif +#endif save_png(buf, gi_unpacked); } @@ -682,7 +682,7 @@ namespace basisu //uint32_t g_color_delta_hist[255 * 3 + 1]; //uint32_t g_color_delta_bad_hist[255 * 3 + 1]; - + // TODO: Split this into multiple methods. bool basisu_backend::encode_image() { @@ -718,7 +718,7 @@ namespace basisu const int COLOR_DELTA_THRESH = 8; const int SEL_DIFF_THRESHOLD = 11; - + for (uint32_t slice_index = 0; slice_index < m_slices.size(); slice_index++) { //const int prev_frame_slice_index = is_video ? find_video_frame(slice_index, -1) : -1; @@ -764,7 +764,7 @@ namespace basisu } // block_x } // block_y - + for (uint32_t block_y = 0; block_y < num_blocks_y; block_y++) { for (uint32_t block_x = 0; block_x < num_blocks_x; block_x++) @@ -842,7 +842,7 @@ namespace basisu const uint32_t cur_inten5 = etc_blk.get_inten_table(0); const etc1_endpoint_palette_entry& cur_endpoints = m_endpoint_palette[m.m_endpoint_index]; - + if (cur_err) { const float endpoint_remap_thresh = maximum(1.0f, m_params.m_endpoint_rdo_quality_thresh); @@ -858,7 +858,7 @@ namespace basisu int best_trial_idx = 0; etc_block trial_etc_blk(etc_blk); - + const int search_dist = minimum(iabs(endpoint_delta) - 1, MAX_ENDPOINT_SEARCH_DIST); for (int d = -search_dist; d < search_dist; d++) { @@ -876,7 +876,7 @@ namespace basisu continue; const etc1_endpoint_palette_entry& p = m_endpoint_palette[m_endpoint_remap_table_new_to_old[trial_idx]]; - + if (m_params.m_compression_level <= 1) { if (p.m_inten5 > cur_inten5) @@ -886,7 +886,7 @@ namespace basisu int delta_g = iabs(cur_endpoints.m_color5.g - p.m_color5.g); int delta_b = iabs(cur_endpoints.m_color5.b - p.m_color5.b); int color_delta = delta_r + delta_g + delta_b; - + if (color_delta > COLOR_DELTA_THRESH) continue; } @@ -924,7 +924,7 @@ namespace basisu const int64_t initial_best_trial_err = INT64_MAX; int64_t best_trial_err = initial_best_trial_err; int best_trial_idx = 0; - + const int search_dist = minimum(iabs(endpoint_delta) - 1, MAX_ENDPOINT_SEARCH_DIST); for (int d = -search_dist; d < search_dist; d++) { @@ -942,7 +942,7 @@ namespace basisu continue; const etc1_endpoint_palette_entry& p = m_endpoint_palette[m_endpoint_remap_table_new_to_old[trial_idx]]; - + if (m_params.m_compression_level <= 1) { if (p.m_inten5 > cur_inten5) @@ -952,7 +952,7 @@ namespace basisu int delta_g = iabs(cur_endpoints.m_color5.g - p.m_color5.g); int delta_b = iabs(cur_endpoints.m_color5.b - p.m_color5.b); int color_delta = delta_r + delta_g + delta_b; - + if (color_delta > COLOR_DELTA_THRESH) continue; } @@ -992,7 +992,7 @@ namespace basisu } #endif // BASISU_SUPPORT_SSE } // if (!g_cpu_supports_sse41) - + } // if (cur_err) } // if ((m_params.m_endpoint_rdo_quality_thresh > 1.0f) && (iabs(endpoint_delta) > 1) && (!block_endpoints_are_referenced(block_x, block_y))) @@ -1011,7 +1011,7 @@ namespace basisu if ((!is_video) || (m.m_endpoint_predictor != basist::CR_ENDPOINT_PRED_INDEX)) { int new_selector_index = m_selector_remap_table_old_to_new[m.m_selector_index]; - + const float selector_remap_thresh = maximum(1.0f, m_params.m_selector_rdo_quality_thresh); //2.5f; int selector_history_buf_index = -1; @@ -1060,7 +1060,7 @@ namespace basisu for (uint32_t p = 0; p < 16; p++) cur_err += color_distance(false, src_pixels.get_ptr()[p], block_colors[pCur_selectors[p]], false); } - + const uint64_t limit_err = (uint64_t)ceilf(cur_err * selector_remap_thresh); // Even if cur_err==limit_err, we still want to scan the history buffer because there may be equivalent entries that are cheaper to code. @@ -1091,7 +1091,7 @@ namespace basisu if (sel_diff >= SEL_DIFF_THRESHOLD) continue; } - + const uint64_t thresh_err = minimum(limit_err, best_trial_err); uint64_t trial_err = 0; @@ -1266,7 +1266,7 @@ namespace basisu //{ // printf("%u, %u, %f\n", g_color_delta_bad_hist[i], g_color_delta_hist[i], g_color_delta_hist[i] ? g_color_delta_bad_hist[i] / (float)g_color_delta_hist[i] : 0); //} - + double total_prep_time = tm.get_elapsed_secs(); debug_printf("basisu_backend::encode_image: Total prep time: %3.2f\n", total_prep_time); @@ -1521,7 +1521,7 @@ namespace basisu if (old_endpoint_was_used[old_endpoint_index]) { const uint32_t new_endpoint_index = m_endpoint_remap_table_old_to_new[old_endpoint_index]; - + new_endpoint_was_used[new_endpoint_index] = true; endpoint_remap_table_new_to_old[new_endpoint_index] = old_endpoint_index; @@ -1660,7 +1660,7 @@ namespace basisu bool basisu_backend::encode_selector_palette() { const basisu_frontend& r = *m_pFront_end; - + histogram delta_selector_pal_histogram(256); for (uint32_t q = 0; q < r.get_total_selector_clusters(); q++) diff --git a/external/basis_universal/encoder/basisu_comp.cpp b/external/basis_universal/encoder/basisu_comp.cpp index 9a52f54fe4..617ff30782 100644 --- a/external/basis_universal/encoder/basisu_comp.cpp +++ b/external/basis_universal/encoder/basisu_comp.cpp @@ -1,5 +1,5 @@ // basisu_comp.cpp -// Copyright (C) 2019-2024 Binomial LLC. All Rights Reserved. +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -25,6 +25,7 @@ #include "basisu_miniz.h" #include "basisu_opencl.h" +#include "basisu_astc_ldr_encode.h" #include "../transcoder/basisu_astc_hdr_core.h" @@ -50,11 +51,207 @@ using namespace buminiz; namespace basisu { + static float uastc_ldr_4x4_lambda_from_quality(float q) + { + q = clamp(q, 0.0f, 1.0f); + + if (q >= 1.0f) + return 0.0f; + + const float lambda_max = 20.0f; + return lambda_max * pow(1.0f - q, 1.3f); + } + + static float uastc_hdr_6x6_lambda_from_quality(float q) + { + // Ideally we would know if it's an upconverted LDR/SDR input, or HDR, then that controls the maximum useful lambda. + q = clamp(q, 0.0f, 1.0f); + + if (q >= 1.0f) + return 0.0f; + + const float lambda_max = 50000.0f; + return lambda_max * pow(1.0f - q, 1.5f); + } + + bool basis_compressor_params::set_format_mode_and_effort(basist::basis_tex_format mode, int effort, bool set_defaults) + { + fmt_debug_printf("set_format_mode_and_effort: mode: {}, effort: {}, set_defaults: {}\n", basist::basis_get_tex_format_name(mode), effort, set_defaults); + + set_format_mode(mode); + + if (effort > 0) + effort = clamp(effort, 0, 10); + + const float feffort = (effort >= 0) ? clamp((float)effort / 10.0f, 0.0f, 1.0f) : 0.0f; + + if (mode == basist::basis_tex_format::cETC1S) + { + if (effort >= 0) + m_etc1s_compression_level = (int)std::round(lerp(0, (float)BASISU_MAX_ETC1S_COMPRESSION_LEVEL, feffort)); + else if (set_defaults) + m_etc1s_compression_level = BASISU_DEFAULT_ETC1S_COMPRESSION_LEVEL; + + fmt_debug_printf("Low-level ETC1S compression (effort) level (0-6): {}\n", m_etc1s_compression_level); + } + else if (mode == basist::basis_tex_format::cUASTC_LDR_4x4) + { + if (effort >= 0) + m_pack_uastc_ldr_4x4_flags = (int)std::round(lerp((float)cPackUASTCLevelFastest, (float)cPackUASTCLevelVerySlow, feffort)); + else if (set_defaults) + m_pack_uastc_ldr_4x4_flags = cPackUASTCLevelDefault; + + fmt_debug_printf("Low-level UASTC LDR 4x4 pack (effort) level (0-4): {}\n", m_pack_uastc_ldr_4x4_flags); + } + else if (mode == basist::basis_tex_format::cUASTC_HDR_4x4) + { + // Set UASTC HDR 4x4 effort level (there is no quality to set - it doesn't support RDO yet). + if (effort >= 0) + m_uastc_hdr_4x4_options.set_quality_level((int)std::round(lerp((float)uastc_hdr_4x4_codec_options::cMinLevel, (float)uastc_hdr_4x4_codec_options::cMaxLevel, feffort))); + else if (set_defaults) + m_uastc_hdr_4x4_options.set_quality_level(uastc_hdr_4x4_codec_options::cDefaultLevel); + + fmt_debug_printf("Low-level UASTC HDR 4x4 quality (actually effort) level (0-4): {}\n", m_uastc_hdr_4x4_options.m_level); + } + else if ((mode == basist::basis_tex_format::cASTC_HDR_6x6) || (mode == basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE)) + { + // Set ASTC HDR 6x6/UASTC HDR 6x6 effort level + if (effort >= 0) + m_astc_hdr_6x6_options.set_user_level(effort); + else if (set_defaults) + m_astc_hdr_6x6_options.set_user_level(astc_6x6_hdr::ASTC_HDR_6X6_DEF_USER_COMP_LEVEL); + + fmt_debug_printf("Low-level UASTC HDR 6x6 master comp (effort) level (0-4): {}, highest comp (effort) level (0-4): {}, num reuse XY deltas: {}, extra patterns flag: {}, brute force partition matching: {}\n", + m_astc_hdr_6x6_options.m_master_comp_level, + m_astc_hdr_6x6_options.m_highest_comp_level, + m_astc_hdr_6x6_options.m_num_reuse_xy_deltas, + m_astc_hdr_6x6_options.m_extra_patterns_flag, + m_astc_hdr_6x6_options.m_brute_force_partition_matching); + } + else if ((mode >= basist::basis_tex_format::cXUASTC_LDR_4x4) && (mode <= basist::basis_tex_format::cASTC_LDR_12x12)) + { + if (effort >= 0) + m_xuastc_ldr_effort_level = effort; + else if (set_defaults) + m_xuastc_ldr_effort_level = astc_ldr::EFFORT_LEVEL_DEF; + + fmt_debug_printf("Low-level XUASTC LDR effort level (0-10): {}\n", m_xuastc_ldr_effort_level); + } + else + { + assert(0); + return false; + } + + return true; + } + + bool basis_compressor_params::set_format_mode_and_quality_effort(basist::basis_tex_format mode, int quality, int effort, bool set_defaults) + { + fmt_debug_printf("set_format_mode_and_quality_effort: mode: {}, quality: {}, effort: {}, set_defaults: {}\n", basist::basis_get_tex_format_name(mode), quality, effort, set_defaults); + + if (!set_format_mode_and_effort(mode, effort, set_defaults)) + return false; + + if (quality > 0) + quality = clamp(quality, 0, 100); + + const float fquality = (quality >= 0) ? clamp((float)quality / 100.0f, 0.0f, 1.0f) : 0.0f; + + if (mode == basist::basis_tex_format::cETC1S) + { + // ETC1S: Map quality and effort to ETC1S quality and effort levels + if (quality >= 0) + m_quality_level = (int)std::round(lerp(0, 255.0f, fquality)); + else if (set_defaults) + m_quality_level = -1; + + fmt_debug_printf("Low-level ETC1S quality level (0-255): {}\n", m_quality_level); + } + else if (mode == basist::basis_tex_format::cUASTC_LDR_4x4) + { + // UASTC LDR 4x4: Map quality to RDO lambda scalar, effort to UASTC LDR 4x4 packing level + if ((quality >= 0) && (quality < 100)) + { + // Enable RDO postprocessing + m_rdo_uastc_ldr_4x4 = true; + + // Attempt to derive a reasonable lambda from quality + m_rdo_uastc_ldr_4x4_quality_scalar = uastc_ldr_4x4_lambda_from_quality(fquality); + } + else if (set_defaults) + { + m_rdo_uastc_ldr_4x4 = false; + + m_rdo_uastc_ldr_4x4_quality_scalar = 1.0f; // the default is 1.0, but the RDO flag isn't enabled + } + + fmt_debug_printf("Low-level UASTC LDR 4x4 RDO flag: {}, lambda setting (0=no extra distortion, higher=more distortion): {}\n", m_rdo_uastc_ldr_4x4, m_rdo_uastc_ldr_4x4_quality_scalar); + } + else if (mode == basist::basis_tex_format::cUASTC_HDR_4x4) + { + // UASTC HDR 4x4: Nothing to do for quality, it doesn't support RDO + if ((quality != -1) && (quality < 100)) + { + fmt_printf("WARNING: UASTC HDR 4x4 codec doesn't have a 'quality' parameter (it doesn't currently support RDO)\n"); + } + } + else if ((mode == basist::basis_tex_format::cASTC_HDR_6x6) || (mode == basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE)) + { + // Set lambda (rate-distortion tradeoff) + if (quality >= 0) + m_astc_hdr_6x6_options.m_lambda = uastc_hdr_6x6_lambda_from_quality(fquality); + else if (set_defaults) + m_astc_hdr_6x6_options.m_lambda = 0.0f; + + fmt_debug_printf("Low-level UASTC HDR 6x6 lambda setting (0=no extra distortion, higher=more distortion): {}\n", m_astc_hdr_6x6_options.m_lambda); + } + else if ((mode >= basist::basis_tex_format::cASTC_LDR_4x4) && (mode <= basist::basis_tex_format::cASTC_LDR_12x12)) + { + // ASTC LDR 4x4-12x12: Nothing to do for quality, it doesn't support RDO + if ((quality != -1) && (quality < 100)) + { + fmt_printf("WARNING: ASTC LDR 4x4-12x12 codec doesn't have a 'quality' parameter (it doesn't currently support RDO)\n"); + } + } + else if ((mode >= basist::basis_tex_format::cXUASTC_LDR_4x4) && (mode <= basist::basis_tex_format::cXUASTC_LDR_12x12)) + { + // XUASTC LDR 4x4-12x12 + if ((quality >= 0) && (quality < 100)) + { + // Enable DCT + lossy supercompression + m_quality_level = quality; + m_xuastc_ldr_use_dct = true; + m_xuastc_ldr_use_lossy_supercompression = true; + } + else if (set_defaults) + { + m_quality_level = -1; + m_xuastc_ldr_use_dct = false; + m_xuastc_ldr_use_lossy_supercompression = false; + } + + fmt_debug_printf("Low-level XUASTC quality level (0-100): {}, Use DCT: {}, Use lossy supercompression: {}\n", m_quality_level, m_xuastc_ldr_use_dct, m_xuastc_ldr_use_lossy_supercompression); + } + else + { + assert(0); + return false; + } + + return true; + } + basis_compressor::basis_compressor() : m_pOpenCL_context(nullptr), m_fmt_mode(basist::basis_tex_format::cETC1S), + m_fmt_mode_block_width(4), + m_fmt_mode_block_height(4), + m_total_slice_orig_texels(0), m_basis_file_size(0), m_basis_bits_per_texel(0.0f), + m_ktx2_file_size(0), + m_ktx2_bits_per_texel(0.0f), m_total_blocks(0), m_hdr_image_scale(1.0f), m_ldr_to_hdr_upconversion_nit_multiplier(1.0f), @@ -63,7 +260,7 @@ namespace basisu m_opencl_failed(false) { debug_printf("basis_compressor::basis_compressor\n"); - + assert(g_library_initialized); } @@ -195,7 +392,7 @@ namespace basisu bool basis_compressor::init(const basis_compressor_params ¶ms) { debug_printf("basis_compressor::init\n"); - + if (!g_library_initialized) { error_printf("basis_compressor::init: basisu_encoder_init() MUST be called before using any encoder functionality!\n"); @@ -207,7 +404,7 @@ namespace basisu error_printf("basis_compressor::init: A non-null job_pool pointer must be specified\n"); return false; } - + m_params = params; if ((m_params.m_compute_stats) && (!m_params.m_validate_output_data)) @@ -217,17 +414,34 @@ namespace basisu m_ldr_to_hdr_upconversion_nit_multiplier = 1.0f; m_upconverted_any_ldr_images = false; + m_total_slice_orig_texels = 0; + m_basis_file_size = 0; + m_basis_bits_per_texel = 0.0f; + m_ktx2_file_size = 0; + m_ktx2_bits_per_texel = 0.0f; + check_for_hdr_inputs(); + if (m_params.m_hdr) + { + if ((m_params.m_debug) && (m_params.m_ktx2_and_basis_srgb_transfer_function) && (m_params.m_ktx2_and_basis_srgb_transfer_function.was_changed())) + { + debug_printf("Warning: m_ktx2_and_basis_srgb_transfer_function being forced to false in HDR mode (we always write linear KTX2/.basis files in HDR mode)\n"); + } + + // Always slam m_ktx2_and_basis_srgb_transfer_function on HDR inputs. We always write linear to KTX2 and .basis for HDR outputs. + m_params.m_ktx2_and_basis_srgb_transfer_function = false; + } + if (m_params.m_debug) { - debug_printf("basis_compressor::init:\n"); + debug_printf("\nbasis_compressor::init:\n"); #define PRINT_BOOL_VALUE(v) fmt_debug_printf("{}: {} {}\n", BASISU_STRINGIZE2(v), static_cast(m_params.v), m_params.v.was_changed()); #define PRINT_INT_VALUE(v) fmt_debug_printf("{}: {} {}\n", BASISU_STRINGIZE2(v), static_cast(m_params.v), m_params.v.was_changed()); #define PRINT_UINT_VALUE(v) fmt_debug_printf("{}: {} {}\n", BASISU_STRINGIZE2(v), static_cast(m_params.v), m_params.v.was_changed()); #define PRINT_FLOAT_VALUE(v) fmt_debug_printf("{}: {} {}\n", BASISU_STRINGIZE2(v), static_cast(m_params.v), m_params.v.was_changed()); - + fmt_debug_printf("Source LDR images: {}, HDR images: {}, filenames: {}, alpha filenames: {}, LDR mipmap images: {}, HDR mipmap images: {}\n", (uint64_t)m_params.m_source_images.size(), (uint64_t)m_params.m_source_images_hdr.size(), (uint64_t)m_params.m_source_filenames.size(), (uint64_t)m_params.m_source_alpha_filenames.size(), @@ -263,9 +477,9 @@ namespace basisu fmt_debug_printf("m_hdr_mode: cASTC_HDR_6X6\n"); break; } - case hdr_modes::cASTC_HDR_6X6_INTERMEDIATE: + case hdr_modes::cUASTC_HDR_6X6_INTERMEDIATE: { - fmt_debug_printf("m_hdr_mode: cASTC_HDR_6X6_INTERMEDIATE\n"); + fmt_debug_printf("m_hdr_mode: cUASTC_HDR_6X6_INTERMEDIATE\n"); break; } default: @@ -274,12 +488,13 @@ namespace basisu } PRINT_BOOL_VALUE(m_uastc); + PRINT_INT_VALUE(m_xuastc_or_astc_ldr_basis_tex_format); PRINT_BOOL_VALUE(m_use_opencl); PRINT_BOOL_VALUE(m_y_flip); PRINT_BOOL_VALUE(m_debug); PRINT_BOOL_VALUE(m_validate_etc1s); PRINT_BOOL_VALUE(m_debug_images); - PRINT_INT_VALUE(m_compression_level); + PRINT_INT_VALUE(m_etc1s_compression_level); PRINT_BOOL_VALUE(m_perceptual); PRINT_BOOL_VALUE(m_no_endpoint_rdo); PRINT_BOOL_VALUE(m_no_selector_rdo); @@ -296,10 +511,10 @@ namespace basisu PRINT_BOOL_VALUE(m_renormalize); PRINT_BOOL_VALUE(m_multithreading); PRINT_BOOL_VALUE(m_disable_hierarchical_endpoint_codebooks); - + PRINT_FLOAT_VALUE(m_endpoint_rdo_thresh); PRINT_FLOAT_VALUE(m_selector_rdo_thresh); - + PRINT_BOOL_VALUE(m_mip_gen); PRINT_BOOL_VALUE(m_mip_renormalize); PRINT_BOOL_VALUE(m_mip_wrapping); @@ -312,14 +527,14 @@ namespace basisu debug_printf("m_max_endpoint_clusters: %u\n", m_params.m_etc1s_max_endpoint_clusters); debug_printf("m_max_selector_clusters: %u\n", m_params.m_etc1s_max_selector_clusters); - debug_printf("m_etc1s_quality_level: %i\n", m_params.m_etc1s_quality_level); + debug_printf("m_quality_level: %i\n", m_params.m_quality_level); debug_printf("UASTC HDR 4x4 quality level: %u\n", m_params.m_uastc_hdr_4x4_options.m_level); debug_printf("m_tex_type: %u\n", m_params.m_tex_type); debug_printf("m_userdata0: 0x%X, m_userdata1: 0x%X\n", m_params.m_userdata0, m_params.m_userdata1); debug_printf("m_us_per_frame: %i (%f fps)\n", m_params.m_us_per_frame, m_params.m_us_per_frame ? 1.0f / (m_params.m_us_per_frame / 1000000.0f) : 0); debug_printf("m_pack_uastc_ldr_4x4_flags: 0x%X\n", m_params.m_pack_uastc_ldr_4x4_flags); - + PRINT_BOOL_VALUE(m_rdo_uastc_ldr_4x4); PRINT_FLOAT_VALUE(m_rdo_uastc_ldr_4x4_quality_scalar); PRINT_INT_VALUE(m_rdo_uastc_ldr_4x4_dict_size); @@ -333,7 +548,7 @@ namespace basisu PRINT_INT_VALUE(m_resample_width); PRINT_INT_VALUE(m_resample_height); PRINT_FLOAT_VALUE(m_resample_factor); - + debug_printf("Has global codebooks: %u\n", m_params.m_pGlobal_codebooks ? 1 : 0); if (m_params.m_pGlobal_codebooks) { @@ -344,7 +559,7 @@ namespace basisu debug_printf("KTX2 UASTC supercompression: %u\n", m_params.m_ktx2_uastc_supercompression); debug_printf("KTX2 Zstd supercompression level: %i\n", (int)m_params.m_ktx2_zstd_supercompression_level); - debug_printf("KTX2 sRGB transfer func: %u\n", (int)m_params.m_ktx2_srgb_transfer_func); + debug_printf("KTX2/basis sRGB transfer function: %u\n", (int)m_params.m_ktx2_and_basis_srgb_transfer_function); debug_printf("Total KTX2 key values: %u\n", m_params.m_ktx2_key_values.size()); for (uint32_t i = 0; i < m_params.m_ktx2_key_values.size(); i++) { @@ -353,21 +568,45 @@ namespace basisu } PRINT_BOOL_VALUE(m_validate_output_data); + PRINT_UINT_VALUE(m_transcode_flags); PRINT_BOOL_VALUE(m_ldr_hdr_upconversion_srgb_to_linear); PRINT_FLOAT_VALUE(m_ldr_hdr_upconversion_nit_multiplier); debug_printf("Allow UASTC HDR 4x4 uber mode: %u\n", m_params.m_uastc_hdr_4x4_options.m_allow_uber_mode); debug_printf("UASTC HDR 4x4 ultra quant: %u\n", m_params.m_uastc_hdr_4x4_options.m_ultra_quant); PRINT_BOOL_VALUE(m_hdr_favor_astc); - + + PRINT_INT_VALUE(m_xuastc_ldr_effort_level); + PRINT_BOOL_VALUE(m_xuastc_ldr_blurring); + PRINT_BOOL_VALUE(m_xuastc_ldr_use_dct); + PRINT_BOOL_VALUE(m_xuastc_ldr_use_lossy_supercompression); + PRINT_BOOL_VALUE(m_xuastc_ldr_force_disable_subsets); + PRINT_BOOL_VALUE(m_xuastc_ldr_force_disable_rgb_dual_plane); + PRINT_INT_VALUE(m_xuastc_ldr_syntax); + + debug_printf("XUASTC LDR channel weights: "); + for (uint32_t i = 0; i < 4; i++) + fmt_debug_printf("{} ", m_params.m_xuastc_ldr_channel_weights[i]); + debug_printf("\n"); + + PRINT_FLOAT_VALUE(m_ls_min_psnr); + PRINT_FLOAT_VALUE(m_ls_thresh_psnr); + PRINT_FLOAT_VALUE(m_ls_thresh_edge_psnr); + PRINT_FLOAT_VALUE(m_ls_min_alpha_psnr); + PRINT_FLOAT_VALUE(m_ls_thresh_alpha_psnr); + PRINT_FLOAT_VALUE(m_ls_thresh_edge_alpha_psnr); + #undef PRINT_BOOL_VALUE #undef PRINT_INT_VALUE #undef PRINT_UINT_VALUE #undef PRINT_FLOAT_VALUE + + fmt_printf("m_format_mode: {}\n", (uint32_t)m_params.get_format_mode()); + fmt_printf("\n"); } if (!sanity_check_input_params()) return false; - + if ((m_params.m_use_opencl) && opencl_is_available() && !m_pOpenCL_context && !m_opencl_failed) { m_pOpenCL_context = opencl_create_context(); @@ -378,14 +617,17 @@ namespace basisu return true; } - void basis_compressor::pick_format_mode() + bool basis_compressor::pick_format_mode() { - // Unfortunately due to the legacy of this code and backwards compat this is more complex than I would like. + // Unfortunately due to the legacy of this code and backwards API compatibility this is more complex than I would like. m_fmt_mode = basist::basis_tex_format::cETC1S; + m_fmt_mode_block_width = 4; + m_fmt_mode_block_height = 4; if (m_params.m_hdr) { assert(m_params.m_uastc); + assert(m_params.m_xuastc_or_astc_ldr_basis_tex_format == -1); switch (m_params.m_hdr_mode) { @@ -394,9 +636,13 @@ namespace basisu break; case hdr_modes::cASTC_HDR_6X6: m_fmt_mode = basist::basis_tex_format::cASTC_HDR_6x6; + m_fmt_mode_block_width = 6; + m_fmt_mode_block_height = 6; break; - case hdr_modes::cASTC_HDR_6X6_INTERMEDIATE: - m_fmt_mode = basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE; + case hdr_modes::cUASTC_HDR_6X6_INTERMEDIATE: + m_fmt_mode = basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE; + m_fmt_mode_block_width = 6; + m_fmt_mode_block_height = 6; break; default: assert(0); @@ -405,7 +651,30 @@ namespace basisu } else if (m_params.m_uastc) { - m_fmt_mode = basist::basis_tex_format::cUASTC4x4; + if (m_params.m_xuastc_or_astc_ldr_basis_tex_format == -1) + { + // UASTC LDR 4x4 + m_fmt_mode = basist::basis_tex_format::cUASTC_LDR_4x4; + } + else + { + // XUASTC LDR 4x4-12x12 or ASTC LDR 4x4-12x12 + m_fmt_mode = static_cast(static_cast(m_params.m_xuastc_or_astc_ldr_basis_tex_format)); + + if (!basis_tex_format_is_xuastc_ldr(m_fmt_mode) && !basis_tex_format_is_astc_ldr(m_fmt_mode)) + { + assert(0); + error_printf("basis_compressor::pick_format_mode: m_xuastc_or_astc_ldr_basis_tex_format is invalid\n"); + return false; + } + + basist::get_basis_tex_format_block_size(m_fmt_mode, m_fmt_mode_block_width, m_fmt_mode_block_height); + } + } + else + { + // ETC1S + assert(m_params.m_xuastc_or_astc_ldr_basis_tex_format == -1); } if (m_params.m_debug) @@ -415,8 +684,8 @@ namespace basisu case basist::basis_tex_format::cETC1S: fmt_debug_printf("Format Mode: cETC1S\n"); break; - case basist::basis_tex_format::cUASTC4x4: - fmt_debug_printf("Format Mode: cUASTC4x4\n"); + case basist::basis_tex_format::cUASTC_LDR_4x4: + fmt_debug_printf("Format Mode: cUASTC_LDR_4x4\n"); break; case basist::basis_tex_format::cUASTC_HDR_4x4: fmt_debug_printf("Format Mode: cUASTC_HDR_4x4\n"); @@ -424,16 +693,56 @@ namespace basisu case basist::basis_tex_format::cASTC_HDR_6x6: fmt_debug_printf("Format Mode: cASTC_HDR_6x6\n"); break; - case basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE: - fmt_debug_printf("Format Mode: cASTC_HDR_6x6_INTERMEDIATE\n"); + case basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE: + fmt_debug_printf("Format Mode: cUASTC_HDR_6x6_INTERMEDIATE\n"); + break; + + case basist::basis_tex_format::cXUASTC_LDR_4x4: + case basist::basis_tex_format::cXUASTC_LDR_5x4: + case basist::basis_tex_format::cXUASTC_LDR_5x5: + case basist::basis_tex_format::cXUASTC_LDR_6x5: + case basist::basis_tex_format::cXUASTC_LDR_6x6: + case basist::basis_tex_format::cXUASTC_LDR_8x5: + case basist::basis_tex_format::cXUASTC_LDR_8x6: + case basist::basis_tex_format::cXUASTC_LDR_10x5: + case basist::basis_tex_format::cXUASTC_LDR_10x6: + case basist::basis_tex_format::cXUASTC_LDR_8x8: + case basist::basis_tex_format::cXUASTC_LDR_10x8: + case basist::basis_tex_format::cXUASTC_LDR_10x10: + case basist::basis_tex_format::cXUASTC_LDR_12x10: + case basist::basis_tex_format::cXUASTC_LDR_12x12: + { + fmt_debug_printf("Format Mode: cXUASTC_LDR_{}x{}\n", m_fmt_mode_block_width, m_fmt_mode_block_height); + break; + } + case basist::basis_tex_format::cASTC_LDR_4x4: + case basist::basis_tex_format::cASTC_LDR_5x4: + case basist::basis_tex_format::cASTC_LDR_5x5: + case basist::basis_tex_format::cASTC_LDR_6x5: + case basist::basis_tex_format::cASTC_LDR_6x6: + case basist::basis_tex_format::cASTC_LDR_8x5: + case basist::basis_tex_format::cASTC_LDR_8x6: + case basist::basis_tex_format::cASTC_LDR_10x5: + case basist::basis_tex_format::cASTC_LDR_10x6: + case basist::basis_tex_format::cASTC_LDR_8x8: + case basist::basis_tex_format::cASTC_LDR_10x8: + case basist::basis_tex_format::cASTC_LDR_10x10: + case basist::basis_tex_format::cASTC_LDR_12x10: + case basist::basis_tex_format::cASTC_LDR_12x12: + { + fmt_debug_printf("Format Mode: cASTC_LDR_{}x{}\n", m_fmt_mode_block_width, m_fmt_mode_block_height); break; + } + default: assert(0); break; } } - } + return true; + } + basis_compressor::error_code basis_compressor::process() { debug_printf("basis_compressor::process\n"); @@ -441,8 +750,9 @@ namespace basisu if (!read_dds_source_images()) return cECFailedReadingSourceImages; - // Note: After here m_params.m_hdr, m_params.m_uastc and m_fmt_mode cannot be changed. - pick_format_mode(); + // Note: After here m_params.m_hdr, m_params.m_uastc and m_fmt_mode, m_fmt_mode_block_width/height cannot be changed. + if (!pick_format_mode()) + return cECFailedInvalidParameters; if (!read_source_images()) return cECFailedReadingSourceImages; @@ -459,6 +769,7 @@ namespace basisu } } + // Some modes/codecs require extracting source blocks up front. if (!extract_source_blocks()) return cECFailedFrontEnd; @@ -468,21 +779,21 @@ namespace basisu { // UASTC 4x4 HDR if (m_params.m_status_output) - printf("Mode: UASTC 4x4 HDR Level %u\n", m_params.m_uastc_hdr_4x4_options.m_level); + printf("Mode: UASTC 4x4 HDR Effort Level (0-4): %u\n", m_params.m_uastc_hdr_4x4_options.m_level); error_code ec = encode_slices_to_uastc_4x4_hdr(); if (ec != cECSuccess) return ec; } - else + else { - assert((m_params.m_hdr_mode == hdr_modes::cASTC_HDR_6X6) || (m_params.m_hdr_mode == hdr_modes::cASTC_HDR_6X6_INTERMEDIATE)); + assert((m_params.m_hdr_mode == hdr_modes::cASTC_HDR_6X6) || (m_params.m_hdr_mode == hdr_modes::cUASTC_HDR_6X6_INTERMEDIATE)); // ASTC 6x6 HDR if (m_params.m_status_output) { - fmt_printf("Mode: ASTC 6x6 HDR {}, Base Level: {}, Highest Level: {}, Lambda: {}, REC 2020: {}\n", - (m_params.m_hdr_mode == hdr_modes::cASTC_HDR_6X6_INTERMEDIATE) ? "Intermediate" : "", + fmt_printf("Mode: ASTC 6x6 HDR {}, Base Effort Level (0-4): {}, Highest Effort Level (0-4): {}, Lambda: {}, REC 2020: {}\n", + (m_params.m_hdr_mode == hdr_modes::cUASTC_HDR_6X6_INTERMEDIATE) ? "Intermediate" : "", m_params.m_astc_hdr_6x6_options.m_master_comp_level, m_params.m_astc_hdr_6x6_options.m_highest_comp_level, m_params.m_astc_hdr_6x6_options.m_lambda, m_params.m_astc_hdr_6x6_options.m_rec2020_bt2100_color_gamut); } @@ -494,11 +805,54 @@ namespace basisu } else if (m_params.m_uastc) { - // UASTC 4x4 LDR - if (m_params.m_status_output) - printf("Mode: UASTC LDR 4x4 Level %u\n", m_params.m_pack_uastc_ldr_4x4_flags & cPackUASTCLevelMask); + error_code ec = cECFailedEncodeUASTC; + + if (basis_tex_format_is_xuastc_ldr(m_fmt_mode) || basis_tex_format_is_astc_ldr(m_fmt_mode)) + { + // XUASTC LDR 4x4-12x12 or ASTC LDR 4x4-12x12 + if (m_params.m_status_output) + { + uint32_t block_width = 0, block_height = 0; + basist::get_basis_tex_format_block_size(m_fmt_mode, block_width, block_height); + + if (basis_tex_format_is_xuastc_ldr(m_fmt_mode)) + { + fmt_printf("Mode: XUASTC LDR {}x{}, Effort Level (0-10): {}, Disable Subsets: {}, Disable RGB Dual Plane: {}\nWeight grid DCT: {}, DCT quality level (1-100): {}, Lossy supercompression: {}, sRGB8 ASTC decode profile: {}, Syntax: {}, Channel weights: {} {} {} {}\n", + block_width, block_height, (int)m_params.m_xuastc_ldr_effort_level, (bool)m_params.m_xuastc_ldr_force_disable_subsets, (bool)m_params.m_xuastc_ldr_force_disable_rgb_dual_plane, + (bool)m_params.m_xuastc_ldr_use_dct, + (bool)m_params.m_xuastc_ldr_use_dct ? m_params.m_quality_level : 0, + (bool)m_params.m_xuastc_ldr_use_lossy_supercompression, + (bool)m_params.m_ktx2_and_basis_srgb_transfer_function, + (int)m_params.m_xuastc_ldr_syntax, + m_params.m_xuastc_ldr_channel_weights[0], m_params.m_xuastc_ldr_channel_weights[1], m_params.m_xuastc_ldr_channel_weights[2], m_params.m_xuastc_ldr_channel_weights[3]); + } + else + { + fmt_printf("Mode: ASTC LDR {}x{}, Effort Level (0-10): {}, Disable Subsets: {}, Disable RGB Dual Plane: {}, sRGB8 ASTC decode profile: {}, Syntax: {}, Channel weights: {} {} {} {}\n", + block_width, block_height, + (int)m_params.m_xuastc_ldr_effort_level, (bool)m_params.m_xuastc_ldr_force_disable_subsets, (bool)m_params.m_xuastc_ldr_force_disable_rgb_dual_plane, + (bool)m_params.m_ktx2_and_basis_srgb_transfer_function, + (int)m_params.m_xuastc_ldr_syntax, + m_params.m_xuastc_ldr_channel_weights[0], m_params.m_xuastc_ldr_channel_weights[1], m_params.m_xuastc_ldr_channel_weights[2], m_params.m_xuastc_ldr_channel_weights[3]); + } + } + + ec = encode_slices_to_xuastc_or_astc_ldr(); + } + else + { + // UASTC LDR 4x4 + if (m_params.m_status_output) + { + if (m_params.m_rdo_uastc_ldr_4x4) + fmt_printf("Mode: UASTC LDR 4x4 Effort Level (0-4): {}, using RDO lambda: {}\n", m_params.m_pack_uastc_ldr_4x4_flags & cPackUASTCLevelMask, m_params.m_rdo_uastc_ldr_4x4_quality_scalar); + else + printf("Mode: UASTC LDR 4x4 Effort Level (0-4): %u\n", m_params.m_pack_uastc_ldr_4x4_flags & cPackUASTCLevelMask); + } + + ec = encode_slices_to_uastc_4x4_ldr(); + } - error_code ec = encode_slices_to_uastc_4x4_ldr(); if (ec != cECSuccess) return ec; } @@ -506,8 +860,8 @@ namespace basisu { // ETC1S if (m_params.m_status_output) - printf("Mode: ETC1S Quality %i, Level %i\n", m_params.m_etc1s_quality_level, (int)m_params.m_compression_level); - + printf("Mode: ETC1S Quality (1-255): %i, Comp Level (Effort, 0-6): %i\n", m_params.m_quality_level, (int)m_params.m_etc1s_compression_level); + if (!process_frontend()) return cECFailedFrontEnd; @@ -546,14 +900,14 @@ namespace basisu if (m_params.m_hdr_mode == hdr_modes::cASTC_HDR_6X6) m_uastc_backend_output.m_tex_format = basist::basis_tex_format::cASTC_HDR_6x6; - else if (m_params.m_hdr_mode == hdr_modes::cASTC_HDR_6X6_INTERMEDIATE) - m_uastc_backend_output.m_tex_format = basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE; + else if (m_params.m_hdr_mode == hdr_modes::cUASTC_HDR_6X6_INTERMEDIATE) + m_uastc_backend_output.m_tex_format = basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE; else { assert(0); return cECFailedEncodeUASTC; } - + m_uastc_backend_output.m_etc1s = false; m_uastc_backend_output.m_srgb = false; m_uastc_backend_output.m_slice_desc = m_slice_descs; @@ -561,13 +915,13 @@ namespace basisu m_uastc_backend_output.m_slice_image_crcs.resize(m_slice_descs.size()); astc_6x6_hdr::astc_hdr_6x6_global_config global_cfg(m_params.m_astc_hdr_6x6_options); - + global_cfg.m_image_stats = m_params.m_compute_stats; global_cfg.m_debug_images = m_params.m_debug_images; global_cfg.m_output_images = m_params.m_debug_images; global_cfg.m_debug_output = m_params.m_debug; global_cfg.m_status_output = m_params.m_status_output || m_params.m_debug; - + for (uint32_t slice_index = 0; slice_index < m_slice_descs.size(); slice_index++) { gpu_image& dst_tex = m_uastc_slice_textures[slice_index]; @@ -578,7 +932,7 @@ namespace basisu const imagef& source_image = m_slice_images_hdr[slice_index]; assert(source_image.get_width() && source_image.get_height()); - + uint8_vec intermediate_tex_data, astc_tex_data; global_cfg.m_debug_image_prefix = m_params.m_astc_hdr_6x6_options.m_debug_image_prefix; @@ -586,7 +940,7 @@ namespace basisu global_cfg.m_output_image_prefix = m_params.m_astc_hdr_6x6_options.m_output_image_prefix; global_cfg.m_output_image_prefix += fmt_string("slice_{}_", slice_index); - + if (m_params.m_debug) fmt_debug_printf("----------------------------------------------------------------------------\n"); @@ -613,7 +967,7 @@ namespace basisu } else { - assert(m_params.m_hdr_mode == hdr_modes::cASTC_HDR_6X6_INTERMEDIATE); + assert(m_params.m_hdr_mode == hdr_modes::cUASTC_HDR_6X6_INTERMEDIATE); dst_buf.resize(intermediate_tex_data.size_in_bytes()); memcpy(&dst_buf[0], intermediate_tex_data.get_ptr(), intermediate_tex_data.size_in_bytes()); @@ -648,7 +1002,7 @@ namespace basisu m_params.m_uastc_hdr_4x4_options.m_r_err_scale = 1.0f; m_params.m_uastc_hdr_4x4_options.m_g_err_scale = 1.0f; } - + const float DEFAULT_BC6H_ERROR_WEIGHT = .65f;// .85f; const float LOWEST_BC6H_ERROR_WEIGHT = .1f; m_params.m_uastc_hdr_4x4_options.m_bc6h_err_weight = m_params.m_hdr_favor_astc ? LOWEST_BC6H_ERROR_WEIGHT : DEFAULT_BC6H_ERROR_WEIGHT; @@ -657,7 +1011,7 @@ namespace basisu any_failures.store(false); astc_hdr_4x4_block_stats enc_stats; - + struct uastc_blk_desc { uint32_t m_solid_flag; @@ -681,7 +1035,7 @@ namespace basisu return false; } - + bool operator== (const uastc_blk_desc& desc) const { if (this == &desc) @@ -709,6 +1063,10 @@ namespace basisu std::map unique_block_descs; std::mutex unique_block_desc_mutex; + std::mutex status_output_mutex; + uint32_t total_blocks_processed = 0; + float last_percentage_printed = 0; + for (uint32_t slice_index = 0; slice_index < m_slice_descs.size(); slice_index++) { gpu_image& tex = m_uastc_slice_textures[slice_index]; @@ -719,18 +1077,16 @@ namespace basisu const uint32_t num_blocks_y = tex.get_blocks_y(); const uint32_t total_blocks = tex.get_total_blocks(); const imagef& source_image = m_slice_images_hdr[slice_index]; - - std::atomic total_blocks_processed; - total_blocks_processed.store(0); - + const uint32_t N = 256; for (uint32_t block_index_iter = 0; block_index_iter < total_blocks; block_index_iter += N) { const uint32_t first_index = block_index_iter; const uint32_t last_index = minimum(total_blocks, block_index_iter + N); - - m_params.m_pJob_pool->add_job([this, first_index, last_index, num_blocks_x, num_blocks_y, total_blocks, &source_image, - &tex, &total_blocks_processed, &any_failures, &enc_stats, &unique_block_descs, &unique_block_desc_mutex] + + m_params.m_pJob_pool->add_job([this, first_index, last_index, num_blocks_x, num_blocks_y, total_blocks, &source_image, + &tex, &any_failures, &enc_stats, &unique_block_descs, &unique_block_desc_mutex, + &status_output_mutex, &total_blocks_processed, &last_percentage_printed] { BASISU_NOTE_UNUSED(num_blocks_y); @@ -750,7 +1106,7 @@ namespace basisu source_image.extract_block_clamped(&block_pixels[0], block_x * 4, block_y * 4, 4, 4); basist::astc_blk& dest_block = *(basist::astc_blk*)tex.get_block_ptr(block_x, block_y); - + float rgb_pixels[16 * 3]; basist::half_float rgb_pixels_half[16 * 3]; for (uint32_t i = 0; i < 16; i++) @@ -764,7 +1120,7 @@ namespace basisu rgb_pixels[i * 3 + 2] = block_pixels[i][2]; rgb_pixels_half[i * 3 + 2] = float_to_half_non_neg_no_nan_inf(block_pixels[i][2]); } - + bool status = astc_hdr_4x4_enc_block(&rgb_pixels[0], rgb_pixels_half, m_params.m_uastc_hdr_4x4_options, all_results); if (!status) { @@ -774,10 +1130,10 @@ namespace basisu double best_err = 1e+30f; int best_result_index = -1; - + const double bc6h_err_weight = m_params.m_uastc_hdr_4x4_options.m_bc6h_err_weight; const double astc_err_weight = (1.0f - bc6h_err_weight); - + for (uint32_t i = 0; i < all_results.size(); i++) { basist::half_float unpacked_bc6h_block[4 * 4 * 3]; @@ -795,9 +1151,9 @@ namespace basisu } const astc_hdr_4x4_pack_results& best_results = all_results[best_result_index]; - + astc_hdr_4x4_pack_results_to_block(dest_block, best_results); - + // Verify that this block is valid UASTC HDR and we can successfully transcode it to BC6H. // (Well, except in fastest mode.) if (m_params.m_uastc_hdr_4x4_options.m_level > 0) @@ -830,12 +1186,12 @@ namespace basisu blk_desc.m_weight_ise_range = best_results.m_best_blk.m_weight_ise_range; blk_desc.m_endpoint_ise_range = best_results.m_best_blk.m_endpoint_ise_range; } - + { std::lock_guard lck(unique_block_desc_mutex); - + auto res = unique_block_descs.insert(std::make_pair(blk_desc, uastc_blk_desc_stats())); - + (res.first)->second.m_count++; #ifdef UASTC_HDR_DEBUG_SAVE_CATEGORIZED_BLOCKS (res.first)->second.m_blks.push_back(dest_block); @@ -843,17 +1199,35 @@ namespace basisu } } - total_blocks_processed++; + } // block_index + + if (m_params.m_status_output) + { + float percent_done = 0; + bool print_flag = false; - uint32_t val = total_blocks_processed; - if (((val & 1023) == 1023) && m_params.m_status_output) { - debug_printf("basis_compressor::encode_slices_to_uastc_4x4_hdr: %3.1f%% done\n", static_cast(val) * 100.0f / total_blocks); + std::lock_guard lck(status_output_mutex); + + total_blocks_processed += (last_index - first_index) + 1; + + percent_done = ((float)total_blocks_processed * 100.0f) / (float)total_blocks; + + if ((percent_done >= 100.0f) || (percent_done >= (last_percentage_printed + 5.0f))) + { + last_percentage_printed = percent_done; + + print_flag = true; + } } + + // minor print race here, doesn't matter + if (print_flag) + debug_printf("basis_compressor::encode_slices_to_uastc_4x4_hdr: %3.1f%% done\n", percent_done); } }); - + } // block_index_iter m_params.m_pJob_pool->wait_for_all(); @@ -867,7 +1241,7 @@ namespace basisu m_uastc_backend_output.m_slice_image_crcs[slice_index] = basist::crc16(tex.get_ptr(), tex.get_size_in_bytes(), 0); } // slice_index - + debug_printf("basis_compressor::encode_slices_to_uastc_4x4_hdr: Total time: %3.3f secs\n", tm.get_elapsed_secs()); if (m_params.m_debug) @@ -914,17 +1288,183 @@ namespace basisu debug_printf(" }\n"); } #endif - + c++; } printf("\n"); - + enc_stats.print(); } return cECSuccess; } + // XUASTC 4x4-12x12 or ASTC 4x4-12x12 + basis_compressor::error_code basis_compressor::encode_slices_to_xuastc_or_astc_ldr() + { + if (m_params.m_debug) + debug_printf("basis_compressor::encode_slices_to_xuastc_or_astc_ldr\n"); + + m_uastc_slice_textures.resize(m_slice_descs.size()); + + const texture_format tex_fmt = basist::basis_get_texture_format_from_xuastc_or_astc_ldr_basis_tex_format(m_fmt_mode); + const basist::transcoder_texture_format transcoder_tex_fmt = basist::basis_get_transcoder_texture_format_from_xuastc_or_astc_ldr_basis_tex_format(m_fmt_mode); + + uint32_t block_width = 0, block_height = 0; + block_width = basist::basis_get_block_width(transcoder_tex_fmt); + block_height = basist::basis_get_block_height(transcoder_tex_fmt); + +#if defined(_DEBUG) || defined(DEBUG) + // sanity checking + { + uint32_t alt_block_width = 0, alt_block_height = 0; + get_basis_tex_format_block_size(m_fmt_mode, alt_block_width, alt_block_height); + assert((block_width == alt_block_width) && (block_height == alt_block_height)); + } +#endif + + for (uint32_t slice_index = 0; slice_index < m_slice_descs.size(); slice_index++) + m_uastc_slice_textures[slice_index].init(tex_fmt, m_slice_descs[slice_index].m_orig_width, m_slice_descs[slice_index].m_orig_height); + + m_uastc_backend_output.m_tex_format = m_fmt_mode; + + m_uastc_backend_output.m_etc1s = false; + m_uastc_backend_output.m_srgb = m_params.m_ktx2_and_basis_srgb_transfer_function; + m_uastc_backend_output.m_slice_desc = m_slice_descs; + m_uastc_backend_output.m_slice_image_data.resize(m_slice_descs.size()); + m_uastc_backend_output.m_slice_image_crcs.resize(m_slice_descs.size()); + + astc_ldr::astc_ldr_encode_config cfg; + cfg.m_astc_block_width = block_width; + cfg.m_astc_block_height = block_height; + cfg.m_block_blurring_p1 = m_params.m_xuastc_ldr_blurring; // experimental, not recommended, very slow + cfg.m_block_blurring_p2 = m_params.m_xuastc_ldr_blurring; // experimental, not recommended, very slow + cfg.m_effort_level = clamp(m_params.m_xuastc_ldr_effort_level, astc_ldr::EFFORT_LEVEL_MIN, astc_ldr::EFFORT_LEVEL_MAX); + cfg.m_force_disable_subsets = m_params.m_xuastc_ldr_force_disable_subsets; + cfg.m_force_disable_rgb_dual_plane = m_params.m_xuastc_ldr_force_disable_rgb_dual_plane; + cfg.m_astc_decode_mode_srgb = m_params.m_ktx2_and_basis_srgb_transfer_function; + + cfg.m_compressed_syntax = (basist::astc_ldr_t::xuastc_ldr_syntax)(int)m_params.m_xuastc_ldr_syntax; + if (cfg.m_compressed_syntax >= basist::astc_ldr_t::xuastc_ldr_syntax::cTotal) + { + error_printf("basis_compressor::encode_slices_to_xuastc_or_astc_ldr: Invalid XUASTC LDR syntax\n"); + return cECFailedInvalidParameters; + } + + if (basist::basis_tex_format_is_xuastc_ldr(m_fmt_mode)) + { + if (m_params.m_quality_level >= 0) + { + // Enable weight grid DCT + cfg.m_dct_quality = static_cast(clamp(m_params.m_quality_level, astc_ldr::DCT_QUALITY_MIN, astc_ldr::DCT_QUALITY_MAX)); + cfg.m_use_dct = m_params.m_xuastc_ldr_use_dct; + } + else + { + // No DCT quality level specified, but they wanted DCT - display warning + if (cfg.m_use_dct) + { + printf("Warning: m_use_dct enabled, but m_quality_level was -1 (not set). Not using DCT. Quality level must range from 1-100.\n"); + } + } + } + + cfg.m_lossy_supercompression = m_params.m_xuastc_ldr_use_lossy_supercompression; + + for (uint32_t i = 0; i < 4; i++) + cfg.m_comp_weights[i] = m_params.m_xuastc_ldr_channel_weights[i]; + + cfg.m_replacement_min_psnr = m_params.m_ls_min_psnr; + cfg.m_psnr_trial_diff_thresh = m_params.m_ls_thresh_psnr; + cfg.m_psnr_trial_diff_thresh_edge = m_params.m_ls_thresh_edge_psnr; + + cfg.m_replacement_min_psnr_alpha = m_params.m_ls_min_alpha_psnr; + cfg.m_psnr_trial_diff_thresh_alpha = m_params.m_ls_thresh_alpha_psnr; + cfg.m_psnr_trial_diff_thresh_edge_alpha = m_params.m_ls_thresh_edge_alpha_psnr; + + cfg.m_debug_output = m_params.m_debug; + cfg.m_debug_images = m_params.m_debug_images; + + for (uint32_t slice_index = 0; slice_index < m_slice_descs.size(); slice_index++) + { + gpu_image& dst_tex = m_uastc_slice_textures[slice_index]; + + basisu_backend_slice_desc& slice_desc = m_slice_descs[slice_index]; + (void)slice_desc; + + const image& slice_source_image = m_slice_images[slice_index]; + const image* pSource_image = &slice_source_image; + + image temp_image; + if ((slice_source_image.get_width() != slice_desc.m_orig_width) || (slice_source_image.get_height() != slice_desc.m_orig_height)) + { + // Copy to actual/original dimensions so PSNR statistics are calculated correctly. (There's no need to pad the image to multiples of the block dimensions.) + temp_image = slice_source_image; + temp_image.crop(slice_desc.m_orig_width, slice_desc.m_orig_height); + pSource_image = &temp_image; + } + + cfg.m_debug_file_prefix = fmt_string("slice_{}_", slice_index); + + if (m_params.m_debug) + fmt_debug_printf("----------------------------------------------------------------------------\n"); + + uint8_vec intermediate_tex_data; + vector2D coded_log_blocks; + + bool comp_status = astc_ldr::compress_image(*pSource_image, intermediate_tex_data, coded_log_blocks, cfg, *m_params.m_pJob_pool); + if (!comp_status) + return cECFailedEncodeUASTC; + + if (m_params.m_debug) + fmt_debug_printf("----------------------------------------------------------------------------\n"); + + const uint32_t num_blocks_x = dst_tex.get_blocks_x(); + const uint32_t num_blocks_y = dst_tex.get_blocks_y(); + + assert(coded_log_blocks.get_width() == num_blocks_x); + assert(coded_log_blocks.get_height() == num_blocks_y); + + for (uint32_t by = 0; by < num_blocks_y; by++) + { + for (uint32_t bx = 0; bx < num_blocks_x; bx++) + { + const astc_helpers::log_astc_block& log_blk = coded_log_blocks(bx, by); + + bool pack_status = astc_helpers::pack_astc_block(*static_cast(dst_tex.get_block_ptr(bx, by)), log_blk); + if (!pack_status) + { + error_printf("basis_compressor::encode_slices_to_xuastc_or_astc_ldr: pack_astc_block() failed!\n"); + return cECFailedEncodeUASTC; + } + + } // bx + } // by + + uint8_vec& dst_buf = m_uastc_backend_output.m_slice_image_data[slice_index]; + + if (basis_tex_format_is_astc_ldr(m_fmt_mode)) + { + // Plain ASTC LDR 4x4-12x12 + dst_buf.resize(dst_tex.get_size_in_bytes()); + memcpy(&dst_buf[0], dst_tex.get_ptr(), dst_tex.get_size_in_bytes()); + } + else + { + // Supercompressed XUASTC LDR 4x4-12x12 + assert(intermediate_tex_data.size_in_bytes()); + + dst_buf.resize(intermediate_tex_data.size_in_bytes()); + memcpy(&dst_buf[0], intermediate_tex_data.get_ptr(), intermediate_tex_data.size_in_bytes()); + } + + m_uastc_backend_output.m_slice_image_crcs[slice_index] = basist::crc16(dst_buf.get_ptr(), dst_buf.size_in_bytes(), 0); + + } // slice_index + + return cECSuccess; + } + basis_compressor::error_code basis_compressor::encode_slices_to_uastc_4x4_ldr() { debug_printf("basis_compressor::encode_slices_to_uastc_4x4_ldr\n"); @@ -933,12 +1473,12 @@ namespace basisu for (uint32_t slice_index = 0; slice_index < m_slice_descs.size(); slice_index++) m_uastc_slice_textures[slice_index].init(texture_format::cUASTC4x4, m_slice_descs[slice_index].m_orig_width, m_slice_descs[slice_index].m_orig_height); - m_uastc_backend_output.m_tex_format = basist::basis_tex_format::cUASTC4x4; + m_uastc_backend_output.m_tex_format = basist::basis_tex_format::cUASTC_LDR_4x4; m_uastc_backend_output.m_etc1s = false; m_uastc_backend_output.m_slice_desc = m_slice_descs; m_uastc_backend_output.m_slice_image_data.resize(m_slice_descs.size()); m_uastc_backend_output.m_slice_image_crcs.resize(m_slice_descs.size()); - + for (uint32_t slice_index = 0; slice_index < m_slice_descs.size(); slice_index++) { gpu_image& tex = m_uastc_slice_textures[slice_index]; @@ -950,19 +1490,21 @@ namespace basisu const uint32_t total_blocks = tex.get_total_blocks(); const image& source_image = m_slice_images[slice_index]; - std::atomic total_blocks_processed; - total_blocks_processed.store(0); - + std::mutex status_output_mutex; + uint32_t total_blocks_processed = 0; + float last_percentage_printed = 0; + const uint32_t N = 256; for (uint32_t block_index_iter = 0; block_index_iter < total_blocks; block_index_iter += N) { const uint32_t first_index = block_index_iter; const uint32_t last_index = minimum(total_blocks, block_index_iter + N); - - m_params.m_pJob_pool->add_job([this, first_index, last_index, num_blocks_x, num_blocks_y, total_blocks, &source_image, &tex, &total_blocks_processed] + + m_params.m_pJob_pool->add_job([this, first_index, last_index, num_blocks_x, num_blocks_y, total_blocks, &source_image, &tex, + &status_output_mutex, &total_blocks_processed, &last_percentage_printed] { BASISU_NOTE_UNUSED(num_blocks_y); - + uint32_t uastc_flags = m_params.m_pack_uastc_ldr_4x4_flags; if ((m_params.m_rdo_uastc_ldr_4x4) && (m_params.m_rdo_uastc_ldr_4x4_favor_simpler_modes_in_rdo_mode)) uastc_flags |= cPackUASTCFavorSimplerModes; @@ -980,14 +1522,31 @@ namespace basisu encode_uastc(&block_pixels[0][0].r, dest_block, uastc_flags); - total_blocks_processed++; + } // block_index + + if (m_params.m_status_output) + { + float percent_done = 0; + bool print_flag = false; - uint32_t val = total_blocks_processed; - if (((val & 16383) == 16383) && m_params.m_status_output) { - debug_printf("basis_compressor::encode_slices_to_uastc_4x4_ldr: %3.1f%% done\n", static_cast(val) * 100.0f / total_blocks); + std::lock_guard lck(status_output_mutex); + + total_blocks_processed += (last_index - first_index) + 1; + + percent_done = ((float)total_blocks_processed * 100.0f) / (float)total_blocks; + + if ((percent_done >= 100.0f) || (percent_done >= (last_percentage_printed + 5.0f))) + { + last_percentage_printed = percent_done; + + print_flag = true; + } } + // minor print race here, doesn't matter + if (print_flag) + debug_printf("basis_compressor::encode_slices_to_uastc_4x4_ldr: %3.1f%% done\n", percent_done); } }); @@ -1005,7 +1564,7 @@ namespace basisu rdo_params.m_lz_dict_size = m_params.m_rdo_uastc_ldr_4x4_dict_size; rdo_params.m_smooth_block_max_error_scale = m_params.m_rdo_uastc_ldr_4x4_max_smooth_block_error_scale; rdo_params.m_max_smooth_block_std_dev = m_params.m_rdo_uastc_ldr_4x4_smooth_block_max_std_dev; - + bool status = uastc_rdo(tex.get_total_blocks(), (basist::uastc_block*)tex.get_ptr(), (const color_rgba *)m_source_blocks[slice_desc.m_first_block_index].m_pixels, rdo_params, m_params.m_pack_uastc_ldr_4x4_flags, m_params.m_rdo_uastc_ldr_4x4_multithreading ? m_params.m_pJob_pool : nullptr, (m_params.m_rdo_uastc_ldr_4x4_multithreading && m_params.m_pJob_pool) ? basisu::minimum(4, (uint32_t)m_params.m_pJob_pool->get_total_threads()) : 0); @@ -1017,11 +1576,11 @@ namespace basisu m_uastc_backend_output.m_slice_image_data[slice_index].resize(tex.get_size_in_bytes()); memcpy(&m_uastc_backend_output.m_slice_image_data[slice_index][0], tex.get_ptr(), tex.get_size_in_bytes()); - + m_uastc_backend_output.m_slice_image_crcs[slice_index] = basist::crc16(tex.get_ptr(), tex.get_size_in_bytes(), 0); - + } // slice_index - + return cECSuccess; } @@ -1057,8 +1616,8 @@ namespace basisu pSource_image = &mips[level - 1]; } - bool status = image_resample(*pSource_image, level_img, - //m_params.m_mip_filter.c_str(), + bool status = image_resample(*pSource_image, level_img, + //m_params.m_mip_filter.c_str(), "box", // TODO: negative lobes in the filter are causing negative colors, try Mitchell m_params.m_mip_scale, m_params.m_mip_wrapping, 0, has_alpha ? 4 : 3); if (!status) @@ -1113,12 +1672,12 @@ namespace basisu image &level_img = *enlarge_vector(mips, 1); level_img.resize(level_width, level_height); - - int result = stbir_resize_uint8_generic( + + int result = stbir_resize_uint8_generic( (const uint8_t *)img.get_ptr(), img.get_width(), img.get_height(), img.get_pitch() * sizeof(color_rgba), (uint8_t *)level_img.get_ptr(), level_img.get_width(), level_img.get_height(), level_img.get_pitch() * sizeof(color_rgba), has_alpha ? 4 : 3, has_alpha ? 3 : STBIR_ALPHA_CHANNEL_NONE, m_params.m_mip_premultiplied ? STBIR_FLAG_ALPHA_PREMULTIPLIED : 0, - m_params.m_mip_wrapping ? STBIR_EDGE_WRAP : STBIR_EDGE_CLAMP, filter, m_params.m_mip_srgb ? STBIR_COLORSPACE_SRGB : STBIR_COLORSPACE_LINEAR, + m_params.m_mip_wrapping ? STBIR_EDGE_WRAP : STBIR_EDGE_CLAMP, filter, m_params.m_mip_srgb ? STBIR_COLORSPACE_SRGB : STBIR_COLORSPACE_LINEAR, nullptr); if (result == 0) @@ -1126,7 +1685,7 @@ namespace basisu error_printf("basis_compressor::generate_mipmaps: stbir_resize_uint8_generic() failed!\n"); return false; } - + if (m_params.m_mip_renormalize) level_img.renormalize_normal_map(); } @@ -1250,7 +1809,7 @@ namespace basisu if (m_params.m_source_mipmap_images.size() || m_params.m_source_mipmap_images_hdr.size()) return true; - + // See if any input filenames are .DDS bool any_dds = false, all_dds = true; for (uint32_t i = 0; i < m_params.m_source_filenames.size(); i++) @@ -1319,7 +1878,7 @@ namespace basisu ldr_mips.erase_index(0U); m_params.m_source_mipmap_images.back().swap(ldr_mips); - + any_mipmaps = true; } } @@ -1339,7 +1898,7 @@ namespace basisu hdr_mips.erase_index(0U); m_params.m_source_mipmap_images_hdr.back().swap(hdr_mips); - + any_mipmaps = true; } @@ -1363,7 +1922,7 @@ namespace basisu error_printf("HDR mode enabled, but only LDR .DDS files were loaded. HDR mode requires half or float (HDR) .DDS inputs.\n"); return false; } - + return true; } @@ -1371,7 +1930,7 @@ namespace basisu { debug_printf("basis_compressor::read_source_images\n"); - const uint32_t total_source_files = m_params.m_read_source_images ? (uint32_t)m_params.m_source_filenames.size() : + const uint32_t total_source_files = m_params.m_read_source_images ? (uint32_t)m_params.m_source_filenames.size() : (m_params.m_hdr ? (uint32_t)m_params.m_source_images_hdr.size() : (uint32_t)m_params.m_source_images.size()); if (!total_source_files) @@ -1395,7 +1954,7 @@ namespace basisu basisu::vector source_images_hdr; basisu::vector source_filenames; - + // TODO: Note HDR images don't support alpha here, currently. // First load all source images, and determine if any have an alpha channel. @@ -1530,7 +2089,7 @@ namespace basisu for (uint32_t x = 0; x < file_image_hdr.get_width(); x++) { const vec4F& c = file_image_hdr(x, y); - + // For now, alpha is always 1.0f in UASTC HDR. file_image_hdr(x, y).set(c[m_params.m_swizzle[0]], c[m_params.m_swizzle[1]], c[m_params.m_swizzle[2]], 1.0f); // c[m_params.m_swizzle[3]]); } @@ -1648,7 +2207,7 @@ namespace basisu source_filenames.push_back(pSource_filename); } - // Check if the caller has generated their own mipmaps. + // Check if the caller has generated their own mipmaps. if (m_params.m_hdr) { if (m_params.m_source_mipmap_images_hdr.size()) @@ -1661,7 +2220,7 @@ namespace basisu } } } - else + else { if (m_params.m_source_mipmap_images.size()) { @@ -1702,23 +2261,23 @@ namespace basisu for (uint32_t source_file_index = 0; source_file_index < total_source_files; source_file_index++) { const std::string &source_filename = source_filenames[source_file_index]; - + basisu::vector slices; basisu::vector slices_hdr; - + slices.reserve(32); slices_hdr.reserve(32); - + // The first (largest) mipmap level. image *pFile_image = source_images.size() ? &source_images[source_file_index] : nullptr; imagef *pFile_image_hdr = source_images_hdr.size() ? &source_images_hdr[source_file_index] : nullptr; - + // Reserve a slot for mip0. if (m_params.m_hdr) slices_hdr.resize(1); else slices.resize(1); - + if ((!m_params.m_hdr) && (m_params.m_source_mipmap_images.size())) { // User-provided mipmaps for each layer or image in the texture array. @@ -1800,10 +2359,10 @@ namespace basisu uint_vec mip_indices(m_params.m_hdr ? slices_hdr.size() : slices.size()); for (uint32_t i = 0; i < (m_params.m_hdr ? slices_hdr.size() : slices.size()); i++) mip_indices[i] = i; - + if ((!m_params.m_hdr) && (m_any_source_image_has_alpha) && (!m_params.m_uastc)) { - // For ETC1S, if source has alpha, then even mips will have RGB, and odd mips will have alpha in RGB. + // For ETC1S, if source has alpha, then even mips will have RGB, and odd mips will have alpha in RGB. basisu::vector alpha_slices; uint_vec new_mip_indices; @@ -1822,7 +2381,7 @@ namespace basisu lvl_a(x, y).set_noclamp_rgba(a, a, a, 255); } } - + lvl_rgb.set_alpha(255); alpha_slices.push_back(lvl_rgb); @@ -1844,7 +2403,7 @@ namespace basisu { assert(slices.size() == mip_indices.size()); } - + for (uint32_t slice_index = 0; slice_index < (m_params.m_hdr ? slices_hdr.size() : slices.size()); slice_index++) { image *pSlice_image = m_params.m_hdr ? nullptr : &slices[slice_index]; @@ -1903,10 +2462,10 @@ namespace basisu m_stats[dest_image_index].m_width = orig_width; m_stats[dest_image_index].m_height = orig_height; - debug_printf("****** Slice %u: mip %u, alpha_slice: %u, filename: \"%s\", original: %ux%u actual: %ux%u\n", - m_slice_descs.size() - 1, mip_indices[slice_index], is_alpha_slice, source_filename.c_str(), - orig_width, orig_height, - m_params.m_hdr ? pSlice_image_hdr->get_width() : pSlice_image->get_width(), + debug_printf("****** Slice %u: mip %u, alpha_slice: %u, filename: \"%s\", original: %ux%u actual: %ux%u\n", + m_slice_descs.size() - 1, mip_indices[slice_index], is_alpha_slice, source_filename.c_str(), + orig_width, orig_height, + m_params.m_hdr ? pSlice_image_hdr->get_width() : pSlice_image->get_width(), m_params.m_hdr ? pSlice_image_hdr->get_height() : pSlice_image->get_height()); basisu_backend_slice_desc& slice_desc = m_slice_descs[dest_image_index]; @@ -1969,7 +2528,7 @@ namespace basisu error_printf("Too many slices!\n"); return false; } - + // Basic sanity check on the slices for (uint32_t i = 1; i < m_slice_descs.size(); i++) { @@ -1979,7 +2538,7 @@ namespace basisu // Make sure images are in order int image_delta = (int)slice_desc.m_source_file_index - (int)prev_slice_desc.m_source_file_index; if (image_delta > 1) - return false; + return false; // Make sure mipmap levels are in order if (!image_delta) @@ -2002,8 +2561,8 @@ namespace basisu if (m_params.m_status_output) { printf("Slice: %u, alpha: %u, orig width/height: %ux%u, width/height: %ux%u, first_block: %u, image_index: %u, mip_level: %u, iframe: %u\n", - i, slice_desc.m_alpha, slice_desc.m_orig_width, slice_desc.m_orig_height, - slice_desc.m_width, slice_desc.m_height, + i, slice_desc.m_alpha, slice_desc.m_orig_width, slice_desc.m_orig_height, + slice_desc.m_width, slice_desc.m_height, slice_desc.m_first_block_index, slice_desc.m_source_file_index, slice_desc.m_mip_index, slice_desc.m_iframe); } @@ -2058,20 +2617,20 @@ namespace basisu } // Do some basic validation for 2D arrays, cubemaps, video, and volumes. - bool basis_compressor::validate_texture_type_constraints() + bool basis_compressor::validate_texture_type_constraints() { debug_printf("basis_compressor::validate_texture_type_constraints\n"); // In 2D mode anything goes (each image may have a different resolution and # of mipmap levels). if (m_params.m_tex_type == basist::cBASISTexType2D) return true; - + uint32_t total_basis_images = 0; for (uint32_t slice_index = 0; slice_index < (m_params.m_hdr ? m_slice_images_hdr.size() : m_slice_images.size()); slice_index++) { const basisu_backend_slice_desc &slice_desc = m_slice_descs[slice_index]; - + total_basis_images = maximum(total_basis_images, slice_desc.m_source_file_index + 1); } @@ -2094,7 +2653,7 @@ namespace basisu const basisu_backend_slice_desc &slice_desc = m_slice_descs[slice_index]; image_mipmap_levels[slice_desc.m_source_file_index] = maximum(image_mipmap_levels[slice_desc.m_source_file_index], slice_desc.m_mip_index + 1); - + if (slice_desc.m_mip_index != 0) continue; @@ -2127,9 +2686,13 @@ namespace basisu debug_printf("basis_compressor::extract_source_blocks\n"); // No need to extract blocks in 6x6 mode, but the 4x4 compressors want 4x4 blocks. - if ((m_fmt_mode == basist::basis_tex_format::cASTC_HDR_6x6) || (m_fmt_mode == basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE)) + if ((m_fmt_mode == basist::basis_tex_format::cASTC_HDR_6x6) || (m_fmt_mode == basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE)) return true; + // No need to extract blocks in XUASTC/ASTC LDR mode either. + if (basis_tex_format_is_xuastc_ldr(m_fmt_mode) || basis_tex_format_is_astc_ldr(m_fmt_mode)) + return true; + if (m_params.m_hdr) m_source_blocks_hdr.resize(m_total_blocks); else @@ -2184,7 +2747,7 @@ namespace basisu bool basis_compressor::process_frontend() { debug_printf("basis_compressor::process_frontend\n"); - + #if 0 // TODO basis_etc1_pack_params pack_params; @@ -2235,21 +2798,21 @@ namespace basisu error_printf("Too many selector clusters! (%u but max is %u)\n", selector_clusters, basisu_frontend::cMaxSelectorClusters); return false; } - - if (m_params.m_etc1s_quality_level != -1) + + if (m_params.m_quality_level != -1) { - const float quality = saturate(m_params.m_etc1s_quality_level / 255.0f); - + const float quality = saturate(m_params.m_quality_level / 255.0f); + const float bits_per_endpoint_cluster = 14.0f; const float max_desired_endpoint_cluster_bits_per_texel = 1.0f; // .15f int max_endpoints = static_cast((max_desired_endpoint_cluster_bits_per_texel * total_texels) / bits_per_endpoint_cluster); - + const float mid = 128.0f / 255.0f; float color_endpoint_quality = quality; const float endpoint_split_point = 0.5f; - + // In v1.2 and in previous versions, the endpoint codebook size at quality 128 was 3072. This wasn't quite large enough. const int ENDPOINT_CODEBOOK_MID_QUALITY_CODEBOOK_SIZE = 4800; const int MAX_ENDPOINT_CODEBOOK_SIZE = 8192; @@ -2260,7 +2823,7 @@ namespace basisu max_endpoints = clamp(max_endpoints, 256, ENDPOINT_CODEBOOK_MID_QUALITY_CODEBOOK_SIZE); max_endpoints = minimum(max_endpoints, m_total_blocks); - + if (max_endpoints < 64) max_endpoints = 64; endpoint_clusters = clamp((uint32_t)(.5f + lerp(32, static_cast(max_endpoints), color_endpoint_quality)), 32, basisu_frontend::cMaxEndpointClusters); @@ -2271,12 +2834,12 @@ namespace basisu max_endpoints = clamp(max_endpoints, 256, MAX_ENDPOINT_CODEBOOK_SIZE); max_endpoints = minimum(max_endpoints, m_total_blocks); - + if (max_endpoints < ENDPOINT_CODEBOOK_MID_QUALITY_CODEBOOK_SIZE) max_endpoints = ENDPOINT_CODEBOOK_MID_QUALITY_CODEBOOK_SIZE; endpoint_clusters = clamp((uint32_t)(.5f + lerp(ENDPOINT_CODEBOOK_MID_QUALITY_CODEBOOK_SIZE, static_cast(max_endpoints), color_endpoint_quality)), 32, basisu_frontend::cMaxEndpointClusters); } - + float bits_per_selector_cluster = 14.0f; const float max_desired_selector_cluster_bits_per_texel = 1.0f; // .15f @@ -2294,18 +2857,18 @@ namespace basisu debug_printf("Max endpoints: %u, max selectors: %u\n", endpoint_clusters, selector_clusters); - if (m_params.m_etc1s_quality_level >= 223) + if (m_params.m_quality_level >= 223) { if (!m_params.m_selector_rdo_thresh.was_changed()) { if (!m_params.m_endpoint_rdo_thresh.was_changed()) m_params.m_endpoint_rdo_thresh *= .25f; - + if (!m_params.m_selector_rdo_thresh.was_changed()) m_params.m_selector_rdo_thresh *= .25f; } } - else if (m_params.m_etc1s_quality_level >= 192) + else if (m_params.m_quality_level >= 192) { if (!m_params.m_endpoint_rdo_thresh.was_changed()) m_params.m_endpoint_rdo_thresh *= .5f; @@ -2313,7 +2876,7 @@ namespace basisu if (!m_params.m_selector_rdo_thresh.was_changed()) m_params.m_selector_rdo_thresh *= .5f; } - else if (m_params.m_etc1s_quality_level >= 160) + else if (m_params.m_quality_level >= 160) { if (!m_params.m_endpoint_rdo_thresh.was_changed()) m_params.m_endpoint_rdo_thresh *= .75f; @@ -2321,18 +2884,18 @@ namespace basisu if (!m_params.m_selector_rdo_thresh.was_changed()) m_params.m_selector_rdo_thresh *= .75f; } - else if (m_params.m_etc1s_quality_level >= 129) + else if (m_params.m_quality_level >= 129) { float l = (quality - 129 / 255.0f) / ((160 - 129) / 255.0f); if (!m_params.m_endpoint_rdo_thresh.was_changed()) m_params.m_endpoint_rdo_thresh *= lerp(1.0f, .75f, l); - + if (!m_params.m_selector_rdo_thresh.was_changed()) m_params.m_selector_rdo_thresh *= lerp(1.0f, .75f, l); } } - + basisu_frontend::params p; p.m_num_source_blocks = m_total_blocks; p.m_pSource_blocks = &m_source_blocks[0]; @@ -2341,14 +2904,14 @@ namespace basisu p.m_perceptual = m_params.m_perceptual; p.m_debug_stats = m_params.m_debug; p.m_debug_images = m_params.m_debug_images; - p.m_compression_level = m_params.m_compression_level; + p.m_compression_level = m_params.m_etc1s_compression_level; p.m_tex_type = m_params.m_tex_type; p.m_multithreaded = m_params.m_multithreading; p.m_disable_hierarchical_endpoint_codebooks = m_params.m_disable_hierarchical_endpoint_codebooks; p.m_validate = m_params.m_validate_etc1s; p.m_pJob_pool = m_params.m_pJob_pool; p.m_pGlobal_codebooks = m_params.m_pGlobal_codebooks; - + // Don't keep trying to use OpenCL if it ever fails. p.m_pOpenCL_context = !m_opencl_failed ? m_pOpenCL_context : nullptr; @@ -2357,7 +2920,7 @@ namespace basisu error_printf("basisu_frontend::init() failed!\n"); return false; } - + m_frontend.compress(); if (m_frontend.get_opencl_failed()) @@ -2368,18 +2931,18 @@ namespace basisu for (uint32_t i = 0; i < m_slice_descs.size(); i++) { char filename[1024]; -#ifdef _WIN32 +#ifdef _WIN32 sprintf_s(filename, sizeof(filename), "rdo_frontend_output_output_blocks_%u.png", i); #else snprintf(filename, sizeof(filename), "rdo_frontend_output_output_blocks_%u.png", i); -#endif +#endif m_frontend.dump_debug_image(filename, m_slice_descs[i].m_first_block_index, m_slice_descs[i].m_num_blocks_x, m_slice_descs[i].m_num_blocks_y, true); #ifdef _WIN32 sprintf_s(filename, sizeof(filename), "rdo_frontend_output_api_%u.png", i); #else snprintf(filename, sizeof(filename), "rdo_frontend_output_api_%u.png", i); -#endif +#endif m_frontend.dump_debug_image(filename, m_slice_descs[i].m_first_block_index, m_slice_descs[i].m_num_blocks_x, m_slice_descs[i].m_num_blocks_y, false); } } @@ -2428,7 +2991,7 @@ namespace basisu for (uint32_t block_x = 0; block_x < num_blocks_x; block_x++) memcpy(m_best_etc1s_images[i].get_block_ptr(block_x, block_y, 0), &m_frontend.get_etc1s_block(slice_desc.m_first_block_index + block_x + block_y * num_blocks_x), sizeof(etc_block)); - m_best_etc1s_images[i].unpack(m_best_etc1s_images_unpacked[i]); + m_best_etc1s_images[i].unpack(m_best_etc1s_images_unpacked[i], false); } return true; @@ -2442,7 +3005,7 @@ namespace basisu backend_params.m_debug = m_params.m_debug; backend_params.m_debug_images = m_params.m_debug_images; backend_params.m_etc1s = true; - backend_params.m_compression_level = m_params.m_compression_level; + backend_params.m_compression_level = m_params.m_etc1s_compression_level; if (!m_params.m_no_endpoint_rdo) backend_params.m_endpoint_rdo_quality_thresh = m_params.m_endpoint_rdo_thresh; @@ -2484,7 +3047,7 @@ namespace basisu m_output_basis_file = comp_data; uint32_t total_orig_pixels = 0; - + for (uint32_t i = 0; i < m_slice_descs.size(); i++) { const basisu_backend_slice_desc& slice_desc = m_slice_descs[i]; @@ -2492,22 +3055,22 @@ namespace basisu total_orig_pixels += slice_desc.m_orig_width * slice_desc.m_orig_height; } - m_basis_file_size = (uint32_t)comp_data.size(); + m_total_slice_orig_texels = total_orig_pixels; + m_basis_file_size = comp_data.size(); m_basis_bits_per_texel = total_orig_pixels ? (comp_data.size() * 8.0f) / total_orig_pixels : 0; - debug_printf("Total .basis output file size: %u, %3.3f bits/texel\n", comp_data.size(), comp_data.size() * 8.0f / total_orig_pixels); + fmt_debug_printf("Total .basis output file size: {}, {3.3} bits/texel\n", m_basis_file_size, m_basis_bits_per_texel); // HDR 6x6 TODO - // HACK HACK const bool is_hdr_6x6 = m_params.m_hdr && (m_params.m_hdr_mode != hdr_modes::cUASTC_HDR_4X4); - + if (m_params.m_validate_output_data) { interval_timer tm; tm.start(); basist::basisu_transcoder_init(); - + debug_printf("basist::basisu_transcoder_init: Took %f ms\n", tm.get_elapsed_ms()); // Verify the compressed data by transcoding it to ASTC (or ETC1)/BC7 and validating the CRC's. @@ -2548,35 +3111,62 @@ namespace basisu return false; } - double start_transcoding_time = tm.get_elapsed_secs(); - - debug_printf("basisu_compressor::start_transcoding() took %3.3fms\n", start_transcoding_time * 1000.0f); - - double total_time_etc1s_or_astc = 0; + double start_transcoding_time = tm.get_elapsed_secs(); + + debug_printf("basisu_compressor::start_transcoding() took %3.3fms\n", start_transcoding_time * 1000.0f); + + double total_time_etc1s_or_astc = 0; + + // Select formats to transcode to + basisu::texture_format tex_format; + basist::block_format blk_format; + + if (m_params.m_hdr) + { + // HDR + tex_format = texture_format::cBC6HUnsigned; + blk_format = basist::block_format::cBC6H; + } + else if (m_fmt_mode == basist::basis_tex_format::cUASTC_LDR_4x4) + { + // UASTC LDR 4x4 + tex_format = texture_format::cUASTC4x4; + blk_format = basist::block_format::cUASTC_4x4; + } + else if (basis_tex_format_is_xuastc_ldr(m_fmt_mode) || basis_tex_format_is_astc_ldr(m_fmt_mode)) + { + // XUASTC LDR 4x4-12x12 or ASTC LDR 4x4-12x12 + basist::transcoder_texture_format transcoder_fmt = basist::basis_get_transcoder_texture_format_from_xuastc_or_astc_ldr_basis_tex_format(m_fmt_mode); + + tex_format = basist::basis_get_texture_format_from_xuastc_or_astc_ldr_basis_tex_format(m_fmt_mode); + blk_format = basist::xuastc_get_block_format(transcoder_fmt); + } + else + { + // ETC1S + tex_format = texture_format::cETC1; + blk_format = basist::block_format::cETC1; + } for (uint32_t slice_iter = 0; slice_iter < m_slice_descs.size(); slice_iter++) { - // Select either BC6H, UASTC LDR 4x4, or ETC1 - basisu::texture_format tex_format = m_params.m_hdr ? texture_format::cBC6HUnsigned : (m_params.m_uastc ? texture_format::cUASTC4x4 : texture_format::cETC1); - basist::block_format blk_format = m_params.m_hdr ? basist::block_format::cBC6H : (m_params.m_uastc ? basist::block_format::cUASTC_4x4 : basist::block_format::cETC1); - gpu_image decoded_texture; decoded_texture.init( tex_format, - m_slice_descs[slice_iter].m_width, m_slice_descs[slice_iter].m_height); - - tm.start(); - - const uint32_t block_size_x = basisu::get_block_width(tex_format); - const uint32_t block_size_y = basisu::get_block_height(tex_format); - const uint32_t num_dst_blocks_x = (m_slice_descs[slice_iter].m_orig_width + block_size_x - 1) / block_size_x; - const uint32_t num_dst_blocks_y = (m_slice_descs[slice_iter].m_orig_height + block_size_y - 1) / block_size_y; + m_slice_descs[slice_iter].m_orig_width, m_slice_descs[slice_iter].m_orig_height); + + const uint32_t dst_block_size_x = basisu::get_block_width(tex_format); + const uint32_t dst_block_size_y = basisu::get_block_height(tex_format); + const uint32_t num_dst_blocks_x = (m_slice_descs[slice_iter].m_orig_width + dst_block_size_x - 1) / dst_block_size_x; + const uint32_t num_dst_blocks_y = (m_slice_descs[slice_iter].m_orig_height + dst_block_size_y - 1) / dst_block_size_y; const uint32_t total_dst_blocks = num_dst_blocks_x * num_dst_blocks_y; + + const uint32_t bytes_per_block = decoded_texture.get_bytes_per_block(); - uint32_t bytes_per_block = m_params.m_uastc ? 16 : 8; + tm.start(); if (!decoder.transcode_slice(&comp_data[0], (uint32_t)comp_data.size(), slice_iter, - reinterpret_cast(decoded_texture.get_ptr()), total_dst_blocks, blk_format, bytes_per_block)) + reinterpret_cast(decoded_texture.get_ptr()), total_dst_blocks, blk_format, bytes_per_block, m_params.m_transcode_flags)) { error_printf("Transcoding failed on slice %u!\n", slice_iter); return false; @@ -2606,15 +3196,15 @@ namespace basisu if (is_hdr_6x6) { assert(basist::basis_is_format_supported(basist::transcoder_texture_format::cTFASTC_HDR_6x6_RGBA, basist::basis_tex_format::cASTC_HDR_6x6)); - assert(basist::basis_is_format_supported(basist::transcoder_texture_format::cTFASTC_HDR_6x6_RGBA, basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE)); + assert(basist::basis_is_format_supported(basist::transcoder_texture_format::cTFASTC_HDR_6x6_RGBA, basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE)); for (uint32_t i = 0; i < m_slice_descs.size(); i++) { gpu_image decoded_texture; - decoded_texture.init(texture_format::cASTC_HDR_6x6, m_slice_descs[i].m_width, m_slice_descs[i].m_height); + decoded_texture.init(texture_format::cASTC_HDR_6x6, m_slice_descs[i].m_orig_width, m_slice_descs[i].m_orig_height); if (!decoder.transcode_slice(&comp_data[0], (uint32_t)comp_data.size(), i, - reinterpret_cast(decoded_texture.get_ptr()), m_slice_descs[i].m_num_blocks_x * m_slice_descs[i].m_num_blocks_y, basist::block_format::cASTC_HDR_6x6, 16)) + reinterpret_cast(decoded_texture.get_ptr()), m_slice_descs[i].m_num_blocks_x * m_slice_descs[i].m_num_blocks_y, basist::block_format::cASTC_HDR_6x6, 16, m_params.m_transcode_flags)) { error_printf("Transcoding failed to ASTC HDR on slice %u!\n", i); return false; @@ -2630,10 +3220,10 @@ namespace basisu for (uint32_t i = 0; i < m_slice_descs.size(); i++) { gpu_image decoded_texture; - decoded_texture.init(texture_format::cASTC_HDR_4x4, m_slice_descs[i].m_width, m_slice_descs[i].m_height); + decoded_texture.init(texture_format::cASTC_HDR_4x4, m_slice_descs[i].m_orig_width, m_slice_descs[i].m_orig_height); if (!decoder.transcode_slice(&comp_data[0], (uint32_t)comp_data.size(), i, - reinterpret_cast(decoded_texture.get_ptr()), m_slice_descs[i].m_num_blocks_x * m_slice_descs[i].m_num_blocks_y, basist::block_format::cASTC_HDR_4x4, 16)) + reinterpret_cast(decoded_texture.get_ptr()), m_slice_descs[i].m_num_blocks_x * m_slice_descs[i].m_num_blocks_y, basist::block_format::cASTC_HDR_4x4, 16, m_params.m_transcode_flags)) { error_printf("Transcoding failed to ASTC HDR on slice %u!\n", i); return false; @@ -2645,16 +3235,19 @@ namespace basisu } else { - if (basist::basis_is_format_supported(basist::transcoder_texture_format::cTFBC7_RGBA, basist::basis_tex_format::cUASTC4x4) && + if (basist::basis_is_format_supported(basist::transcoder_texture_format::cTFBC7_RGBA, basist::basis_tex_format::cUASTC_LDR_4x4) && basist::basis_is_format_supported(basist::transcoder_texture_format::cTFBC7_RGBA, basist::basis_tex_format::cETC1S)) { for (uint32_t i = 0; i < m_slice_descs.size(); i++) { gpu_image decoded_texture; - decoded_texture.init(texture_format::cBC7, m_slice_descs[i].m_width, m_slice_descs[i].m_height); + decoded_texture.init(texture_format::cBC7, m_slice_descs[i].m_orig_width, m_slice_descs[i].m_orig_height); + + const uint32_t num_bc7_blocks_x = decoded_texture.get_blocks_x(); + const uint32_t num_bc7_blocks_y = decoded_texture.get_blocks_y(); if (!decoder.transcode_slice(&comp_data[0], (uint32_t)comp_data.size(), i, - reinterpret_cast(decoded_texture.get_ptr()), m_slice_descs[i].m_num_blocks_x * m_slice_descs[i].m_num_blocks_y, basist::block_format::cBC7, 16)) + decoded_texture.get_ptr(), num_bc7_blocks_x * num_bc7_blocks_y, basist::block_format::cBC7, 16, m_params.m_transcode_flags)) { error_printf("Transcoding failed to BC7 on slice %u!\n", i); return false; @@ -2671,25 +3264,37 @@ namespace basisu { if (m_params.m_hdr) { - // BC6H bool status = m_decoded_output_textures[i].unpack_hdr(m_decoded_output_textures_bc6h_hdr_unpacked[i]); - assert(status); - BASISU_NOTE_UNUSED(status); - - // ASTC HDR + if (!status) + { + error_printf("Unpacking failed on slice %u!\n", i); + return false; + } + status = m_decoded_output_textures_astc_hdr[i].unpack_hdr(m_decoded_output_textures_astc_hdr_unpacked[i]); - assert(status); + if (!status) + { + error_printf("Unpacking failed on slice %u!\n", i); + return false; + } } else { - bool status = m_decoded_output_textures[i].unpack(m_decoded_output_textures_unpacked[i]); - assert(status); - BASISU_NOTE_UNUSED(status); + bool status = m_decoded_output_textures[i].unpack(m_decoded_output_textures_unpacked[i], m_params.m_ktx2_and_basis_srgb_transfer_function); + if (!status) + { + error_printf("Unpacking failed on slice %u!\n", i); + return false; + } if (m_decoded_output_textures_bc7[i].get_pixel_width()) { - status = m_decoded_output_textures_bc7[i].unpack(m_decoded_output_textures_unpacked_bc7[i]); - assert(status); + status = m_decoded_output_textures_bc7[i].unpack(m_decoded_output_textures_unpacked_bc7[i], m_params.m_ktx2_and_basis_srgb_transfer_function); + if (!status) + { + error_printf("Unpacking failed on slice %u!\n", i); + return false; + } } } } @@ -2703,6 +3308,7 @@ namespace basisu if (!is_hdr_6x6) { + // Sanity check decoded output texture sizes for (uint32_t slice_index = 0; slice_index < m_slice_descs.size(); slice_index++) { const basisu_backend_slice_desc& slice_desc = m_slice_descs[slice_index]; @@ -2715,7 +3321,7 @@ namespace basisu } } // if (m_params.m_validate_output_data) - + return true; } @@ -2756,7 +3362,7 @@ namespace basisu { const std::string filename(string_format("%s_compressive_tonemapped.png", pBasename)); image compressive_tonemapped_img; - + bool status = tonemap_image_compressive(compressive_tonemapped_img, hdr_img); if (!status) { @@ -2802,7 +3408,7 @@ namespace basisu if (m_params.m_status_output) { - printf("Wrote output .basis/.ktx2 file \"%s\"\n", output_filename.c_str()); + printf("Wrote compressed output file \"%s\"\n", output_filename.c_str()); } } @@ -2824,7 +3430,7 @@ namespace basisu uint32_t total_texels = 0; for (uint32_t i = 0; i < m_slice_descs.size(); i++) total_texels += (m_slice_descs[i].m_orig_width * m_slice_descs[i].m_orig_height); - + m_basis_bits_per_texel = ((float)comp_size * 8.0f) / total_texels; fmt_debug_printf("Output file size: {}, {3.2} bits/texel, LZ compressed file size: {}, {3.2} bits/texel\n", @@ -2833,7 +3439,7 @@ namespace basisu } m_stats.resize(m_slice_descs.size()); - + if (m_params.m_validate_output_data) { if (m_params.m_hdr) @@ -2906,7 +3512,7 @@ namespace basisu im.calc(m_slice_images_hdr[slice_index], m_decoded_output_textures_astc_hdr_unpacked[slice_index], 0, 3, true, true); s.m_basis_rgb_avg_log2_psnr = (float)im.m_psnr; - + if (m_params.m_print_stats) { printf("\nASTC Log2 RGB: "); @@ -2924,7 +3530,7 @@ namespace basisu printf("\n"); } } - + if (m_params.m_debug_images) { std::string out_basename; @@ -2940,9 +3546,9 @@ namespace basisu { gpu_image bc6h_tex(m_decoded_output_textures[slice_index]); bc6h_tex.override_dimensions(slice_desc.m_orig_width, slice_desc.m_orig_height); - + std::string filename(out_basename + "_bc6h.dds"); - write_compressed_texture_file(filename.c_str(), bc6h_tex, true); + write_compressed_texture_file(filename.c_str(), bc6h_tex, false); printf("Wrote .DDS file %s\n", filename.c_str()); } @@ -2950,11 +3556,11 @@ namespace basisu { gpu_image astc_tex(m_decoded_output_textures_astc_hdr[slice_index]); astc_tex.override_dimensions(slice_desc.m_orig_width, slice_desc.m_orig_height); - + std::string filename1(out_basename + "_astc.astc"); - + uint32_t block_width = 4, block_height = 4; - if ((m_params.m_hdr_mode == hdr_modes::cASTC_HDR_6X6) || (m_params.m_hdr_mode == hdr_modes::cASTC_HDR_6X6_INTERMEDIATE)) + if ((m_params.m_hdr_mode == hdr_modes::cASTC_HDR_6X6) || (m_params.m_hdr_mode == hdr_modes::cUASTC_HDR_6X6_INTERMEDIATE)) { block_width = 6; block_height = 6; @@ -2964,7 +3570,7 @@ namespace basisu printf("Wrote .ASTC file %s\n", filename1.c_str()); std::string filename2(out_basename + "_astc.ktx"); - write_compressed_texture_file(filename2.c_str(), astc_tex, true); + write_compressed_texture_file(filename2.c_str(), astc_tex, false); printf("Wrote .KTX file %s\n", filename2.c_str()); } @@ -2972,7 +3578,7 @@ namespace basisu { imagef astc_img(m_decoded_output_textures_astc_hdr_unpacked[slice_index]); astc_img.resize(slice_desc.m_orig_width, slice_desc.m_orig_height); - + std::string filename(out_basename + "_unpacked_astc.exr"); write_exr(filename.c_str(), astc_img, 3, 0); printf("Wrote .EXR file %s\n", filename.c_str()); @@ -3007,50 +3613,50 @@ namespace basisu printf("Slice: %u\n", slice_index); image_stats& s = m_stats[slice_index]; - + image_metrics em; // ---- .basis stats em.calc(m_slice_images[slice_index], m_decoded_output_textures_unpacked[slice_index], 0, 3); if (m_params.m_print_stats) - em.print(".basis RGB Avg: "); + em.print("RGB Avg: "); s.m_basis_rgb_avg_psnr = (float)em.m_psnr; em.calc(m_slice_images[slice_index], m_decoded_output_textures_unpacked[slice_index], 0, 4); if (m_params.m_print_stats) - em.print(".basis RGBA Avg: "); + em.print("RGBA Avg: "); s.m_basis_rgba_avg_psnr = (float)em.m_psnr; em.calc(m_slice_images[slice_index], m_decoded_output_textures_unpacked[slice_index], 0, 1); if (m_params.m_print_stats) - em.print(".basis R Avg: "); + em.print("R Avg: "); em.calc(m_slice_images[slice_index], m_decoded_output_textures_unpacked[slice_index], 1, 1); if (m_params.m_print_stats) - em.print(".basis G Avg: "); + em.print("G Avg: "); em.calc(m_slice_images[slice_index], m_decoded_output_textures_unpacked[slice_index], 2, 1); if (m_params.m_print_stats) - em.print(".basis B Avg: "); + em.print("B Avg: "); - if (m_params.m_uastc) + //if (m_params.m_uastc) { em.calc(m_slice_images[slice_index], m_decoded_output_textures_unpacked[slice_index], 3, 1); if (m_params.m_print_stats) - em.print(".basis A Avg: "); + em.print("A Avg: "); s.m_basis_a_avg_psnr = (float)em.m_psnr; } em.calc(m_slice_images[slice_index], m_decoded_output_textures_unpacked[slice_index], 0, 0); if (m_params.m_print_stats) - em.print(".basis 709 Luma: "); + em.print("709 Luma: "); s.m_basis_luma_709_psnr = static_cast(em.m_psnr); s.m_basis_luma_709_ssim = static_cast(em.m_ssim); em.calc(m_slice_images[slice_index], m_decoded_output_textures_unpacked[slice_index], 0, 0, true, true); if (m_params.m_print_stats) - em.print(".basis 601 Luma: "); + em.print("601 Luma: "); s.m_basis_luma_601_psnr = static_cast(em.m_psnr); if (m_slice_descs.size() == 1) @@ -3058,8 +3664,8 @@ namespace basisu const uint32_t output_size = comp_size ? (uint32_t)comp_size : (uint32_t)comp_data.size(); if (m_params.m_print_stats) { - debug_printf(".basis RGB PSNR per bit/texel*10000: %3.3f\n", 10000.0f * s.m_basis_rgb_avg_psnr / ((output_size * 8.0f) / (slice_desc.m_orig_width * slice_desc.m_orig_height))); - debug_printf(".basis Luma 709 PSNR per bit/texel*10000: %3.3f\n", 10000.0f * s.m_basis_luma_709_psnr / ((output_size * 8.0f) / (slice_desc.m_orig_width * slice_desc.m_orig_height))); + debug_printf("RGB PSNR per bit/texel*10000: %3.3f\n", 10000.0f * s.m_basis_rgb_avg_psnr / ((output_size * 8.0f) / (slice_desc.m_orig_width * slice_desc.m_orig_height))); + debug_printf("Luma 709 PSNR per bit/texel*10000: %3.3f\n", 10000.0f * s.m_basis_luma_709_psnr / ((output_size * 8.0f) / (slice_desc.m_orig_width * slice_desc.m_orig_height))); } } @@ -3067,45 +3673,45 @@ namespace basisu { // ---- BC7 stats em.calc(m_slice_images[slice_index], m_decoded_output_textures_unpacked_bc7[slice_index], 0, 3); - //if (m_params.m_print_stats) - // em.print("BC7 RGB Avg: "); + if (m_params.m_print_stats) + em.print("BC7 RGB Avg: "); s.m_bc7_rgb_avg_psnr = (float)em.m_psnr; em.calc(m_slice_images[slice_index], m_decoded_output_textures_unpacked_bc7[slice_index], 0, 4); - //if (m_params.m_print_stats) - // em.print("BC7 RGBA Avg: "); + if (m_params.m_print_stats) + em.print("BC7 RGBA Avg: "); s.m_bc7_rgba_avg_psnr = (float)em.m_psnr; em.calc(m_slice_images[slice_index], m_decoded_output_textures_unpacked_bc7[slice_index], 0, 1); - //if (m_params.m_print_stats) - // em.print("BC7 R Avg: "); + if (m_params.m_print_stats) + em.print("BC7 R Avg: "); em.calc(m_slice_images[slice_index], m_decoded_output_textures_unpacked_bc7[slice_index], 1, 1); - //if (m_params.m_print_stats) - // em.print("BC7 G Avg: "); + if (m_params.m_print_stats) + em.print("BC7 G Avg: "); em.calc(m_slice_images[slice_index], m_decoded_output_textures_unpacked_bc7[slice_index], 2, 1); - //if (m_params.m_print_stats) - // em.print("BC7 B Avg: "); + if (m_params.m_print_stats) + em.print("BC7 B Avg: "); - if (m_params.m_uastc) + //if (m_params.m_uastc) { em.calc(m_slice_images[slice_index], m_decoded_output_textures_unpacked_bc7[slice_index], 3, 1); - //if (m_params.m_print_stats) - // em.print("BC7 A Avg: "); + if (m_params.m_print_stats) + em.print("BC7 A Avg: "); s.m_bc7_a_avg_psnr = (float)em.m_psnr; } em.calc(m_slice_images[slice_index], m_decoded_output_textures_unpacked_bc7[slice_index], 0, 0); - //if (m_params.m_print_stats) - // em.print("BC7 709 Luma: "); + if (m_params.m_print_stats) + em.print("BC7 709 Luma: "); s.m_bc7_luma_709_psnr = static_cast(em.m_psnr); s.m_bc7_luma_709_ssim = static_cast(em.m_ssim); em.calc(m_slice_images[slice_index], m_decoded_output_textures_unpacked_bc7[slice_index], 0, 0, true, true); - //if (m_params.m_print_stats) - // em.print("BC7 601 Luma: "); + if (m_params.m_print_stats) + em.print("BC7 601 Luma: "); s.m_bc7_luma_601_psnr = static_cast(em.m_psnr); } @@ -3146,10 +3752,10 @@ namespace basisu { gpu_image best_etc1s_gpu_image(m_best_etc1s_images[slice_index]); best_etc1s_gpu_image.override_dimensions(slice_desc.m_orig_width, slice_desc.m_orig_height); - write_compressed_texture_file((out_basename + "_best_etc1s.ktx").c_str(), best_etc1s_gpu_image, true); + write_compressed_texture_file((out_basename + "_best_etc1s.ktx").c_str(), best_etc1s_gpu_image, m_params.m_ktx2_and_basis_srgb_transfer_function); image best_etc1s_unpacked; - best_etc1s_gpu_image.unpack(best_etc1s_unpacked); + best_etc1s_gpu_image.unpack(best_etc1s_unpacked, m_params.m_ktx2_and_basis_srgb_transfer_function); save_png(out_basename + "_best_etc1s.png", best_etc1s_unpacked); } } @@ -3160,7 +3766,7 @@ namespace basisu { gpu_image decoded_etc1s_or_astc(m_decoded_output_textures[slice_index]); decoded_etc1s_or_astc.override_dimensions(slice_desc.m_orig_width, slice_desc.m_orig_height); - write_compressed_texture_file((out_basename + "_transcoded_etc1s_or_astc.ktx").c_str(), decoded_etc1s_or_astc, true); + write_compressed_texture_file((out_basename + "_transcoded_etc1s_or_astc.ktx").c_str(), decoded_etc1s_or_astc, m_params.m_ktx2_and_basis_srgb_transfer_function); image temp(m_decoded_output_textures_unpacked[slice_index]); temp.crop(slice_desc.m_orig_width, slice_desc.m_orig_height); @@ -3172,21 +3778,28 @@ namespace basisu { gpu_image decoded_bc7(m_decoded_output_textures_bc7[slice_index]); decoded_bc7.override_dimensions(slice_desc.m_orig_width, slice_desc.m_orig_height); - write_compressed_texture_file((out_basename + "_transcoded_bc7.ktx").c_str(), decoded_bc7, true); + write_compressed_texture_file((out_basename + "_transcoded_bc7.ktx").c_str(), decoded_bc7, m_params.m_ktx2_and_basis_srgb_transfer_function); image temp(m_decoded_output_textures_unpacked_bc7[slice_index]); temp.crop(slice_desc.m_orig_width, slice_desc.m_orig_height); save_png(out_basename + "_transcoded_bc7.png", temp); } } + + if ((m_params.m_debug) && (m_decoded_output_textures_bc7[slice_index].get_pixel_width())) + { + const gpu_image& decoded_bc7 = m_decoded_output_textures_bc7[slice_index]; + + create_bc7_debug_images(slice_desc.m_orig_width, slice_desc.m_orig_height, decoded_bc7.get_ptr(), m_params.m_debug_images ? out_basename.c_str() : nullptr); + } } } // if (m_params.m_hdr) } // if (m_params.m_validate_output_data) - + return true; } - + // Make sure all the mip 0's have the same dimensions and number of mipmap levels, or we can't encode the KTX2 file. bool basis_compressor::validate_ktx2_constraints() { @@ -3227,20 +3840,80 @@ namespace basisu return true; } + + // KTX2 DFD base definitions // colorModel=KTX2_KDF_DF_MODEL_ETC1S (0xA3) // LDR ETC1S texture data in a custom format, with global codebooks - static uint8_t g_ktx2_etc1s_nonalpha_dfd[44] = { 0x2C,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x2,0x0,0x28,0x0,0xA3,0x1,0x2,0x0,0x3,0x3,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x3F,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0xFF,0xFF,0xFF,0xFF }; - static uint8_t g_ktx2_etc1s_alpha_dfd[60] = { 0x3C,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x2,0x0,0x38,0x0,0xA3,0x1,0x2,0x0,0x3,0x3,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x3F,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0xFF,0xFF,0xFF,0xFF,0x40,0x0,0x3F,0xF,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0xFF,0xFF,0xFF,0xFF }; + static uint8_t g_ktx2_etc1s_nonalpha_dfd[44] = + { + 0x2C,0x0,0x0,0x0, // 0 totalSize + 0x0,0x0,0x0,0x0, // 1 descriptorType/vendorId + 0x2,0x0,0x28,0x0, // 2 descriptorBlockSize/versionNumber + 0xA3,0x1,0x2,0x0, // 3 flags, transferFunction, colorPrimaries, colorModel (KTX2_KDF_DF_MODEL_UASTC_HDR_6X6_INTERMEDIATE) + 0x3,0x3,0x0,0x0, // 4 texelBlockDimension0-texelBlockDimension3 + 0x0,0x0,0x0,0x0, // 5 bytesPlane0-bytesPlane3 + 0x0,0x0,0x0,0x0, // 6 bytesPlane4-bytesPlane7 + 0x0,0x0,0x3F,0x0, // 7 bitOffset/bitLength/channelType and Qualifer flags (KHR_DF_SAMPLE_DATATYPE_FLOAT etc.) + 0x0,0x0,0x0,0x0, // 8 samplePosition0-samplePosition3 + 0x0,0x0,0x0,0x0, // 9 sampleLower (0) + 0xFF,0xFF,0xFF,0xFF // 10 sampleHigher (0xFF) + }; + static uint8_t g_ktx2_etc1s_alpha_dfd[60] = + { + 0x3C,0x0,0x0,0x0, + 0x0,0x0,0x0,0x0, + 0x2,0x0,0x38,0x0, + 0xA3,0x1,0x2,0x0, + 0x3,0x3,0x0,0x0, + 0x0,0x0,0x0,0x0, + 0x0,0x0,0x0,0x0, + 0x0,0x0,0x3F,0x0, + 0x0,0x0,0x0,0x0, + 0x0,0x0,0x0,0x0, + 0xFF,0xFF,0xFF,0xFF, + 0x40,0x0,0x3F,0xF, + 0x0,0x0,0x0,0x0, + 0x0,0x0,0x0,0x0, + 0xFF,0xFF,0xFF,0xFF + }; + // colorModel=KTX2_KDF_DF_MODEL_UASTC_LDR_4X4 (0xA6) // LDR UASTC 4x4 texture data in a custom block format - static uint8_t g_ktx2_uastc_ldr_4x4_nonalpha_dfd[44] = { 0x2C,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x2,0x0,0x28,0x0,0xA6,0x1,0x2,0x0,0x3,0x3,0x0,0x0,0x10,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x7F,0x4,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0xFF,0xFF,0xFF,0xFF }; - static uint8_t g_ktx2_uastc_ldr_4x4_alpha_dfd[44] = { 0x2C,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x2,0x0,0x28,0x0,0xA6,0x1,0x2,0x0,0x3,0x3,0x0,0x0,0x10,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x7F,0x3,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0xFF,0xFF,0xFF,0xFF }; + static uint8_t g_ktx2_uastc_ldr_4x4_nonalpha_dfd[44] = + { + 0x2C,0x0,0x0,0x0, + 0x0,0x0,0x0,0x0, + 0x2,0x0,0x28,0x0, + 0xA6,0x1,0x2,0x0, + 0x3,0x3,0x0,0x0, + 0x10,0x0,0x0,0x0, + 0x0,0x0,0x0,0x0, + 0x0,0x0,0x7F,0x4, + 0x0,0x0,0x0,0x0, + 0x0,0x0,0x0,0x0, + 0xFF,0xFF,0xFF,0xFF + }; + + static uint8_t g_ktx2_uastc_ldr_4x4_alpha_dfd[44] = + { + 0x2C,0x0,0x0,0x0, + 0x0,0x0,0x0,0x0, + 0x2,0x0,0x28,0x0, + 0xA6,0x1,0x2,0x0, + 0x3,0x3,0x0,0x0, + 0x10,0x0,0x0,0x0, + 0x0,0x0,0x0,0x0, + 0x0,0x0,0x7F,0x3, + 0x0,0x0,0x0,0x0, + 0x0,0x0,0x0,0x0, + 0xFF,0xFF,0xFF,0xFF + }; // colorModel=KTX2_KDF_DF_MODEL_UASTC_HDR_4X4 (0xA7) // Standard ASTC HDR 4x4 texture data but constrained for easy transcoding to BC6H, either highest quality or RDO optimized. - static uint8_t g_ktx2_uastc_hdr_4x4_nonalpha_dfd[44] = + static uint8_t g_ktx2_uastc_hdr_4x4_nonalpha_dfd[44] = { 0x2C,0x0,0x0,0x0, // 0 totalSize 0x0,0x0,0x0,0x0, // 1 descriptorType/vendorId @@ -3249,7 +3922,7 @@ namespace basisu 0x3,0x3,0x0,0x0, // 4 texelBlockDimension0-texelBlockDimension3 0x10,0x0,0x0,0x0, // 5 bytesPlane0-bytesPlane3 0x0,0x0,0x0,0x0, // 6 bytesPlane4-bytesPlane7 - 0x0,0x0,0x7F,0x80, // 7 bitLength/bitOffset/channelType and Qualifer flags (KHR_DF_SAMPLE_DATATYPE_FLOAT etc.) + 0x0,0x0,0x7F,0x80, // 7 bitOffset/bitLength/channelType and Qualifer flags (KHR_DF_SAMPLE_DATATYPE_FLOAT etc.) 0x0,0x0,0x0,0x0, // 8 samplePosition0-samplePosition3 0x0,0x0,0x0,0x0, // 9 sampleLower (0.0) 0x00, 0x00, 0x80, 0x3F // 10 sampleHigher (1.0) @@ -3266,35 +3939,85 @@ namespace basisu 0x5,0x5,0x0,0x0, // 4 texelBlockDimension0-texelBlockDimension3 0x10,0x0,0x0,0x0, // 5 bytesPlane0-bytesPlane3 0x0,0x0,0x0,0x0, // 6 bytesPlane4-bytesPlane7 - 0x0,0x0,0x7F,0x80, // 7 bitLength/bitOffset/channelType and Qualifer flags (KHR_DF_SAMPLE_DATATYPE_FLOAT etc.) + 0x0,0x0,0x7F,0x80, // 7 bitOffset/bitLength/channelType and Qualifer flags (KHR_DF_SAMPLE_DATATYPE_FLOAT etc.) 0x0,0x0,0x0,0x0, // 8 samplePosition0-samplePosition3 0x0,0x0,0x0,0x0, // 9 sampleLower (0.0) 0x00, 0x00, 0x80, 0x3F // 10 sampleHigher (1.0) }; - // colorModel=KTX2_KDF_DF_MODEL_ASTC_HDR_6X6_INTERMEDIATE (0xA8) + // colorModel=KTX2_KDF_DF_MODEL_UASTC_HDR_6X6_INTERMEDIATE (0xA8) // Our custom intermediate format that when decoded directly outputs ASTC HDR 6x6 - static uint8_t g_ktx2_astc_hdr_6x6_intermediate_nonalpha_dfd[44] = + static uint8_t g_ktx2_uastc_hdr_6x6_intermediate_nonalpha_dfd[44] = { 0x2C,0x0,0x0,0x0, // 0 totalSize 0x0,0x0,0x0,0x0, // 1 descriptorType/vendorId 0x2,0x0,0x28,0x0, // 2 descriptorBlockSize/versionNumber - 0xA8,0x1,0x1,0x0, // 3 flags, transferFunction, colorPrimaries, colorModel (KTX2_KDF_DF_MODEL_ASTC_HDR_6X6_INTERMEDIATE) + 0xA8,0x1,0x1,0x0, // 3 flags, transferFunction, colorPrimaries, colorModel (KTX2_KDF_DF_MODEL_UASTC_HDR_6X6_INTERMEDIATE) 0x5,0x5,0x0,0x0, // 4 texelBlockDimension0-texelBlockDimension3 0x10,0x0,0x0,0x0, // 5 bytesPlane0-bytesPlane3 0x0,0x0,0x0,0x0, // 6 bytesPlane4-bytesPlane7 - 0x0,0x0,0x7F,0x80, // 7 bitLength/bitOffset/channelType and Qualifer flags (KHR_DF_SAMPLE_DATATYPE_FLOAT etc.) + 0x0,0x0,0x7F,0x80, // 7 bitOffset/bitLength/channelType and Qualifer flags (KHR_DF_SAMPLE_DATATYPE_FLOAT etc.) 0x0,0x0,0x0,0x0, // 8 samplePosition0-samplePosition3 0x0,0x0,0x0,0x0, // 9 sampleLower (0.0) 0x00, 0x00, 0x80, 0x3F // 10 sampleHigher (1.0) }; + // colorModel=KTX2_KDF_DF_MODEL_XUASTC_LDR_INTERMEDIATE (0xA9) + // Custom supercompressed intermediate format, decodes directly to standard ASTC LDR 4x4-12x12. + static uint8_t g_ktx2_xuastc_ldr_intermediate_dfd[44] = + { + 0x2C,0x0,0x0,0x0, // 0 totalSize + 0x0,0x0,0x0,0x0, // 1 descriptorType/vendorId + 0x2,0x0,0x28,0x0, // 2 descriptorBlockSize/versionNumber + (uint8_t)basist::KTX2_KDF_DF_MODEL_XUASTC_LDR_INTERMEDIATE,0x1,0x1,0x0, // 3 flags, transferFunction, colorPrimaries, colorModel (KTX2_KDF_DF_MODEL_UASTC_HDR_6X6_INTERMEDIATE) + 0x3,0x3,0x0,0x0, // 4 texelBlockDimension0-texelBlockDimension3 + 0x10,0x0,0x0,0x0, // 5 bytesPlane0-bytesPlane3 + 0x0,0x0,0x0,0x0, // 6 bytesPlane4-bytesPlane7 + 0x0,0x0,0x7F,0x00, // 7 bitOffset/bitLength/channelType and Qualifer flags (KHR_DF_SAMPLE_DATATYPE_FLOAT etc.) + 0x0,0x0,0x0,0x0, // 8 samplePosition0-samplePosition3 + 0x0,0x0,0x0,0x0, // 9 sampleLower (0) + 0xFF,0xFF,0xFF,0xFF // 10 sampleHigher (0xFF) + }; + + // ASTC LDR 4x4 + static uint8_t g_ktx2_astc_ldr_dfd[44] = + { + 0x2C,0x0,0x0,0x0, // 0 totalSize + 0x0,0x0,0x0,0x0, // 1 descriptorType/vendorId + 0x2,0x0,0x28,0x0, // 2 descriptorBlockSize/versionNumber + (uint8_t)basist::KTX2_KDF_DF_MODEL_ASTC,0x1,0x1,0x0, // 3 flags, transferFunction, colorPrimaries, colorModel + 0x3,0x3,0x0,0x0, // 4 texelBlockDimension0-texelBlockDimension3 + 0x10,0x0,0x0,0x0, // 5 bytesPlane0-bytesPlane3 + 0x0,0x0,0x0,0x0, // 6 bytesPlane4-bytesPlane7 + 0x0,0x0,0x7F,0x00, // 7 bitOffset/bitLength/channelType and Qualifer flags (KHR_DF_SAMPLE_DATATYPE_FLOAT etc.), channelID=KHR_DF_CHANNEL_ASTC_DATA + 0x0,0x0,0x0,0x0, // 8 samplePosition0-samplePosition3 + 0x0,0x0,0x0,0x0, // 9 sampleLower (0.0) + 0xFF,0xFF,0xFF,0xFF // 10 sampleHigher (0xFF) + }; + bool basis_compressor::get_dfd(uint8_vec &dfd, const basist::ktx2_header &header) { - const uint8_t* pDFD; - uint32_t dfd_len; + const uint8_t* pDFD = nullptr; + uint32_t dfd_len = 0; - if (m_params.m_uastc) + const bool is_xuastc_ldr = basis_tex_format_is_xuastc_ldr(m_fmt_mode); + const bool is_astc_ldr = basis_tex_format_is_astc_ldr(m_fmt_mode); + + // TODO: This was writen before m_fmt_mode existed, refactor to use that exclusively instead. + + if (is_xuastc_ldr) + { + // XUASTC LDR 4x4-12x12 + pDFD = g_ktx2_xuastc_ldr_intermediate_dfd; + dfd_len = sizeof(g_ktx2_xuastc_ldr_intermediate_dfd); + } + else if (is_astc_ldr) + { + // ASTC LDR 4x4-12x12 + pDFD = g_ktx2_astc_ldr_dfd; + dfd_len = sizeof(g_ktx2_astc_ldr_dfd); + } + else if (m_params.m_uastc) { if (m_params.m_hdr) { @@ -3302,20 +4025,26 @@ namespace basisu { case hdr_modes::cUASTC_HDR_4X4: { + assert(m_fmt_mode == basist::basis_tex_format::cUASTC_HDR_4x4); + pDFD = g_ktx2_uastc_hdr_4x4_nonalpha_dfd; dfd_len = sizeof(g_ktx2_uastc_hdr_4x4_nonalpha_dfd); break; } case hdr_modes::cASTC_HDR_6X6: { + assert(m_fmt_mode == basist::basis_tex_format::cASTC_HDR_6x6); + pDFD = g_ktx2_astc_hdr_6x6_nonalpha_dfd; dfd_len = sizeof(g_ktx2_astc_hdr_6x6_nonalpha_dfd); break; } - case hdr_modes::cASTC_HDR_6X6_INTERMEDIATE: + case hdr_modes::cUASTC_HDR_6X6_INTERMEDIATE: { - pDFD = g_ktx2_astc_hdr_6x6_intermediate_nonalpha_dfd; - dfd_len = sizeof(g_ktx2_astc_hdr_6x6_intermediate_nonalpha_dfd); + assert(m_fmt_mode == basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE); + + pDFD = g_ktx2_uastc_hdr_6x6_intermediate_nonalpha_dfd; + dfd_len = sizeof(g_ktx2_uastc_hdr_6x6_intermediate_nonalpha_dfd); break; } default: @@ -3328,11 +4057,15 @@ namespace basisu // Must be LDR UASTC 4x4 else if (m_any_source_image_has_alpha) { + assert(m_fmt_mode == basist::basis_tex_format::cUASTC_LDR_4x4); + pDFD = g_ktx2_uastc_ldr_4x4_alpha_dfd; dfd_len = sizeof(g_ktx2_uastc_ldr_4x4_alpha_dfd); } else { + assert(m_fmt_mode == basist::basis_tex_format::cUASTC_LDR_4x4); + pDFD = g_ktx2_uastc_ldr_4x4_nonalpha_dfd; dfd_len = sizeof(g_ktx2_uastc_ldr_4x4_nonalpha_dfd); } @@ -3341,6 +4074,7 @@ namespace basisu { // Must be ETC1S. assert(!m_params.m_hdr); + assert(m_fmt_mode == basist::basis_tex_format::cETC1S); if (m_any_source_image_has_alpha) { @@ -3353,32 +4087,40 @@ namespace basisu dfd_len = sizeof(g_ktx2_etc1s_nonalpha_dfd); } } - + assert(dfd_len >= 44); dfd.resize(dfd_len); memcpy(dfd.data(), pDFD, dfd_len); + // Now modify the DFD DWORD's directly uint32_t dfd_bits = basisu::read_le_dword(dfd.data() + 3 * sizeof(uint32_t)); - // Color primaries - if ((m_params.m_hdr) && (m_params.m_astc_hdr_6x6_options.m_rec2020_bt2100_color_gamut)) + // Color primaries - TODO: Move this option outside of the m_astc_hdr_6x6_options struct. + //if ((m_params.m_hdr) && (m_params.m_astc_hdr_6x6_options.m_rec2020_bt2100_color_gamut)) + if (m_params.m_astc_hdr_6x6_options.m_rec2020_bt2100_color_gamut) { dfd_bits &= ~(0xFF << 8); dfd_bits |= (basist::KTX2_DF_PRIMARIES_BT2020 << 8); } - - // Transfer function + + // Write the transfer function (linear vs. sRGB) - crucial so any decoders/transcoders know which ASTC decoding profile was used during encoding. dfd_bits &= ~(0xFF << 16); if (m_params.m_hdr) { - // TODO: In HDR mode, always write linear for now. + if (m_params.m_ktx2_and_basis_srgb_transfer_function) + { + debug_printf("WARNING: In HDR mode but m_ktx2_and_basis_srgb_transfer_function was set to true, which is being ignored while writing the KTX2 DFD transfer function field\n"); + } + + // TODO: In HDR mode, always write linear, as a sRGB transfer function doesn't make sense for HDR. dfd_bits |= (basist::KTX2_KHR_DF_TRANSFER_LINEAR << 16); } else { - if (m_params.m_ktx2_srgb_transfer_func) + // set the KTX2 DFD transfer function + if (m_params.m_ktx2_and_basis_srgb_transfer_function) dfd_bits |= (basist::KTX2_KHR_DF_TRANSFER_SRGB << 16); else dfd_bits |= (basist::KTX2_KHR_DF_TRANSFER_LINEAR << 16); @@ -3386,6 +4128,7 @@ namespace basisu basisu::write_le_dword(dfd.data() + 3 * sizeof(uint32_t), dfd_bits); + // If supercompressed, manipulate the plane bits to match the khronos ktx2 tool's output if (header.m_supercompression_scheme != basist::KTX2_SS_NONE) { uint32_t plane_bits = basisu::read_le_dword(dfd.data() + 5 * sizeof(uint32_t)); @@ -3401,16 +4144,34 @@ namespace basisu if (m_params.m_uastc) { dfd_chan0 &= ~(0xF << 24); - - // TODO: Allow the caller to override this - if (m_any_source_image_has_alpha) + + // TODO: Allow the caller to override this. Derive from swizzle? + // Only do this for UASTC LDR 4x4 or XUASTC LDR 4x4-12x12 - and now also ASTC LDR 4x4-12x12, which isn't quite standard, but we need some way of determining if the ASTC data has alpha by examining the KTX2 DFD. + if ((m_any_source_image_has_alpha) && + ((m_fmt_mode == basist::basis_tex_format::cUASTC_LDR_4x4) || basist::basis_tex_format_is_xuastc_ldr(m_fmt_mode) || basis_tex_format_is_astc_ldr(m_fmt_mode))) + { dfd_chan0 |= (basist::KTX2_DF_CHANNEL_UASTC_RGBA << 24); + } else + { + // basist::KTX2_DF_CHANNEL_UASTC_RGB==0 dfd_chan0 |= (basist::KTX2_DF_CHANNEL_UASTC_RGB << 24); + } } basisu::write_le_dword(dfd.data() + 7 * sizeof(uint32_t), dfd_chan0); + if ((is_xuastc_ldr) || (is_astc_ldr)) + { + // Write XUASTC/ASTC LDR block dimensions + uint32_t texelBlockDimensions = basisu::read_le_dword(dfd.data() + 4 * sizeof(uint32_t)); + + texelBlockDimensions &= ~0xFFFF; + texelBlockDimensions |= ((m_fmt_mode_block_width - 1) | ((m_fmt_mode_block_height - 1) << 8)); + + basisu::write_le_dword(dfd.data() + 4 * sizeof(uint32_t), texelBlockDimensions); + } + return true; } @@ -3418,6 +4179,8 @@ namespace basisu { //bool needs_global_data = false; bool can_use_zstd = false; + bool is_xuastc_ldr = false; + bool is_astc_ldr = false; switch (m_fmt_mode) { @@ -3426,7 +4189,7 @@ namespace basisu //needs_global_data = true; break; } - case basist::basis_tex_format::cUASTC4x4: + case basist::basis_tex_format::cUASTC_LDR_4x4: { can_use_zstd = true; break; @@ -3441,11 +4204,50 @@ namespace basisu can_use_zstd = true; break; } - case basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE: + case basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE: { //needs_global_data = true; break; } + case basist::basis_tex_format::cXUASTC_LDR_4x4: + case basist::basis_tex_format::cXUASTC_LDR_5x4: + case basist::basis_tex_format::cXUASTC_LDR_5x5: + case basist::basis_tex_format::cXUASTC_LDR_6x5: + case basist::basis_tex_format::cXUASTC_LDR_6x6: + case basist::basis_tex_format::cXUASTC_LDR_8x5: + case basist::basis_tex_format::cXUASTC_LDR_8x6: + case basist::basis_tex_format::cXUASTC_LDR_10x5: + case basist::basis_tex_format::cXUASTC_LDR_10x6: + case basist::basis_tex_format::cXUASTC_LDR_8x8: + case basist::basis_tex_format::cXUASTC_LDR_10x8: + case basist::basis_tex_format::cXUASTC_LDR_10x10: + case basist::basis_tex_format::cXUASTC_LDR_12x10: + case basist::basis_tex_format::cXUASTC_LDR_12x12: + { + // has built-in compression, no need for Zstd + is_xuastc_ldr = true; + break; + } + case basist::basis_tex_format::cASTC_LDR_4x4: + case basist::basis_tex_format::cASTC_LDR_5x4: + case basist::basis_tex_format::cASTC_LDR_5x5: + case basist::basis_tex_format::cASTC_LDR_6x5: + case basist::basis_tex_format::cASTC_LDR_6x6: + case basist::basis_tex_format::cASTC_LDR_8x5: + case basist::basis_tex_format::cASTC_LDR_8x6: + case basist::basis_tex_format::cASTC_LDR_10x5: + case basist::basis_tex_format::cASTC_LDR_10x6: + case basist::basis_tex_format::cASTC_LDR_8x8: + case basist::basis_tex_format::cASTC_LDR_10x8: + case basist::basis_tex_format::cASTC_LDR_10x10: + case basist::basis_tex_format::cASTC_LDR_12x10: + case basist::basis_tex_format::cASTC_LDR_12x12: + { + // plain ASTC LDR 4x4-12x12 - can use Zstd + is_astc_ldr = true; + can_use_zstd = true; + break; + } default: assert(0); //fmt_debug_printf("HERE 1\n"); @@ -3456,7 +4258,6 @@ namespace basisu { if ((m_params.m_ktx2_uastc_supercompression != basist::KTX2_SS_NONE) && (m_params.m_ktx2_uastc_supercompression != basist::KTX2_SS_ZSTANDARD)) { - //fmt_debug_printf("HERE 2\n"); return false; } } @@ -3507,17 +4308,52 @@ namespace basisu header.m_vk_format = basist::KTX2_FORMAT_ASTC_6x6_SFLOAT_BLOCK; else { - assert(m_params.m_hdr_mode == hdr_modes::cASTC_HDR_6X6_INTERMEDIATE); + assert(m_params.m_hdr_mode == hdr_modes::cUASTC_HDR_6X6_INTERMEDIATE); header.m_vk_format = basist::KTX2_VK_FORMAT_UNDEFINED; } } else { - // Either ETC1S or UASTC LDR 4x4. - assert((m_fmt_mode == basist::basis_tex_format::cETC1S) || (m_fmt_mode == basist::basis_tex_format::cUASTC4x4)); + // Either ETC1S, UASTC LDR 4x4, or XUASTC/ASTC LDR 4x4-12x12. + assert((m_fmt_mode == basist::basis_tex_format::cETC1S) || (m_fmt_mode == basist::basis_tex_format::cUASTC_LDR_4x4) || is_xuastc_ldr || is_astc_ldr); + + if (is_astc_ldr) + { + // Get the correct Vulkan format (UNORM or sRGB). + uint32_t fmt = 0; + + assert((basist::KTX2_FORMAT_ASTC_4x4_UNORM_BLOCK + 1) == basist::KTX2_FORMAT_ASTC_4x4_SRGB_BLOCK); + + switch (m_fmt_mode) + { + case basist::basis_tex_format::cASTC_LDR_4x4: fmt = basist::KTX2_FORMAT_ASTC_4x4_UNORM_BLOCK; break; + case basist::basis_tex_format::cASTC_LDR_5x4: fmt = basist::KTX2_FORMAT_ASTC_5x4_UNORM_BLOCK; break; + case basist::basis_tex_format::cASTC_LDR_5x5: fmt = basist::KTX2_FORMAT_ASTC_5x5_UNORM_BLOCK; break; + case basist::basis_tex_format::cASTC_LDR_6x5: fmt = basist::KTX2_FORMAT_ASTC_6x5_UNORM_BLOCK; break; + case basist::basis_tex_format::cASTC_LDR_6x6: fmt = basist::KTX2_FORMAT_ASTC_6x6_UNORM_BLOCK; break; + case basist::basis_tex_format::cASTC_LDR_8x5: fmt = basist::KTX2_FORMAT_ASTC_8x5_UNORM_BLOCK; break; + case basist::basis_tex_format::cASTC_LDR_8x6: fmt = basist::KTX2_FORMAT_ASTC_8x6_UNORM_BLOCK; break; + case basist::basis_tex_format::cASTC_LDR_10x5: fmt = basist::KTX2_FORMAT_ASTC_10x5_UNORM_BLOCK; break; + case basist::basis_tex_format::cASTC_LDR_10x6: fmt = basist::KTX2_FORMAT_ASTC_10x6_UNORM_BLOCK; break; + case basist::basis_tex_format::cASTC_LDR_8x8: fmt = basist::KTX2_FORMAT_ASTC_8x8_UNORM_BLOCK; break; + case basist::basis_tex_format::cASTC_LDR_10x8: fmt = basist::KTX2_FORMAT_ASTC_10x8_UNORM_BLOCK; break; + case basist::basis_tex_format::cASTC_LDR_10x10: fmt = basist::KTX2_FORMAT_ASTC_10x10_UNORM_BLOCK; break; + case basist::basis_tex_format::cASTC_LDR_12x10: fmt = basist::KTX2_FORMAT_ASTC_12x10_UNORM_BLOCK; break; + case basist::basis_tex_format::cASTC_LDR_12x12: fmt = basist::KTX2_FORMAT_ASTC_12x12_UNORM_BLOCK; break; + default: + assert(0); + return false; + } + assert(fmt); - header.m_vk_format = basist::KTX2_VK_FORMAT_UNDEFINED; + header.m_vk_format = fmt + (m_params.m_ktx2_and_basis_srgb_transfer_function ? 1 : 0); + } + else + { + // A supercompressed format, i.e. not a standard format. + header.m_vk_format = basist::KTX2_VK_FORMAT_UNDEFINED; + } } header.m_type_size = 1; @@ -3605,7 +4441,7 @@ namespace basisu uint8_vec ktx2_global_data; - // Create ETC1S global supercompressed data + // Create global supercompressed data if (m_fmt_mode == basist::basis_tex_format::cETC1S) { basist::ktx2_etc1s_global_data_header etc1s_global_data_header; @@ -3659,10 +4495,11 @@ namespace basisu header.m_supercompression_scheme = basist::KTX2_SS_BASISLZ; } - else if (m_fmt_mode == basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE) + else if ((m_fmt_mode == basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE) || (is_xuastc_ldr)) { - basisu::vector image_descs(total_levels * total_layers * total_faces); - memset((void *)image_descs.data(), 0, image_descs.size_in_bytes()); + // The global data for UASTC HDR 6x6 INTERMEDIATE and XUASTC LDR is an array of ktx2_slice_offset_len_desc's, which the transcoder needs to locate the variable length compressed slice data. + basisu::vector slice_offset_len_descs(total_levels * total_layers * total_faces); + memset((void *)slice_offset_len_descs.data(), 0, slice_offset_len_descs.size_in_bytes()); for (uint32_t slice_index = 0; slice_index < m_slice_descs.size(); slice_index++) { @@ -3680,19 +4517,19 @@ namespace basisu const uint32_t output_image_index = level_index * (total_layers * total_faces) + layer_index * total_faces + face_index; - image_descs[output_image_index].m_rgb_slice_byte_length = m_uastc_backend_output.m_slice_image_data[slice_index].size(); - image_descs[output_image_index].m_rgb_slice_byte_offset = slice_level_offsets[slice_index]; + slice_offset_len_descs[output_image_index].m_slice_byte_length = m_uastc_backend_output.m_slice_image_data[slice_index].size(); + slice_offset_len_descs[output_image_index].m_slice_byte_offset = slice_level_offsets[slice_index]; } // slice_index - append_vector(ktx2_global_data, (const uint8_t*)image_descs.data(), image_descs.size_in_bytes()); + append_vector(ktx2_global_data, (const uint8_t*)slice_offset_len_descs.data(), slice_offset_len_descs.size_in_bytes()); header.m_supercompression_scheme = basist::KTX2_SS_BASISLZ; } - + // Key values basist::ktx2_transcoder::key_value_vec key_values(m_params.m_ktx2_key_values); - + basist::ktx2_add_key_value(key_values, "KTXwriter", fmt_string("Basis Universal {}", BASISU_LIB_VERSION_STRING)); if (m_params.m_hdr) @@ -3713,11 +4550,10 @@ namespace basisu uint8_vec key_value_data; - // DFD + // DFD (Data Format Descriptor) uint8_vec dfd; if (!get_dfd(dfd, header)) { - //fmt_debug_printf("HERE 7\n"); return false; } @@ -3729,20 +4565,17 @@ namespace basisu { if (key_values[i].m_key.size() < 2) { - //fmt_debug_printf("HERE 8\n"); return false; } if (key_values[i].m_key.back() != 0) { - //fmt_debug_printf("HERE 9\n"); return false; } const uint64_t total_len = (uint64_t)key_values[i].m_key.size() + (uint64_t)key_values[i].m_value.size(); if (total_len >= UINT32_MAX) { - //fmt_debug_printf("HERE 10\n"); return false; } @@ -3764,7 +4597,7 @@ namespace basisu #if BASISU_DISABLE_KTX2_ALIGNMENT_WORKAROUND break; #endif - + // Hack to ensure the KVD block ends on a 16 byte boundary, because we have no other official way of aligning the data. uint32_t kvd_end_file_offset = kvd_file_offset + (uint32_t)key_value_data.size(); uint32_t bytes_needed_to_pad = (16 - (kvd_end_file_offset & 15)) & 15; @@ -3777,22 +4610,21 @@ namespace basisu assert(!pass); if (pass) { - //fmt_debug_printf("HERE 11\n"); return false; } if (bytes_needed_to_pad < 6) bytes_needed_to_pad += 16; - // Just add the padding. It's likely not necessary anymore, but can't really hurt. + // Just add the padding. It's likely not necessary anymore, but can't really hurt other than a tiny increase in file size. //printf("WARNING: Due to a KTX2 validator bug related to mipPadding, we must insert a dummy key into the KTX2 file of %u bytes\n", bytes_needed_to_pad); - - // We're not good - need to add a dummy key large enough to force file alignment so the mip level array gets aligned. + + // We're not good - need to add a dummy key large enough to force file alignment so the mip level array gets aligned. // We can't just add some bytes before the mip level array because ktx2check will see that as extra data in the file that shouldn't be there in ktxValidator::validateDataSize(). key_values.enlarge(1); for (uint32_t i = 0; i < (bytes_needed_to_pad - 4 - 1 - 1); i++) key_values.back().m_key.push_back(127); - + key_values.back().m_key.push_back(0); key_values.back().m_value.push_back(0); @@ -3800,13 +4632,13 @@ namespace basisu key_values.sort(); key_value_data.resize(0); - + // Try again } basisu::vector level_index_array(total_levels); memset((void *)level_index_array.data(), 0, level_index_array.size_in_bytes()); - + m_output_ktx2_file.clear(); m_output_ktx2_file.reserve(m_output_basis_file.size()); @@ -3815,8 +4647,8 @@ namespace basisu // Level index array append_vector(m_output_ktx2_file, (const uint8_t*)level_index_array.data(), level_index_array.size_in_bytes()); - - // DFD + + // Write DFD const uint8_t* pDFD = dfd.data(); uint32_t dfd_len = (uint32_t)dfd.size(); @@ -3824,7 +4656,7 @@ namespace basisu header.m_dfd_byte_length = dfd_len; append_vector(m_output_ktx2_file, pDFD, dfd_len); - // Key value data + // Write Key value data if (key_value_data.size()) { assert(kvd_file_offset == m_output_ktx2_file.size()); @@ -3834,7 +4666,7 @@ namespace basisu append_vector(m_output_ktx2_file, key_value_data); } - // Global Supercompressed Data + // Write Global Supercompressed Data if (ktx2_global_data.size()) { uint32_t ofs = m_output_ktx2_file.size() & 7; @@ -3848,14 +4680,13 @@ namespace basisu append_vector(m_output_ktx2_file, ktx2_global_data); } - // mipPadding + // Write mipPadding if (header.m_supercompression_scheme == basist::KTX2_SS_NONE) { - // We currently can't do this or the validator will incorrectly give an error. uint32_t ofs = m_output_ktx2_file.size() & 15; uint32_t padding = (16 - ofs) & 15; - // Make sure we're always aligned here (due to a validator bug). + // Make sure we're always aligned here (due to an old validator bug, which has been fixed). if (padding) { printf("Warning: KTX2 mip level data is not 16-byte aligned. This may trigger a ktx2check validation bug. Writing %u bytes of mipPadding.\n", padding); @@ -3869,8 +4700,7 @@ namespace basisu for (int level = total_levels - 1; level >= 0; level--) { level_index_array[level].m_byte_length = compressed_level_data_bytes[level].size(); - - //if (m_params.m_uastc) + if (can_use_zstd) { level_index_array[level].m_uncompressed_byte_length = level_data_bytes[level].size(); @@ -3879,7 +4709,7 @@ namespace basisu level_index_array[level].m_byte_offset = m_output_ktx2_file.size(); append_vector(m_output_ktx2_file, compressed_level_data_bytes[level]); } - + // Write final header memcpy(m_output_ktx2_file.data(), &header, sizeof(header)); @@ -3894,7 +4724,10 @@ namespace basisu total_orig_pixels += slice_desc.m_orig_width * slice_desc.m_orig_height; } - debug_printf("Total .ktx2 output file size: %u, %3.3f bits/texel\n", m_output_ktx2_file.size(), ((float)m_output_ktx2_file.size() * 8.0f) / total_orig_pixels); + m_ktx2_file_size = m_output_ktx2_file.size(); + m_ktx2_bits_per_texel = total_orig_pixels ? (m_ktx2_file_size * 8.0f) / total_orig_pixels : 0; + + fmt_debug_printf("Total .ktx2 output file size: {}, {3.3} bits/texel\n", m_ktx2_file_size, m_ktx2_bits_per_texel); return true; } @@ -3921,7 +4754,7 @@ namespace basisu std::atomic result; result.store(true); - + std::atomic opencl_failed; opencl_failed.store(false); @@ -3936,19 +4769,19 @@ namespace basisu tm.start(); basis_compressor c; - + // Dummy job pool job_pool task_jpool(1); params.m_pJob_pool = &task_jpool; // TODO: Remove this flag entirely - params.m_multithreading = true; - + params.m_multithreading = true; + // Stop using OpenCL if a failure ever occurs. if (opencl_failed) params.m_use_opencl = false; bool status = c.init(params); - + if (c.get_opencl_failed()) opencl_failed.store(true); @@ -3977,7 +4810,7 @@ namespace basisu else { results.m_error_code = basis_compressor::cECFailedInitializing; - + result = false; } @@ -3994,32 +4827,58 @@ namespace basisu return result; } - static void* basis_compress( + void* basis_compress_internal( basist::basis_tex_format mode, - const basisu::vector *pSource_images, - const basisu::vector *pSource_images_hdr, - uint32_t flags_and_quality, float uastc_rdo_quality, + const basisu::vector* pSource_images, + const basisu::vector* pSource_images_hdr, + uint32_t flags_and_quality, float uastc_rdo_or_dct_quality, size_t* pSize, - image_stats* pStats) + image_stats* pStats, + int quality_level, int effort_level) { assert((pSource_images != nullptr) || (pSource_images_hdr != nullptr)); assert(!((pSource_images != nullptr) && (pSource_images_hdr != nullptr))); + if ((quality_level != -1) && (uastc_rdo_or_dct_quality != 0.0f)) + { + fmt_debug_printf("basis_compress_internal: quality_level is not -1, but uastc_rdo_or_dct_quality isn't 0!\n"); + + // Can't use both old and new-style quality control methods + uastc_rdo_or_dct_quality = 0.0f; + } + + if (!pSize) + { + error_printf("basis_compress: Need pSize parameter!\n"); + assert(0); + return nullptr; + } + + // Can't provide both LDR and HDR images + if ( ((pSource_images) && (pSource_images->size() != 0)) && + ((pSource_images_hdr) && (pSource_images_hdr->size() != 0)) + ) + { + error_printf("basis_compress: Can't provide both LDR and HDR source images!\n"); + assert(0); + return nullptr; + } + // Check input parameters if (pSource_images) { - if ((!pSource_images->size()) || (!pSize)) + if (!pSource_images->size()) { - error_printf("basis_compress: Invalid parameter\n"); + error_printf("basis_compress: No source LDR images\n"); assert(0); return nullptr; } } else { - if ((!pSource_images_hdr->size()) || (!pSize)) + if (!pSource_images_hdr->size()) { - error_printf("basis_compress: Invalid parameter\n"); + error_printf("basis_compress: No source HDR images\n"); assert(0); return nullptr; } @@ -4030,100 +4889,196 @@ namespace basisu // Initialize a job pool uint32_t num_threads = 1; if (flags_and_quality & cFlagThreaded) - num_threads = basisu::maximum(1, std::thread::hardware_concurrency()); + num_threads = basisu::maximum(1, get_num_hardware_threads()); job_pool jp(num_threads); // Initialize the compressor parameter struct basis_compressor_params comp_params; + + // Set the codec (basist::basis_tex_format) we'll be using. comp_params.set_format_mode(mode); comp_params.m_pJob_pool = &jp; comp_params.m_y_flip = (flags_and_quality & cFlagYFlip) != 0; + + // Set debug related parameters comp_params.m_debug = (flags_and_quality & cFlagDebug) != 0; comp_params.m_debug_images = (flags_and_quality & cFlagDebugImages) != 0; - // Copy the largest mipmap level - if (pSource_images) + // Set texture type: 2D, 2D array, cubemap array etc. + comp_params.m_tex_type = (basist::basis_texture_type)((flags_and_quality >> cFlagTextureTypeShift) & cFlagTextureTypeMask); + + if (comp_params.m_tex_type != basist::basis_texture_type::cBASISTexType2D) { - comp_params.m_source_images.resize(1); - comp_params.m_source_images[0] = (*pSource_images)[0]; - - // Copy the smaller mipmap levels, if any - if (pSource_images->size() > 1) + // 2D array, cubemap array, or texture video. Assume any extra images the user has supplied are actually cubemap faces, o4r array layers, or texture video frames. + // We assume the dimensions are correct here and let the compressor validate them. + // TODO: This simplified API doesn't allow the user to also specify the mipmap levels here. + if (pSource_images) { - comp_params.m_source_mipmap_images.resize(1); - comp_params.m_source_mipmap_images[0].resize(pSource_images->size() - 1); - - for (uint32_t i = 1; i < pSource_images->size(); i++) - comp_params.m_source_mipmap_images[0][i - 1] = (*pSource_images)[i]; + for (uint32_t i = 0; i < pSource_images->size(); i++) + comp_params.m_source_images.push_back((*pSource_images)[i]); + } + else + { + for (uint32_t i = 0; i < pSource_images_hdr->size(); i++) + comp_params.m_source_images_hdr.push_back((*pSource_images_hdr)[i]); } } else { - comp_params.m_source_images_hdr.resize(1); - comp_params.m_source_images_hdr[0] = (*pSource_images_hdr)[0]; + // Plain 2D mode. Assume any extra images the user has supplied are precomputed mipmap levels of the correct dimensions. + // Copy the largest mipmap level and mipmaps. We assume the dimensions are correct here and let the compressor validate them. + if (pSource_images) + { + comp_params.m_source_images.resize(1); + comp_params.m_source_images[0] = (*pSource_images)[0]; + + // Copy the smaller mipmap levels, if any + if (pSource_images->size() > 1) + { + comp_params.m_source_mipmap_images.resize(1); + comp_params.m_source_mipmap_images[0].resize(pSource_images->size() - 1); - // Copy the smaller mipmap levels, if any - if (pSource_images_hdr->size() > 1) + for (uint32_t i = 1; i < pSource_images->size(); i++) + comp_params.m_source_mipmap_images[0][i - 1] = (*pSource_images)[i]; + } + } + else { - comp_params.m_source_mipmap_images_hdr.resize(1); - comp_params.m_source_mipmap_images_hdr[0].resize(pSource_images_hdr->size() - 1); + comp_params.m_source_images_hdr.resize(1); + comp_params.m_source_images_hdr[0] = (*pSource_images_hdr)[0]; + + // Copy the smaller mipmap levels, if any + if (pSource_images_hdr->size() > 1) + { + comp_params.m_source_mipmap_images_hdr.resize(1); + comp_params.m_source_mipmap_images_hdr[0].resize(pSource_images_hdr->size() - 1); - for (uint32_t i = 1; i < pSource_images->size(); i++) - comp_params.m_source_mipmap_images_hdr[0][i - 1] = (*pSource_images_hdr)[i]; + for (uint32_t i = 1; i < pSource_images->size(); i++) + comp_params.m_source_mipmap_images_hdr[0][i - 1] = (*pSource_images_hdr)[i]; + } } } - + comp_params.m_multithreading = (flags_and_quality & cFlagThreaded) != 0; comp_params.m_use_opencl = (flags_and_quality & cFlagUseOpenCL) != 0; comp_params.m_write_output_basis_or_ktx2_files = false; - comp_params.m_perceptual = (flags_and_quality & cFlagSRGB) != 0; - comp_params.m_mip_srgb = comp_params.m_perceptual; + // sRGB handling - set parameters consistently + // sRGB here controls the error metrics, KTX2/.basis transfer function fields, and mipmap filtering + const bool srgb_flag = (flags_and_quality & cFlagSRGB) != 0; + + // Use sRGB colorspace metrics, channel weights + comp_params.m_perceptual = srgb_flag; + + // This will be written to the KTX2 DFD, .basis file header, also controls the ASTC profile decoding mode for ASTC LDR 4x4 - 12x12 and XUASTC LDR 4x4 - 12x12. + comp_params.m_ktx2_and_basis_srgb_transfer_function = srgb_flag; + + // Correct for sRGB transfer function during mipmapping + comp_params.m_mip_srgb = srgb_flag; + comp_params.m_mip_gen = (flags_and_quality & (cFlagGenMipsWrap | cFlagGenMipsClamp)) != 0; comp_params.m_mip_wrapping = (flags_and_quality & cFlagGenMipsWrap) != 0; - if (mode == basist::basis_tex_format::cUASTC4x4) + if (mode == basist::basis_tex_format::cUASTC_LDR_4x4) { + // Set pack level from flags comp_params.m_pack_uastc_ldr_4x4_flags = flags_and_quality & cPackUASTCLevelMask; - comp_params.m_rdo_uastc_ldr_4x4 = (flags_and_quality & cFlagUASTCRDO) != 0; - comp_params.m_rdo_uastc_ldr_4x4_quality_scalar = uastc_rdo_quality; + + // Now optionally enable UASTC LDR 4x4 RDO. + // We used to look at the (flags_and_quality & cFlagUASTCRDO) != 0; flag to determine if we'll be using RDO here. + // The flag isn't necessary, we'll now just examine uastc_rdo_or_dct_quality and decide to enable it. + if (uastc_rdo_or_dct_quality > 0.0f) + { + comp_params.m_rdo_uastc_ldr_4x4 = true; + comp_params.m_rdo_uastc_ldr_4x4_quality_scalar = uastc_rdo_or_dct_quality; + } } else if (mode == basist::basis_tex_format::cETC1S) { - comp_params.m_etc1s_quality_level = basisu::maximum(1, flags_and_quality & 255); + // Set ETC1S quality level (codebook sizes) from flags. + comp_params.m_quality_level = basisu::maximum(1, flags_and_quality & 255); } + else if (basist::basis_tex_format_is_xuastc_ldr(mode) || basist::basis_tex_format_is_astc_ldr(mode)) + { + // Set ASTC LDR/UASTC LDR 4x4-12x12 effort level + comp_params.m_xuastc_ldr_effort_level = flags_and_quality & 255; + + // Optionally enable weight grid DCT for XUASTC. + // Valid XUASTC LDR weight grid DCT quality levels are 1-100. + if (basist::basis_tex_format_is_xuastc_ldr(mode) && (uastc_rdo_or_dct_quality != 0.0f)) + { + if ((uastc_rdo_or_dct_quality >= (float)BASISU_XUASTC_QUALITY_MIN) && (uastc_rdo_or_dct_quality < (float)BASISU_XUASTC_QUALITY_MAX)) + { + // Enable weight grid DCT usage, set quality level. + comp_params.m_xuastc_ldr_use_dct = true; + comp_params.m_quality_level = (int)uastc_rdo_or_dct_quality; + + // Also enable bounded lossy distortion mode in the normally lossless supercompressor for extra savings. + comp_params.m_xuastc_ldr_use_lossy_supercompression = true; + } + else + { + // Invalid quality level + assert(0); + return nullptr; + } + } + if (basist::basis_tex_format_is_xuastc_ldr(mode)) + { + // Set XUASTC LDR syntax + comp_params.m_xuastc_ldr_syntax = (flags_and_quality >> cFlagXUASTCLDRSyntaxShift) & cFlagXUASTCLDRSyntaxMask; + if (comp_params.m_xuastc_ldr_syntax >= (int)basist::astc_ldr_t::xuastc_ldr_syntax::cTotal) + { + error_printf("basis_compress: basis_compressor::init() failed - invalid XUASTC LDR syntax\n"); + return nullptr; + } + } + } + comp_params.m_create_ktx2_file = (flags_and_quality & cFlagKTX2) != 0; - + if (comp_params.m_create_ktx2_file) { // Set KTX2 specific parameters. if ((flags_and_quality & cFlagKTX2UASTCSuperCompression) && (comp_params.m_uastc)) comp_params.m_ktx2_uastc_supercompression = basist::KTX2_SS_ZSTANDARD; - - comp_params.m_ktx2_srgb_transfer_func = comp_params.m_perceptual; } - + comp_params.m_compute_stats = (pStats != nullptr); comp_params.m_print_stats = (flags_and_quality & cFlagPrintStats) != 0; comp_params.m_status_output = (flags_and_quality & cFlagPrintStatus) != 0; if (mode == basist::basis_tex_format::cUASTC_HDR_4x4) { + // Set UASTC HDR 4x4 effort level comp_params.m_uastc_hdr_4x4_options.set_quality_level(flags_and_quality & cPackUASTCLevelMask); } - else if ((mode == basist::basis_tex_format::cASTC_HDR_6x6) || (mode == basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE)) + else if ((mode == basist::basis_tex_format::cASTC_HDR_6x6) || (mode == basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE)) { + // Set ASTC HDR 6x6/UASTC HDR 6x6 effort level comp_params.m_astc_hdr_6x6_options.set_user_level(flags_and_quality & cPackUASTCLevelMask); - comp_params.m_astc_hdr_6x6_options.m_lambda = uastc_rdo_quality; - comp_params.m_astc_hdr_6x6_options.m_rec2020_bt2100_color_gamut = (flags_and_quality & cFlagREC2020) != 0; + + // Set lambda (rate-distortion tradeoff) + comp_params.m_astc_hdr_6x6_options.m_lambda = uastc_rdo_or_dct_quality; } + // TODO: REC2020 isn't specific to HDR 6x6 anymore, it's always used for KTX2 files. + // This will be written to the KTX2 DFD. + comp_params.m_astc_hdr_6x6_options.m_rec2020_bt2100_color_gamut = (flags_and_quality & cFlagREC2020) != 0; + comp_params.m_validate_output_data = (flags_and_quality & cFlagValidateOutput) != 0; + + // Now set the unified quality/effort level, if they've specified it. + // This will override some of the lower-level options set above, or leave them alone if -1. + if ((quality_level != -1) || (effort_level != -1)) + { + comp_params.set_format_mode_and_quality_effort(mode, quality_level, effort_level, false); + } // Create the compressor, initialize it, and process the input basis_compressor comp; @@ -4156,6 +5111,7 @@ namespace basisu error_printf("basis_compress: Out of memory\n"); return nullptr; } + memcpy(pFile_data, pFile_data_vec->get_ptr(), pFile_data_vec->size()); *pSize = pFile_data_vec->size(); @@ -4171,27 +5127,47 @@ namespace basisu void* basis_compress( basist::basis_tex_format mode, const basisu::vector& source_images, - uint32_t flags_and_quality, float uastc_rdo_quality, + uint32_t flags_and_quality, float uastc_rdo_or_dct_quality, + size_t* pSize, + image_stats* pStats) + { + return basis_compress_internal(mode, &source_images, nullptr, flags_and_quality, uastc_rdo_or_dct_quality, pSize, pStats, -1, -1); + } + + void* basis_compress2( + basist::basis_tex_format mode, + const basisu::vector& source_images, + uint32_t flags_and_quality, int quality_level, int effort_level, size_t* pSize, image_stats* pStats) { - return basis_compress(mode, &source_images, nullptr, flags_and_quality, uastc_rdo_quality, pSize, pStats); + return basis_compress_internal(mode, &source_images, nullptr, flags_and_quality, 0.0f, pSize, pStats, quality_level, effort_level); } void* basis_compress( basist::basis_tex_format mode, const basisu::vector& source_images_hdr, - uint32_t flags_and_quality, float lambda, + uint32_t flags_and_quality, float uastc_rdo_or_dct_quality, + size_t* pSize, + image_stats* pStats) + { + return basis_compress_internal(mode, nullptr, &source_images_hdr, flags_and_quality, uastc_rdo_or_dct_quality, pSize, pStats, -1, -1); + } + + void* basis_compress2( + basist::basis_tex_format mode, + const basisu::vector& source_images_hdr, + uint32_t flags_and_quality, int quality_level, int effort_level, size_t* pSize, image_stats* pStats) { - return basis_compress(mode, nullptr, &source_images_hdr, flags_and_quality, lambda, pSize, pStats); + return basis_compress_internal(mode, nullptr, &source_images_hdr, flags_and_quality, 0.0f, pSize, pStats, quality_level, effort_level); } void* basis_compress( basist::basis_tex_format mode, const uint8_t* pImageRGBA, uint32_t width, uint32_t height, uint32_t pitch_in_pixels, - uint32_t flags_and_quality, float uastc_rdo_quality, + uint32_t flags_and_quality, float uastc_rdo_or_dct_quality, size_t* pSize, image_stats* pStats) { @@ -4219,9 +5195,43 @@ namespace basisu for (uint32_t y = 0; y < height; y++) memcpy(source_image[0].get_ptr() + y * width, (const color_rgba*)pImageRGBA + y * pitch_in_pixels, width * sizeof(color_rgba)); - return basis_compress(mode, source_image, flags_and_quality, uastc_rdo_quality, pSize, pStats); + return basis_compress(mode, source_image, flags_and_quality, uastc_rdo_or_dct_quality, pSize, pStats); } + void* basis_compress2( + basist::basis_tex_format mode, + const uint8_t* pImageRGBA, uint32_t width, uint32_t height, uint32_t pitch_in_pixels, + uint32_t flags_and_quality, int quality_level, int effort_level, + size_t* pSize, + image_stats* pStats) + { + if (!pitch_in_pixels) + pitch_in_pixels = width; + + if ((!pImageRGBA) || (!width) || (!height) || (pitch_in_pixels < width) || (!pSize)) + { + error_printf("basis_compress: Invalid parameter\n"); + assert(0); + return nullptr; + } + + *pSize = 0; + + if ((width > BASISU_MAX_SUPPORTED_TEXTURE_DIMENSION) || (height > BASISU_MAX_SUPPORTED_TEXTURE_DIMENSION)) + { + error_printf("basis_compress: Image too large\n"); + return nullptr; + } + + // Copy the source image + basisu::vector source_image(1); + source_image[0].crop(width, height, width, g_black_color, false); + for (uint32_t y = 0; y < height; y++) + memcpy(source_image[0].get_ptr() + y * width, (const color_rgba*)pImageRGBA + y * pitch_in_pixels, width * sizeof(color_rgba)); + + return basis_compress2(mode, source_image, flags_and_quality, quality_level, effort_level, pSize, pStats); + } + void basis_free_data(void* p) { free(p); @@ -4241,7 +5251,7 @@ namespace basisu const uint32_t W = 1024, H = 1024; basisu::vector images; image& img = images.enlarge(1)->resize(W, H); - + const uint32_t NUM_RAND_LETTERS = 6000;// 40000; rand r; @@ -4283,7 +5293,7 @@ namespace basisu error_printf("basis_benchmark_etc1s_opencl: basis_compress() failed (CPU)!\n"); return false; } - + best_cpu_time = minimum(best_cpu_time, cpu_time); basis_free_data(pComp_data); @@ -4326,8 +5336,11 @@ namespace basisu } printf("Best GPU time: %3.3f\n", best_gpu_time); - + return best_gpu_time < best_cpu_time; } } // namespace basisu + + + diff --git a/external/basis_universal/encoder/basisu_comp.h b/external/basis_universal/encoder/basisu_comp.h index e761eacf7f..f2113dcb78 100644 --- a/external/basis_universal/encoder/basisu_comp.h +++ b/external/basis_universal/encoder/basisu_comp.h @@ -1,5 +1,5 @@ // basisu_comp.h -// Copyright (C) 2019-2024 Binomial LLC. All Rights Reserved. +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -20,9 +20,10 @@ #include "basisu_uastc_enc.h" #include "basisu_uastc_hdr_4x4_enc.h" #include "basisu_astc_hdr_6x6_enc.h" +#include "basisu_astc_ldr_encode.h" -#define BASISU_LIB_VERSION 160 -#define BASISU_LIB_VERSION_STRING "1.60" +#define BASISU_LIB_VERSION 200 +#define BASISU_LIB_VERSION_STRING "2.00" #ifndef BASISD_SUPPORT_KTX2 #error BASISD_SUPPORT_KTX2 is undefined @@ -43,10 +44,10 @@ namespace basisu const uint32_t BASISU_MAX_SUPPORTED_TEXTURE_DIMENSION = 16384; // Allow block's color distance to increase by 1.5 while searching for an alternative nearby endpoint. - const float BASISU_DEFAULT_ENDPOINT_RDO_THRESH = 1.5f; - + const float BASISU_DEFAULT_ENDPOINT_RDO_THRESH = 1.5f; + // Allow block's color distance to increase by 1.25 while searching the selector history buffer for a close enough match. - const float BASISU_DEFAULT_SELECTOR_RDO_THRESH = 1.25f; + const float BASISU_DEFAULT_SELECTOR_RDO_THRESH = 1.25f; const int BASISU_DEFAULT_QUALITY = 128; const float BASISU_DEFAULT_HYBRID_SEL_CB_QUALITY_THRESH = 2.0f; @@ -54,6 +55,8 @@ namespace basisu const uint32_t BASISU_MAX_IMAGE_DIMENSION = 16384; const uint32_t BASISU_QUALITY_MIN = 1; const uint32_t BASISU_QUALITY_MAX = 255; + const uint32_t BASISU_XUASTC_QUALITY_MIN = 1; + const uint32_t BASISU_XUASTC_QUALITY_MAX = 100; const uint32_t BASISU_MAX_ENDPOINT_CLUSTERS = basisu_frontend::cMaxEndpointClusters; const uint32_t BASISU_MAX_SELECTOR_CLUSTERS = basisu_frontend::cMaxSelectorClusters; @@ -75,7 +78,7 @@ namespace basisu m_filename.clear(); m_width = 0; m_height = 0; - + m_basis_rgb_avg_psnr = 0.0f; m_basis_rgb_avg_log2_psnr = 0.0f; @@ -94,7 +97,7 @@ namespace basisu m_bc7_luma_709_psnr = 0.0f; m_bc7_luma_601_psnr = 0.0f; m_bc7_luma_709_ssim = 0.0f; - + m_best_etc1s_rgb_avg_psnr = 0.0f; m_best_etc1s_luma_709_psnr = 0.0f; m_best_etc1s_luma_601_psnr = 0.0f; @@ -128,8 +131,8 @@ namespace basisu float m_bc7_luma_709_psnr; float m_bc7_luma_601_psnr; float m_bc7_luma_709_ssim; - - // LDR: Highest achievable quality ETC1S statistics + + // LDR: Highest achievable quality ETC1S statistics, for development/comparison float m_best_etc1s_rgb_avg_psnr; float m_best_etc1s_luma_709_psnr; float m_best_etc1s_luma_601_psnr; @@ -141,11 +144,11 @@ namespace basisu enum class hdr_modes { // standard but constrained ASTC HDR 4x4 tex data that can be rapidly transcoded to BC6H - cUASTC_HDR_4X4, + cUASTC_HDR_4X4, // standard RDO optimized or non-RDO (highest quality) ASTC HDR 6x6 tex data that can be rapidly re-encoded to BC6H cASTC_HDR_6X6, // a custom intermediate format based off ASTC HDR that can be rapidly decoded straight to ASTC HDR or re-encoded to BC6H - cASTC_HDR_6X6_INTERMEDIATE, + cUASTC_HDR_6X6_INTERMEDIATE, cTotal }; @@ -230,17 +233,21 @@ namespace basisu bool m_changed; }; + // Low-level direct compressor parameters. + // Also see basis_compress() below for a simplified C-style interface. struct basis_compressor_params { basis_compressor_params() : - m_compression_level((int)BASISU_DEFAULT_COMPRESSION_LEVEL, 0, (int)BASISU_MAX_COMPRESSION_LEVEL), + m_xuastc_or_astc_ldr_basis_tex_format(-1, -1, INT_MAX), + // Note the ETC1S default compression/effort level is 2, not the command line default of 1. + m_etc1s_compression_level((int)BASISU_DEFAULT_ETC1S_COMPRESSION_LEVEL, 0, (int)BASISU_MAX_ETC1S_COMPRESSION_LEVEL), m_selector_rdo_thresh(BASISU_DEFAULT_SELECTOR_RDO_THRESH, 0.0f, 1e+10f), m_endpoint_rdo_thresh(BASISU_DEFAULT_ENDPOINT_RDO_THRESH, 0.0f, 1e+10f), m_mip_scale(1.0f, .000125f, 4.0f), m_mip_smallest_dimension(1, 1, 16384), m_etc1s_max_endpoint_clusters(512), m_etc1s_max_selector_clusters(512), - m_etc1s_quality_level(-1), + m_quality_level(-1), m_pack_uastc_ldr_4x4_flags(cPackUASTCLevelDefault), m_rdo_uastc_ldr_4x4_quality_scalar(1.0f, 0.001f, 50.0f), m_rdo_uastc_ldr_4x4_dict_size(BASISU_RDO_UASTC_DICT_SIZE_DEFAULT, BASISU_RDO_UASTC_DICT_SIZE_MIN, BASISU_RDO_UASTC_DICT_SIZE_MAX), @@ -253,8 +260,14 @@ namespace basisu m_resample_factor(0.0f, .00125f, 100.0f), m_ktx2_uastc_supercompression(basist::KTX2_SS_NONE), m_ktx2_zstd_supercompression_level(6, INT_MIN, INT_MAX), + m_transcode_flags(0, 0, UINT32_MAX), m_ldr_hdr_upconversion_nit_multiplier(0.0f, 0.0f, basist::MAX_HALF_FLOAT), m_ldr_hdr_upconversion_black_bias(0.0f, 0.0f, 1.0f), + m_xuastc_ldr_effort_level(astc_ldr::EFFORT_LEVEL_DEF, astc_ldr::EFFORT_LEVEL_MIN, astc_ldr::EFFORT_LEVEL_MAX), + m_xuastc_ldr_syntax((int)basist::astc_ldr_t::xuastc_ldr_syntax::cFullZStd, (int)basist::astc_ldr_t::xuastc_ldr_syntax::cFullArith, (int)basist::astc_ldr_t::xuastc_ldr_syntax::cFullZStd), + m_ls_min_psnr(35.0f, 0.0f, 100.0f), m_ls_min_alpha_psnr(38.0f, 0.0f, 100.0f), + m_ls_thresh_psnr(1.5f, 0.0f, 100.0f), m_ls_thresh_alpha_psnr(0.75f, 0.0f, 100.0f), + m_ls_thresh_edge_psnr(1.0f, 0.0f, 100.00f), m_ls_thresh_edge_alpha_psnr(0.5f, 0.0f, 100.00f), m_pJob_pool(nullptr) { clear(); @@ -262,9 +275,12 @@ namespace basisu void clear() { + m_format_mode = basist::basis_tex_format::cETC1S; + m_uastc.clear(); m_hdr.clear(); m_hdr_mode = hdr_modes::cUASTC_HDR_4X4; + m_xuastc_or_astc_ldr_basis_tex_format = -1; m_use_opencl.clear(); m_status_output.clear(); @@ -286,7 +302,7 @@ namespace basisu m_selector_rdo_thresh.clear(); m_read_source_images.clear(); m_write_output_basis_or_ktx2_files.clear(); - m_compression_level.clear(); + m_etc1s_compression_level.clear(); m_compute_stats.clear(); m_print_stats.clear(); m_check_for_alpha.clear(); @@ -301,7 +317,7 @@ namespace basisu m_no_endpoint_rdo.clear(); m_endpoint_rdo_thresh.clear(); - + m_mip_gen.clear(); m_mip_scale.clear(); m_mip_filter = "kaiser"; @@ -315,7 +331,7 @@ namespace basisu m_etc1s_max_endpoint_clusters = 0; m_etc1s_max_selector_clusters = 0; - m_etc1s_quality_level = -1; + m_quality_level = -1; m_tex_type = basist::cBASISTexType2D; m_userdata0 = 0; @@ -342,37 +358,62 @@ namespace basisu m_ktx2_uastc_supercompression = basist::KTX2_SS_NONE; m_ktx2_key_values.clear(); m_ktx2_zstd_supercompression_level.clear(); - m_ktx2_srgb_transfer_func.clear(); + m_ktx2_and_basis_srgb_transfer_function.clear(); m_validate_output_data.clear(); + m_transcode_flags.clear(); m_ldr_hdr_upconversion_srgb_to_linear.clear(); m_hdr_favor_astc.clear(); - + m_uastc_hdr_4x4_options.init(); m_astc_hdr_6x6_options.clear(); m_ldr_hdr_upconversion_nit_multiplier.clear(); m_ldr_hdr_upconversion_black_bias.clear(); + m_xuastc_ldr_effort_level.clear(); + m_xuastc_ldr_use_dct.clear(); + m_xuastc_ldr_use_lossy_supercompression.clear(); + m_xuastc_ldr_force_disable_subsets.clear(); + m_xuastc_ldr_force_disable_rgb_dual_plane.clear(); + m_xuastc_ldr_syntax.clear(); + + m_ls_min_psnr.clear(); + m_ls_min_alpha_psnr.clear(); + m_ls_thresh_psnr.clear(); + m_ls_thresh_alpha_psnr.clear(); + m_ls_thresh_edge_psnr.clear(); + m_ls_thresh_edge_alpha_psnr.clear(); + for (uint32_t i = 0; i < 4; i++) + m_xuastc_ldr_channel_weights[i] = 1; + m_xuastc_ldr_blurring.clear(); + m_pJob_pool = nullptr; } - + // Configures the compressor's mode by setting the proper parameters (which were preserved for backwards compatibility with old code). - void set_format_mode(basist::basis_tex_format m) + // This is by far the preferred way of controlling which codec mode the compressor will select. + void set_format_mode(basist::basis_tex_format mode) { - switch (m) + m_format_mode = mode; + + switch (mode) { case basist::basis_tex_format::cETC1S: { + // ETC1S + m_xuastc_or_astc_ldr_basis_tex_format = -1; m_hdr = false; m_uastc = false; m_hdr_mode = hdr_modes::cUASTC_HDR_4X4; // doesn't matter break; } - case basist::basis_tex_format::cUASTC4x4: + case basist::basis_tex_format::cUASTC_LDR_4x4: { + // UASTC LDR 4x4 + m_xuastc_or_astc_ldr_basis_tex_format = -1; m_hdr = false; m_uastc = true; m_hdr_mode = hdr_modes::cUASTC_HDR_4X4; // doesn't matter @@ -380,6 +421,8 @@ namespace basisu } case basist::basis_tex_format::cUASTC_HDR_4x4: { + // UASTC HDR 4x4 + m_xuastc_or_astc_ldr_basis_tex_format = -1; m_hdr = true; m_uastc = true; m_hdr_mode = hdr_modes::cUASTC_HDR_4X4; @@ -387,16 +430,56 @@ namespace basisu } case basist::basis_tex_format::cASTC_HDR_6x6: { + // ASTC HDR 6x6 + m_xuastc_or_astc_ldr_basis_tex_format = -1; m_hdr = true; m_uastc = true; m_hdr_mode = hdr_modes::cASTC_HDR_6X6; break; } - case basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE: + case basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE: { + // UASTC HDR 6x6 + m_xuastc_or_astc_ldr_basis_tex_format = -1; m_hdr = true; m_uastc = true; - m_hdr_mode = hdr_modes::cASTC_HDR_6X6_INTERMEDIATE; + m_hdr_mode = hdr_modes::cUASTC_HDR_6X6_INTERMEDIATE; + break; + } + case basist::basis_tex_format::cXUASTC_LDR_4x4: + case basist::basis_tex_format::cXUASTC_LDR_5x4: + case basist::basis_tex_format::cXUASTC_LDR_5x5: + case basist::basis_tex_format::cXUASTC_LDR_6x5: + case basist::basis_tex_format::cXUASTC_LDR_6x6: + case basist::basis_tex_format::cXUASTC_LDR_8x5: + case basist::basis_tex_format::cXUASTC_LDR_8x6: + case basist::basis_tex_format::cXUASTC_LDR_10x5: + case basist::basis_tex_format::cXUASTC_LDR_10x6: + case basist::basis_tex_format::cXUASTC_LDR_8x8: + case basist::basis_tex_format::cXUASTC_LDR_10x8: + case basist::basis_tex_format::cXUASTC_LDR_10x10: + case basist::basis_tex_format::cXUASTC_LDR_12x10: + case basist::basis_tex_format::cXUASTC_LDR_12x12: + case basist::basis_tex_format::cASTC_LDR_4x4: + case basist::basis_tex_format::cASTC_LDR_5x4: + case basist::basis_tex_format::cASTC_LDR_5x5: + case basist::basis_tex_format::cASTC_LDR_6x5: + case basist::basis_tex_format::cASTC_LDR_6x6: + case basist::basis_tex_format::cASTC_LDR_8x5: + case basist::basis_tex_format::cASTC_LDR_8x6: + case basist::basis_tex_format::cASTC_LDR_10x5: + case basist::basis_tex_format::cASTC_LDR_10x6: + case basist::basis_tex_format::cASTC_LDR_8x8: + case basist::basis_tex_format::cASTC_LDR_10x8: + case basist::basis_tex_format::cASTC_LDR_10x10: + case basist::basis_tex_format::cASTC_LDR_12x10: + case basist::basis_tex_format::cASTC_LDR_12x12: + { + // ASTC LDR 4x4-12x12 or XUASTC LDR 4x4-12x12 + m_xuastc_or_astc_ldr_basis_tex_format = (int)mode; + m_hdr = false; + m_uastc = true; + m_hdr_mode = hdr_modes::cUASTC_HDR_4X4; // doesn't matter break; } default: @@ -405,39 +488,84 @@ namespace basisu } } - // By default we generate LDR ETC1S data. - // if m_uastc is true but m_hdr is not true, we generate UASTC 4x4 LDR data (8bpp with or without RDO). - // if m_uastc is true and m_hdr is true, we generate 4x4 or 6x6 HDR data (either standard ASTC, constrained ASTC, RDO ASTC, or intermediate), controlled by m_hdr_mode. + // Like set_format_mode() but also sets the effort and quality parameters appropriately for the selected mode. + // "Effort" (perf. vs. highest achievable quality) and "quality" (quality vs. bitrate) parameters are now mode dependent. + // Effort ranges from [0,10] and quality ranges from [1,100], unless they are -1 in which case you get the codec's default settings. + bool set_format_mode_and_effort(basist::basis_tex_format mode, int effort = -1, bool set_defaults = true); + bool set_format_mode_and_quality_effort(basist::basis_tex_format mode, int quality = -1, int effort = -1, bool set_defaults = true); + + // Sets all the sRGB-related options (m_perceptual, m_mip_srgb, m_ktx2_and_basis_srgb_transfer_function) to the specified value. + void set_srgb_options(bool srgb_flag) + { + m_perceptual = srgb_flag; + m_mip_srgb = srgb_flag; + m_ktx2_and_basis_srgb_transfer_function = srgb_flag; + } + + // Simpler helpers - I wish this was easier, but backwards API compat is also valuable. + bool is_etc1s() const + { + return !m_uastc; + } + + bool is_uastc_ldr_4x4() const + { + return m_uastc && !m_hdr && (m_xuastc_or_astc_ldr_basis_tex_format == -1); + } + bool is_uastc_hdr_4x4() const + { + return m_uastc && m_hdr && (m_hdr_mode == hdr_modes::cUASTC_HDR_4X4); + } + + // By default we generate LDR ETC1S data. + // Ideally call set_format_mode() above instead of directly manipulating the below fields. These individual parameters are for backwards API compatibility. + // - If m_uastc is false you get ETC1S (the default). + // - If m_uastc is true, and m_hdr is not true, and m_xuastc_or_astc_ldr_basis_tex_format==-1, we generate UASTC 4x4 LDR data (8bpp with or without RDO). + // - If m_uastc is true, and m_hdr is not true, and m_xuastc_or_astc_ldr_basis_tex_format!=-1, we generate XUASTC 4x4-12x12 or ASTC 4x4-12x12 LDR data. + // - If m_uastc is true and m_hdr is true, we generate 4x4 or 6x6 HDR data, controlled by m_hdr_mode. + // True to generate UASTC .basis/.KTX2 file data, otherwise ETC1S. - // Should be true for any non-ETC1S format (UASTC 4x4 LDR, UASTC 4x4 HDR, RDO ASTC 6x6 HDR, and ASTC 6x6 HDR intermediate). + // Should be true for any non-ETC1S format (UASTC 4x4 LDR, UASTC 4x4 HDR, RDO ASTC 6x6 HDR, UASTC 6x6 HDR, or ASTC/XUASTC LDR 4x4-12x12). + // Note: Ideally call set_format_mode() or set_format_mode_and_quality_effort() above instead. + // Many of these individual parameters are for backwards API compatibility. bool_param m_uastc; // Set m_hdr to true to switch to UASTC HDR mode. m_hdr_mode then controls which format is output. // m_hdr_mode then controls which format is output (4x4, 6x6, or 6x6 intermediate). + // Note: Ideally call set_format_mode() instead. This is for backwards API compatibility. bool_param m_hdr; - + // If m_hdr is true, this specifies which mode we operate in (currently UASTC 4x4 HDR or ASTC 6x6 HDR). Defaults to UASTC 4x4 HDR for backwards compatibility. + // Note: Ideally call set_format_mode() instead. This is for backwards API compatibility. hdr_modes m_hdr_mode; + // If not -1: Generate XUASTC or ASTC LDR 4x4-12x12 files in the specified basis_tex_format (which also sets the ASTC block size). If -1 (the default), don't generate XUASTC/ASTC LDR files. + // m_uastc must also be set to true if this is not -1. + // Note: Ideally call set_format_mode() instead. + param m_xuastc_or_astc_ldr_basis_tex_format; // enum basis_tex_format + + // True to enable OpenCL if it's available. The compressor will fall back to CPU encoding if something goes wrong. bool_param m_use_opencl; - // If m_read_source_images is true, m_source_filenames (and optionally m_source_alpha_filenames) contains the filenames of PNG etc. images to read. + // If m_read_source_images is true, m_source_filenames (and optionally m_source_alpha_filenames) contains the filenames of PNG etc. images to read. // Otherwise, the compressor processes the images in m_source_images or m_source_images_hdr. basisu::vector m_source_filenames; basisu::vector m_source_alpha_filenames; - + + // An array of 2D LDR/SDR source images. basisu::vector m_source_images; - + + // An array of 2D HDR source images. basisu::vector m_source_images_hdr; - + // Stores mipmaps starting from level 1. Level 0 is still stored in m_source_images, as usual. // If m_source_mipmaps isn't empty, automatic mipmap generation isn't done. m_source_mipmaps.size() MUST equal m_source_images.size() or the compressor returns an error. // The compressor applies the user-provided swizzling (in m_swizzle) to these images. basisu::vector< basisu::vector > m_source_mipmap_images; basisu::vector< basisu::vector > m_source_mipmap_images_hdr; - + // Filename of the output basis/ktx2 file std::string m_out_filename; @@ -448,20 +576,24 @@ namespace basisu // If true, the compressor will print basis status to stdout during compression. bool_param m_status_output; - + // Output debug information during compression bool_param m_debug; + + // Low-level ETC1S data validation during encoding (slower/development). bool_param m_validate_etc1s; - + // m_debug_images is pretty slow bool_param m_debug_images; - // ETC1S compression level, from 0 to BASISU_MAX_COMPRESSION_LEVEL (higher is slower). + // ETC1S compression effort level, from 0 to BASISU_MAX_ETC1S_COMPRESSION_LEVEL (higher is slower). // This parameter controls numerous internal encoding speed vs. compression efficiency/performance tradeoffs. // Note this is NOT the same as the ETC1S quality level, and most users shouldn't change this. - param m_compression_level; - - // Use perceptual sRGB colorspace metrics instead of linear + param m_etc1s_compression_level; + + // Use perceptual sRGB colorspace metrics instead of linear. + // Note: You probably also want to set m_ktx2_srgb_transfer_func to match. + // Note: This member variable was previously called "m_perceptual". bool_param m_perceptual; // Disable selector RDO, for faster compression but larger files @@ -476,47 +608,53 @@ namespace basisu // Write the output basis/ktx2 file to disk using m_out_filename bool_param m_write_output_basis_or_ktx2_files; - - // Compute and display image metrics + + // Compute and display image metrics bool_param m_compute_stats; // Print stats to stdout, if m_compute_stats is true. bool_param m_print_stats; - + // Check to see if any input image has an alpha channel, if so then the output basis/ktx2 file will have alpha channels bool_param m_check_for_alpha; - + // Always put alpha slices in the output basis/ktx2 file, even when the input doesn't have alpha - bool_param m_force_alpha; - bool_param m_multithreading; + bool_param m_force_alpha; + // True to enable multithreading in various compressors. + // Note currently, some compressors (like ASTC/XUASTC LDR) will utilize threading anyway if the job pool is more than one thread. + bool_param m_multithreading; + // Split the R channel to RGB and the G channel to alpha, then write a basis/ktx2 file with alpha channels uint8_t m_swizzle[4]; + // Renormalize normal map normals after loading image bool_param m_renormalize; // If true the front end will not use 2 level endpoint codebook searching, for slightly higher quality but much slower execution. - // Note some m_compression_level's disable this automatically. + // Note some m_etc1s_compression_level's disable this automatically. bool_param m_disable_hierarchical_endpoint_codebooks; - + // mipmap generation parameters bool_param m_mip_gen; param m_mip_scale; std::string m_mip_filter; bool_param m_mip_srgb; bool_param m_mip_premultiplied; // not currently supported - bool_param m_mip_renormalize; + bool_param m_mip_renormalize; bool_param m_mip_wrapping; bool_param m_mip_fast; param m_mip_smallest_dimension; - - // ETC1S codebook size (quality) control. + + // ETC1S codebook size (quality) control. // If m_etc1s_quality_level != -1, it controls the quality level. It ranges from [1,255] or [BASISU_QUALITY_MIN, BASISU_QUALITY_MAX]. // Otherwise m_max_endpoint_clusters/m_max_selector_clusters controls the codebook sizes directly. uint32_t m_etc1s_max_endpoint_clusters; uint32_t m_etc1s_max_selector_clusters; - int m_etc1s_quality_level; + // Quality level (bitrate vs. distortion tradeoff) control for ETC1S or XUASTC LDR 4x4-12x12 (must not be -1 for DCT to be used in XUASTC LDR 4x4 mode) + int m_quality_level; + // m_tex_type, m_userdata0, m_userdata1, m_framerate - These fields go directly into the .basis file header. basist::basis_texture_type m_tex_type; uint32_t m_userdata0; @@ -527,7 +665,7 @@ namespace basisu // cPackUASTCLevelDefault, etc. uint32_t m_pack_uastc_ldr_4x4_flags; bool_param m_rdo_uastc_ldr_4x4; - param m_rdo_uastc_ldr_4x4_quality_scalar; + param m_rdo_uastc_ldr_4x4_quality_scalar; // RDO lambda for UASTC 4x4 LDR param m_rdo_uastc_ldr_4x4_dict_size; param m_rdo_uastc_ldr_4x4_max_smooth_block_error_scale; param m_rdo_uastc_ldr_4x4_smooth_block_max_std_dev; @@ -536,10 +674,12 @@ namespace basisu bool_param m_rdo_uastc_ldr_4x4_favor_simpler_modes_in_rdo_mode; bool_param m_rdo_uastc_ldr_4x4_multithreading; + // Resample input texture after loading param m_resample_width; param m_resample_height; param m_resample_factor; + // ETC1S global codebook control const basist::basisu_lowlevel_etc1s_transcoder *m_pGlobal_codebooks; // KTX2 specific parameters. @@ -548,19 +688,28 @@ namespace basisu basist::ktx2_supercompression m_ktx2_uastc_supercompression; basist::ktx2_transcoder::key_value_vec m_ktx2_key_values; param m_ktx2_zstd_supercompression_level; - bool_param m_ktx2_srgb_transfer_func; + + // Note: The default for this parameter (which used to be "m_ktx2_srgb_transfer_func") used to be false, now setting this to true and renaming to m_ktx2_and_basis_srgb_transfer_function. + // Also see m_perceptual and m_mip_srgb, which should in most uses be the same. + // This also controls the XUASTC LDR ASTC decode profile (linear vs. sRGB) in the simulated decoder block. + // For XUASTC LDR, it's also still used when generating .basis files vs. .KTX2. + bool_param m_ktx2_and_basis_srgb_transfer_function; // false = linear transfer function, true = sRGB transfer function uastc_hdr_4x4_codec_options m_uastc_hdr_4x4_options; astc_6x6_hdr::astc_hdr_6x6_global_config m_astc_hdr_6x6_options; + // True to try transcoding the generated output after compression to a few formats. bool_param m_validate_output_data; + + // The flags to use while transcoding if m_validate_output_data + param m_transcode_flags; // LDR->HDR upconversion parameters. - // - // If true, LDR images (such as PNG) will be converted to normalized [0,1] linear light (via a sRGB->Linear conversion), or absolute luminance (nits or candelas per meter squared), and then processed as HDR. + // + // If true, LDR images (such as PNG) will be converted to normalized [0,1] linear light (via a sRGB->Linear conversion), or absolute luminance (nits or candelas per meter squared), and then processed as HDR. // Otherwise, LDR images are assumed to already be in linear light (i.e. they don't use the sRGB transfer function). bool_param m_ldr_hdr_upconversion_srgb_to_linear; - + // m_ldr_hdr_upconversion_nit_multiplier is only used when loading SDR/LDR images and compressing to an HDR output format. // By default m_ldr_hdr_upconversion_nit_multiplier is 0. It's an override for the default. // When loading LDR images, a default multiplier of 1.0 will be used in UASTC 4x4 HDR mode. Partially for backwards compatibility with previous library releases, and also because it doesn't really matter with this encoder what the multiplier is. @@ -571,10 +720,35 @@ namespace basisu // Defaults to no bias (0.0f). param m_ldr_hdr_upconversion_black_bias; - // If true, ASTC HDR quality is favored more than BC6H quality. Otherwise it's a rough balance. + // If true, ASTC HDR quality is favored more than BC6H quality by the dual target encoder. Otherwise it's a rough balance. + // UASTC HDR 4x4 bool_param m_hdr_favor_astc; + // XUASTC LDR 4x4-12x12 specific options + param m_xuastc_ldr_effort_level; + bool_param m_xuastc_ldr_use_dct; // set the DCT quality above using m_quality_level, [1,100] + bool_param m_xuastc_ldr_use_lossy_supercompression; // allows the compressor to introduce a bounded amount of distortion if doing so would make smaller files (actually ASTC or XUASTC) + bool_param m_xuastc_ldr_force_disable_subsets; // disable 2-3 subset usage in all effort levels, faster encoding, faster transcoding to BC7, but lower quality) + bool_param m_xuastc_ldr_force_disable_rgb_dual_plane; // disable RGB dual plane usage (still can use dual plane on alpha blocks), for faster transcoding to BC7 but lower quality + param m_xuastc_ldr_syntax; // favor faster decompression over ratio, default is basist::astc_ldr_t::xuastc_ldr_syntax::cFullZstd (fastest transcoding but lower ratio) + uint32_t m_xuastc_ldr_channel_weights[4]; + bool_param m_xuastc_ldr_blurring; // experimental, not recommended, very slow + + // XUASTC Lossy supercompression PSNR threshold parameters + param m_ls_min_psnr, m_ls_min_alpha_psnr; + param m_ls_thresh_psnr, m_ls_thresh_alpha_psnr; + param m_ls_thresh_edge_psnr, m_ls_thresh_edge_alpha_psnr; + + // Job pool, MUST not be nullptr; job_pool *m_pJob_pool; + + // Returns the current format mode as set by set_format_mode() above. + // Because of backwards API compatibility we don't use this directly yet, it's just here to aid the transition to the new API. + basist::basis_tex_format get_format_mode() const { return m_format_mode; } + + private: + // This is set by set_format_mode() above. For backwards API compat we don't use it directly, it's just here to aid the transition to the new API. + basist::basis_tex_format m_format_mode; }; // Important: basisu_encoder_init() MUST be called first before using this class. @@ -588,7 +762,7 @@ namespace basisu // Note it *should* be possible to call init() multiple times with different inputs, but this scenario isn't well tested. Ideally, create 1 object, compress, then delete it. bool init(const basis_compressor_params ¶ms); - + enum error_code { cECSuccess = 0, @@ -602,45 +776,63 @@ namespace basisu cECFailedCreateBasisFile, cECFailedWritingOutput, cECFailedUASTCRDOPostProcess, - cECFailedCreateKTX2File + cECFailedCreateKTX2File, + cECFailedInvalidParameters }; error_code process(); // The output .basis file will always be valid of process() succeeded. const uint8_vec &get_output_basis_file() const { return m_output_basis_file; } - + // The output .ktx2 file will only be valid if m_create_ktx2_file was true and process() succeeded. const uint8_vec& get_output_ktx2_file() const { return m_output_ktx2_file; } const basisu::vector &get_stats() const { return m_stats; } - uint32_t get_basis_file_size() const { return m_basis_file_size; } + // Sum of all slice orig pixels. Intended for statistics display. + uint64_t get_total_slice_orig_texels() const { return m_total_slice_orig_texels; } + + uint64_t get_basis_file_size() const { return m_basis_file_size; } double get_basis_bits_per_texel() const { return m_basis_bits_per_texel; } + uint64_t get_ktx2_file_size() const { return m_ktx2_file_size; } + double get_ktx2_bits_per_texel() const { return m_ktx2_bits_per_texel; } + bool get_any_source_image_has_alpha() const { return m_any_source_image_has_alpha; } bool get_opencl_failed() const { return m_opencl_failed; } - + private: basis_compressor_params m_params; - + opencl_context_ptr m_pOpenCL_context; - basist::basis_tex_format m_fmt_mode; - + // the output mode/codec + basist::basis_tex_format m_fmt_mode; + + // the output mode/codec's block width/height + uint32_t m_fmt_mode_block_width; + uint32_t m_fmt_mode_block_height; + + // Note these images are expanded if necessary (duplicating cols/rows) to account for block dimensions. basisu::vector m_slice_images; basisu::vector m_slice_images_hdr; basisu::vector m_stats; - uint32_t m_basis_file_size; + uint64_t m_total_slice_orig_texels; + + uint64_t m_basis_file_size; double m_basis_bits_per_texel; + uint64_t m_ktx2_file_size; + double m_ktx2_bits_per_texel; + basisu_backend_slice_desc_vec m_slice_descs; uint32_t m_total_blocks; - + basisu_frontend m_frontend; // These are 4x4 blocks. @@ -658,7 +850,7 @@ namespace basisu basisu::vector m_decoded_output_textures; // BC6H in HDR mode basisu::vector m_decoded_output_textures_unpacked; - + basisu::vector m_decoded_output_textures_bc7; basisu::vector m_decoded_output_textures_unpacked_bc7; @@ -669,16 +861,16 @@ namespace basisu uint8_vec m_output_basis_file; uint8_vec m_output_ktx2_file; - + basisu::vector m_uastc_slice_textures; basisu_backend_output m_uastc_backend_output; // The amount the HDR input has to be scaled up in case it had to be rescaled to fit into half floats. - float m_hdr_image_scale; - + float m_hdr_image_scale; + // The upconversion multiplier used to load LDR images in HDR mode. float m_ldr_to_hdr_upconversion_nit_multiplier; - + // True if any loaded source images were LDR and upconverted to HDR. bool m_upconverted_any_ldr_images; @@ -701,48 +893,20 @@ namespace basisu error_code encode_slices_to_astc_6x6_hdr(); error_code encode_slices_to_uastc_4x4_hdr(); error_code encode_slices_to_uastc_4x4_ldr(); + error_code encode_slices_to_xuastc_or_astc_ldr(); bool generate_mipmaps(const imagef& img, basisu::vector& mips, bool has_alpha); bool generate_mipmaps(const image &img, basisu::vector &mips, bool has_alpha); bool validate_texture_type_constraints(); bool validate_ktx2_constraints(); bool get_dfd(uint8_vec& dfd, const basist::ktx2_header& hdr); bool create_ktx2_file(); - void pick_format_mode(); - - uint32_t get_block_width() const - { - if (m_params.m_hdr) - { - switch (m_params.m_hdr_mode) - { - case hdr_modes::cASTC_HDR_6X6: - case hdr_modes::cASTC_HDR_6X6_INTERMEDIATE: - return 6; - default: - break; - } - } - return 4; - } + bool pick_format_mode(); - uint32_t get_block_height() const - { - if (m_params.m_hdr) - { - switch (m_params.m_hdr_mode) - { - case hdr_modes::cASTC_HDR_6X6: - case hdr_modes::cASTC_HDR_6X6_INTERMEDIATE: - return 6; - default: - break; - } - } - return 4; - } + uint32_t get_block_width() const { return m_fmt_mode_block_width; } + uint32_t get_block_height() const { return m_fmt_mode_block_height; } }; - - // Alternative simple C-style wrapper API around the basis_compressor class. + + // Alternative simple C-style wrapper API around the basis_compressor class. // This doesn't expose every encoder feature, but it's enough to get going. // Important: basisu_encoder_init() MUST be called first before calling these functions. // @@ -751,16 +915,22 @@ namespace basisu // OR // pImageRGBA: pointer to a 32-bpp RGBx or RGBA raster image, R first in memory, A last. Top scanline first in memory. // width/height/pitch_in_pixels: dimensions of pImageRGBA - // - // flags_and_quality: Combination of the above flags logically OR'd with the ETC1S or UASTC level, i.e. "cFlagSRGB | cFlagGenMipsClamp | cFlagThreaded | 128" or "cFlagSRGB | cFlagGenMipsClamp | cFlagUASTC | cFlagThreaded | cPackUASTCLevelDefault". + // + // flags_and_quality: Combination of the above flags logically OR'd with the ETC1S or UASTC quality or effort level. + // Note: basis_compress2() variants below accept the new-style "quality_level" (0-100) and "effort_level" (0-10) parameters instead of packing them into flags_and_quality. // In ETC1S mode, the lower 8-bits are the ETC1S quality level which ranges from [1,255] (higher=better quality/larger files) - // In UASTC mode, the lower 8-bits are the UASTC LDR/HDR pack level (see cPackUASTCLevelFastest, etc.). Fastest/lowest quality is 0, so be sure to set it correctly. Valid values are [0,4] for both LDR/HDR. - // In UASTC mode, be sure to set this, otherwise it defaults to 0 (fastest/lowest quality). - // - // uastc_rdo_quality: Float UASTC RDO quality level (0=no change, higher values lower quality but increase compressibility, initially try .5-1.5) - // + // In UASTC LDR 4x4 mode, the lower 8-bits are the UASTC LDR/HDR pack or effort level (see cPackUASTCLevelFastest to cPackUASTCLevelVerySlow). Fastest/lowest quality is 0, so be sure to set it correctly. Valid values are [0,4] for both LDR/HDR. + // In UASTC HDR 4x4 mode, the lower 8-bits are the codec's effort level. Valid range is [uastc_hdr_4x4_codec_options::cMinLevel, uastc_hdr_4x4_codec_options::cMaxLevel]. Higher=better quality, but slower. + // In RDO ASTC HDR 6x6/UASTC HDR 6x6 mode, the lower 8-bits are the codec's effort level. Valid range is [0,astc_6x6_hdr::ASTC_HDR_6X6_MAX_USER_COMP_LEVEL]. Higher levels=better quality, but slower. + // In XUASTC/ASTC LDR 4x4-12x12 mode, the lower 8-bits are the compressor's effort level from [0,10] (astc_ldr_t::EFFORT_LEVEL_MIN, astc_ldr_t::EFFORT_LEVEL_MAX). + // + // float uastc_rdo_or_dct_quality: + // UASTC LDR 4x4 RDO quality level: RDO lambda setting - 0=no change/highest quality. Higher values lower quality but increase compressibility, initially try .5-1.5. + // RDO ASTC 6x6 HDR/UASTC 6x6 HDR: RDO lambda setting. 0=no change/highest quality. Higher values lower quality but increase compressibility, initially try 250-2000 (HDR) or 1000-10000 (LDR/SDR inputs upconverted to HDR). + // In XUASTC/ASTC LDR 4x4-12x12 mode, this is the [1,100] weight grid DCT quality level. + // // pSize: Returns the output data's compressed size in bytes - // + // // Return value is the compressed .basis or .ktx2 file data, or nullptr on failure. Must call basis_free() to free it. enum { @@ -769,36 +939,63 @@ namespace basisu cFlagDebug = 1 << 10, // enable debug output cFlagKTX2 = 1 << 11, // generate a KTX2 file - cFlagKTX2UASTCSuperCompression = 1 << 12, // use KTX2 Zstd supercompression on UASTC files + cFlagKTX2UASTCSuperCompression = 1 << 12, // use KTX2 Zstd supercompression on non-supercompressed formats that support it. - cFlagSRGB = 1 << 13, // input texture is sRGB, use perceptual colorspace metrics, also use sRGB filtering during mipmap gen, and also sets KTX2 output transfer func to sRGB + cFlagSRGB = 1 << 13, // input texture is sRGB, use perceptual colorspace metrics, also use sRGB filtering during mipmap gen, and also sets KTX2/.basis output transfer func to sRGB cFlagGenMipsClamp = 1 << 14, // generate mipmaps with clamp addressing cFlagGenMipsWrap = 1 << 15, // generate mipmaps with wrap addressing - + cFlagYFlip = 1 << 16, // flip source image on Y axis before compression - - cFlagUASTCRDO = 1 << 17, // use RDO postprocessing when generating UASTC files (must set uastc_rdo_quality to the quality scalar) - + + // Note 11/18/2025: cFlagUASTCRDO flag is now ignored. Now if uastc_rdo_or_dct_quality>0 in UASTC LDR 4x4 mode, you automatically get RDO. + //cFlagUASTCRDO = 1 << 17, // use RDO postprocessing when generating UASTC LDR 4x4 files (must set uastc_rdo_or_dct_quality to the quality scalar) + cFlagPrintStats = 1 << 18, // print image stats to stdout cFlagPrintStatus = 1 << 19, // print status to stdout + + cFlagDebugImages = 1 << 20, // enable debug image generation (for development, slower) - cFlagDebugImages = 1 << 20, // enable status output - - cFlagREC2020 = 1 << 21, // ASTC 6x6 modes: treat input as REC 2020 vs. the default 709 - + cFlagREC2020 = 1 << 21, // treat input as REC 2020 vs. the default 709 (for codecs that support this, currently UASTC HDR and ASTC 6x6), bit is always placed into KTX2 DFD + cFlagValidateOutput = 1 << 22, // transcode the output after encoding for testing + + // XUASTC LDR profile: full arith, hybrid or full zstd (see basist::astc_ldr_t::xuastc_ldr_syntax) + cFlagXUASTCLDRSyntaxFullArith = 0 << 23, + cFlagXUASTCLDRSyntaxHybrid = 1 << 23, + cFlagXUASTCLDRSyntaxFullZStd = 2 << 23, + + cFlagXUASTCLDRSyntaxShift = 23, + cFlagXUASTCLDRSyntaxMask = 3, + + // Texture Type: 2D, 2D Array, Cubemap Array, or Texture Video (see enum basis_texture_type). Defaults to plain 2D. + cFlagTextureType2D = 0 << 25, + cFlagTextureType2DArray = 1 << 25, + cFlagTextureTypeCubemapArray = 2 << 25, + cFlagTextureTypeVideoFrames = 3 << 25, + + cFlagTextureTypeShift = 25, + cFlagTextureTypeMask = 3, }; - // This function accepts an array of source images. + void* basis_compress_internal( + basist::basis_tex_format mode, + const basisu::vector* pSource_images, + const basisu::vector* pSource_images_hdr, + uint32_t flags_and_quality, float uastc_rdo_or_dct_quality, + size_t* pSize, + image_stats* pStats, + int quality_level = -1, int effort_level = -1); + + // This function accepts an array of source images. // If more than one image is provided, it's assumed the images form a mipmap pyramid and automatic mipmap generation is disabled. - // Returns a pointer to the compressed .basis or .ktx2 file data. *pSize is the size of the compressed data. + // Returns a pointer to the compressed .basis or .ktx2 file data. *pSize is the size of the compressed data. // Important: The returned block MUST be manually freed using basis_free_data(). // basisu_encoder_init() MUST be called first! - // LDR version. To compress the LDR source image as HDR: Use the cFlagHDR flag. + // LDR version. void* basis_compress( basist::basis_tex_format mode, const basisu::vector &source_images, - uint32_t flags_and_quality, float uastc_rdo_quality, + uint32_t flags_and_quality, float uastc_rdo_or_dct_quality, size_t* pSize, image_stats* pStats = nullptr); @@ -807,7 +1004,7 @@ namespace basisu void* basis_compress( basist::basis_tex_format mode, const basisu::vector& source_images_hdr, - uint32_t flags_and_quality, float lambda, + uint32_t flags_and_quality, float uastc_rdo_or_dct_quality, size_t* pSize, image_stats* pStats = nullptr); @@ -816,7 +1013,30 @@ namespace basisu void* basis_compress( basist::basis_tex_format mode, const uint8_t* pImageRGBA, uint32_t width, uint32_t height, uint32_t pitch_in_pixels, - uint32_t flags_and_quality, float uastc_rdo_quality, + uint32_t flags_and_quality, float uastc_rdo_or_dct_quality, + size_t* pSize, + image_stats* pStats = nullptr); + + // basis_compress2() variants accept the new unified quality_level and effort_level parameters instead of the old flags/float uastc_rdo_or_dct_quality parameter. + // quality_level must be [0,100], effort_level [0,10]. + void* basis_compress2( + basist::basis_tex_format mode, + const basisu::vector& source_images, + uint32_t flags_and_quality, int quality_level, int effort_level, + size_t* pSize, + image_stats* pStats = nullptr); + + void* basis_compress2( + basist::basis_tex_format mode, + const basisu::vector& source_images_hdr, + uint32_t flags_and_quality, int quality_level, int effort_level, + size_t* pSize, + image_stats* pStats = nullptr); + + void* basis_compress2( + basist::basis_tex_format mode, + const uint8_t* pImageRGBA, uint32_t width, uint32_t height, uint32_t pitch_in_pixels, + uint32_t flags_and_quality, int quality_level, int effort_level, size_t* pSize, image_stats* pStats = nullptr); @@ -841,7 +1061,7 @@ namespace basisu double m_basis_bits_per_texel; bool m_any_source_image_has_alpha; - parallel_results() + parallel_results() { clear(); } @@ -857,7 +1077,7 @@ namespace basisu m_any_source_image_has_alpha = false; } }; - + // Compresses an array of input textures across total_threads threads using the basis_compressor class. // Compressing multiple textures at a time is substantially more efficient than just compressing one at a time. // total_threads must be >= 1. @@ -865,5 +1085,6 @@ namespace basisu uint32_t total_threads, const basisu::vector ¶ms_vec, basisu::vector< parallel_results > &results_vec); - + } // namespace basisu + diff --git a/external/basis_universal/encoder/basisu_enc.cpp b/external/basis_universal/encoder/basisu_enc.cpp index cccf66c171..841209b50f 100644 --- a/external/basis_universal/encoder/basisu_enc.cpp +++ b/external/basis_universal/encoder/basisu_enc.cpp @@ -1,5 +1,5 @@ // basisu_enc.cpp -// Copyright (C) 2019-2024 Binomial LLC. All Rights Reserved. +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -23,6 +23,8 @@ #include "basisu_opencl.h" #include "basisu_uastc_hdr_4x4_enc.h" #include "basisu_astc_hdr_6x6_enc.h" +#include "basisu_astc_ldr_common.h" +#include "basisu_astc_ldr_encode.h" #include @@ -58,7 +60,7 @@ namespace basisu #endif fast_linear_to_srgb g_fast_linear_to_srgb; - + uint8_t g_hamming_dist[256] = { 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, @@ -81,7 +83,7 @@ namespace basisu // This is a Public Domain 8x8 font from here: // https://github.com/dhepper/font8x8/blob/master/font8x8_basic.h - const uint8_t g_debug_font8x8_basic[127 - 32 + 1][8] = + const uint8_t g_debug_font8x8_basic[127 - 32 + 1][8] = { { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, // U+0020 ( ) { 0x18, 0x3C, 0x3C, 0x18, 0x18, 0x00, 0x18, 0x00}, // U+0021 (!) @@ -181,9 +183,17 @@ namespace basisu { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00} // U+007F }; + float g_srgb_to_linear_table[256]; + + void init_srgb_to_linear_table() + { + for (int i = 0; i < 256; ++i) + g_srgb_to_linear_table[i] = srgb_to_linear((float)i * (1.0f / 255.0f)); + } + bool g_library_initialized; std::mutex g_encoder_init_mutex; - + // Encoder library initialization (just call once at startup) bool basisu_encoder_init(bool use_opencl, bool opencl_force_serialization) { @@ -193,7 +203,7 @@ namespace basisu return true; detect_sse41(); - + basist::basisu_transcoder_init(); pack_etc1_solid_color_init(); //uastc_init(); @@ -210,7 +220,11 @@ namespace basisu astc_hdr_enc_init(); basist::bc6h_enc_init(); astc_6x6_hdr::global_init(); + astc_ldr::global_init(); + astc_ldr::encoder_init(); + init_srgb_to_linear_table(); + g_library_initialized = true; return true; } @@ -221,7 +235,7 @@ namespace basisu g_library_initialized = false; } - + void error_vprintf(const char* pFmt, va_list args) { const uint32_t BUF_SIZE = 256; @@ -238,10 +252,12 @@ namespace basisu return; } + fflush(stdout); + if (total_chars >= (int)BUF_SIZE) { basisu::vector var_buf(total_chars + 1); - + va_copy(args_copy, args); int total_chars_retry = vsnprintf(var_buf.data(), var_buf.size(), pFmt, args_copy); va_end(args_copy); @@ -277,6 +293,7 @@ namespace basisu void platform_sleep(uint32_t ms) { // TODO + BASISU_NOTE_UNUSED(ms); } #endif @@ -316,7 +333,7 @@ namespace basisu #else #error TODO #endif - + interval_timer::interval_timer() : m_start_time(0), m_stop_time(0), m_started(false), m_stopped(false) { if (!g_timer_freq) @@ -350,7 +367,7 @@ namespace basisu timer_ticks delta = stop_time - m_start_time; return delta * g_timer_freq; } - + void interval_timer::init() { if (!g_timer_freq) @@ -377,7 +394,7 @@ namespace basisu return ticks * g_timer_freq; } - // Note this is linear<->sRGB, NOT REC709 which uses slightly different equations/transfer functions. + // Note this is linear<->sRGB, NOT REC709 which uses slightly different equations/transfer functions. // However the gamuts/white points of REC709 and sRGB are the same. float linear_to_srgb(float l) { @@ -387,7 +404,7 @@ namespace basisu else return saturate(1.055f * powf(l, 1.0f / 2.4f) - .055f); } - + float srgb_to_linear(float s) { assert(s >= 0.0f && s <= 1.0f); @@ -396,21 +413,21 @@ namespace basisu else return saturate(powf((s + .055f) * (1.0f / 1.055f), 2.4f)); } - + const uint32_t MAX_32BIT_ALLOC_SIZE = 250000000; - + bool load_tga(const char* pFilename, image& img) { int w = 0, h = 0, n_chans = 0; uint8_t* pImage_data = read_tga(pFilename, w, h, n_chans); - + if ((!pImage_data) || (!w) || (!h) || ((n_chans != 3) && (n_chans != 4))) { error_printf("Failed loading .TGA image \"%s\"!\n", pFilename); if (pImage_data) free(pImage_data); - + return false; } @@ -426,7 +443,7 @@ namespace basisu return false; } } - + img.resize(w, h); const uint8_t *pSrc = pImage_data; @@ -469,7 +486,7 @@ namespace basisu { interval_timer tm; tm.start(); - + if (!buf_size) return false; @@ -488,7 +505,7 @@ namespace basisu return true; } - + bool load_png(const char* pFilename, image& img) { uint8_vec buffer; @@ -507,9 +524,9 @@ namespace basisu uint8_t *pImage_data = jpgd::decompress_jpeg_image_from_file(pFilename, &width, &height, &actual_comps, 4, jpgd::jpeg_decoder::cFlagLinearChromaFiltering); if (!pImage_data) return false; - + img.init(pImage_data, width, height, 4); - + free(pImage_data); return true; @@ -642,7 +659,7 @@ namespace basisu dst[2] = basist::half_to_float(pSrc_pixel[2]); dst[3] = basist::half_to_float(pSrc_pixel[3]); } - + pSrc_image_h += (width * 4); } @@ -722,7 +739,7 @@ namespace basisu return ((strcasecmp(pExt, "hdr") == 0) || (strcasecmp(pExt, "exr") == 0)); } - + // TODO: move parameters to struct, add a HDR clean flag to eliminate NaN's/Inf's bool load_image_hdr(const char* pFilename, imagef& img, bool ldr_srgb_to_linear, float linear_nit_multiplier, float ldr_black_bias) { @@ -740,7 +757,7 @@ namespace basisu return false; return true; } - + if (strcasecmp(pExt, "exr") == 0) { int n_chans = 0; @@ -760,12 +777,12 @@ namespace basisu return true; } - + bool save_png(const char* pFilename, const image &img, uint32_t image_save_flags, uint32_t grayscale_comp) { if (!img.get_total_pixels()) return false; - + void* pPNG_data = nullptr; size_t PNG_data_size = 0; @@ -783,7 +800,7 @@ namespace basisu else { bool has_alpha = false; - + if ((image_save_flags & cImageSaveIgnoreAlpha) == 0) has_alpha = img.has_alpha(); @@ -800,7 +817,7 @@ namespace basisu pDst[0] = pSrc->r; pDst[1] = pSrc->g; pDst[2] = pSrc->b; - + pSrc++; pDst += 3; } @@ -824,10 +841,35 @@ namespace basisu } free(pPNG_data); - + return status; } + bool save_qoi(const char* pFilename, const image& img, uint32_t qoi_colorspace) + { + assert(img.get_width() && img.get_height()); + + qoi_desc desc; + clear_obj(desc); + + desc.width = img.get_width(); + desc.height = img.get_height(); + desc.channels = 4; + desc.colorspace = (uint8_t)qoi_colorspace; + + int out_len = 0; + void* pData = qoi_encode(img.get_ptr(), &desc, &out_len); + if ((!pData) || (!out_len)) + return false; + + const bool status = write_data_to_file(pFilename, pData, out_len); + + QOI_FREE(pData); + pData = nullptr; + + return status; + } + bool read_file_to_vec(const char* pFilename, uint8_vec& data) { FILE* pFile = nullptr; @@ -838,7 +880,7 @@ namespace basisu #endif if (!pFile) return false; - + fseek(pFile, 0, SEEK_END); #ifdef _WIN32 int64_t filesize = _ftelli64(pFile); @@ -909,7 +951,7 @@ namespace basisu return false; } fseek(pFile, 0, SEEK_SET); - + if (fread(pData, 1, (size_t)len, pFile) != (size_t)len) { fclose(pFile); @@ -942,19 +984,20 @@ namespace basisu return fclose(pFile) != EOF; } - + bool image_resample(const image &src, image &dst, bool srgb, - const char *pFilter, float filter_scale, + const char *pFilter, float filter_scale, bool wrapping, - uint32_t first_comp, uint32_t num_comps) + uint32_t first_comp, uint32_t num_comps, + float filter_scale_y) { assert((first_comp + num_comps) <= 4); const int cMaxComps = 4; - + const uint32_t src_w = src.get_width(), src_h = src.get_height(); const uint32_t dst_w = dst.get_width(), dst_h = dst.get_height(); - + if (maximum(src_w, src_h) > BASISU_RESAMPLER_MAX_DIMENSION) { printf("Image is too large!\n"); @@ -963,17 +1006,19 @@ namespace basisu if (!src_w || !src_h || !dst_w || !dst_h) return false; - + if ((num_comps < 1) || (num_comps > cMaxComps)) return false; - + if ((minimum(dst_w, dst_h) < 1) || (maximum(dst_w, dst_h) > BASISU_RESAMPLER_MAX_DIMENSION)) { printf("Image is too large!\n"); return false; } - if ((src_w == dst_w) && (src_h == dst_h)) + if ( (src_w == dst_w) && (src_h == dst_h) && + (filter_scale == 1.0f) && + ((filter_scale_y < 0.0f) || (filter_scale_y == 1.0f)) ) { dst = src; return true; @@ -997,17 +1042,19 @@ namespace basisu std::vector samples[cMaxComps]; Resampler *resamplers[cMaxComps]; - + resamplers[0] = new Resampler(src_w, src_h, dst_w, dst_h, wrapping ? Resampler::BOUNDARY_WRAP : Resampler::BOUNDARY_CLAMP, 0.0f, 1.0f, - pFilter, nullptr, nullptr, filter_scale, filter_scale, 0, 0); + pFilter, nullptr, nullptr, + filter_scale, (filter_scale_y >= 0.0f) ? filter_scale_y : filter_scale, 0, 0); samples[0].resize(src_w); for (uint32_t i = 1; i < num_comps; ++i) { resamplers[i] = new Resampler(src_w, src_h, dst_w, dst_h, wrapping ? Resampler::BOUNDARY_WRAP : Resampler::BOUNDARY_CLAMP, 0.0f, 1.0f, - pFilter, resamplers[0]->get_clist_x(), resamplers[0]->get_clist_y(), filter_scale, filter_scale, 0, 0); + pFilter, resamplers[0]->get_clist_x(), resamplers[0]->get_clist_y(), + filter_scale, (filter_scale_y >= 0.0f) ? filter_scale_y : filter_scale, 0, 0); samples[i].resize(src_w); } @@ -1057,7 +1104,7 @@ namespace basisu break; const bool linear_flag = !srgb || (comp_index == 3); - + color_rgba *pDst = &dst(0, dst_y); for (uint32_t x = 0; x < dst_w; x++) @@ -1090,7 +1137,7 @@ namespace basisu return true; } - bool image_resample(const imagef& src, imagef& dst, + bool image_resample(const imagef& src, imagef& dst, const char* pFilter, float filter_scale, bool wrapping, uint32_t first_comp, uint32_t num_comps) @@ -1183,7 +1230,7 @@ namespace basisu const float* pOutput_samples = resamplers[c]->get_line(); if (!pOutput_samples) break; - + vec4F* pDst = &dst(0, dst_y); for (uint32_t x = 0; x < dst_w; x++) @@ -1216,9 +1263,9 @@ namespace basisu A[0].m_key = 1; return; } - + A[0].m_key += A[1].m_key; - + int s = 2, r = 0, next; for (next = 1; next < (num_syms - 1); ++next) { @@ -1310,7 +1357,7 @@ namespace basisu for (i = 0; i < num_syms; i++) { uint32_t freq = pSyms0[i].m_key; - + // We scale all input frequencies to 16-bits. assert(freq <= UINT16_MAX); @@ -1501,7 +1548,7 @@ namespace basisu uint32_t total_used = tab.get_total_used_codes(); put_bits(total_used, cHuffmanMaxSymsLog2); - + if (!total_used) return 0; @@ -1565,7 +1612,7 @@ namespace basisu const uint32_t l = syms[i] & 63, e = syms[i] >> 6; put_code(l, ct); - + if (l == cHuffmanSmallZeroRunCode) put_bits(e, cHuffmanSmallZeroRunExtraBits); else if (l == cHuffmanBigZeroRunCode) @@ -1592,7 +1639,7 @@ namespace basisu huffman_encoding_table etab; etab.init(h, 16); - + { bitwise_coder c; c.init(1024); @@ -1727,9 +1774,9 @@ namespace basisu // We now have chosen an entry to place in the picked list, now determine which side it goes on. const uint32_t entry_to_move = m_entries_to_do[best_entry]; - + float side = pick_side(num_syms, entry_to_move, pDist_func, pCtx, dist_func_weight); - + // Put entry_to_move either on the "left" or "right" side of the picked entries if (side <= 0) m_entries_picked.push_back(entry_to_move); @@ -1832,7 +1879,7 @@ namespace basisu } return which_side; } - + void image_metrics::calc(const imagef& a, const imagef& b, uint32_t first_chan, uint32_t total_chans, bool avg_comp_error, bool log) { assert((first_chan < 4U) && (first_chan + total_chans <= 4U)); @@ -1843,16 +1890,19 @@ namespace basisu double max_e = -1e+30f; double sum = 0.0f, sum_sqr = 0.0f; + m_width = width; + m_height = height; + m_has_neg = false; m_any_abnormal = false; m_hf_mag_overflow = false; - + for (uint32_t y = 0; y < height; y++) { for (uint32_t x = 0; x < width; x++) { const vec4F& ca = a(x, y), &cb = b(x, y); - + if (total_chans) { for (uint32_t c = 0; c < total_chans; c++) @@ -1867,7 +1917,7 @@ namespace basisu if (std::isinf(fa) || std::isinf(fb) || std::isnan(fa) || std::isnan(fb)) m_any_abnormal = true; - + const double delta = fabs(fa - fb); max_e = basisu::maximum(max_e, delta); @@ -1902,10 +1952,10 @@ namespace basisu } double ca_l = get_luminance(ca), cb_l = get_luminance(cb); - + double delta = fabs(ca_l - cb_l); max_e = basisu::maximum(max_e, delta); - + if (log) { double log2_delta = log2(basisu::maximum(0.0f, ca_l) + 1.0f) - log2(basisu::maximum(0.0f, cb_l) + 1.0f); @@ -1931,7 +1981,7 @@ namespace basisu m_mean = (float)(sum / total_values); m_mean_squared = (float)(sum_sqr / total_values); m_rms = (float)sqrt(sum_sqr / total_values); - + const double max_val = 1.0f; m_psnr = m_rms ? (float)clamp(log10(max_val / m_rms) * 20.0f, 0.0f, 1000.0f) : 1000.0f; } @@ -1944,12 +1994,15 @@ namespace basisu const uint32_t width = basisu::minimum(a.get_width(), b.get_width()); const uint32_t height = basisu::minimum(a.get_height(), b.get_height()); + m_width = width; + m_height = height; + m_has_neg = false; m_hf_mag_overflow = false; m_any_abnormal = false; uint_vec hist(65536); - + for (uint32_t y = 0; y < height; y++) { for (uint32_t x = 0; x < width; x++) @@ -1960,7 +2013,7 @@ namespace basisu { if ((ca[i] < 0.0f) || (cb[i] < 0.0f)) m_has_neg = true; - + if ((fabs(ca[i]) > basist::MAX_HALF_FLOAT) || (fabs(cb[i]) > basist::MAX_HALF_FLOAT)) m_hf_mag_overflow = true; @@ -2010,10 +2063,13 @@ namespace basisu const uint32_t width = basisu::minimum(a.get_width(), b.get_width()); const uint32_t height = basisu::minimum(a.get_height(), b.get_height()); + m_width = width; + m_height = height; + m_has_neg = false; m_hf_mag_overflow = false; m_any_abnormal = false; - + double sum = 0.0f, sum2 = 0.0f; m_max = 0; @@ -2050,7 +2106,7 @@ namespace basisu } // x } // y - + double total_values = (double)width * (double)height; if (avg_comp_error) total_values *= (double)clamp(total_chans, 1, 4); @@ -2069,12 +2125,17 @@ namespace basisu const uint32_t width = basisu::minimum(a.get_width(), b.get_width()); const uint32_t height = basisu::minimum(a.get_height(), b.get_height()); + m_width = width; + m_height = height; + double hist[256]; clear_obj(hist); m_has_neg = false; m_any_abnormal = false; m_hf_mag_overflow = false; + m_sum_a = 0; + m_sum_b = 0; for (uint32_t y = 0; y < height; y++) { @@ -2085,7 +2146,11 @@ namespace basisu if (total_chans) { for (uint32_t c = 0; c < total_chans; c++) + { hist[iabs(ca[first_chan + c] - cb[first_chan + c])]++; + m_sum_a += ca[first_chan + c]; + m_sum_b += cb[first_chan + c]; + } } else { @@ -2093,6 +2158,12 @@ namespace basisu hist[iabs(ca.get_601_luma() - cb.get_601_luma())]++; else hist[iabs(ca.get_709_luma() - cb.get_709_luma())]++; + + for (uint32_t c = 0; c < 3; c++) + { + m_sum_a += ca[c]; + m_sum_b += cb[c]; + } } } } @@ -2168,63 +2239,7 @@ namespace basisu } } - uint32_t hash_hsieh(const uint8_t *pBuf, size_t len) - { - if (!pBuf || !len) - return 0; - - uint32_t h = static_cast(len); - - const uint32_t bytes_left = len & 3; - len >>= 2; - - while (len--) - { - const uint16_t *pWords = reinterpret_cast(pBuf); - - h += pWords[0]; - - const uint32_t t = (pWords[1] << 11) ^ h; - h = (h << 16) ^ t; - - pBuf += sizeof(uint32_t); - - h += h >> 11; - } - - switch (bytes_left) - { - case 1: - h += *reinterpret_cast(pBuf); - h ^= h << 10; - h += h >> 1; - break; - case 2: - h += *reinterpret_cast(pBuf); - h ^= h << 11; - h += h >> 17; - break; - case 3: - h += *reinterpret_cast(pBuf); - h ^= h << 16; - h ^= (static_cast(pBuf[sizeof(uint16_t)])) << 18; - h += h >> 11; - break; - default: - break; - } - - h ^= h << 3; - h += h >> 5; - h ^= h << 4; - h += h >> 17; - h ^= h << 25; - h += h >> 6; - - return h; - } - - job_pool::job_pool(uint32_t num_threads) : + job_pool::job_pool(uint32_t num_threads) : m_num_active_jobs(0) { m_kill_flag.store(false); @@ -2246,13 +2261,13 @@ namespace basisu job_pool::~job_pool() { debug_printf("job_pool::~job_pool\n"); - + // Notify all workers that they need to die right now. { std::lock_guard lk(m_mutex); m_kill_flag.store(true); } - + m_has_work.notify_all(); #ifdef __EMSCRIPTEN__ @@ -2262,7 +2277,7 @@ namespace basisu break; std::this_thread::sleep_for(std::chrono::milliseconds(50)); } - + // At this point all worker threads should be exiting or exited. // We could call detach(), but this seems to just call join() anyway. #endif @@ -2271,7 +2286,7 @@ namespace basisu for (uint32_t i = 0; i < m_threads.size(); i++) m_threads[i].join(); } - + void job_pool::add_job(const std::function& job) { std::unique_lock lock(m_mutex); @@ -2291,7 +2306,7 @@ namespace basisu std::unique_lock lock(m_mutex); m_queue.emplace_back(std::move(job)); - + const size_t queue_size = m_queue.size(); lock.unlock(); @@ -2340,7 +2355,7 @@ namespace basisu //debug_printf("job_pool::job_thread: starting %u\n", index); m_num_active_workers.fetch_add(1); - + while (!m_kill_flag) { std::unique_lock lock(m_mutex); @@ -2376,9 +2391,9 @@ namespace basisu --m_num_active_jobs; - // Now check if there are no more jobs remaining. + // Now check if there are no more jobs remaining. const bool all_done = m_queue.empty() && !m_num_active_jobs; - + lock.unlock(); if (all_done) @@ -2439,7 +2454,7 @@ namespace basisu // Simple validation if ((hdr.m_cmap != 0) && (hdr.m_cmap != 1)) return nullptr; - + if (hdr.m_cmap) { if ((hdr.m_cmap_bpp == 0) || (hdr.m_cmap_bpp > 32)) @@ -2598,13 +2613,13 @@ namespace basisu bytes_remaining += bytes_to_skip; } } - + width = hdr.m_width; height = hdr.m_height; const uint32_t source_pitch = width * tga_bytes_per_pixel; const uint32_t dest_pitch = width * n_chans; - + uint8_t *pImage = (uint8_t *)malloc(dest_pitch * height); if (!pImage) return nullptr; @@ -2626,7 +2641,7 @@ namespace basisu int pixels_remaining = width; uint8_t *pDst = &input_line_buf[0]; - do + do { if (!run_remaining) { @@ -2811,7 +2826,7 @@ namespace basisu if (!filedata.size() || (filedata.size() > UINT32_MAX)) return nullptr; - + return read_tga(&filedata[0], (uint32_t)filedata.size(), width, height, n_chans); } @@ -2958,13 +2973,13 @@ namespace basisu if (cur_line.size() < 3) return false; - + if (!is_x && !is_y) return false; comp[d] = is_x ? 0 : 1; dir[d] = (is_neg_x || is_neg_y) ? -1 : 1; - + uint32_t& dim = d ? minor_dim : major_dim; cur_line.erase(0, 3); @@ -3002,7 +3017,7 @@ namespace basisu if ((dim < 1) || (dim > MAX_SUPPORTED_DIM)) return false; } - + // temp image: width=minor, height=major img.resize(minor_dim, major_dim); @@ -3030,7 +3045,7 @@ namespace basisu } else { - // c[0]/red is 2.Check GB and E for validity. + // c[0]/red is 2.Check GB and E for validity. color_rgba c; memcpy(&c, &filedata[cur_ofs], 4); @@ -3152,7 +3167,7 @@ namespace basisu // width=minor axis dimension // height=major axis dimension // in file, pixels are emitted in minor order, them major (so major=scanlines in the file) - + imagef final_img; if (comp[0] == 0) // if major axis is X final_img.resize(major_dim, minor_dim); @@ -3169,10 +3184,10 @@ namespace basisu uint32_t dst_x = 0, dst_y = 0; // is the minor dim output x? - if (comp[1] == 0) + if (comp[1] == 0) { // minor axis is x, major is y - + // is minor axis (which is output x) flipped? if (dir[1] < 0) dst_x = minor_dim - 1 - minor_iter; @@ -3231,7 +3246,7 @@ namespace basisu return buf; } - + static uint8_vec& append_string(uint8_vec& buf, const std::string& str) { if (!str.size()) @@ -3248,7 +3263,7 @@ namespace basisu if (max_v < 1e-32f) rgbe.clear(); - else + else { int e; const float scale = frexp(max_v, &e) * 256.0f / max_v; @@ -3261,14 +3276,14 @@ namespace basisu const bool RGBE_FORCE_RAW = false; const bool RGBE_FORCE_OLD_CRUNCH = false; // note must readers (particularly stb_image.h's) don't properly support this, when they should - + bool write_rgbe(uint8_vec &file_data, imagef& img, rgbe_header_info& hdr_info) { if (!img.get_width() || !img.get_height()) return false; const uint32_t width = img.get_width(), height = img.get_height(); - + file_data.resize(0); file_data.reserve(1024 + img.get_width() * img.get_height() * 4); @@ -3301,7 +3316,7 @@ namespace basisu { int prev_r = -1, prev_g = -1, prev_b = -1, prev_e = -1; uint32_t cur_run_len = 0; - + for (uint32_t x = 0; x < width; x++) { color_rgba rgbe; @@ -3314,7 +3329,7 @@ namespace basisu // this ensures rshift stays 0, it's lame but this path is only for testing readers color_rgba f(1, 1, 1, cur_run_len - 1); append_vector(file_data, (const uint8_t*)&f, sizeof(f)); - append_vector(file_data, (const uint8_t*)&rgbe, sizeof(rgbe)); + append_vector(file_data, (const uint8_t*)&rgbe, sizeof(rgbe)); cur_run_len = 0; } } @@ -3324,12 +3339,12 @@ namespace basisu { color_rgba f(1, 1, 1, cur_run_len); append_vector(file_data, (const uint8_t*)&f, sizeof(f)); - + cur_run_len = 0; } - + append_vector(file_data, (const uint8_t*)&rgbe, sizeof(rgbe)); - + prev_r = rgbe[0]; prev_g = rgbe[1]; prev_b = rgbe[2]; @@ -3354,7 +3369,7 @@ namespace basisu { color_rgba rgbe(2, 2, width >> 8, width & 0xFF); append_vector(file_data, (const uint8_t*)&rgbe, sizeof(rgbe)); - + for (uint32_t x = 0; x < width; x++) { float2rgbe(rgbe, img(x, y)); @@ -3366,7 +3381,7 @@ namespace basisu for (uint32_t c = 0; c < 4; c++) { int raw_ofs = -1; - + uint32_t x = 0; while (x < width) { @@ -3381,7 +3396,7 @@ namespace basisu break; run_len++; } - + const uint32_t cost_to_keep_raw = ((raw_ofs != -1) ? 0 : 1) + run_len; // 0 or 1 bytes to start a raw run, then the repeated bytes issued as raw const uint32_t cost_to_take_run = 2 + 1; // 2 bytes to issue the RLE, then 1 bytes to start whatever follows it (raw or RLE) @@ -3405,7 +3420,7 @@ namespace basisu raw_ofs = -1; file_data.push_back(cur_byte); - + x++; } } // x @@ -3424,7 +3439,7 @@ namespace basisu return false; return write_vec_to_file(pFilename, file_data); } - + bool read_exr(const char* pFilename, imagef& img, int& n_chans) { n_chans = 0; @@ -3432,7 +3447,7 @@ namespace basisu int width = 0, height = 0; float* out_rgba = nullptr; const char* err = nullptr; - + int status = LoadEXRWithLayer(&out_rgba, &width, &height, pFilename, nullptr, &err, &n_chans); if (status != 0) { @@ -3451,7 +3466,7 @@ namespace basisu } img.resize(width, height); - + if (n_chans == 1) { const float* pSrc = out_rgba; @@ -3505,16 +3520,16 @@ namespace basisu { assert((n_chans == 1) || (n_chans == 3) || (n_chans == 4)); - const bool linear_hint = (flags & WRITE_EXR_LINEAR_HINT) != 0, + const bool linear_hint = (flags & WRITE_EXR_LINEAR_HINT) != 0, store_float = (flags & WRITE_EXR_STORE_FLOATS) != 0, no_compression = (flags & WRITE_EXR_NO_COMPRESSION) != 0; - + const uint32_t width = img.get_width(), height = img.get_height(); assert(width && height); - + if (!width || !height) return false; - + float_vec layers[4]; float* image_ptrs[4]; for (uint32_t c = 0; c < n_chans; c++) @@ -3543,7 +3558,7 @@ namespace basisu assert(0); return false; } - + for (uint32_t y = 0; y < height; y++) { for (uint32_t x = 0; x < width; x++) @@ -3567,7 +3582,7 @@ namespace basisu image.height = height; header.num_channels = n_chans; - + header.channels = (EXRChannelInfo*)calloc(header.num_channels, sizeof(EXRChannelInfo)); // Must be (A)BGR order, since most of EXR viewers expect this channel order. @@ -3578,37 +3593,37 @@ namespace basisu c = "BGR"[i]; else if (n_chans == 4) c = "ABGR"[i]; - + header.channels[i].name[0] = c; header.channels[i].name[1] = '\0'; header.channels[i].p_linear = linear_hint; } - + header.pixel_types = (int*)calloc(header.num_channels, sizeof(int)); header.requested_pixel_types = (int*)calloc(header.num_channels, sizeof(int)); - + if (!no_compression) header.compression_type = TINYEXR_COMPRESSIONTYPE_ZIP; - for (int i = 0; i < header.num_channels; i++) + for (int i = 0; i < header.num_channels; i++) { // pixel type of input image - header.pixel_types[i] = TINYEXR_PIXELTYPE_FLOAT; + header.pixel_types[i] = TINYEXR_PIXELTYPE_FLOAT; // pixel type of output image to be stored in .EXR - header.requested_pixel_types[i] = store_float ? TINYEXR_PIXELTYPE_FLOAT : TINYEXR_PIXELTYPE_HALF; + header.requested_pixel_types[i] = store_float ? TINYEXR_PIXELTYPE_FLOAT : TINYEXR_PIXELTYPE_HALF; } const char* pErr_msg = nullptr; int ret = SaveEXRImageToFile(&image, &header, pFilename, &pErr_msg); - if (ret != TINYEXR_SUCCESS) + if (ret != TINYEXR_SUCCESS) { error_printf("Save EXR err: %s\n", pErr_msg); FreeEXRErrorMessage(pErr_msg); } - + free(header.channels); free(header.pixel_types); free(header.requested_pixel_types); @@ -3622,7 +3637,7 @@ namespace basisu va_list args; va_start(args, pFmt); -#ifdef _WIN32 +#ifdef _WIN32 vsprintf_s(buf, sizeof(buf), pFmt, args); #else vsnprintf(buf, sizeof(buf), pFmt, args); @@ -3647,7 +3662,7 @@ namespace basisu for (uint32_t x = 0; x < 8; x++) { const uint32_t q = row_bits & (1 << x); - + const color_rgba* pColor = q ? &fg : pBG; if (!pColor) continue; @@ -3667,8 +3682,8 @@ namespace basisu } } } - - // Very basic global Reinhard tone mapping, output converted to sRGB with no dithering, alpha is carried through unchanged. + + // Very basic global Reinhard tone mapping, output converted to sRGB with no dithering, alpha is carried through unchanged. // Only used for debugging/development. void tonemap_image_reinhard(image &ldr_img, const imagef &hdr_img, float exposure, bool add_noise, bool per_component, bool luma_scaling) { @@ -3678,7 +3693,7 @@ namespace basisu rand r; r.seed(128); - + for (uint32_t y = 0; y < height; y++) { for (uint32_t x = 0; x < width; x++) @@ -3713,7 +3728,7 @@ namespace basisu { //Lmapped = L / (1.0f + L); //Lmapped /= L; - + Lmapped = 1.0f / (1.0f + L); } @@ -3933,7 +3948,7 @@ namespace basisu dst_img.set_all(color_rgba(0, 0, 0, 255)); basisu::vector half_img(width * 3 * height); - + uint32_t low_h = UINT32_MAX, high_h = 0; for (uint32_t y = 0; y < height; y++) @@ -3957,7 +3972,7 @@ namespace basisu low_h = minimum(low_h, h); high_h = maximum(high_h, h); - + half_img[(x + y * width) * 3 + i] = (basist::half_float)h; } // i @@ -3974,7 +3989,7 @@ namespace basisu for (uint32_t i = 0; i < 3; i++) { basist::half_float h = half_img[(x + y * width) * 3 + i]; - + float f = (float)(h - low_h) / (float)(high_h - low_h); int iv = basisu::clamp((int)std::round(f * 255.0f), 0, 255); @@ -3988,6 +4003,328 @@ namespace basisu return true; } + bool arith_test() + { + basist::arith_fastbits_f32::init(); + + fmt_printf("random bit test\n"); + + const uint32_t N = 1000; + + // random bit test + for (uint32_t i = 0; i < N; i++) + { + basist::arith::arith_enc enc; + enc.init(4096); + + { + basisu::rand r; + r.seed(i + 1); + uint32_t num_vals = r.irand(1, 20000); + + for (uint32_t j = 0; j < num_vals; j++) + enc.put_bit(r.bit()); + + enc.flush(); + } + + { + basisu::rand r; + r.seed(i + 1); + uint32_t num_vals = r.irand(1, 20000); + + basist::arith::arith_dec dec; + dec.init(enc.get_data_buf().get_ptr(), enc.get_data_buf().size()); + + for (uint32_t j = 0; j < num_vals; j++) + { + uint32_t t = r.bit(); + + uint32_t a = dec.get_bit(); + if (t != a) + { + fmt_printf("error!"); + return false; + } + } + } + } + + fmt_printf("Random bit test OK\n"); + + fmt_printf("random bits test\n"); + + // random bits test + for (uint32_t i = 0; i < N; i++) + { + basist::arith::arith_enc enc; + enc.init(4096); + + { + basisu::rand r; + r.seed(i + 1); + uint32_t num_vals = r.irand(1, 20000); + uint32_t num_bits = r.irand(1, 20); + + for (uint32_t j = 0; j < num_vals; j++) + enc.put_bits(r.urand32() & ((1 << num_bits) - 1), num_bits); + + enc.flush(); + } + + { + basisu::rand r; + r.seed(i + 1); + uint32_t num_vals = r.irand(1, 20000); + uint32_t num_bits = r.irand(1, 20); + + basist::arith::arith_dec dec; + dec.init(enc.get_data_buf().get_ptr(), enc.get_data_buf().size()); + + for (uint32_t j = 0; j < num_vals; j++) + { + uint32_t t = r.urand32() & ((1 << num_bits) - 1); + + uint32_t a = dec.get_bits(num_bits); + if (t != a) + { + fmt_printf("error!"); + return false; + } + } + } + } + + fmt_printf("Random bits test OK\n"); + + fmt_printf("random adaptive bit model test\n"); + + // adaptive bit model random test + for (uint32_t i = 0; i < N; i++) + { + basist::arith::arith_enc enc; + enc.init(4096); + + { + basisu::rand r; + r.seed(i + 1); + uint32_t num_vals = r.irand(1, 20000); + + basist::arith::arith_bit_model bm; + bm.init(); + + for (uint32_t j = 0; j < num_vals; j++) + enc.encode(r.bit(), bm); + + enc.flush(); + } + + { + basisu::rand r; + r.seed(i + 1); + uint32_t num_vals = r.irand(1, 20000); + + basist::arith::arith_dec dec; + dec.init(enc.get_data_buf().get_ptr(), enc.get_data_buf().size()); + + basist::arith::arith_bit_model bm; + bm.init(); + + for (uint32_t j = 0; j < num_vals; j++) + { + uint32_t t = r.bit(); + + uint32_t a = dec.decode_bit(bm); + if (t != a) + { + fmt_printf("error!"); + return false; + } + } + } + } + fmt_printf("Random adaptive bits test OK\n"); + + fmt_printf("random adaptive bit model 0 or 1 run test\n"); + + // adaptive bit model 0 or 1 test + for (uint32_t i = 0; i < N; i++) + { + basist::arith::arith_enc enc; + enc.init(4096); + + { + basisu::rand r; + r.seed(i + 1); + uint32_t num_vals = r.irand(1, 20000); + + basist::arith::arith_bit_model bm; + bm.init(); + + for (uint32_t j = 0; j < num_vals; j++) + enc.encode(i & 1, bm); + + enc.flush(); + } + + { + basisu::rand r; + r.seed(i + 1); + uint32_t num_vals = r.irand(1, 20000); + + basist::arith::arith_dec dec; + dec.init(enc.get_data_buf().get_ptr(), enc.get_data_buf().size()); + + basist::arith::arith_bit_model bm; + bm.init(); + + for (uint32_t j = 0; j < num_vals; j++) + { + uint32_t t = i & 1; + + uint32_t a = dec.decode_bit(bm); + if (t != a) + { + fmt_printf("error!"); + return false; + } + } + } + } + + fmt_printf("Adaptive bit model 0 or 1 run test OK\n"); + + fmt_printf("random adaptive bit model 0 or 1 run 2 test\n"); + + // adaptive bit model 0 or 1 run test + for (uint32_t i = 0; i < N; i++) + { + basist::arith::arith_enc enc; + enc.init(4096); + + { + basisu::rand r; + r.seed(i + 1); + uint32_t num_vals = r.irand(1, 2000); + + basist::arith::arith_bit_model bm; + bm.init(); + + for (uint32_t j = 0; j < num_vals; j++) + { + const uint32_t run_len = r.irand(1, 128); + const uint32_t t = r.bit(); + for (uint32_t k = 0; k < run_len; k++) + enc.encode(t, bm); + } + + if (r.frand(0.0f, 1.0f) < .1f) + { + for (uint32_t q = 0; q < 1000; q++) + enc.encode(0, bm); + } + + enc.flush(); + } + + { + basisu::rand r; + r.seed(i + 1); + uint32_t num_vals = r.irand(1, 2000); + + basist::arith::arith_dec dec; + dec.init(enc.get_data_buf().get_ptr(), enc.get_data_buf().size()); + + basist::arith::arith_bit_model bm; + bm.init(); + + for (uint32_t j = 0; j < num_vals; j++) + { + const uint32_t run_len = r.irand(1, 128); + const uint32_t t = r.bit(); + + for (uint32_t k = 0; k < run_len; k++) + { + uint32_t a = dec.decode_bit(bm); + if (a != t) + { + fmt_printf("adaptive bit model random run test failed!\n"); + return false; + } + } + } + + if (r.frand(0.0f, 1.0f) < .1f) + { + for (uint32_t q = 0; q < 1000; q++) + { + uint32_t d = dec.decode_bit(bm); + if (d != 0) + { + fmt_printf("adaptive bit model random run test failed!\n"); + return false; + } + } + } + } + } + + fmt_printf("Random data model test\n"); + + // random data model test + for (uint32_t i = 0; i < N; i++) + { + basist::arith::arith_enc enc; + enc.init(4096); + + { + basisu::rand r; + r.seed(i + 1); + const uint32_t num_vals = r.irand(1, 60000); + + uint32_t num_syms = r.irand(2, basist::arith::ArithMaxSyms); + + basist::arith::arith_data_model dm; + dm.init(num_syms); + + for (uint32_t j = 0; j < num_vals; j++) + enc.encode(r.irand(0, num_syms - 1), dm); + + enc.flush(); + } + + { + basisu::rand r; + r.seed(i + 1); + uint32_t num_vals = r.irand(1, 60000); + + const uint32_t num_syms = r.irand(2, basist::arith::ArithMaxSyms); + + basist::arith::arith_dec dec; + dec.init(enc.get_data_buf().get_ptr(), enc.get_data_buf().size()); + + basist::arith::arith_data_model dm; + dm.init(num_syms); + + for (uint32_t j = 0; j < num_vals; j++) + { + uint32_t expected = r.irand(0, num_syms - 1); + uint32_t actual = dec.decode_sym(dm); + if (actual != expected) + { + fmt_printf("adaptive data model random test failed!\n"); + return false; + } + } + } + } + + fmt_printf("Adaptive data model random test OK\n"); + + fmt_printf("Overall OK\n"); + return true; + } + static void rasterize_line(image& dst, int xs, int ys, int xe, int ye, int pred, int inc_dec, int e, int e_inc, int e_no_inc, const color_rgba& color) { int start, end, var; @@ -4023,6 +4360,7 @@ namespace basisu } } } + void draw_line(image& dst, int xs, int ys, int xe, int ye, const color_rgba& color) { if (xs > xe) @@ -4084,7 +4422,7 @@ namespace basisu int y = 0; int err = 1 - x; - while (x >= y) + while (x >= y) { dst.set_clipped(cx + x, cy + y, color); dst.set_clipped(cx + y, cy + x, color); @@ -4097,11 +4435,11 @@ namespace basisu ++y; - if (err < 0) + if (err < 0) { err += 2 * y + 1; } - else + else { --x; err += 2 * (y - x) + 1; @@ -4116,4 +4454,270 @@ namespace basisu img(x, y).a = (uint8_t)a; } + // red=3 subsets, blue=2 subsets, green=mode 6, white=mode 7, purple = 2 plane + const color_rgba g_bc7_mode_vis_colors[8] = + { + color_rgba(190, 0, 0, 255), // 0 + color_rgba(0, 0, 255, 255), // 1 + color_rgba(255, 0, 0, 255), // 2 + color_rgba(0, 0, 130, 255), // 3 + color_rgba(255, 0, 255, 255), // 4 + color_rgba(190, 0, 190, 255), // 5 + color_rgba(50, 167, 30, 255), // 6 + color_rgba(255, 255, 255, 255) // 7 + }; + + void create_bc7_debug_images( + uint32_t width, uint32_t height, + const void *pBlocks, + const char *pFilename_prefix) + { + assert(width && height && pBlocks ); + + const uint32_t num_bc7_blocks_x = (width + 3) >> 2; + const uint32_t num_bc7_blocks_y = (height + 3) >> 2; + const uint32_t total_bc7_blocks = num_bc7_blocks_x * num_bc7_blocks_y; + + image bc7_mode_vis(width, height); + + uint32_t bc7_mode_hist[9] = {}; + + uint32_t mode4_index_hist[2] = {}; + uint32_t mode4_rot_hist[4] = {}; + uint32_t mode5_rot_hist[4] = {}; + + uint32_t num_2subsets = 0, num_3subsets = 0, num_dp = 0; + + uint32_t total_solid_bc7_blocks = 0; + uint32_t num_unpack_failures = 0; + + for (uint32_t by = 0; by < num_bc7_blocks_y; by++) + { + const uint32_t base_y = by * 4; + + for (uint32_t bx = 0; bx < num_bc7_blocks_x; bx++) + { + const uint32_t base_x = bx * 4; + + const basist::bc7_block& blk = ((const basist::bc7_block *)pBlocks)[bx + by * num_bc7_blocks_x]; + + color_rgba unpacked_pixels[16]; + bool status = basist::bc7u::unpack_bc7(&blk, (basist::color_rgba*)unpacked_pixels); + if (!status) + num_unpack_failures++; + + int mode_index = basist::bc7u::determine_bc7_mode(&blk); + + bool is_solid = false; + + // assumes our transcoder's analytical BC7 encoder wrote the solid block + if (mode_index == 5) + { + const uint8_t* pBlock_bytes = (const uint8_t *)&blk; + + if (pBlock_bytes[0] == 0b00100000) + { + static const uint8_t s_tail_bytes[8] = { 0xac, 0xaa, 0xaa, 0xaa, 0, 0, 0, 0 }; + if ((pBlock_bytes[8] & ~3) == (s_tail_bytes[0] & ~3)) + { + if (memcmp(pBlock_bytes + 9, s_tail_bytes + 1, 7) == 0) + { + is_solid = true; + } + } + } + } + + total_solid_bc7_blocks += is_solid; + + if ((mode_index == 0) || (mode_index == 2)) + num_3subsets++; + else if ((mode_index == 1) || (mode_index == 3)) + num_2subsets++; + + bc7_mode_hist[mode_index + 1]++; + + if (mode_index == 4) + { + num_dp++; + mode4_index_hist[range_check(basist::bc7u::determine_bc7_mode_4_index_mode(&blk), 0, 1)]++; + mode4_rot_hist[range_check(basist::bc7u::determine_bc7_mode_4_or_5_rotation(&blk), 0, 3)]++; + } + else if (mode_index == 5) + { + num_dp++; + mode5_rot_hist[range_check(basist::bc7u::determine_bc7_mode_4_or_5_rotation(&blk), 0, 3)]++; + } + + color_rgba c((mode_index < 0) ? g_black_color : g_bc7_mode_vis_colors[mode_index]); + + if (is_solid) + c.set(64, 0, 64, 255); + + bc7_mode_vis.fill_box(base_x, base_y, 4, 4, c); + + } // bx + + } // by + + fmt_debug_printf("--------- BC7 statistics:\n"); + fmt_debug_printf("\nTotal BC7 unpack failures: {}\n", num_unpack_failures); + fmt_debug_printf("Total solid blocks: {} {3.2}%\n", total_solid_bc7_blocks, (float)total_solid_bc7_blocks * (float)100.0f / (float)total_bc7_blocks); + + fmt_debug_printf("\nTotal 2-subsets: {} {3.2}%\n", num_2subsets, (float)num_2subsets * 100.0f / (float)total_bc7_blocks); + fmt_debug_printf("Total 3-subsets: {} {3.2}%\n", num_3subsets, (float)num_3subsets * 100.0f / (float)total_bc7_blocks); + fmt_debug_printf("Total Dual Plane: {} {3.2}%\n", num_dp, (float)num_dp * 100.0f / (float)total_bc7_blocks); + + fmt_debug_printf("\nBC7 mode histogram:\n"); + for (int i = -1; i <= 7; i++) + { + fmt_debug_printf(" {}: {} {3.3}%\n", i, bc7_mode_hist[1 + i], (float)bc7_mode_hist[1 + i] * 100.0f / (float)total_bc7_blocks); + } + + fmt_debug_printf("\nMode 4 index bit histogram: {} {3.2}%, {} {3.2}%\n", + mode4_index_hist[0], (float)mode4_index_hist[0] * 100.0f / (float)total_bc7_blocks, + mode4_index_hist[1], (float)mode4_index_hist[1] * 100.0f / (float)total_bc7_blocks); + + fmt_debug_printf("\nMode 4 rotation histogram:\n"); + for (uint32_t i = 0; i < 4; i++) + { + fmt_debug_printf(" {}: {} {3.2}%\n", i, mode4_rot_hist[i], (float)mode4_rot_hist[i] * 100.0f / (float)total_bc7_blocks); + } + + fmt_debug_printf("\nMode 5 rotation histogram:\n"); + for (uint32_t i = 0; i < 4; i++) + { + fmt_debug_printf(" {}: {} {3.2}%\n", i, mode5_rot_hist[i], (float)mode5_rot_hist[i] * 100.0f / (float)total_bc7_blocks); + } + + if (pFilename_prefix) + { + std::string mode_vis_filename(std::string(pFilename_prefix) + "bc7_mode_vis.png"); + save_png(mode_vis_filename, bc7_mode_vis); + + fmt_debug_printf("Wrote BC7 mode visualization to PNG file {}\n", mode_vis_filename); + } + + fmt_debug_printf("--------- End BC7 statistics\n"); + fmt_debug_printf("\n"); + } + + static inline float edge(const vec2F& a, const vec2F& b, const vec2F& pos) + { + return (pos[0] - a[0]) * (b[1] - a[1]) - (pos[1] - a[1]) * (b[0] - a[0]); + } + + void draw_tri2(image& dst, const image* pTex, const tri2& tri, bool alpha_blend) + { + assert(dst.get_total_pixels()); + + float area = edge(tri.p0, tri.p1, tri.p2); + if (std::fabs(area) < 1e-6f) + return; + + const float oo_area = 1.0f / area; + + int minx = (int)std::floor(basisu::minimum(tri.p0[0], tri.p1[0], tri.p2[0] )); + int miny = (int)std::floor(basisu::minimum(tri.p0[1], tri.p1[1], tri.p2[1] )); + + int maxx = (int)std::ceil(basisu::maximum(tri.p0[0], tri.p1[0], tri.p2[0])); + int maxy = (int)std::ceil(basisu::maximum(tri.p0[1], tri.p1[1], tri.p2[1])); + + auto clamp8 = [&](float fv) { int v = (int)(fv + .5f); if (v < 0) v = 0; else if (v > 255) v = 255; return (uint8_t)v; }; + + if ((maxx < 0) || (maxy < 0)) + return; + if ((minx >= (int)dst.get_width()) || (miny >= (int)dst.get_height())) + return; + + if (minx < 0) + minx = 0; + if (maxx >= (int)dst.get_width()) + maxx = dst.get_width() - 1; + if (miny < 0) + miny = 0; + if (maxy >= (int)dst.get_height()) + maxy = dst.get_height() - 1; + + vec4F tex(1.0f); + + for (int y = miny; y <= maxy; ++y) + { + assert((y >= 0) && (y < (int)dst.get_height())); + + for (int x = minx; x <= maxx; ++x) + { + assert((x >= 0) && (x < (int)dst.get_width())); + + vec2F p{ (float)x + 0.5f, (float)y + 0.5f }; + + float w0 = edge(tri.p1, tri.p2, p) * oo_area; + float w1 = edge(tri.p2, tri.p0, p) * oo_area; + float w2 = edge(tri.p0, tri.p1, p) * oo_area; + + if ((w0 < 0) || (w1 < 0) || (w2 < 0)) + continue; + + float u = tri.t0[0] * w0 + tri.t1[0] * w1 + tri.t2[0] * w2; + float v = tri.t0[1] * w0 + tri.t1[1] * w1 + tri.t2[1] * w2; + + if (pTex) + tex = pTex->get_filtered_vec4F(u * float(pTex->get_width()), v * float(pTex->get_height())) * (1.0f / 255.0f); + + float r = (float)tri.c0.r * w0 + (float)tri.c1.r * w1 + (float)tri.c2.r * w2; + float g = (float)tri.c0.g * w0 + (float)tri.c1.g * w1 + (float)tri.c2.g * w2; + float b = (float)tri.c0.b * w0 + (float)tri.c1.b * w1 + (float)tri.c2.b * w2; + float a = (float)tri.c0.a * w0 + (float)tri.c1.a * w1 + (float)tri.c2.a * w2; + + r *= tex[0]; + g *= tex[1]; + b *= tex[2]; + a *= tex[3]; + + if (alpha_blend) + { + color_rgba dst_color(dst(x, y)); + + const float fa = (float)a * (1.0f / 255.0f); + + r = lerp((float)dst_color[0], r, fa); + g = lerp((float)dst_color[1], g, fa); + b = lerp((float)dst_color[2], b, fa); + a = lerp((float)dst_color[3], a, fa); + + dst(x, y) = color_rgba(clamp8(r), clamp8(g), clamp8(b), clamp8(a)); + } + else + { + dst(x, y) = color_rgba(clamp8(r), clamp8(g), clamp8(b), clamp8(a)); + } + + } // x + } // y + } + + // macro sent by CMakeLists.txt file when (TARGET_WASM AND WASM_THREADING) +#if BASISU_WASI_THREADS + // Default to 8 - seems reasonable. + static int g_num_wasi_threads = 8; +#else + static int g_num_wasi_threads = 0; +#endif + + void set_num_wasi_threads(uint32_t num_threads) + { + g_num_wasi_threads = num_threads; + } + + int get_num_hardware_threads() + { +#ifdef __wasi__ + int num_threads = g_num_wasi_threads; +#else + int num_threads = std::thread::hardware_concurrency(); +#endif + + return num_threads; + } + } // namespace basisu diff --git a/external/basis_universal/encoder/basisu_enc.h b/external/basis_universal/encoder/basisu_enc.h index 0c644e8a28..d0b77415e0 100644 --- a/external/basis_universal/encoder/basisu_enc.h +++ b/external/basis_universal/encoder/basisu_enc.h @@ -1,5 +1,5 @@ // basisu_enc.h -// Copyright (C) 2019-2024 Binomial LLC. All Rights Reserved. +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -40,6 +40,7 @@ namespace basisu { extern uint8_t g_hamming_dist[256]; extern const uint8_t g_debug_font8x8_basic[127 - 32 + 1][8]; + extern float g_srgb_to_linear_table[256]; // sRGB EOTF->linear light [0,1], 1=~100 nits // true if basisu_encoder_init() has been called and returned. extern bool g_library_initialized; @@ -61,7 +62,7 @@ namespace basisu void error_vprintf(const char* pFmt, va_list args); void error_printf(const char *pFmt, ...); - + template inline void fmt_error_printf(const char* pFmt, Args&&... args) { @@ -72,7 +73,7 @@ namespace basisu } void platform_sleep(uint32_t ms); - + // Helpers inline uint8_t clamp255(int32_t i) @@ -92,21 +93,27 @@ namespace basisu return val << shift; } - inline int32_t clampi(int32_t value, int32_t low, int32_t high) - { - if (value < low) - value = low; - else if (value > high) - value = high; - return value; + inline int32_t clampi(int32_t value, int32_t low, int32_t high) + { + if (value < low) + value = low; + else if (value > high) + value = high; + return value; } inline uint8_t mul_8(uint32_t v, uint32_t a) { - v = v * a + 128; + v = v * a + 128; return (uint8_t)((v + (v >> 8)) >> 8); } + inline int fast_roundf_pos_int(float x) + { + assert(x >= 0.0f); + return (int)(x + 0.5f); + } + inline int fast_roundf_int(float x) { return (x >= 0.0f) ? (int)(x + 0.5f) : (int)(x - 0.5f); @@ -163,7 +170,7 @@ namespace basisu return bits; } - + // Open interval inline int bounds_check(int v, int l, int h) { (void)v; (void)l; (void)h; assert(v >= l && v < h); return v; } inline uint32_t bounds_check(uint32_t v, uint32_t l, uint32_t h) { (void)v; (void)l; (void)h; assert(v >= l && v < h); return v; } @@ -211,10 +218,10 @@ namespace basisu return -1; return (int)res; } - + // Hashing - - inline uint32_t bitmix32c(uint32_t v) + + inline uint32_t bitmix32c(uint32_t v) { v = (v + 0x7ed55d16) + (v << 12); v = (v ^ 0xc761c23c) ^ (v >> 19); @@ -225,7 +232,7 @@ namespace basisu return v; } - inline uint32_t bitmix32(uint32_t v) + inline uint32_t bitmix32(uint32_t v) { v -= (v << 6); v ^= (v >> 17); @@ -246,29 +253,7 @@ namespace basisu seed = seed ^ (seed >> 15); return seed; } - - uint32_t hash_hsieh(const uint8_t* pBuf, size_t len); - - template - struct bit_hasher - { - inline std::size_t operator()(const Key& k) const - { - return hash_hsieh(reinterpret_cast(&k), sizeof(k)); - } - }; - - struct string_hasher - { - inline std::size_t operator()(const std::string& k) const - { - size_t l = k.size(); - if (!l) - return 0; - return hash_hsieh(reinterpret_cast(k.c_str()), l); - } - }; - + class running_stat { public: @@ -351,7 +336,7 @@ namespace basisu }; // Linear algebra - + template class vec { @@ -456,7 +441,7 @@ namespace basisu inline const T *get_ptr() const { return reinterpret_cast(&m_v[0]); } inline T *get_ptr() { return reinterpret_cast(&m_v[0]); } - + inline vec operator- () const { vec res; for (uint32_t i = 0; i < N; i++) res.m_v[i] = -m_v[i]; return res; } inline vec operator+ () const { return *this; } inline vec &operator+= (const vec &other) { for (uint32_t i = 0; i < N; i++) m_v[i] += other.m_v[i]; return *this; } @@ -465,17 +450,19 @@ namespace basisu inline vec &operator*=(const vec &other) { for (uint32_t i = 0; i < N; i++) m_v[i] *= other.m_v[i]; return *this; } inline vec &operator/= (T s) { for (uint32_t i = 0; i < N; i++) m_v[i] /= s; return *this; } inline vec &operator*= (T s) { for (uint32_t i = 0; i < N; i++) m_v[i] *= s; return *this; } - + friend inline vec operator+(const vec &lhs, const vec &rhs) { vec res; for (uint32_t i = 0; i < N; i++) res.m_v[i] = lhs.m_v[i] + rhs.m_v[i]; return res; } friend inline vec operator-(const vec &lhs, const vec &rhs) { vec res; for (uint32_t i = 0; i < N; i++) res.m_v[i] = lhs.m_v[i] - rhs.m_v[i]; return res; } friend inline vec operator*(const vec &lhs, T val) { vec res; for (uint32_t i = 0; i < N; i++) res.m_v[i] = lhs.m_v[i] * val; return res; } friend inline vec operator*(T val, const vec &rhs) { vec res; for (uint32_t i = 0; i < N; i++) res.m_v[i] = val * rhs.m_v[i]; return res; } friend inline vec operator/(const vec &lhs, T val) { vec res; for (uint32_t i = 0; i < N; i++) res.m_v[i] = lhs.m_v[i] / val; return res; } friend inline vec operator/(const vec &lhs, const vec &rhs) { vec res; for (uint32_t i = 0; i < N; i++) res.m_v[i] = lhs.m_v[i] / rhs.m_v[i]; return res; } - + static inline T dot_product(const vec &lhs, const vec &rhs) { T res = lhs.m_v[0] * rhs.m_v[0]; for (uint32_t i = 1; i < N; i++) res += lhs.m_v[i] * rhs.m_v[i]; return res; } + static inline T dot_product3(const vec& lhs, const vec& rhs) { T res = lhs.m_v[0] * rhs.m_v[0]; for (uint32_t i = 1; i < minimum(3u, N); i++) res += lhs.m_v[i] * rhs.m_v[i]; return res; } inline T dot(const vec &rhs) const { return dot_product(*this, rhs); } + inline T dot3(const vec& rhs) const { return dot_product3(*this, rhs); } inline T norm() const { return dot_product(*this, *this); } inline T length() const { return sqrt(norm()); } @@ -546,7 +533,7 @@ namespace basisu template struct bitwise_copyable< vec > { enum { cFlag = true }; }; template struct bitwise_movable< vec > { enum { cFlag = true }; }; - + template class matrix { @@ -810,7 +797,7 @@ namespace basisu } #undef BASISU_GET_KEY - + // Very simple job pool with no dependencies. class job_pool { @@ -820,24 +807,24 @@ namespace basisu // num_threads is the TOTAL number of job pool threads, including the calling thread! So 2=1 new thread, 3=2 new threads, etc. job_pool(uint32_t num_threads); ~job_pool(); - + void add_job(const std::function& job); void add_job(std::function&& job); void wait_for_all(); size_t get_total_threads() const { return 1 + m_threads.size(); } - + private: std::vector m_threads; std::vector > m_queue; - + std::mutex m_mutex; std::condition_variable m_has_work; std::condition_variable m_no_more_jobs; - + uint32_t m_num_active_jobs; - + std::atomic m_kill_flag; std::atomic m_num_active_workers; @@ -882,7 +869,7 @@ namespace basisu return *this; } }; - + class color_rgba { public: @@ -1006,7 +993,7 @@ namespace basisu inline const uint8_t &operator[] (uint32_t index) const { assert(index < 4); return m_comps[index]; } inline uint8_t &operator[] (uint32_t index) { assert(index < 4); return m_comps[index]; } - + inline void clear() { m_comps[0] = 0; @@ -1042,7 +1029,7 @@ namespace basisu } inline int get_601_luma() const { return (19595U * m_comps[0] + 38470U * m_comps[1] + 7471U * m_comps[2] + 32768U) >> 16U; } - inline int get_709_luma() const { return (13938U * m_comps[0] + 46869U * m_comps[1] + 4729U * m_comps[2] + 32768U) >> 16U; } + inline int get_709_luma() const { return (13938U * m_comps[0] + 46869U * m_comps[1] + 4729U * m_comps[2] + 32768U) >> 16U; } inline int get_luma(bool luma_601) const { return luma_601 ? get_601_luma() : get_709_luma(); } inline uint32_t get_bgra_uint32() const { return b | (g << 8) | (r << 16) | (a << 24); } @@ -1114,7 +1101,7 @@ namespace basisu return color_distance(e1, e2, alpha); } } - + inline uint32_t color_distance(bool perceptual, const color_rgba &e1, const color_rgba &e2, bool alpha) { if (perceptual) @@ -1127,7 +1114,7 @@ namespace basisu int delta_l = dr * 14 + dg * 45 + db * 5; int delta_cr = dr * 64 - delta_l; int delta_cb = db * 64 - delta_l; - + // not >> 6, so the output is scaled by 7 bits, not 6 (to match the original function which scaled by 7, but had rare overflow issues) uint32_t id = ((uint32_t)(delta_l * delta_l) >> 5U) + ((((uint32_t)(delta_cr * delta_cr) >> 5U) * 26U) >> 7U) + @@ -1151,7 +1138,7 @@ namespace basisu if (alpha) { int da = (e1.a - e2.a) << 7; - + // This shouldn't overflow if da is 255 or -255: 29.99 bits after squaring. uint32_t ea = ((uint32_t)(da * da) >> 7U); id += ea; @@ -1160,7 +1147,7 @@ namespace basisu // Make sure it can't overflow assert((((int64_t)da * (int64_t)da) >> 7) == ea); #endif - + } return id; @@ -1222,7 +1209,7 @@ namespace basisu return true; } - + inline std::string string_tolower(const std::string& s) { std::string result(s); @@ -1267,7 +1254,7 @@ namespace basisu char fname_buf[_MAX_FNAME] = { 0 }; char ext_buf[_MAX_EXT] = { 0 }; - errno_t error = _splitpath_s(p, + errno_t error = _splitpath_s(p, pDrive ? drive_buf : NULL, pDrive ? _MAX_DRIVE : 0, pDir ? dir_buf : NULL, pDir ? _MAX_DIR : 0, pFilename ? fname_buf : NULL, pFilename ? _MAX_FNAME : 0, @@ -1299,7 +1286,7 @@ namespace basisu if ((pDir->size()) && (pDir->back() != '/')) *pDir += "/"; } - + if (pFilename) { *pFilename = pBaseName; @@ -1326,7 +1313,7 @@ namespace basisu return (c == '/'); #endif } - + inline bool is_drive_separator(char c) { #ifdef _WIN32 @@ -1354,7 +1341,7 @@ namespace basisu string_combine_path(dst, p, q); string_combine_path(dst, dst.c_str(), r); } - + inline void string_combine_path_and_extension(std::string &dst, const char *p, const char *q, const char *r, const char *pExt) { string_combine_path(dst, p, q, r); @@ -1556,7 +1543,7 @@ namespace basisu codebook.resize(0); codebook.reserve(max_clusters); - + uint32_t node_index = 0; while (true) @@ -1567,7 +1554,7 @@ namespace basisu { codebook.resize(codebook.size() + 1); codebook.back() = cur.m_training_vecs; - + if (node_stack.empty()) break; @@ -1575,7 +1562,7 @@ namespace basisu node_stack.pop_back(); continue; } - + node_stack.push_back(cur.m_right_index); node_index = cur.m_left_index; } @@ -1616,7 +1603,7 @@ namespace basisu assert(node.is_leaf()); var_heap.delete_top(); - + if (node.m_training_vecs.size() > 1) { if (split_node(node_index, var_heap, l_children, r_children)) @@ -1705,7 +1692,7 @@ namespace basisu m_nodes[node_index].m_left_index = l_child_index; m_nodes[node_index].m_right_index = r_child_index; - + m_nodes[node_index].m_codebook_index = m_next_codebook_index; m_next_codebook_index++; @@ -1719,7 +1706,7 @@ namespace basisu if ((l_child.m_var <= 0.0f) && (l_child.m_training_vecs.size() > 1)) { TrainingVectorType v(m_training_vecs[l_child.m_training_vecs[0]].first); - + for (uint32_t i = 1; i < l_child.m_training_vecs.size(); i++) { if (!(v == m_training_vecs[l_child.m_training_vecs[i]].first)) @@ -1746,10 +1733,10 @@ namespace basisu if ((l_child.m_var > 0.0f) && (l_child.m_training_vecs.size() > 1)) var_heap.add_heap(l_child_index, l_child.m_var); - + if ((r_child.m_var > 0.0f) && (r_child.m_training_vecs.size() > 1)) var_heap.add_heap(r_child_index, r_child.m_var); - + return true; } @@ -1845,7 +1832,7 @@ namespace basisu for (uint32_t i = 0; i < node.m_training_vecs.size(); i++) { const TrainingVectorType& v = m_training_vecs[node.m_training_vecs[i]].first; - + l = TrainingVectorType::component_min(l, v); h = TrainingVectorType::component_max(h, v); } @@ -1926,8 +1913,8 @@ namespace basisu const uint32_t cMaxIters = 6; for (uint32_t iter = 0; iter < cMaxIters; iter++) { - l_children.resize(0); - r_children.resize(0); + l_children.resize(0); + r_children.resize(0); TrainingVectorType new_l_child(cZero), new_r_child(cZero); @@ -1975,11 +1962,12 @@ namespace basisu r_weight = 0; TrainingVectorType firstVec; + firstVec.clear(); for (uint32_t i = 0; i < node.m_training_vecs.size(); i++) { const TrainingVectorType& v = m_training_vecs[node.m_training_vecs[i]].first; const uint64_t weight = m_training_vecs[node.m_training_vecs[i]].second; - + if ((!i) || (v == firstVec)) { firstVec = v; @@ -2081,7 +2069,7 @@ namespace basisu } Quantizer quantizers[cMaxThreads]; - + bool success_flags[cMaxThreads]; clear_obj(success_flags); @@ -2180,12 +2168,12 @@ namespace basisu // rg 6/24/2025 - Cross platform determinism #if 0 - typedef std::unordered_map < typename Quantizer::training_vec_type, weighted_block_group, + typedef std::unordered_map < typename Quantizer::training_vec_type, weighted_block_group, training_vec_bit_hasher> group_hash; #else typedef std::map< typename Quantizer::training_vec_type, weighted_block_group > group_hash; #endif - + //interval_timer tm; //tm.start(); @@ -2197,7 +2185,7 @@ namespace basisu #endif weighted_block_group g; - + if (even_odd_input_pairs_equal) { g.m_indices.resize(2); @@ -2282,7 +2270,7 @@ namespace basisu typename group_hash::const_iterator group_iter = unique_vec_iters[group_index]; const uint_vec& training_vec_indices = group_iter->second.m_indices; - + append_vector(codebook.back(), training_vec_indices); } } @@ -2359,7 +2347,7 @@ namespace basisu const double inv_total = 1.0f / total; const double neg_inv_log2 = -1.0f / log(2.0f); - + double e = 0.0f; for (uint32_t i = 0; i < m_hist.size(); i++) if (m_hist[i]) @@ -2368,7 +2356,7 @@ namespace basisu return e; } }; - + struct sym_freq { uint32_t m_key; @@ -2378,7 +2366,7 @@ namespace basisu sym_freq *canonical_huffman_radix_sort_syms(uint32_t num_syms, sym_freq *pSyms0, sym_freq *pSyms1); void canonical_huffman_calculate_minimum_redundancy(sym_freq *A, int num_syms); void canonical_huffman_enforce_max_code_size(int *pNum_codes, int code_list_len, int max_code_size); - + class huffman_encoding_table { public: @@ -2399,7 +2387,7 @@ namespace basisu bool init(uint32_t num_syms, const uint16_t *pFreq, uint32_t max_code_size); bool init(uint32_t num_syms, const uint32_t *pSym_freq, uint32_t max_code_size); - + inline const uint16_vec &get_codes() const { return m_codes; } inline const uint8_vec &get_code_sizes() const { return m_code_sizes; } @@ -2430,7 +2418,7 @@ namespace basisu m_bytes(other.m_bytes), m_bit_buffer(other.m_bit_buffer), m_bit_buffer_size(other.m_bit_buffer_size), - m_total_bits(other.m_total_bits) + m_total_bits(other.m_total_bits) { } @@ -2512,7 +2500,7 @@ namespace basisu m_bit_buffer = 0; m_bit_buffer_size = 0; - + return 8; } @@ -2561,7 +2549,7 @@ namespace basisu if (v < u) return put_bits(v, k); - + uint32_t x = v + u; assert((x >> 1) >= u); @@ -2573,20 +2561,20 @@ namespace basisu inline uint32_t put_rice(uint32_t v, uint32_t m) { assert(m); - + const uint64_t start_bits = m_total_bits; uint32_t q = v >> m, r = v & ((1 << m) - 1); // rice coding sanity check assert(q <= 64); - + for (; q > 16; q -= 16) put_bits(0xFFFF, 16); put_bits((1 << q) - 1, q); put_bits(r << 1, m + 1); - + return (uint32_t)(m_total_bits - start_bits); } @@ -2596,13 +2584,13 @@ namespace basisu const uint32_t chunk_size = 1 << chunk_bits; const uint32_t chunk_mask = chunk_size - 1; - + uint32_t total_bits = 0; for ( ; ; ) { uint32_t next_v = v >> chunk_bits; - + total_bits += put_bits((v & chunk_mask) | (next_v ? chunk_size : 0), chunk_bits + 1); if (!next_v) break; @@ -2619,11 +2607,11 @@ namespace basisu { for (uint32_t i = 0; i < other.m_bytes.size(); i++) put_bits(other.m_bytes[i], 8); - + if (other.m_bit_buffer_size) put_bits(other.m_bit_buffer, other.m_bit_buffer_size); } - + private: uint8_vec m_bytes; uint32_t m_bit_buffer, m_bit_buffer_size; @@ -2653,7 +2641,7 @@ namespace basisu inline void init(uint32_t bits_per_sym, uint32_t total_syms_per_group) { assert((bits_per_sym * total_syms_per_group) <= 16 && total_syms_per_group >= 1 && bits_per_sym >= 1); - + m_bits_per_sym = bits_per_sym; m_total_syms_per_group = total_syms_per_group; m_cur_sym_bits = 0; @@ -2707,7 +2695,7 @@ namespace basisu return true; } - + inline uint32_t emit_next_sym(bitwise_coder &c) { uint32_t bits = 0; @@ -2737,7 +2725,7 @@ namespace basisu bool huffman_test(int rand_seed); // VQ index reordering - + class palette_index_reorderer { public: @@ -2758,7 +2746,7 @@ namespace basisu typedef float(*pEntry_dist_func)(uint32_t i, uint32_t j, void *pCtx); void init(uint32_t num_indices, const uint32_t *pIndices, uint32_t num_syms, pEntry_dist_func pDist_func, void *pCtx, float dist_func_weight); - + // Table remaps old to new symbol indices inline const uint_vec &get_remap_table() const { return m_remap_table; } @@ -2779,12 +2767,12 @@ namespace basisu class image { public: - image() : + image() : m_width(0), m_height(0), m_pitch(0) { } - image(uint32_t w, uint32_t h, uint32_t p = UINT32_MAX) : + image(uint32_t w, uint32_t h, uint32_t p = UINT32_MAX) : m_width(0), m_height(0), m_pitch(0) { resize(w, h, p); @@ -2850,7 +2838,7 @@ namespace basisu image &clear() { - m_width = 0; + m_width = 0; m_height = 0; m_pitch = 0; clear_vector(m_pixels); @@ -2878,7 +2866,7 @@ namespace basisu void init(const uint8_t *pImage, uint32_t width, uint32_t height, uint32_t comps) { assert(comps >= 1 && comps <= 4); - + resize(width, height); for (uint32_t y = 0; y < height; y++) @@ -2970,7 +2958,7 @@ namespace basisu p = w; clear(); - + if ((!p) || (!w) || (!h)) return *this; @@ -3049,8 +3037,8 @@ namespace basisu y = wrap_v ? posmod(y, m_height) : clamp(y, 0, m_height - 1); return m_pixels[x + y * m_pitch]; } - - inline image &set_clipped(int x, int y, const color_rgba &c) + + inline image &set_clipped(int x, int y, const color_rgba &c) { if ((static_cast(x) < m_width) && (static_cast(y) < m_height)) (*this)(x, y) = c; @@ -3214,7 +3202,7 @@ namespace basisu } void debug_text(uint32_t x_ofs, uint32_t y_ofs, uint32_t x_scale, uint32_t y_scale, const color_rgba &fg, const color_rgba *pBG, bool alpha_only, const char* p, ...); - + // bilinear filtering vec4F get_filtered_vec4F(float x, float y) const { @@ -3257,7 +3245,7 @@ namespace basisu return result; } - + private: uint32_t m_width, m_height, m_pitch; // all in pixels color_rgba_vec m_pixels; @@ -3269,7 +3257,7 @@ namespace basisu inline bool is_solid_block(uint32_t n, const color_rgba* pPixels) { assert(n); - + if (n <= 1) return true; @@ -3300,12 +3288,12 @@ namespace basisu class imagef { public: - imagef() : + imagef() : m_width(0), m_height(0), m_pitch(0) { } - imagef(uint32_t w, uint32_t h, uint32_t p = UINT32_MAX) : + imagef(uint32_t w, uint32_t h, uint32_t p = UINT32_MAX) : m_width(0), m_height(0), m_pitch(0) { resize(w, h, p); @@ -3365,7 +3353,7 @@ namespace basisu imagef &clear() { - m_width = 0; + m_width = 0; m_height = 0; m_pitch = 0; clear_vector(m_pixels); @@ -3421,7 +3409,7 @@ namespace basisu set_clipped(x + ix, y + iy, c); return *this; } - + imagef &crop(uint32_t w, uint32_t h, uint32_t p = UINT32_MAX, const vec4F &background = vec4F(0,0,0,1)) { if (p == UINT32_MAX) @@ -3440,7 +3428,7 @@ namespace basisu cur_state.swap(m_pixels); m_pixels.resize(p * h); - + for (uint32_t y = 0; y < h; y++) { for (uint32_t x = 0; x < w; x++) @@ -3503,8 +3491,8 @@ namespace basisu y = wrap_v ? posmod(y, m_height) : clamp(y, 0, m_height - 1); return m_pixels[x + y * m_pitch]; } - - inline imagef &set_clipped(int x, int y, const vec4F &c) + + inline imagef &set_clipped(int x, int y, const vec4F &c) { if ((static_cast(x) < m_width) && (static_cast(y) < m_height)) (*this)(x, y) = c; @@ -3589,7 +3577,7 @@ namespace basisu { float &p = c[s]; union { float f; uint32_t u; } x; x.f = p; - + if ((std::isnan(p)) || (std::isinf(p)) || (x.u == 0x80000000)) { if (std::isnan(p)) @@ -3634,14 +3622,14 @@ namespace basisu fprintf(stderr, "One or more input pixels was negative -- setting these pixel components to 0 because ASTC HDR doesn't support signed values.\n"); neg_msg = true; } - + status = false; } if (p > highest_mag) { p = highest_mag; - + if (!clamp_msg) { fprintf(stderr, "One or more input pixels had to be clamped to %f.\n", highest_mag); @@ -3705,7 +3693,7 @@ namespace basisu return result; } - + private: uint32_t m_width, m_height, m_pitch; // all in pixels vec4F_vec m_pixels; @@ -3767,15 +3755,17 @@ namespace basisu }; extern fast_linear_to_srgb g_fast_linear_to_srgb; - + // Image metrics - + class image_metrics { public: // TODO: Add ssim + uint32_t m_width, m_height; double m_max, m_mean, m_mean_squared, m_rms, m_psnr, m_ssim; bool m_has_neg, m_hf_mag_overflow, m_any_abnormal; + uint64_t m_sum_a, m_sum_b; image_metrics() { @@ -3784,6 +3774,8 @@ namespace basisu void clear() { + m_width = 0; + m_height = 0; m_max = 0; m_mean = 0; m_mean_squared = 0; @@ -3793,10 +3785,23 @@ namespace basisu m_has_neg = false; m_hf_mag_overflow = false; m_any_abnormal = false; + m_sum_a = 0; + m_sum_b = 0; + } + + void print(const char *pPrefix = nullptr) + { + //fmt_printf("{}Max: {3.3} Mean: {3.3} RMS: {3.3} PSNR: {2.3} dB, Sums: {} {}, Dim: {}x{}\n", pPrefix ? pPrefix : "", m_max, m_mean, m_rms, m_psnr, m_sum_a, m_sum_b, m_width, m_height); + fmt_printf("{}Max: {3.3} Mean: {3.3} RMS: {3.3} PSNR: {2.3} dB\n", pPrefix ? pPrefix : "", m_max, m_mean, m_rms, m_psnr); } - void print(const char *pPrefix = nullptr) { printf("%sMax: %3.3f Mean: %3.3f RMS: %3.3f PSNR: %2.3f dB\n", pPrefix ? pPrefix : "", m_max, m_mean, m_rms, m_psnr); } - void print_hp(const char* pPrefix = nullptr) { printf("%sMax: %3.6f Mean: %3.6f RMS: %3.6f PSNR: %2.6f dB, Any Neg: %u, Half float overflow: %u, Any NaN/Inf: %u\n", pPrefix ? pPrefix : "", m_max, m_mean, m_rms, m_psnr, m_has_neg, m_hf_mag_overflow, m_any_abnormal); } + void print_hp(const char* pPrefix = nullptr) + { + //fmt_printf("{}Max: {3.6} Mean: {3.6} RMS: {3.6} PSNR: {2.6} dB, Any Neg: {}, Half float overflow: {}, Any NaN/Inf: {}, Sums: {} {}, Dim: {}x{}\n", + // pPrefix ? pPrefix : "", m_max, m_mean, m_rms, m_psnr, m_has_neg, m_hf_mag_overflow, m_any_abnormal, m_sum_a, m_sum_b, m_width, m_height); + fmt_printf("{}Max: {3.6} Mean: {3.6} RMS: {3.6} PSNR: {2.6} dB, Any Neg: {}, Half float overflow: {}, Any NaN/Inf: {}\n", + pPrefix ? pPrefix : "", m_max, m_mean, m_rms, m_psnr, m_has_neg, m_hf_mag_overflow, m_any_abnormal); + } void calc(const imagef& a, const imagef& b, uint32_t first_chan = 0, uint32_t total_chans = 0, bool avg_comp_error = true, bool log = false); void calc_half(const imagef& a, const imagef& b, uint32_t first_chan, uint32_t total_chans, bool avg_comp_error); @@ -3820,7 +3825,7 @@ namespace basisu bool load_jpg(const char *pFilename, image& img); bool load_jpg(const uint8_t* pBuf, size_t buf_size, image& img); inline bool load_jpg(const std::string &filename, image &img) { return load_jpg(filename.c_str(), img); } - + // Currently loads .PNG, .TGA, or .JPG bool load_image(const char* pFilename, image& img); inline bool load_image(const std::string &filename, image &img) { return load_image(filename.c_str(), img); } @@ -3829,9 +3834,9 @@ namespace basisu // Supports .HDR and most (but not all) .EXR's (see TinyEXR). bool load_image_hdr(const char* pFilename, imagef& img, bool ldr_srgb_to_linear = true, float linear_nit_multiplier = 1.0f, float ldr_black_bias = 0.0f); - + inline bool load_image_hdr(const std::string& filename, imagef& img, bool ldr_srgb_to_linear = true, float linear_nit_multiplier = 1.0f, float ldr_black_bias = 0.0f) - { + { return load_image_hdr(filename.c_str(), img, ldr_srgb_to_linear, linear_nit_multiplier, ldr_black_bias); } @@ -3849,7 +3854,7 @@ namespace basisu uint8_t *read_tga(const uint8_t *pBuf, uint32_t buf_size, int &width, int &height, int &n_chans); uint8_t *read_tga(const char *pFilename, int &width, int &height, int &n_chans); - + struct rgbe_header_info { std::string m_program; @@ -3861,13 +3866,13 @@ namespace basisu double m_exposure; // watts/steradian/m^2. bool m_has_exposure; - void clear() - { - m_program.clear(); - m_gamma = 1.0f; - m_has_gamma = false; - m_exposure = 1.0f; - m_has_exposure = false; + void clear() + { + m_program.clear(); + m_gamma = 1.0f; + m_has_gamma = false; + m_exposure = 1.0f; + m_has_exposure = false; } }; @@ -3879,7 +3884,7 @@ namespace basisu bool read_exr(const char* pFilename, imagef& img, int& n_chans); bool read_exr(const void* pMem, size_t mem_size, imagef& img); - + enum { WRITE_EXR_LINEAR_HINT = 1, // hint for lossy comp. methods: exr_perceptual_treatment_t, logarithmic or linear, defaults to logarithmic @@ -3889,7 +3894,7 @@ namespace basisu // Supports 1 (Y), 3 (RGB), or 4 (RGBA) channel images. bool write_exr(const char* pFilename, const imagef& img, uint32_t n_chans, uint32_t flags); - + enum { cImageSaveGrayscale = 1, @@ -3899,25 +3904,28 @@ namespace basisu bool save_png(const char* pFilename, const image& img, uint32_t image_save_flags = 0, uint32_t grayscale_comp = 0); inline bool save_png(const std::string &filename, const image &img, uint32_t image_save_flags = 0, uint32_t grayscale_comp = 0) { return save_png(filename.c_str(), img, image_save_flags, grayscale_comp); } + bool save_qoi(const char* pFilename, const image& img, uint32_t qoi_colorspace = 0); + inline bool save_qoi(const std::string& filename, const image& img, uint32_t qoi_colorspace = 0) { return save_qoi(filename.c_str(), img, qoi_colorspace); } + bool read_file_to_vec(const char* pFilename, uint8_vec& data); - bool read_file_to_data(const char* pFilename, void *pData, size_t len); + bool read_file_to_data(const char* pFilename, void *pData, size_t len); bool write_data_to_file(const char* pFilename, const void* pData, size_t len); - + inline bool write_vec_to_file(const char* pFilename, const uint8_vec& v) { return v.size() ? write_data_to_file(pFilename, &v[0], v.size()) : write_data_to_file(pFilename, "", 0); } - + bool image_resample(const image &src, image &dst, bool srgb = false, - const char *pFilter = "lanczos4", float filter_scale = 1.0f, + const char *pFilter = "lanczos4", float filter_scale = 1.0f, bool wrapping = false, - uint32_t first_comp = 0, uint32_t num_comps = 4); + uint32_t first_comp = 0, uint32_t num_comps = 4, float filter_scale_y = -1.0f); - bool image_resample(const imagef& src, imagef& dst, + bool image_resample(const imagef& src, imagef& dst, const char* pFilter = "lanczos4", float filter_scale = 1.0f, bool wrapping = false, uint32_t first_comp = 0, uint32_t num_comps = 4); - + // Timing - + typedef uint64_t timer_ticks; class interval_timer @@ -3930,7 +3938,7 @@ namespace basisu double get_elapsed_secs() const; inline double get_elapsed_ms() const { return 1000.0f* get_elapsed_secs(); } - + static void init(); static inline timer_ticks get_ticks_per_sec() { return g_freq; } static timer_ticks get_ticks(); @@ -4006,7 +4014,7 @@ namespace basisu void tonemap_image_reinhard(image& ldr_img, const imagef& hdr_img, float exposure, bool add_noise = false, bool per_component = true, bool luma_scaling = false); bool tonemap_image_compressive(image& dst_img, const imagef& hdr_test_img); bool tonemap_image_compressive2(image& dst_img, const imagef& hdr_test_img); - + // Intersection enum eClear { cClear = 0 }; enum eInitExpand { cInitExpand = 0 }; @@ -4297,9 +4305,9 @@ namespace basisu BASISU_FORCE_INLINE float fast_half_to_float_pos_not_inf_or_nan(basist::half_float h) { assert(!basist::half_is_signed(h) && !basist::is_half_inf_or_nan(h)); - + // add 112 to the exponent (112+half float's exp bias of 15=float32's bias of 127) - static const fu32 K = { 0x77800000 }; + static const fu32 K = { 0x77800000 }; fu32 o; o.u = h << 13; @@ -4315,7 +4323,7 @@ namespace basisu // Sutract 112 from the exponent, to change the bias from 127 to 15. static const fu32 g_f_to_h{ 0x7800000 }; - + fu32 fu; fu.f = minimum((float)basist::MAX_HALF_FLOAT, fabsf(f)) * g_f_to_h.f; @@ -4327,17 +4335,17 @@ namespace basisu { assert(!isnan(f) && !isinf(f)); assert((f >= 0.0f) && (f <= basist::MAX_HALF_FLOAT)); - + // Sutract 112 from the exponent, to change the bias from 127 to 15. static const fu32 g_f_to_h{ 0x7800000 }; fu32 fu; fu.f = f * g_f_to_h.f; - + return (basist::half_float)((fu.u >> (23 - 10)) & 0x7FFF); } - + inline basist::half_float fast_float_to_half_no_clamp_neg_nan_or_inf(float f) { assert(!isnan(f) && !isinf(f)); @@ -4363,6 +4371,28 @@ namespace basisu return (basist::half_float)h; } + bool arith_test(); + + void set_image_alpha(image& img, uint32_t a); + + void create_bc7_debug_images( + uint32_t width, uint32_t height, + const void* pBlocks, + const char* pFilename_prefix); + + struct tri2 + { + vec2F p0, p1, p2; + vec2F t0, t1, t2; + color_rgba c0, c1, c2; + }; + + // simple non-perspective correct triangle rasterizer with texture mapping, useful for generating randomized test data + void draw_tri2(image& dst, const image* pTex, const tri2& tri, bool alpha_blend); + + void set_num_wasi_threads(uint32_t num_threads); + int get_num_hardware_threads(); + } // namespace basisu #include "basisu_math.h" diff --git a/external/basis_universal/encoder/basisu_etc.cpp b/external/basis_universal/encoder/basisu_etc.cpp index abc78e360d..5bae228b3a 100644 --- a/external/basis_universal/encoder/basisu_etc.cpp +++ b/external/basis_universal/encoder/basisu_etc.cpp @@ -1,5 +1,5 @@ // basis_etc.cpp -// Copyright (C) 2019-2024 Binomial LLC. All Rights Reserved. +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -39,7 +39,7 @@ namespace basisu { -16,-48,-64,-80,8,40,56,72 }, { -16,-40,-64,-80,8,32,56,72 }, { -16,-32,-64,-80,8,24,56,72 }, { -16,-40,-56,-80,8,32,48,72 }, { -24,-32,-56,-80,16,24,48,72 }, { -8,-16,-24,-80,0,8,16,72 }, { -32,-48,-64,-72,24,40,56,64 }, { -24,-40,-56,-72,16,32,48,64 } }; - + // Given an ETC1 diff/inten_table/selector, and an 8-bit desired color, this table encodes the best packed_color in the low byte, and the abs error in the high byte. static uint16_t g_etc1_inverse_lookup[2 * 8 * 4][256]; // [ diff/inten_table/selector][desired_color ] @@ -113,7 +113,7 @@ namespace basisu static uint32_t etc1_decode_value(uint32_t diff, uint32_t inten, uint32_t selector, uint32_t packed_c) { - const uint32_t limit = diff ? 32 : 16; + const uint32_t limit = diff ? 32 : 16; BASISU_NOTE_UNUSED(limit); assert((diff < 2) && (inten < 8) && (selector < 4) && (packed_c < limit)); int c; @@ -261,7 +261,7 @@ namespace basisu return best_error; } - + const uint32_t BASISU_ETC1_CLUSTER_FIT_ORDER_TABLE_SIZE = 165; static const struct { uint8_t m_v[4]; } g_cluster_fit_order_tab[BASISU_ETC1_CLUSTER_FIT_ORDER_TABLE_SIZE] = @@ -300,7 +300,7 @@ namespace basisu { { 2, 1, 2, 3 } },{ { 4, 1, 0, 3 } },{ { 3, 1, 1, 3 } },{ { 1, 1, 2, 4 } },{ { 2, 1, 0, 5 } }, { { 1, 0, 1, 6 } },{ { 0, 2, 1, 5 } },{ { 0, 2, 0, 6 } },{ { 1, 1, 1, 5 } },{ { 1, 1, 0, 6 } } }; - + const int g_etc1_inten_tables[cETC1IntenModifierValues][cETC1SelectorValues] = { { -8, -2, 2, 8 }, { -17, -5, 5, 17 }, { -29, -9, 9, 29 }, { -42, -13, 13, 42 }, @@ -600,7 +600,7 @@ namespace basisu const int y3 = pInten_modifer_table[3]; pDst[3].set(ir + y3, ig + y3, ib + y3, 255); } - + bool unpack_etc1(const etc_block& block, color_rgba *pDst, bool preserve_alpha) { const bool diff_flag = block.get_diff_bit(); @@ -723,7 +723,7 @@ namespace basisu { return (n << 4) | n; } - + uint64_t etc_block::evaluate_etc1_error(const color_rgba* pBlock_pixels, bool perceptual, int subblock_index) const { color_rgba unpacked_block[16]; @@ -772,7 +772,7 @@ namespace basisu } } } - + bool etc1_optimizer::compute() { assert(m_pResult->m_pSelectors); @@ -811,26 +811,26 @@ namespace basisu #if defined(DEBUG) || defined(_DEBUG) { - // Ultimate sanity check on the returned error. + // Ultimate sanity check on the returned error. // If this check fails, it likely means the SSE code diverged from C++ somehow, or there was an overflow somewhere. color_rgba block_colors[4]; m_best_solution.m_coords.get_block_colors(block_colors); const color_rgba* pSrc_pixels = m_pParams->m_pSrc_pixels; uint64_t actual_error = 0; - + bool perceptual; if (m_pParams->m_quality >= cETCQualityMedium) perceptual = m_pParams->m_perceptual; else perceptual = (m_pParams->m_quality == cETCQualityFast) ? false : m_pParams->m_perceptual; - + for (uint32_t i = 0; i < n; i++) actual_error += color_distance(perceptual, pSrc_pixels[i], block_colors[pSelectors[i]], false); - + assert(actual_error == m_best_solution.m_error); } -#endif +#endif m_pResult->m_error = m_best_solution.m_error; @@ -1015,10 +1015,10 @@ namespace basisu m_luma.resize(n); m_sorted_luma_indices.resize(n); m_sorted_luma.resize(n); - + int min_r = 255, min_g = 255, min_b = 255; int max_r = 0, max_g = 0, max_b = 0; - + for (uint32_t i = 0; i < n; i++) { const color_rgba& c = m_pParams->m_pSrc_pixels[i]; @@ -1056,7 +1056,7 @@ namespace basisu m_pSorted_luma = &m_sorted_luma[0]; m_pSorted_luma_indices = &m_sorted_luma_indices[0]; - + for (uint32_t i = 0; i < n; i++) m_pSorted_luma[i] = m_luma[m_pSorted_luma_indices[i]]; } @@ -1072,7 +1072,7 @@ namespace basisu bool etc1_optimizer::check_for_redundant_solution(const etc1_solution_coordinates& coords) { // Hash first 3 bytes of color (RGB) - uint32_t kh = hash_hsieh((uint8_t*)&coords.m_unscaled_color.r, 3); + uint32_t kh = basist::hash_hsieh((uint8_t*)&coords.m_unscaled_color.r, 3); uint32_t h0 = kh & cSolutionsTriedHashMask; uint32_t h1 = (kh >> cSolutionsTriedHashBits) & cSolutionsTriedHashMask; @@ -1087,7 +1087,7 @@ namespace basisu return true; } - + static uint8_t g_eval_dist_tables[8][256] = { // 99% threshold @@ -1178,7 +1178,7 @@ namespace basisu uint64_t total_error = 0; const color_rgba* pSrc_pixels = m_pParams->m_pSrc_pixels; - + if (!g_cpu_supports_sse41) { for (uint32_t c = 0; c < n; c++) @@ -1235,6 +1235,7 @@ namespace basisu perceptual_distance_rgb_4_N_sse41((int64_t*)&total_error, pSelectors_to_use, block_colors, pSrc_pixels, n, trial_solution.m_error); else linear_distance_rgb_4_N_sse41((int64_t*)&total_error, pSelectors_to_use, block_colors, pSrc_pixels, n, trial_solution.m_error); + for (uint32_t i = 0; i < n; i++) m_temp_selectors[i] = pSelectors_to_use[i]; } @@ -1258,7 +1259,7 @@ namespace basisu } trial_solution.m_coords.m_unscaled_color = coords.m_unscaled_color; trial_solution.m_coords.m_color4 = m_pParams->m_use_color4; - + #if BASISU_DEBUG_ETC_ENCODER_DEEPER printf("Eval done: %u error: %I64u best error so far: %I64u\n", (trial_solution.m_error < pBest_solution->m_error), trial_solution.m_error, pBest_solution->m_error); #endif @@ -1272,7 +1273,7 @@ namespace basisu success = true; } } - + return success; } @@ -1303,14 +1304,14 @@ namespace basisu } const color_rgba base_color(coords.get_scaled_color()); - + const uint32_t n = m_pParams->m_num_src_pixels; assert(trial_solution.m_selectors.size() == n); trial_solution.m_error = UINT64_MAX; - + const bool perceptual = (m_pParams->m_quality == cETCQualityFast) ? false : m_pParams->m_perceptual; - + for (int inten_table = cETC1IntenModifierValues - 1; inten_table >= 0; --inten_table) { const int* pInten_table = g_etc1_inten_tables[inten_table]; @@ -1330,10 +1331,10 @@ namespace basisu // 0 1 2 3 // 01 12 23 const uint32_t block_inten_midpoints[3] = { block_inten[0] + block_inten[1], block_inten[1] + block_inten[2], block_inten[2] + block_inten[3] }; - + uint64_t total_error = 0; const color_rgba* pSrc_pixels = m_pParams->m_pSrc_pixels; - + if (perceptual) { if ((m_pSorted_luma[n - 1] * 2) < block_inten_midpoints[0]) diff --git a/external/basis_universal/encoder/basisu_frontend.cpp b/external/basis_universal/encoder/basisu_frontend.cpp index 102f1bdc37..d721b37d15 100644 --- a/external/basis_universal/encoder/basisu_frontend.cpp +++ b/external/basis_universal/encoder/basisu_frontend.cpp @@ -1,5 +1,5 @@ // basisu_frontend.cpp -// Copyright (C) 2019-2024 Binomial LLC. All Rights Reserved. +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. // -// TODO: +// TODO: // This code originally supported full ETC1 and ETC1S, so there's some legacy stuff to be cleaned up in here. // Add endpoint tiling support (where we force adjacent blocks to use the same endpoints during quantization), for a ~10% or more increase in bitrate at same SSIM. The backend already supports this. // @@ -40,20 +40,20 @@ namespace basisu const uint32_t BASISU_ENDPOINT_PARENT_CODEBOOK_SIZE = 16; const uint32_t BASISU_SELECTOR_PARENT_CODEBOOK_SIZE_COMP_LEVEL_01 = 32; const uint32_t BASISU_SELECTOR_PARENT_CODEBOOK_SIZE_COMP_LEVEL_DEFAULT = 16; - + // TODO - How to handle internal verifies in the basisu lib static inline void handle_verify_failure(int line) { error_printf("basisu_frontend: verify check failed at line %i!\n", line); abort(); } - + bool basisu_frontend::init(const params &p) { debug_printf("basisu_frontend::init: Multithreaded: %u, Job pool total threads: %u, NumEndpointClusters: %u, NumSelectorClusters: %u, Perceptual: %u, CompressionLevel: %u\n", p.m_multithreaded, p.m_pJob_pool ? p.m_pJob_pool->get_total_threads() : 0, p.m_max_endpoint_clusters, p.m_max_selector_clusters, p.m_perceptual, p.m_compression_level); - + if ((p.m_max_endpoint_clusters < 1) || (p.m_max_endpoint_clusters > cMaxEndpointClusters)) return false; if ((p.m_max_selector_clusters < 1) || (p.m_max_selector_clusters > cMaxSelectorClusters)) @@ -61,9 +61,9 @@ namespace basisu m_source_blocks.resize(0); append_vector(m_source_blocks, p.m_pSource_blocks, p.m_num_source_blocks); - + m_params = p; - + if (m_params.m_pOpenCL_context) { BASISU_ASSUME(sizeof(cl_pixel_block) == sizeof(pixel_block)); @@ -80,7 +80,7 @@ namespace basisu m_encoded_blocks.resize(m_params.m_num_source_blocks); memset(&m_encoded_blocks[0], 0, m_encoded_blocks.size() * sizeof(m_encoded_blocks[0])); - + m_num_endpoint_codebook_iterations = 1; m_num_selector_codebook_iterations = 1; @@ -150,7 +150,7 @@ namespace basisu if (m_params.m_disable_hierarchical_endpoint_codebooks) m_use_hierarchical_endpoint_codebooks = false; - debug_printf("Endpoint refinement: %u, Hierarchical endpoint codebooks: %u, Hierarchical selector codebooks: %u, Endpoint codebook iters: %u, Selector codebook iters: %u\n", + debug_printf("Endpoint refinement: %u, Hierarchical endpoint codebooks: %u, Hierarchical selector codebooks: %u, Endpoint codebook iters: %u, Selector codebook iters: %u\n", m_endpoint_refinement, m_use_hierarchical_endpoint_codebooks, m_use_hierarchical_selector_codebooks, m_num_endpoint_codebook_iterations, m_num_selector_codebook_iterations); return true; @@ -238,7 +238,7 @@ namespace basisu { BASISU_FRONTEND_VERIFY(validate_endpoint_cluster_hierarchy(false)); } - + eliminate_redundant_or_empty_endpoint_clusters(); if (m_params.m_validate) @@ -252,7 +252,7 @@ namespace basisu if (early_out) break; } - + if (m_params.m_validate) { BASISU_FRONTEND_VERIFY(check_etc1s_constraints()); @@ -268,13 +268,13 @@ namespace basisu if (m_use_hierarchical_selector_codebooks) compute_selector_clusters_within_each_parent_cluster(); - + if (m_params.m_compression_level == 0) { create_optimized_selector_codebook(0); find_optimal_selector_clusters_for_each_block(); - + introduce_special_selector_clusters(); } else @@ -295,7 +295,7 @@ namespace basisu } } } - + optimize_selector_codebook(); if (m_params.m_debug_stats) @@ -321,7 +321,7 @@ namespace basisu const basist::basisu_lowlevel_etc1s_transcoder::endpoint_vec& endpoints = pTranscoder->get_endpoints(); const basist::basisu_lowlevel_etc1s_transcoder::selector_vec& selectors = pTranscoder->get_selectors(); - + m_endpoint_cluster_etc_params.resize(endpoints.size()); for (uint32_t i = 0; i < endpoints.size(); i++) { @@ -420,7 +420,7 @@ namespace basisu const uint32_t last_index = minimum(m_total_blocks, first_index + N); m_params.m_pJob_pool->add_job([this, first_index, last_index, pass] { - + for (uint32_t block_index = first_index; block_index < last_index; block_index++) { const etc_block& blk = pass ? m_encoded_blocks[block_index] : m_etc1_blocks_etc1s[block_index]; @@ -433,7 +433,7 @@ namespace basisu uint64_t best_err = UINT64_MAX; uint32_t best_index = 0; etc_block best_block(trial_blk); - + for (uint32_t i = 0; i < m_endpoint_cluster_etc_params.size(); i++) { if (m_endpoint_cluster_etc_params[i].m_inten_table[0] > blk.get_inten_table(0)) @@ -496,6 +496,7 @@ namespace basisu const uint32_t block_endpoint_index = m_block_endpoint_clusters_indices[block_index][0]; etc_block trial_blk; + clear_obj(trial_blk); trial_blk.set_block_color5_etc1s(m_endpoint_cluster_etc_params[block_endpoint_index].m_color_unscaled[0]); trial_blk.set_inten_tables_etc1s(m_endpoint_cluster_etc_params[block_endpoint_index].m_inten_table[0]); trial_blk.set_flip_bit(true); @@ -546,7 +547,7 @@ namespace basisu m_selector_cluster_block_indices.resize(selectors.size()); for (uint32_t block_index = 0; block_index < m_etc1_blocks_etc1s.size(); block_index++) m_selector_cluster_block_indices[m_block_selector_cluster_index[block_index]].push_back(block_index); - + return true; } @@ -580,9 +581,9 @@ namespace basisu const uint32_t new_selector_cluster_index = m_optimized_cluster_selectors.size_u32(); m_optimized_cluster_selectors.push_back(blk); - + vector_ensure_element_is_valid(m_selector_cluster_block_indices, new_selector_cluster_index); - + for (uint32_t block_index = 0; block_index < m_total_blocks; block_index++) { if (m_orig_encoded_blocks[block_index].get_raw_selector_bits() != blk.get_raw_selector_bits()) @@ -590,7 +591,7 @@ namespace basisu // See if using flat selectors actually decreases the block's error. const uint32_t old_selector_cluster_index = m_block_selector_cluster_index[block_index]; - + etc_block cur_blk; const uint32_t endpoint_cluster_index = get_subblock_endpoint_cluster_index(block_index, 0); cur_blk.set_block_color5_etc1s(get_endpoint_cluster_unscaled_color(endpoint_cluster_index, false)); @@ -606,10 +607,10 @@ namespace basisu if (new_err >= cur_err) continue; - + // Change the block to use the new cluster m_block_selector_cluster_index[block_index] = new_selector_cluster_index; - + m_selector_cluster_block_indices[new_selector_cluster_index].push_back(block_index); block_relocated_flags[block_index] = true; @@ -685,7 +686,7 @@ namespace basisu old_to_new[i] = (find_res.first)->second; continue; } - + old_to_new[i] = total_new_entries++; new_to_old.push_back(i); } @@ -714,7 +715,7 @@ namespace basisu { new_selector_cluster_indices[m_block_selector_cluster_index[i]].push_back(i); } - + m_optimized_cluster_selectors.swap(new_optimized_cluster_selectors); m_selector_cluster_block_indices.swap(new_selector_cluster_indices); @@ -725,7 +726,7 @@ namespace basisu for (uint32_t j = 0; j < m_selector_clusters_within_each_parent_cluster[i].size(); j++) m_selector_clusters_within_each_parent_cluster[i][j] = old_to_new[m_selector_clusters_within_each_parent_cluster[i][j]]; } - + debug_printf("optimize_selector_codebook: Before: %u After: %u\n", orig_total_selector_clusters, total_new_entries); } @@ -735,11 +736,11 @@ namespace basisu interval_timer tm; tm.start(); - + m_etc1_blocks_etc1s.resize(m_total_blocks); bool use_cpu = true; - + if (m_params.m_pOpenCL_context) { uint32_t total_perms = 64; @@ -747,9 +748,9 @@ namespace basisu total_perms = 4; else if (m_params.m_compression_level == 1) total_perms = 16; - else if (m_params.m_compression_level == BASISU_MAX_COMPRESSION_LEVEL) + else if (m_params.m_compression_level == BASISU_MAX_ETC1S_COMPRESSION_LEVEL) total_perms = OPENCL_ENCODE_ETC1S_MAX_PERMS; - + bool status = opencl_encode_etc1s_blocks(m_params.m_pOpenCL_context, m_etc1_blocks_etc1s.data(), m_params.m_perceptual, total_perms); if (status) use_cpu = false; @@ -760,7 +761,7 @@ namespace basisu m_opencl_failed = true; } } - + if (use_cpu) { const uint32_t N = 4096; @@ -783,7 +784,7 @@ namespace basisu optimizer_params.m_quality = cETCQualityFast; else if (m_params.m_compression_level == 1) optimizer_params.m_quality = cETCQualityMedium; - else if (m_params.m_compression_level == BASISU_MAX_COMPRESSION_LEVEL) + else if (m_params.m_compression_level == BASISU_MAX_ETC1S_COMPRESSION_LEVEL) optimizer_params.m_quality = cETCQualityUber; optimizer_params.m_num_src_pixels = 16; @@ -817,16 +818,16 @@ namespace basisu m_params.m_pJob_pool->wait_for_all(); } // use_cpu - + debug_printf("init_etc1_images: Elapsed time: %3.3f secs\n", tm.get_elapsed_secs()); } void basisu_frontend::init_endpoint_training_vectors() { debug_printf("init_endpoint_training_vectors\n"); - + vec6F_quantizer::array_of_weighted_training_vecs &training_vecs = m_endpoint_clusterizer.get_training_vecs(); - + training_vecs.resize(m_total_blocks * 2); const uint32_t N = 16384; @@ -838,12 +839,12 @@ namespace basisu m_params.m_pJob_pool->add_job( [this, first_index, last_index, &training_vecs] { for (uint32_t block_index = first_index; block_index < last_index; block_index++) - { + { const etc_block &blk = m_etc1_blocks_etc1s[block_index]; color_rgba block_colors[2]; blk.get_block_low_high_colors(block_colors, 0); - + vec6F v; v[0] = block_colors[0].r * (1.0f / 255.0f); v[1] = block_colors[0].g * (1.0f / 255.0f); @@ -851,7 +852,7 @@ namespace basisu v[3] = block_colors[1].r * (1.0f / 255.0f); v[4] = block_colors[1].g * (1.0f / 255.0f); v[5] = block_colors[1].b * (1.0f / 255.0f); - + training_vecs[block_index * 2 + 0] = std::make_pair(v, 1); training_vecs[block_index * 2 + 1] = std::make_pair(v, 1); @@ -870,7 +871,7 @@ namespace basisu const uint32_t parent_codebook_size = (m_params.m_max_endpoint_clusters >= 256) ? BASISU_ENDPOINT_PARENT_CODEBOOK_SIZE : 0; uint32_t max_threads = 0; - max_threads = m_params.m_multithreaded ? minimum(std::thread::hardware_concurrency(), cMaxCodebookCreationThreads) : 0; + max_threads = m_params.m_multithreaded ? minimum(get_num_hardware_threads(), cMaxCodebookCreationThreads) : 0; if (m_params.m_pJob_pool) max_threads = minimum((int)m_params.m_pJob_pool->get_total_threads(), max_threads); @@ -919,12 +920,12 @@ namespace basisu for (uint32_t cluster_index = 0; cluster_index < m_endpoint_clusters.size(); cluster_index++) { const uint_vec &cluster = m_endpoint_clusters[cluster_index]; - + uint32_t parent_cluster_index = 0; for (uint32_t j = 0; j < cluster.size(); j++) { const uint32_t block_index = cluster[j] >> 1; - + BASISU_FRONTEND_VERIFY(block_index < m_block_parent_endpoint_cluster.size()); if (!j) @@ -938,7 +939,7 @@ namespace basisu } } } - + if (m_params.m_debug_stats) debug_printf("Total endpoint clusters: %u, parent clusters: %u\n", m_endpoint_clusters.size_u32(), m_endpoint_parent_clusters.size_u32()); } @@ -996,7 +997,7 @@ namespace basisu BASISU_FRONTEND_VERIFY(cluster_indices.size()); vector_sort(cluster_indices); - + auto last = std::unique(cluster_indices.begin(), cluster_indices.end()); cluster_indices.erase(last, cluster_indices.end()); } @@ -1009,8 +1010,8 @@ namespace basisu const uint32_t N = 512; for (uint32_t cluster_index_iter = 0; cluster_index_iter < m_endpoint_clusters.size(); cluster_index_iter += N) { - const uint32_t first_index = cluster_index_iter; - const uint32_t last_index = minimum(m_endpoint_clusters.size_u32(), cluster_index_iter + N); + const uint32_t first_index = cluster_index_iter; + const uint32_t last_index = minimum(m_endpoint_clusters.size_u32(), cluster_index_iter + N); m_params.m_pJob_pool->add_job( [this, first_index, last_index] { @@ -1039,7 +1040,7 @@ namespace basisu const endpoint_cluster_etc_params &etc_params = m_endpoint_cluster_etc_params[cluster_index]; assert(etc_params.m_valid); - + color_rgba block_colors[4]; etc_block::get_block_colors5(block_colors, etc_params.m_color_unscaled[0], etc_params.m_inten_table[0], true); @@ -1071,7 +1072,7 @@ namespace basisu quant_err.m_cluster_subblock_index = cluster_indices_iter; quant_err.m_block_index = block_index; quant_err.m_subblock_index = subblock_index; - + { std::lock_guard lock(m_lock); @@ -1088,7 +1089,7 @@ namespace basisu vector_sort(m_subblock_endpoint_quant_err_vec); } - + void basisu_frontend::introduce_new_endpoint_clusters() { debug_printf("introduce_new_endpoint_clusters\n"); @@ -1159,9 +1160,9 @@ namespace basisu BASISU_FRONTEND_VERIFY(cluster_sizes[subblock_to_move.m_cluster_index] >= 2); cluster_sizes[subblock_to_move.m_cluster_index] -= 2; - + ignore_cluster.insert(subblock_to_move.m_cluster_index); - + total_new_clusters++; num_new_endpoint_clusters--; @@ -1197,23 +1198,23 @@ namespace basisu inline std::size_t operator()(const color_rgba& k) const { uint32_t v = *(const uint32_t*)&k; - + //return bitmix32(v); - + //v ^= (v << 10); //v ^= (v >> 12); - + return v; } }; - + // Given each endpoint cluster, gather all the block pixels which are in that cluster and compute optimized ETC1S endpoints for them. // TODO: Don't optimize endpoint clusters which haven't changed. // If step>=1, we check to ensure the new endpoint values actually decrease quantization error. void basisu_frontend::generate_endpoint_codebook(uint32_t step) { debug_printf("generate_endpoint_codebook\n"); - + interval_timer tm; tm.start(); @@ -1226,7 +1227,7 @@ namespace basisu const uint32_t total_clusters = (uint32_t)m_endpoint_clusters.size(); basisu::vector pixel_clusters(total_clusters); - + std::vector input_pixels; input_pixels.reserve(m_total_blocks * 16); @@ -1259,7 +1260,7 @@ namespace basisu pixel_weights.resize(pixel_weights.size() + total_pixels); uint64_t dst_ofs = first_pixel_index; - + uint64_t total_r = 0, total_g = 0, total_b = 0; for (uint32_t cluster_indices_iter = 0; cluster_indices_iter < cluster_indices.size(); cluster_indices_iter++) { @@ -1311,7 +1312,7 @@ namespace basisu const uint64_t first_pixel_index = input_pixels.size(); uint32_t prev_color = 0, cur_weight = 0; - + for (uint32_t i = 0; i < colors.size(); i++) { uint32_t cur_color = pSorted[i]; @@ -1359,7 +1360,7 @@ namespace basisu uint32_t *pPrev_weight = nullptr; color_rgba prev_color; - + { color_rgba cur_color = pBlock_pixels[0]; auto res = color_hasher.insert(cur_color, 0); @@ -1371,7 +1372,7 @@ namespace basisu prev_color = cur_color; pPrev_weight = &(res.first)->second; } - + for (uint32_t i = 1; i < 16; i++) { color_rgba cur_color = pBlock_pixels[i]; @@ -1404,9 +1405,9 @@ namespace basisu input_pixels.resize(first_pixel_index + total_unique_pixels); pixel_weights.resize(first_pixel_index + total_unique_pixels); - + uint32_t j = 0; - + for (auto it = color_hasher.begin(); it != color_hasher.end(); ++it, ++j) { input_pixels[first_pixel_index + j] = it->first; @@ -1438,7 +1439,7 @@ namespace basisu uint32_t total_perms = 64; if (m_params.m_compression_level <= 1) total_perms = 16; - else if (m_params.m_compression_level == BASISU_MAX_COMPRESSION_LEVEL) + else if (m_params.m_compression_level == BASISU_MAX_ETC1S_COMPRESSION_LEVEL) total_perms = OPENCL_ENCODE_ETC1S_MAX_PERMS; basisu::vector output_blocks(total_clusters); @@ -1456,7 +1457,7 @@ namespace basisu for (uint32_t old_cluster_index = 0; old_cluster_index < m_endpoint_clusters.size(); old_cluster_index++) { const uint32_t new_cluster_index = sorted_cluster_indices_old_to_new[old_cluster_index]; - + const etc_block& blk = output_blocks[new_cluster_index]; endpoint_cluster_etc_params& prev_etc_params = m_endpoint_cluster_etc_params[old_cluster_index]; @@ -1464,7 +1465,7 @@ namespace basisu prev_etc_params.m_valid = true; etc_block::unpack_color5(prev_etc_params.m_color_unscaled[0], blk.get_base5_color(), false); prev_etc_params.m_inten_table[0] = blk.get_inten_table(0); - prev_etc_params.m_color_error[0] = 0; // dummy value - we don't actually use this + prev_etc_params.m_color_error[0] = 0; // dummy value - we don't actually use this } use_cpu = false; @@ -1518,7 +1519,7 @@ namespace basisu { etc1_optimizer optimizer; - etc1_solution_coordinates solutions[2]; + //etc1_solution_coordinates solutions[2]; etc1_optimizer::params cluster_optimizer_params; cluster_optimizer_params.m_num_src_pixels = total_pixels; @@ -1529,7 +1530,7 @@ namespace basisu if (m_params.m_compression_level <= 1) cluster_optimizer_params.m_quality = cETCQualityMedium; - else if (m_params.m_compression_level == BASISU_MAX_COMPRESSION_LEVEL) + else if (m_params.m_compression_level == BASISU_MAX_ETC1S_COMPRESSION_LEVEL) cluster_optimizer_params.m_quality = cETCQualityUber; etc1_optimizer::results cluster_optimizer_results; @@ -1647,7 +1648,7 @@ namespace basisu uint32_t basisu_frontend::refine_endpoint_clusterization() { debug_printf("refine_endpoint_clusterization\n"); - + if (m_use_hierarchical_endpoint_codebooks) compute_endpoint_clusters_within_each_parent_cluster(); @@ -1668,9 +1669,9 @@ namespace basisu } // cluster_indices_iter } - + //---------------------------------------------------------- - + // Create a new endpoint clusterization interval_timer tm; @@ -1687,7 +1688,7 @@ namespace basisu const uint32_t total_parent_clusters = (uint32_t)m_endpoint_clusters_within_each_parent_cluster.size(); basisu::vector cl_block_info_structs(m_total_blocks); - + // the size of each parent cluster, in total clusters uint_vec parent_cluster_sizes(total_parent_clusters); for (uint32_t i = 0; i < total_parent_clusters; i++) @@ -1701,7 +1702,7 @@ namespace basisu cur_ofs += parent_cluster_sizes[i]; } - + // Note: total_actual_endpoint_clusters is not necessarly equal to m_endpoint_clusters.size(), because clusters may live in multiple parent clusters after the first refinement step. BASISU_FRONTEND_VERIFY(cur_ofs >= m_endpoint_clusters.size()); const uint32_t total_actual_endpoint_clusters = cur_ofs; @@ -1727,11 +1728,11 @@ namespace basisu cl_endpoint_cluster_structs[dst_ofs + j].m_cluster_index = (uint16_t)endpoint_cluster_index; } } - + for (uint32_t block_index = 0; block_index < m_total_blocks; block_index++) { const uint32_t block_parent_endpoint_cluster_index = m_block_parent_endpoint_cluster[block_index]; - + cl_block_info_structs[block_index].m_num_clusters = (uint16_t)(parent_cluster_sizes[block_parent_endpoint_cluster_index]); cl_block_info_structs[block_index].m_first_cluster_ofs = (uint16_t)(first_parent_cluster_ofs[block_parent_endpoint_cluster_index]); @@ -1746,7 +1747,7 @@ namespace basisu uint_vec sorted_block_indices(m_total_blocks); indirect_sort(m_total_blocks, sorted_block_indices.data(), block_cluster_indices.data()); - + bool status = opencl_refine_endpoint_clusterization( m_params.m_pOpenCL_context, cl_block_info_structs.data(), @@ -1902,7 +1903,7 @@ namespace basisu break; } } // j - + best_cluster_indices[block_index] = best_cluster_index; } // block_index @@ -1912,9 +1913,9 @@ namespace basisu } // block_index_iter m_params.m_pJob_pool->wait_for_all(); - + } // use_cpu - + debug_printf("refine_endpoint_clusterization time: %3.3f secs\n", tm.get_elapsed_secs()); basisu::vector > optimized_endpoint_clusters(m_endpoint_clusters.size()); @@ -1957,7 +1958,7 @@ namespace basisu basisu::vector > new_endpoint_clusters(m_endpoint_clusters.size()); basisu::vector new_subblock_etc_params(m_endpoint_clusters.size()); - + for (uint32_t i = 0; i < m_endpoint_clusters.size(); i++) { uint32_t j = sorted_endpoint_cluster_indices[i]; @@ -1972,7 +1973,7 @@ namespace basisu new_endpoint_clusters.resize(0); new_subblock_etc_params.resize(0); - + for (int i = 0; i < (int)m_endpoint_clusters.size(); ) { if (!m_endpoint_clusters[i].size()) @@ -1990,7 +1991,7 @@ namespace basisu new_endpoint_clusters.push_back(m_endpoint_clusters[i]); new_subblock_etc_params.push_back(m_endpoint_cluster_etc_params[i]); - + for (int k = i + 1; k < j; k++) { append_vector(new_endpoint_clusters.back(), m_endpoint_clusters[k]); @@ -1998,7 +1999,7 @@ namespace basisu i = j; } - + if (m_endpoint_clusters.size() != new_endpoint_clusters.size()) { if (m_params.m_debug_stats) @@ -2013,7 +2014,7 @@ namespace basisu void basisu_frontend::create_initial_packed_texture() { debug_printf("create_initial_packed_texture\n"); - + interval_timer tm; tm.start(); @@ -2026,7 +2027,7 @@ namespace basisu for (uint32_t block_index = 0; block_index < m_total_blocks; block_index++) { uint32_t cluster0 = m_block_endpoint_clusters_indices[block_index][0]; - + const color_rgba& color_unscaled = m_endpoint_cluster_etc_params[cluster0].m_color_unscaled[0]; uint32_t inten = m_endpoint_cluster_etc_params[cluster0].m_inten_table[0]; @@ -2088,7 +2089,7 @@ namespace basisu m_params.m_pJob_pool->wait_for_all(); } // use_cpu - + m_orig_encoded_blocks = m_encoded_blocks; debug_printf("Elapsed time: %3.3f secs\n", tm.get_elapsed_secs()); @@ -2105,7 +2106,7 @@ namespace basisu for (uint32_t cluster_indices_iter = 0; cluster_indices_iter < cluster_indices.size(); cluster_indices_iter++) { const uint32_t block_index = cluster_indices[cluster_indices_iter]; - + block_selector_cluster_indices[block_index] = cluster_index; } // cluster_indices_iter @@ -2130,7 +2131,7 @@ namespace basisu BASISU_FRONTEND_VERIFY(cluster_indices.size()); vector_sort(cluster_indices); - + auto last = std::unique(cluster_indices.begin(), cluster_indices.end()); cluster_indices.erase(last, cluster_indices.end()); } @@ -2139,11 +2140,11 @@ namespace basisu void basisu_frontend::generate_selector_clusters() { debug_printf("generate_selector_clusters\n"); - + typedef tree_vector_quant vec16F_clusterizer; - + vec16F_clusterizer::array_of_weighted_training_vecs training_vecs(m_total_blocks); - + const uint32_t N = 4096; for (uint32_t block_index_iter = 0; block_index_iter < m_total_blocks; block_index_iter += N) { @@ -2171,10 +2172,10 @@ namespace basisu const uint32_t cColorDistToWeight = 300; const uint32_t cMaxWeight = 4096; uint32_t weight = clamp(dist / cColorDistToWeight, 1, cMaxWeight); - + training_vecs[block_index].first = v; training_vecs[block_index].second = weight; - + } // block_index } ); @@ -2192,7 +2193,7 @@ namespace basisu debug_printf("Using selector parent codebook size %u\n", parent_codebook_size); uint32_t max_threads = 0; - max_threads = m_params.m_multithreaded ? minimum(std::thread::hardware_concurrency(), cMaxCodebookCreationThreads) : 0; + max_threads = m_params.m_multithreaded ? minimum(get_num_hardware_threads(), cMaxCodebookCreationThreads) : 0; if (m_params.m_pJob_pool) max_threads = minimum((int)m_params.m_pJob_pool->get_total_threads(), max_threads); @@ -2235,7 +2236,7 @@ namespace basisu for (uint32_t cluster_index = 0; cluster_index < m_selector_cluster_block_indices.size(); cluster_index++) { const uint_vec &cluster = m_selector_cluster_block_indices[cluster_index]; - + uint32_t parent_cluster_index = 0; for (uint32_t j = 0; j < cluster.size(); j++) { @@ -2267,7 +2268,7 @@ namespace basisu debug_printf("Total selector clusters (from m_selector_cluster_block_indices.size()): %u\n", (uint32_t)m_selector_cluster_block_indices.size()); m_optimized_cluster_selectors.resize(total_selector_clusters); - + // For each selector codebook entry, and for each of the 4x4 selectors, determine which selector minimizes the error across all the blocks that use that quantized selector. const uint32_t N = 256; for (uint32_t cluster_index_iter = 0; cluster_index_iter < total_selector_clusters; cluster_index_iter += N) @@ -2351,7 +2352,7 @@ namespace basisu m_params.m_pJob_pool->wait_for_all(); debug_printf("Elapsed time: %3.3f secs\n", tm.get_elapsed_secs()); - + if (m_params.m_debug_images) { uint32_t max_selector_cluster_size = 0; @@ -2377,7 +2378,7 @@ namespace basisu uint32_t block_index = cluster_block_indices[i]; const etc_block &blk = m_orig_encoded_blocks[block_index]; - + for (uint32_t y = 0; y < 4; y++) for (uint32_t x = 0; x < 4; x++) selector_cluster_vis.set_clipped(x_spacer_len + x + 5 * i, selector_cluster_index * 5 + y, color_rgba((blk.get_selector(x, y) * 255) / 3)); @@ -2399,7 +2400,7 @@ namespace basisu interval_timer tm; tm.start(); - + if (m_params.m_validate) { // Sanity checks @@ -2414,7 +2415,7 @@ namespace basisu } m_block_selector_cluster_index.resize(m_total_blocks); - + if (m_params.m_compression_level == 0) { // Just leave the blocks in their original selector clusters. @@ -2435,7 +2436,7 @@ namespace basisu return; } - + bool use_cpu = true; if ((m_params.m_pOpenCL_context) && m_use_hierarchical_selector_codebooks) @@ -2444,17 +2445,17 @@ namespace basisu basisu::vector selector_structs; selector_structs.reserve(m_optimized_cluster_selectors.size()); - + uint_vec parent_selector_cluster_offsets(num_parent_clusters); uint_vec selector_cluster_indices; selector_cluster_indices.reserve(m_optimized_cluster_selectors.size()); - + uint32_t cur_ofs = 0; for (uint32_t parent_index = 0; parent_index < num_parent_clusters; parent_index++) { parent_selector_cluster_offsets[parent_index] = cur_ofs; - + for (uint32_t j = 0; j < m_selector_clusters_within_each_parent_cluster[parent_index].size(); j++) { const uint32_t selector_cluster_index = m_selector_clusters_within_each_parent_cluster[parent_index][j]; @@ -2464,7 +2465,7 @@ namespace basisu sel_bits |= (m_optimized_cluster_selectors[selector_cluster_index].get_selector(p & 3, p >> 2) << (p * 2)); selector_structs.enlarge(1)->m_packed_selectors = sel_bits; - + selector_cluster_indices.push_back(selector_cluster_index); } @@ -2472,7 +2473,7 @@ namespace basisu } const uint32_t total_input_selectors = cur_ofs; - + basisu::vector block_structs(m_total_blocks); for (uint32_t i = 0; i < m_total_blocks; i++) { @@ -2496,7 +2497,7 @@ namespace basisu selector_cluster_indices.data(), output_selector_cluster_indices.data(), m_params.m_perceptual); - + if (!status) { error_printf("basisu_frontend::find_optimal_selector_clusters_for_each_block: opencl_find_optimal_selector_clusters_for_each_block() failed! Using CPU.\n"); @@ -2510,7 +2511,7 @@ namespace basisu m_selector_cluster_block_indices[i].resize(0); m_selector_cluster_block_indices[i].reserve(128); } - + for (uint32_t block_index = 0; block_index < m_total_blocks; block_index++) { etc_block& blk = m_encoded_blocks[block_index]; @@ -2542,7 +2543,7 @@ namespace basisu } } } - + const uint32_t N = 2048; for (uint32_t block_index_iter = 0; block_index_iter < m_total_blocks; block_index_iter += N) { @@ -2550,13 +2551,13 @@ namespace basisu const uint32_t last_index = minimum(m_total_blocks, first_index + N); m_params.m_pJob_pool->add_job( [this, first_index, last_index, &unpacked_optimized_cluster_selectors] { - + int prev_best_cluster_index = 0; for (uint32_t block_index = first_index; block_index < last_index; block_index++) { const pixel_block& block = get_source_pixel_block(block_index); - + etc_block& blk = m_encoded_blocks[block_index]; if ((block_index > first_index) && (block == get_source_pixel_block(block_index - 1))) @@ -2564,18 +2565,18 @@ namespace basisu blk.set_raw_selector_bits(m_optimized_cluster_selectors[prev_best_cluster_index].get_raw_selector_bits()); m_block_selector_cluster_index[block_index] = prev_best_cluster_index; - + continue; } - + const color_rgba* pBlock_pixels = block.get_ptr(); - + color_rgba trial_block_colors[4]; blk.get_block_colors_etc1s(trial_block_colors); // precompute errors for the i-th block pixel and selector sel: [sel][i] uint32_t trial_errors[4][16]; - + if (m_params.m_perceptual) { for (uint32_t sel = 0; sel < 4; ++sel) @@ -2652,7 +2653,7 @@ namespace basisu for (uint32_t cluster_iter = 0; cluster_iter < total_clusters; cluster_iter++) { const uint32_t cluster_index = m_use_hierarchical_selector_codebooks ? (*pCluster_indices)[cluster_iter] : cluster_iter; - + const uint8_t* pSels = &unpacked_optimized_cluster_selectors[cluster_index * 16]; uint64_t trial_err = (uint64_t)trial_errors[pSels[0]][0] + trial_errors[pSels[1]][1] + trial_errors[pSels[2]][2] + trial_errors[pSels[3]][3]; @@ -2685,7 +2686,7 @@ namespace basisu m_block_selector_cluster_index[block_index] = best_cluster_index; prev_best_cluster_index = best_cluster_index; - + } // block_index } ); @@ -2693,7 +2694,7 @@ namespace basisu } // block_index_iter m_params.m_pJob_pool->wait_for_all(); - + for (uint32_t i = 0; i < m_selector_cluster_block_indices.size(); i++) { m_selector_cluster_block_indices[i].resize(0); @@ -2707,7 +2708,7 @@ namespace basisu vector_ensure_element_is_valid(m_selector_cluster_block_indices, best_cluster_index); m_selector_cluster_block_indices[best_cluster_index].push_back(block_index); } - + } // if (use_cpu) debug_printf("Elapsed time: %3.3f secs\n", tm.get_elapsed_secs()); @@ -2717,7 +2718,7 @@ namespace basisu uint32_t basisu_frontend::refine_block_endpoints_given_selectors() { debug_printf("refine_block_endpoints_given_selectors\n"); - + for (int block_index = 0; block_index < static_cast(m_total_blocks); block_index++) { //uint32_t selector_cluster = m_block_selector_cluster_index(block_x, block_y); @@ -2790,7 +2791,7 @@ namespace basisu total_subblocks_examined += total_pixels / 8; etc1_optimizer optimizer; - etc1_solution_coordinates solutions[2]; + //etc1_solution_coordinates solutions[2]; etc1_optimizer::params cluster_optimizer_params; cluster_optimizer_params.m_num_src_pixels = total_pixels; @@ -2898,7 +2899,7 @@ namespace basisu if (m_params.m_debug_stats) debug_printf("Total subblock endpoints refined: %u (%3.1f%%)\n", total_subblocks_refined, total_subblocks_refined * 100.0f / total_subblocks_examined); - + return total_subblocks_refined; } @@ -2990,7 +2991,7 @@ namespace basisu } // The backend has remapped the block endpoints while optimizing the output symbols for better rate distortion performance, so let's go and reoptimize the endpoint codebook. - // This is currently the only place where the backend actually goes and changes the quantization and calls the frontend to fix things up. + // This is currently the only place where the backend actually goes and changes the quantization and calls the frontend to fix things up. // This is basically a bottom up clusterization stage, where some leaves can be combined. void basisu_frontend::reoptimize_remapped_endpoints(const uint_vec &new_block_endpoints, int_vec &old_to_new_endpoint_cluster_indices, bool optimize_final_codebook, uint_vec *pBlock_selector_indices) { @@ -3002,12 +3003,12 @@ namespace basisu basisu::vector cluster_valid(new_endpoint_cluster_block_indices.size()); basisu::vector cluster_improved(new_endpoint_cluster_block_indices.size()); - + const uint32_t N = 256; for (uint32_t cluster_index_iter = 0; cluster_index_iter < new_endpoint_cluster_block_indices.size(); cluster_index_iter += N) { - const uint32_t first_index = cluster_index_iter; - const uint32_t last_index = minimum((uint32_t)new_endpoint_cluster_block_indices.size(), cluster_index_iter + N); + const uint32_t first_index = cluster_index_iter; + const uint32_t last_index = minimum((uint32_t)new_endpoint_cluster_block_indices.size(), cluster_index_iter + N); m_params.m_pJob_pool->add_job( [this, first_index, last_index, &cluster_improved, &cluster_valid, &new_endpoint_cluster_block_indices, &pBlock_selector_indices ] { @@ -3027,13 +3028,13 @@ namespace basisu blk.set_block_color5_etc1s(get_endpoint_cluster_unscaled_color(cluster_index, false)); blk.set_inten_tables_etc1s(get_endpoint_cluster_inten_table(cluster_index, false)); blk.set_flip_bit(true); - + uint64_t cur_err = 0; for (uint32_t cluster_block_indices_iter = 0; cluster_block_indices_iter < cluster_block_indices.size(); cluster_block_indices_iter++) { const uint32_t block_index = cluster_block_indices[cluster_block_indices_iter]; - + const color_rgba *pBlock_pixels = get_source_pixel_block(block_index).get_ptr(); memcpy(&cluster_pixels[cluster_block_indices_iter * 16], pBlock_pixels, 16 * sizeof(color_rgba)); @@ -3045,17 +3046,17 @@ namespace basisu blk.set_raw_selector_bits(blk_selectors.get_raw_selector_bits()); cur_err += blk.evaluate_etc1_error(pBlock_pixels, m_params.m_perceptual); - + for (uint32_t y = 0; y < 4; y++) for (uint32_t x = 0; x < 4; x++) force_selectors[cluster_block_indices_iter * 16 + x + y * 4] = static_cast(blk_selectors.get_selector(x, y)); } endpoint_cluster_etc_params new_endpoint_cluster_etc_params; - + { etc1_optimizer optimizer; - etc1_solution_coordinates solutions[2]; + //etc1_solution_coordinates solutions[2]; etc1_optimizer::params cluster_optimizer_params; cluster_optimizer_params.m_num_src_pixels = total_pixels; @@ -3065,7 +3066,7 @@ namespace basisu cluster_optimizer_params.m_perceptual = m_params.m_perceptual; cluster_optimizer_params.m_pForce_selectors = &force_selectors[0]; - if (m_params.m_compression_level == BASISU_MAX_COMPRESSION_LEVEL) + if (m_params.m_compression_level == BASISU_MAX_ETC1S_COMPRESSION_LEVEL) cluster_optimizer_params.m_quality = cETCQualityUber; else cluster_optimizer_params.m_quality = cETCQualitySlow; @@ -3091,7 +3092,7 @@ namespace basisu if (new_endpoint_cluster_etc_params.m_color_error[0] < cur_err) { m_endpoint_cluster_etc_params[cluster_index] = new_endpoint_cluster_etc_params; - + cluster_improved[cluster_index] = true; } @@ -3104,13 +3105,13 @@ namespace basisu } // cluster_index_iter m_params.m_pJob_pool->wait_for_all(); - + uint32_t total_unused_clusters = 0; uint32_t total_improved_clusters = 0; - + old_to_new_endpoint_cluster_indices.resize(m_endpoint_clusters.size()); vector_set_all(old_to_new_endpoint_cluster_indices, -1); - + int total_new_endpoint_clusters = 0; for (uint32_t old_cluster_index = 0; old_cluster_index < m_endpoint_clusters.size(); old_cluster_index++) @@ -3145,7 +3146,7 @@ namespace basisu for (uint32_t block_index = 0; block_index < new_block_endpoints.size(); block_index++) { const uint32_t old_endpoint_cluster_index = new_block_endpoints[block_index]; - + const int new_endpoint_cluster_index = old_to_new_endpoint_cluster_indices[old_endpoint_cluster_index]; BASISU_FRONTEND_VERIFY(new_endpoint_cluster_index >= 0); @@ -3158,13 +3159,13 @@ namespace basisu new_endpoint_cluster_etc_params[new_endpoint_cluster_index].m_subblocks.push_back(block_index * 2 + 0); new_endpoint_cluster_etc_params[new_endpoint_cluster_index].m_subblocks.push_back(block_index * 2 + 1); - + m_block_endpoint_clusters_indices[block_index][0] = new_endpoint_cluster_index; m_block_endpoint_clusters_indices[block_index][1] = new_endpoint_cluster_index; } debug_printf("basisu_frontend::reoptimize_remapped_endpoints: stage 2\n"); - + m_endpoint_clusters = new_endpoint_clusters; m_endpoint_cluster_etc_params = new_endpoint_cluster_etc_params; @@ -3200,7 +3201,7 @@ namespace basisu debug_printf("Final (post-RDO) endpoint clusters: %u\n", m_endpoint_clusters.size()); } - + //debug_printf("validate_output: %u\n", validate_output()); } @@ -3228,7 +3229,7 @@ namespace basisu // If the endpoint cluster lives in more than one parent node, that's wrong. if (subblock_parent_indices[subblock_index] != -1) return false; - + subblock_parent_indices[subblock_index] = parent_index; } } @@ -3252,7 +3253,7 @@ namespace basisu if (subblock_cluster_indices[subblock_index] != -1) return false; - + subblock_cluster_indices[subblock_index] = cluster_index; // There are transformations on the endpoint clusters that can break the strict tree requirement @@ -3266,7 +3267,7 @@ namespace basisu } } } - + // Make sure all endpoint clusters are present in the parent cluster. for (uint32_t i = 0; i < subblock_cluster_indices.size(); i++) { @@ -3291,7 +3292,7 @@ namespace basisu #define CHECK(x) BASISU_FRONTEND_VERIFY(x); CHECK(get_output_block(block_index).get_flip_bit() == true); - + const bool diff_flag = get_diff_flag(block_index); CHECK(diff_flag == true); @@ -3305,11 +3306,11 @@ namespace basisu // basisu only supports ETC1S, so these must be equal. CHECK(endpoint_cluster0_index == endpoint_cluster1_index); - + CHECK(blk.set_block_color5_check(get_endpoint_cluster_unscaled_color(endpoint_cluster0_index, false), get_endpoint_cluster_unscaled_color(endpoint_cluster1_index, false))); CHECK(get_endpoint_cluster_color_is_used(endpoint_cluster0_index, false)); - + blk.set_inten_table(0, get_endpoint_cluster_inten_table(endpoint_cluster0_index, false)); blk.set_inten_table(1, get_endpoint_cluster_inten_table(endpoint_cluster1_index, false)); @@ -3329,7 +3330,7 @@ namespace basisu CHECK(rdo_output_block.get_base5_color() == blk.get_base5_color()); CHECK(rdo_output_block.get_delta3_color() == blk.get_delta3_color()); CHECK(rdo_output_block.get_raw_selector_bits() == blk.get_raw_selector_bits()); - + #undef CHECK } @@ -3376,9 +3377,10 @@ namespace basisu } image img; - g.unpack(img); + g.unpack(img, false); save_png(pFilename, img); } } // namespace basisu + diff --git a/external/basis_universal/encoder/basisu_frontend.h b/external/basis_universal/encoder/basisu_frontend.h index 18ff5b6675..a5aadb34ac 100644 --- a/external/basis_universal/encoder/basisu_frontend.h +++ b/external/basis_universal/encoder/basisu_frontend.h @@ -1,5 +1,5 @@ // basisu_frontend.h -// Copyright (C) 2019-2024 Binomial LLC. All Rights Reserved. +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -37,8 +37,10 @@ namespace basisu uint32_t &operator[] (uint32_t i) { assert(i < 2); return m_comps[i]; } }; - const uint32_t BASISU_DEFAULT_COMPRESSION_LEVEL = 2; - const uint32_t BASISU_MAX_COMPRESSION_LEVEL = 6; + // rg [11/25/25] - The command line tool defaults to ETC1S level 1, but the API 2. Changing this breaks backwards compatibility for anyone using the API and our test suite. + const uint32_t BASISU_DEFAULT_ETC1S_COMPRESSION_LEVEL = 2; + + const uint32_t BASISU_MAX_ETC1S_COMPRESSION_LEVEL = 6; class basisu_frontend { @@ -61,7 +63,7 @@ namespace basisu enum { cMaxEndpointClusters = 16128, - + cMaxSelectorClusters = 16128, }; @@ -72,7 +74,7 @@ namespace basisu m_pSource_blocks(NULL), m_max_endpoint_clusters(256), m_max_selector_clusters(256), - m_compression_level(BASISU_DEFAULT_COMPRESSION_LEVEL), + m_compression_level(BASISU_DEFAULT_ETC1S_COMPRESSION_LEVEL), m_perceptual(true), m_debug_stats(false), m_debug_images(false), @@ -101,12 +103,12 @@ namespace basisu bool m_validate; bool m_multithreaded; bool m_disable_hierarchical_endpoint_codebooks; - + basist::basis_texture_type m_tex_type; const basist::basisu_lowlevel_etc1s_transcoder *m_pGlobal_codebooks; - + opencl_context_ptr m_pOpenCL_context; - + job_pool *m_pJob_pool; }; @@ -143,12 +145,12 @@ namespace basisu uint32_t get_total_selector_clusters() const { return static_cast(m_selector_cluster_block_indices.size()); } uint32_t get_block_selector_cluster_index(uint32_t block_index) const { return m_block_selector_cluster_index[block_index]; } const etc_block &get_selector_cluster_selector_bits(uint32_t cluster_index) const { return m_optimized_cluster_selectors[cluster_index]; } - + // Returns block indices using each selector cluster const uint_vec &get_selector_cluster_block_indices(uint32_t selector_cluster_index) const { return m_selector_cluster_block_indices[selector_cluster_index]; } void dump_debug_image(const char *pFilename, uint32_t first_block, uint32_t num_blocks_x, uint32_t num_blocks_y, bool output_blocks); - + void reoptimize_remapped_endpoints(const uint_vec &new_block_endpoints, int_vec &old_to_new_endpoint_cluster_indices, bool optimize_final_codebook, uint_vec *pBlock_selector_indices = nullptr); bool get_opencl_failed() const { return m_opencl_failed; } @@ -170,15 +172,15 @@ namespace basisu // The quantized ETC1S texture. etc_block_vec m_encoded_blocks; - + // Quantized blocks after endpoint quant, but before selector quant - etc_block_vec m_orig_encoded_blocks; - + etc_block_vec m_orig_encoded_blocks; + // Full quality ETC1S texture etc_block_vec m_etc1_blocks_etc1s; - + typedef vec<6, float> vec6F; - + // Endpoint clusterizer typedef tree_vector_quant vec6F_quantizer; vec6F_quantizer m_endpoint_clusterizer; @@ -187,16 +189,16 @@ namespace basisu basisu::vector m_endpoint_clusters; // Array of subblock indices for each parent endpoint cluster - // Note: Initially, each endpoint cluster will only live in a single parent cluster, in a shallow tree. + // Note: Initially, each endpoint cluster will only live in a single parent cluster, in a shallow tree. // As the endpoint clusters are manipulated this constraint gets broken. basisu::vector m_endpoint_parent_clusters; - + // Each block's parent endpoint cluster index - uint8_vec m_block_parent_endpoint_cluster; + uint8_vec m_block_parent_endpoint_cluster; // Array of endpoint cluster indices for each parent endpoint cluster basisu::vector m_endpoint_clusters_within_each_parent_cluster; - + struct endpoint_cluster_etc_params { endpoint_cluster_etc_params() @@ -266,13 +268,13 @@ namespace basisu }; typedef basisu::vector cluster_subblock_etc_params_vec; - - // Each endpoint cluster's ETC1S parameters + + // Each endpoint cluster's ETC1S parameters cluster_subblock_etc_params_vec m_endpoint_cluster_etc_params; // The endpoint cluster index used by each ETC1 subblock. basisu::vector m_block_endpoint_clusters_indices; - + // The block(s) within each selector cluster // Note: If you add anything here that uses selector cluster indicies, be sure to update optimize_selector_codebook()! basisu::vector m_selector_cluster_block_indices; @@ -282,13 +284,13 @@ namespace basisu // The block(s) within each parent selector cluster. basisu::vector m_selector_parent_cluster_block_indices; - + // Each block's parent selector cluster uint8_vec m_block_parent_selector_cluster; // Array of selector cluster indices for each parent selector cluster basisu::vector m_selector_clusters_within_each_parent_cluster; - + // Each block's selector cluster index basisu::vector m_block_selector_cluster_index; diff --git a/external/basis_universal/encoder/basisu_gpu_texture.cpp b/external/basis_universal/encoder/basisu_gpu_texture.cpp index 983d6a868d..59a2a174d9 100644 --- a/external/basis_universal/encoder/basisu_gpu_texture.cpp +++ b/external/basis_universal/encoder/basisu_gpu_texture.cpp @@ -1,5 +1,5 @@ // basisu_gpu_texture.cpp -// Copyright (C) 2019-2024 Binomial LLC. All Rights Reserved. +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -29,13 +29,13 @@ namespace basisu //------------------------------------------------------------------------------------------------ // ETC2 EAC - void unpack_etc2_eac(const void *pBlock_bits, color_rgba *pPixels) + void unpack_etc2_eac(const void* pBlock_bits, color_rgba* pPixels) { static_assert(sizeof(eac_a8_block) == 8, "sizeof(eac_a8_block) == 8"); - const eac_a8_block *pBlock = static_cast(pBlock_bits); + const eac_a8_block* pBlock = static_cast(pBlock_bits); - const int8_t *pTable = g_etc2_eac_tables[pBlock->m_table]; + const int8_t* pTable = g_etc2_eac_tables[pBlock->m_table]; const uint64_t selector_bits = pBlock->get_selector_bits(); @@ -73,10 +73,10 @@ namespace basisu uint8_t m_high_color[cTotalEndpointBytes]; uint8_t m_selectors[cTotalSelectorBytes]; - inline uint32_t get_high_color() const { return m_high_color[0] | (m_high_color[1] << 8U); } + inline uint32_t get_high_color() const { return m_high_color[0] | (m_high_color[1] << 8U); } inline uint32_t get_low_color() const { return m_low_color[0] | (m_low_color[1] << 8U); } - static void unpack_color(uint32_t c, uint32_t &r, uint32_t &g, uint32_t &b) + static void unpack_color(uint32_t c, uint32_t& r, uint32_t& g, uint32_t& b) { r = (c >> 11) & 31; g = (c >> 5) & 63; @@ -91,11 +91,11 @@ namespace basisu }; // Returns true if the block uses 3 color punchthrough alpha mode. - bool unpack_bc1(const void *pBlock_bits, color_rgba *pPixels, bool set_alpha) + bool unpack_bc1(const void* pBlock_bits, color_rgba* pPixels, bool set_alpha) { static_assert(sizeof(bc1_block) == 8, "sizeof(bc1_block) == 8"); - const bc1_block *pBlock = static_cast(pBlock_bits); + const bc1_block* pBlock = static_cast(pBlock_bits); const uint32_t l = pBlock->get_low_color(); const uint32_t h = pBlock->get_high_color(); @@ -147,11 +147,11 @@ namespace basisu return used_punchthrough; } - bool unpack_bc1_nv(const void *pBlock_bits, color_rgba *pPixels, bool set_alpha) + bool unpack_bc1_nv(const void* pBlock_bits, color_rgba* pPixels, bool set_alpha) { static_assert(sizeof(bc1_block) == 8, "sizeof(bc1_block) == 8"); - const bc1_block *pBlock = static_cast(pBlock_bits); + const bc1_block* pBlock = static_cast(pBlock_bits); const uint32_t l = pBlock->get_low_color(); const uint32_t h = pBlock->get_high_color(); @@ -182,19 +182,19 @@ namespace basisu if (l > h) { c[2].r = (uint8_t)(((2 * r0 + r1) * 22) / 8); - c[2].g = (uint8_t)(((256 * c[0].g + gdiff/4 + 128 + gdiff * 80) / 256)); + c[2].g = (uint8_t)(((256 * c[0].g + gdiff / 4 + 128 + gdiff * 80) / 256)); c[2].b = (uint8_t)(((2 * b0 + b1) * 22) / 8); c[2].a = 0xFF; c[3].r = (uint8_t)(((2 * r1 + r0) * 22) / 8); - c[3].g = (uint8_t)((256 * c[1].g - gdiff/4 + 128 - gdiff * 80) / 256); + c[3].g = (uint8_t)((256 * c[1].g - gdiff / 4 + 128 - gdiff * 80) / 256); c[3].b = (uint8_t)(((2 * b1 + b0) * 22) / 8); c[3].a = 0xFF; } else { c[2].r = (uint8_t)(((r0 + r1) * 33) / 8); - c[2].g = (uint8_t)((256 * c[0].g + gdiff/4 + 128 + gdiff * 128) / 256); + c[2].g = (uint8_t)((256 * c[0].g + gdiff / 4 + 128 + gdiff * 128) / 256); c[2].b = (uint8_t)(((b0 + b1) * 33) / 8); c[2].a = 0xFF; @@ -229,9 +229,9 @@ namespace basisu static inline int interp_5_6_amd(int c0, int c1) { assert(c0 < 256 && c1 < 256); return (c0 * 43 + c1 * 21 + 32) >> 6; } static inline int interp_half_5_6_amd(int c0, int c1) { assert(c0 < 256 && c1 < 256); return (c0 + c1 + 1) >> 1; } - bool unpack_bc1_amd(const void *pBlock_bits, color_rgba *pPixels, bool set_alpha) + bool unpack_bc1_amd(const void* pBlock_bits, color_rgba* pPixels, bool set_alpha) { - const bc1_block *pBlock = static_cast(pBlock_bits); + const bc1_block* pBlock = static_cast(pBlock_bits); const uint32_t l = pBlock->get_low_color(); const uint32_t h = pBlock->get_high_color(); @@ -310,7 +310,7 @@ namespace basisu return (selector_bits >> (((y * 4) + x) * cBC4SelectorBits)) & (cMaxSelectorValues - 1); } - static inline uint32_t get_block_values6(uint8_t *pDst, uint32_t l, uint32_t h) + static inline uint32_t get_block_values6(uint8_t* pDst, uint32_t l, uint32_t h) { pDst[0] = static_cast(l); pDst[1] = static_cast(h); @@ -323,7 +323,7 @@ namespace basisu return 6; } - static inline uint32_t get_block_values8(uint8_t *pDst, uint32_t l, uint32_t h) + static inline uint32_t get_block_values8(uint8_t* pDst, uint32_t l, uint32_t h) { pDst[0] = static_cast(l); pDst[1] = static_cast(h); @@ -336,7 +336,7 @@ namespace basisu return 8; } - static inline uint32_t get_block_values(uint8_t *pDst, uint32_t l, uint32_t h) + static inline uint32_t get_block_values(uint8_t* pDst, uint32_t l, uint32_t h) { if (l > h) return get_block_values8(pDst, l, h); @@ -345,11 +345,11 @@ namespace basisu } }; - void unpack_bc4(const void *pBlock_bits, uint8_t *pPixels, uint32_t stride) + void unpack_bc4(const void* pBlock_bits, uint8_t* pPixels, uint32_t stride) { static_assert(sizeof(bc4_block) == 8, "sizeof(bc4_block) == 8"); - const bc4_block *pBlock = static_cast(pBlock_bits); + const bc4_block* pBlock = static_cast(pBlock_bits); uint8_t sel_values[8]; bc4_block::get_block_values(sel_values, pBlock->get_low_alpha(), pBlock->get_high_alpha()); @@ -366,11 +366,11 @@ namespace basisu } // Returns false if the block uses 3-color punchthrough alpha mode, which isn't supported on some GPU's for BC3. - bool unpack_bc3(const void *pBlock_bits, color_rgba *pPixels) + bool unpack_bc3(const void* pBlock_bits, color_rgba* pPixels) { bool success = true; - if (unpack_bc1((const uint8_t *)pBlock_bits + sizeof(bc4_block), pPixels, true)) + if (unpack_bc1((const uint8_t*)pBlock_bits + sizeof(bc4_block), pPixels, true)) success = false; unpack_bc4(pBlock_bits, &pPixels[0].a, sizeof(color_rgba)); @@ -379,10 +379,10 @@ namespace basisu } // writes RG - void unpack_bc5(const void *pBlock_bits, color_rgba *pPixels) + void unpack_bc5(const void* pBlock_bits, color_rgba* pPixels) { unpack_bc4(pBlock_bits, &pPixels[0].r, sizeof(color_rgba)); - unpack_bc4((const uint8_t *)pBlock_bits + sizeof(bc4_block), &pPixels[0].g, sizeof(color_rgba)); + unpack_bc4((const uint8_t*)pBlock_bits + sizeof(bc4_block), &pPixels[0].g, sizeof(color_rgba)); } //------------------------------------------------------------------------------------------------ @@ -439,323 +439,6 @@ namespace basisu } } - //------------------------------------------------------------------------------------------------ - // BC7 mode 0-7 decompression. - // Instead of one monster routine to unpack all the BC7 modes, we're lumping the 3 subset, 2 subset, 1 subset, and dual plane modes together into simple shared routines. - - static inline uint32_t bc7_dequant(uint32_t val, uint32_t pbit, uint32_t val_bits) { assert(val < (1U << val_bits)); assert(pbit < 2); assert(val_bits >= 4 && val_bits <= 8); const uint32_t total_bits = val_bits + 1; val = (val << 1) | pbit; val <<= (8 - total_bits); val |= (val >> total_bits); assert(val <= 255); return val; } - static inline uint32_t bc7_dequant(uint32_t val, uint32_t val_bits) { assert(val < (1U << val_bits)); assert(val_bits >= 4 && val_bits <= 8); val <<= (8 - val_bits); val |= (val >> val_bits); assert(val <= 255); return val; } - - static inline uint32_t bc7_interp2(uint32_t l, uint32_t h, uint32_t w) { assert(w < 4); return (l * (64 - basist::g_bc7_weights2[w]) + h * basist::g_bc7_weights2[w] + 32) >> 6; } - static inline uint32_t bc7_interp3(uint32_t l, uint32_t h, uint32_t w) { assert(w < 8); return (l * (64 - basist::g_bc7_weights3[w]) + h * basist::g_bc7_weights3[w] + 32) >> 6; } - static inline uint32_t bc7_interp4(uint32_t l, uint32_t h, uint32_t w) { assert(w < 16); return (l * (64 - basist::g_bc7_weights4[w]) + h * basist::g_bc7_weights4[w] + 32) >> 6; } - static inline uint32_t bc7_interp(uint32_t l, uint32_t h, uint32_t w, uint32_t bits) - { - assert(l <= 255 && h <= 255); - switch (bits) - { - case 2: return bc7_interp2(l, h, w); - case 3: return bc7_interp3(l, h, w); - case 4: return bc7_interp4(l, h, w); - default: - break; - } - return 0; - } - - bool unpack_bc7_mode0_2(uint32_t mode, const void* pBlock_bits, color_rgba* pPixels) - { - //const uint32_t SUBSETS = 3; - const uint32_t ENDPOINTS = 6; - const uint32_t COMPS = 3; - const uint32_t WEIGHT_BITS = (mode == 0) ? 3 : 2; - const uint32_t ENDPOINT_BITS = (mode == 0) ? 4 : 5; - const uint32_t PBITS = (mode == 0) ? 6 : 0; - const uint32_t WEIGHT_VALS = 1 << WEIGHT_BITS; - - uint32_t bit_offset = 0; - const uint8_t* pBuf = static_cast(pBlock_bits); - - if (read_bits32(pBuf, bit_offset, mode + 1) != (1U << mode)) return false; - - const uint32_t part = read_bits32(pBuf, bit_offset, (mode == 0) ? 4 : 6); - - color_rgba endpoints[ENDPOINTS]; - for (uint32_t c = 0; c < COMPS; c++) - for (uint32_t e = 0; e < ENDPOINTS; e++) - endpoints[e][c] = (uint8_t)read_bits32(pBuf, bit_offset, ENDPOINT_BITS); - - uint32_t pbits[6]; - for (uint32_t p = 0; p < PBITS; p++) - pbits[p] = read_bits32(pBuf, bit_offset, 1); - - uint32_t weights[16]; - for (uint32_t i = 0; i < 16; i++) - weights[i] = read_bits32(pBuf, bit_offset, ((!i) || (i == basist::g_bc7_table_anchor_index_third_subset_1[part]) || (i == basist::g_bc7_table_anchor_index_third_subset_2[part])) ? (WEIGHT_BITS - 1) : WEIGHT_BITS); - - assert(bit_offset == 128); - - for (uint32_t e = 0; e < ENDPOINTS; e++) - for (uint32_t c = 0; c < 4; c++) - endpoints[e][c] = (uint8_t)((c == 3) ? 255 : (PBITS ? bc7_dequant(endpoints[e][c], pbits[e], ENDPOINT_BITS) : bc7_dequant(endpoints[e][c], ENDPOINT_BITS))); - - color_rgba block_colors[3][8]; - for (uint32_t s = 0; s < 3; s++) - for (uint32_t i = 0; i < WEIGHT_VALS; i++) - { - for (uint32_t c = 0; c < 3; c++) - block_colors[s][i][c] = (uint8_t)bc7_interp(endpoints[s * 2 + 0][c], endpoints[s * 2 + 1][c], i, WEIGHT_BITS); - block_colors[s][i][3] = 255; - } - - for (uint32_t i = 0; i < 16; i++) - pPixels[i] = block_colors[basist::g_bc7_partition3[part * 16 + i]][weights[i]]; - - return true; - } - - bool unpack_bc7_mode1_3_7(uint32_t mode, const void* pBlock_bits, color_rgba* pPixels) - { - //const uint32_t SUBSETS = 2; - const uint32_t ENDPOINTS = 4; - const uint32_t COMPS = (mode == 7) ? 4 : 3; - const uint32_t WEIGHT_BITS = (mode == 1) ? 3 : 2; - const uint32_t ENDPOINT_BITS = (mode == 7) ? 5 : ((mode == 1) ? 6 : 7); - const uint32_t PBITS = (mode == 1) ? 2 : 4; - const uint32_t SHARED_PBITS = (mode == 1) ? true : false; - const uint32_t WEIGHT_VALS = 1 << WEIGHT_BITS; - - uint32_t bit_offset = 0; - const uint8_t* pBuf = static_cast(pBlock_bits); - - if (read_bits32(pBuf, bit_offset, mode + 1) != (1U << mode)) return false; - - const uint32_t part = read_bits32(pBuf, bit_offset, 6); - - color_rgba endpoints[ENDPOINTS]; - for (uint32_t c = 0; c < COMPS; c++) - for (uint32_t e = 0; e < ENDPOINTS; e++) - endpoints[e][c] = (uint8_t)read_bits32(pBuf, bit_offset, ENDPOINT_BITS); - - uint32_t pbits[4]; - for (uint32_t p = 0; p < PBITS; p++) - pbits[p] = read_bits32(pBuf, bit_offset, 1); - - uint32_t weights[16]; - for (uint32_t i = 0; i < 16; i++) - weights[i] = read_bits32(pBuf, bit_offset, ((!i) || (i == basist::g_bc7_table_anchor_index_second_subset[part])) ? (WEIGHT_BITS - 1) : WEIGHT_BITS); - - assert(bit_offset == 128); - - for (uint32_t e = 0; e < ENDPOINTS; e++) - for (uint32_t c = 0; c < 4; c++) - endpoints[e][c] = (uint8_t)((c == ((mode == 7U) ? 4U : 3U)) ? 255 : bc7_dequant(endpoints[e][c], pbits[SHARED_PBITS ? (e >> 1) : e], ENDPOINT_BITS)); - - color_rgba block_colors[2][8]; - for (uint32_t s = 0; s < 2; s++) - for (uint32_t i = 0; i < WEIGHT_VALS; i++) - { - for (uint32_t c = 0; c < COMPS; c++) - block_colors[s][i][c] = (uint8_t)bc7_interp(endpoints[s * 2 + 0][c], endpoints[s * 2 + 1][c], i, WEIGHT_BITS); - block_colors[s][i][3] = (COMPS == 3) ? 255 : block_colors[s][i][3]; - } - - for (uint32_t i = 0; i < 16; i++) - pPixels[i] = block_colors[basist::g_bc7_partition2[part * 16 + i]][weights[i]]; - - return true; - } - - bool unpack_bc7_mode4_5(uint32_t mode, const void* pBlock_bits, color_rgba* pPixels) - { - const uint32_t ENDPOINTS = 2; - const uint32_t COMPS = 4; - const uint32_t WEIGHT_BITS = 2; - const uint32_t A_WEIGHT_BITS = (mode == 4) ? 3 : 2; - const uint32_t ENDPOINT_BITS = (mode == 4) ? 5 : 7; - const uint32_t A_ENDPOINT_BITS = (mode == 4) ? 6 : 8; - //const uint32_t WEIGHT_VALS = 1 << WEIGHT_BITS; - //const uint32_t A_WEIGHT_VALS = 1 << A_WEIGHT_BITS; - - uint32_t bit_offset = 0; - const uint8_t* pBuf = static_cast(pBlock_bits); - - if (read_bits32(pBuf, bit_offset, mode + 1) != (1U << mode)) return false; - - const uint32_t comp_rot = read_bits32(pBuf, bit_offset, 2); - const uint32_t index_mode = (mode == 4) ? read_bits32(pBuf, bit_offset, 1) : 0; - - color_rgba endpoints[ENDPOINTS]; - for (uint32_t c = 0; c < COMPS; c++) - for (uint32_t e = 0; e < ENDPOINTS; e++) - endpoints[e][c] = (uint8_t)read_bits32(pBuf, bit_offset, (c == 3) ? A_ENDPOINT_BITS : ENDPOINT_BITS); - - const uint32_t weight_bits[2] = { index_mode ? A_WEIGHT_BITS : WEIGHT_BITS, index_mode ? WEIGHT_BITS : A_WEIGHT_BITS }; - - uint32_t weights[16], a_weights[16]; - - for (uint32_t i = 0; i < 16; i++) - (index_mode ? a_weights : weights)[i] = read_bits32(pBuf, bit_offset, weight_bits[index_mode] - ((!i) ? 1 : 0)); - - for (uint32_t i = 0; i < 16; i++) - (index_mode ? weights : a_weights)[i] = read_bits32(pBuf, bit_offset, weight_bits[1 - index_mode] - ((!i) ? 1 : 0)); - - assert(bit_offset == 128); - - for (uint32_t e = 0; e < ENDPOINTS; e++) - for (uint32_t c = 0; c < 4; c++) - endpoints[e][c] = (uint8_t)bc7_dequant(endpoints[e][c], (c == 3) ? A_ENDPOINT_BITS : ENDPOINT_BITS); - - color_rgba block_colors[8]; - for (uint32_t i = 0; i < (1U << weight_bits[0]); i++) - for (uint32_t c = 0; c < 3; c++) - block_colors[i][c] = (uint8_t)bc7_interp(endpoints[0][c], endpoints[1][c], i, weight_bits[0]); - - for (uint32_t i = 0; i < (1U << weight_bits[1]); i++) - block_colors[i][3] = (uint8_t)bc7_interp(endpoints[0][3], endpoints[1][3], i, weight_bits[1]); - - for (uint32_t i = 0; i < 16; i++) - { - pPixels[i] = block_colors[weights[i]]; - pPixels[i].a = block_colors[a_weights[i]].a; - if (comp_rot >= 1) - std::swap(pPixels[i].a, pPixels[i].m_comps[comp_rot - 1]); - } - - return true; - } - - struct bc7_mode_6 - { - struct - { - uint64_t m_mode : 7; - uint64_t m_r0 : 7; - uint64_t m_r1 : 7; - uint64_t m_g0 : 7; - uint64_t m_g1 : 7; - uint64_t m_b0 : 7; - uint64_t m_b1 : 7; - uint64_t m_a0 : 7; - uint64_t m_a1 : 7; - uint64_t m_p0 : 1; - } m_lo; - - union - { - struct - { - uint64_t m_p1 : 1; - uint64_t m_s00 : 3; - uint64_t m_s10 : 4; - uint64_t m_s20 : 4; - uint64_t m_s30 : 4; - - uint64_t m_s01 : 4; - uint64_t m_s11 : 4; - uint64_t m_s21 : 4; - uint64_t m_s31 : 4; - - uint64_t m_s02 : 4; - uint64_t m_s12 : 4; - uint64_t m_s22 : 4; - uint64_t m_s32 : 4; - - uint64_t m_s03 : 4; - uint64_t m_s13 : 4; - uint64_t m_s23 : 4; - uint64_t m_s33 : 4; - - } m_hi; - - uint64_t m_hi_bits; - }; - }; - - bool unpack_bc7_mode6(const void *pBlock_bits, color_rgba *pPixels) - { - static_assert(sizeof(bc7_mode_6) == 16, "sizeof(bc7_mode_6) == 16"); - - const bc7_mode_6 &block = *static_cast(pBlock_bits); - - if (block.m_lo.m_mode != (1 << 6)) - return false; - - const uint32_t r0 = (uint32_t)((block.m_lo.m_r0 << 1) | block.m_lo.m_p0); - const uint32_t g0 = (uint32_t)((block.m_lo.m_g0 << 1) | block.m_lo.m_p0); - const uint32_t b0 = (uint32_t)((block.m_lo.m_b0 << 1) | block.m_lo.m_p0); - const uint32_t a0 = (uint32_t)((block.m_lo.m_a0 << 1) | block.m_lo.m_p0); - const uint32_t r1 = (uint32_t)((block.m_lo.m_r1 << 1) | block.m_hi.m_p1); - const uint32_t g1 = (uint32_t)((block.m_lo.m_g1 << 1) | block.m_hi.m_p1); - const uint32_t b1 = (uint32_t)((block.m_lo.m_b1 << 1) | block.m_hi.m_p1); - const uint32_t a1 = (uint32_t)((block.m_lo.m_a1 << 1) | block.m_hi.m_p1); - - color_rgba vals[16]; - for (uint32_t i = 0; i < 16; i++) - { - const uint32_t w = basist::g_bc7_weights4[i]; - const uint32_t iw = 64 - w; - vals[i].set_noclamp_rgba( - (r0 * iw + r1 * w + 32) >> 6, - (g0 * iw + g1 * w + 32) >> 6, - (b0 * iw + b1 * w + 32) >> 6, - (a0 * iw + a1 * w + 32) >> 6); - } - - pPixels[0] = vals[block.m_hi.m_s00]; - pPixels[1] = vals[block.m_hi.m_s10]; - pPixels[2] = vals[block.m_hi.m_s20]; - pPixels[3] = vals[block.m_hi.m_s30]; - - pPixels[4] = vals[block.m_hi.m_s01]; - pPixels[5] = vals[block.m_hi.m_s11]; - pPixels[6] = vals[block.m_hi.m_s21]; - pPixels[7] = vals[block.m_hi.m_s31]; - - pPixels[8] = vals[block.m_hi.m_s02]; - pPixels[9] = vals[block.m_hi.m_s12]; - pPixels[10] = vals[block.m_hi.m_s22]; - pPixels[11] = vals[block.m_hi.m_s32]; - - pPixels[12] = vals[block.m_hi.m_s03]; - pPixels[13] = vals[block.m_hi.m_s13]; - pPixels[14] = vals[block.m_hi.m_s23]; - pPixels[15] = vals[block.m_hi.m_s33]; - - return true; - } - - bool unpack_bc7(const void *pBlock, color_rgba *pPixels) - { - const uint32_t first_byte = static_cast(pBlock)[0]; - - for (uint32_t mode = 0; mode <= 7; mode++) - { - if (first_byte & (1U << mode)) - { - switch (mode) - { - case 0: - case 2: - return unpack_bc7_mode0_2(mode, pBlock, pPixels); - case 1: - case 3: - case 7: - return unpack_bc7_mode1_3_7(mode, pBlock, pPixels); - case 4: - case 5: - return unpack_bc7_mode4_5(mode, pBlock, pPixels); - case 6: - return unpack_bc7_mode6(pBlock, pPixels); - default: - break; - } - } - } - - return false; - } - static inline int bc6h_sign_extend(int val, int bits) { assert((bits >= 1) && (bits < 32)); @@ -1105,7 +788,7 @@ namespace basisu return false; if (pBlock->m_hi.m_alpha == 1) return false; - + color_rgba colors[4]; colors[0].r = pBlock->m_hi.m_r0; @@ -1155,7 +838,7 @@ namespace basisu for (uint32_t i = 0; i < 16; i++) { const uint32_t sel = (pBlock->m_sels[4 + (i >> 2)] >> ((i & 3) * 2)) & 3; - + const uint32_t x = i & 3; const uint32_t y = i >> 2; pPixels[4 + x + y * 8] = block1_colors[sel]; @@ -1216,7 +899,7 @@ namespace basisu { return color_rgba((col[0] << 3) | (col[0] >> 2), (col[1] << 3) | (col[1] >> 2), (col[2] << 3) | (col[2] >> 2), 255); } - + static color_rgba convert_rgba_5554_to_8888(const color_rgba& col) { return color_rgba((col[0] << 3) | (col[0] >> 2), (col[1] << 3) | (col[1] >> 2), (col[2] << 3) | (col[2] >> 2), (col[3] << 4) | col[3]); @@ -1239,10 +922,10 @@ namespace basisu { // colora=554 color_rgba color_a(pBlock->m_opaque_color_data.m_red_a, pBlock->m_opaque_color_data.m_green_a, (pBlock->m_opaque_color_data.m_blue_a << 1) | (pBlock->m_opaque_color_data.m_blue_a >> 3), 255); - + // colora=555 color_rgba color_b(pBlock->m_opaque_color_data.m_red_b, pBlock->m_opaque_color_data.m_green_b, pBlock->m_opaque_color_data.m_blue_b, 255); - + colors[0] = convert_rgb_555_to_888(color_a); colors[3] = convert_rgb_555_to_888(color_b); @@ -1251,11 +934,11 @@ namespace basisu } else { - // colora=4433 + // colora=4433 color_rgba color_a( - (pBlock->m_trans_color_data.m_red_a << 1) | (pBlock->m_trans_color_data.m_red_a >> 3), + (pBlock->m_trans_color_data.m_red_a << 1) | (pBlock->m_trans_color_data.m_red_a >> 3), (pBlock->m_trans_color_data.m_green_a << 1) | (pBlock->m_trans_color_data.m_green_a >> 3), - (pBlock->m_trans_color_data.m_blue_a << 2) | (pBlock->m_trans_color_data.m_blue_a >> 1), + (pBlock->m_trans_color_data.m_blue_a << 2) | (pBlock->m_trans_color_data.m_blue_a >> 1), pBlock->m_trans_color_data.m_alpha_a << 1); //colorb=4443 @@ -1331,9 +1014,9 @@ namespace basisu for (uint32_t x = 0; x < 4; x++) { const uint32_t shift = 45 - ((y + x * 4) * 3); - + const uint32_t sel = (uint32_t)((sels >> shift) & 7); - + int val = base + g_etc2_eac_tables[table][sel] * mul; val = clamp(val, 0, 2047); @@ -1362,9 +1045,10 @@ namespace basisu { basist::unpack_uastc(*static_cast(p), (basist::color32 *)pPixels, false); } - + // Unpacks to RGBA, R, RG, or A. LDR GPU texture formats only. - bool unpack_block(texture_format fmt, const void* pBlock, color_rgba* pPixels) + // astc_srgb: if true, ASTC LDR formats are decoded in sRGB decode mode, otherwise L8. + bool unpack_block(texture_format fmt, const void* pBlock, color_rgba* pPixels, bool astc_srgb) { switch (fmt) { @@ -1400,7 +1084,7 @@ namespace basisu } case texture_format::cBC7: { - return unpack_bc7(pBlock, pPixels); + return basist::bc7u::unpack_bc7(pBlock, reinterpret_cast(pPixels)); } // Full ETC2 color blocks (planar/T/H modes) is currently unsupported in basisu, but we do support ETC2 with alpha (using ETC1 for color) case texture_format::cETC2_RGB: @@ -1433,14 +1117,32 @@ namespace basisu return false; } case texture_format::cASTC_LDR_4x4: - { - const bool astc_srgb = false; - bool status = basisu_astc::astc::decompress_ldr(reinterpret_cast(pPixels), static_cast(pBlock), astc_srgb, 4, 4); + case texture_format::cASTC_LDR_5x4: + case texture_format::cASTC_LDR_5x5: + case texture_format::cASTC_LDR_6x5: + case texture_format::cASTC_LDR_6x6: + case texture_format::cASTC_LDR_8x5: + case texture_format::cASTC_LDR_8x6: + case texture_format::cASTC_LDR_10x5: + case texture_format::cASTC_LDR_10x6: + case texture_format::cASTC_LDR_8x8: + case texture_format::cASTC_LDR_10x8: + case texture_format::cASTC_LDR_10x10: + case texture_format::cASTC_LDR_12x10: + case texture_format::cASTC_LDR_12x12: + { + const uint32_t block_width = get_block_width(fmt), block_height = get_block_height(fmt); + + assert(get_astc_ldr_texture_format(block_width, block_height) == fmt); + assert(astc_helpers::is_valid_block_size(block_width, block_height)); + + // TODO: Allow caller to use the Android decoder, too. + bool status = basisu_astc::astc::decompress_ldr(reinterpret_cast(pPixels), static_cast(pBlock), astc_srgb, block_width, block_height); assert(status); if (!status) return false; - + break; } case texture_format::cATC_RGB: @@ -1532,7 +1234,7 @@ namespace basisu #else // Use our decoder basist::half_float half_block[16][4]; - + astc_helpers::log_astc_block log_blk; if (!astc_helpers::unpack_block(pBlock, log_blk, 4, 4)) return false; @@ -1577,8 +1279,8 @@ namespace basisu assert(0); return false; } - - bool gpu_image::unpack(image& img) const + + bool gpu_image::unpack(image& img, bool astc_srgb) const { img.resize(get_pixel_width(), get_pixel_height()); img.set_all(g_black_color); @@ -1589,11 +1291,11 @@ namespace basisu if ((m_fmt == texture_format::cPVRTC1_4_RGB) || (m_fmt == texture_format::cPVRTC1_4_RGBA)) { pvrtc4_image pi(m_width, m_height); - + if (get_total_blocks() != pi.get_total_blocks()) return false; - - memcpy(&pi.get_blocks()[0], get_ptr(), get_size_in_bytes()); + + memcpy((void *)&pi.get_blocks()[0], (const void *)get_ptr(), get_size_in_bytes()); pi.deswizzle(); @@ -1615,7 +1317,7 @@ namespace basisu { const void* pBlock = get_block_ptr(bx, by); - if (!unpack_block(m_fmt, pBlock, pixels)) + if (!unpack_block(m_fmt, pBlock, pixels, astc_srgb)) success = false; img.set_block_clipped(pixels, bx * m_block_width, by * m_block_height, m_block_width, m_block_height); @@ -1662,14 +1364,14 @@ namespace basisu return success; } - + // KTX1 texture file writing static const uint8_t g_ktx_file_id[12] = { 0xAB, 0x4B, 0x54, 0x58, 0x20, 0x31, 0x31, 0xBB, 0x0D, 0x0A, 0x1A, 0x0A }; // KTX/GL enums enum { - KTX_ENDIAN = 0x04030201, + KTX_ENDIAN = 0x04030201, KTX_OPPOSITE_ENDIAN = 0x01020304, KTX_ETC1_RGB8_OES = 0x8D64, KTX_RED = 0x1903, @@ -1689,7 +1391,7 @@ namespace basisu KTX_COMPRESSED_RGB_BPTC_UNSIGNED_FLOAT = 0x8E8F, KTX_COMPRESSED_RGB_PVRTC_4BPPV1_IMG = 0x8C00, KTX_COMPRESSED_RGBA_PVRTC_4BPPV1_IMG = 0x8C02, - + KTX_COMPRESSED_RGBA_ASTC_4x4_KHR = 0x93B0, KTX_COMPRESSED_RGBA_ASTC_5x4_KHR = 0x93B1, KTX_COMPRESSED_RGBA_ASTC_5x5_KHR = 0x93B2, @@ -1731,7 +1433,7 @@ namespace basisu KTX_COMPRESSED_R11_EAC = 0x9270, KTX_COMPRESSED_RG11_EAC = 0x9272 }; - + struct ktx_header { uint8_t m_identifier[12]; @@ -1753,7 +1455,7 @@ namespace basisu }; // Input is a texture array of mipmapped gpu_image's: gpu_images[array_index][level_index] - bool create_ktx_texture_file(uint8_vec &ktx_data, const basisu::vector& gpu_images, bool cubemap_flag) + bool create_ktx_texture_file(uint8_vec &ktx_data, const basisu::vector& gpu_images, bool cubemap_flag, bool astc_srgb_flag) { if (!gpu_images.size()) { @@ -1773,7 +1475,7 @@ namespace basisu return false; } } - + for (uint32_t array_index = 0; array_index < gpu_images.size(); array_index++) { const gpu_image_vec &levels = gpu_images[array_index]; @@ -1905,18 +1607,101 @@ namespace basisu { internal_fmt = KTX_COMPRESSED_RGBA_ASTC_6x6_KHR; // TODO: should we write RGB? We don't support generating HDR 6x6 with alpha. - base_internal_fmt = KTX_RGBA; + base_internal_fmt = KTX_RGBA; break; } // We use different enums for HDR vs. LDR ASTC, but internally they are both just ASTC. - case texture_format::cASTC_LDR_4x4: case texture_format::cASTC_HDR_4x4: - case texture_format::cUASTC_HDR_4x4: // UASTC_HDR is just HDR-only ASTC + case texture_format::cUASTC_HDR_4x4: // UASTC_HDR 4x4 is just HDR-only ASTC { internal_fmt = KTX_COMPRESSED_RGBA_ASTC_4x4_KHR; base_internal_fmt = KTX_RGBA; break; } + case texture_format::cASTC_LDR_4x4: + { + internal_fmt = !astc_srgb_flag ? KTX_COMPRESSED_RGBA_ASTC_4x4_KHR : KTX_COMPRESSED_SRGB8_ALPHA8_ASTC_4x4_KHR; + base_internal_fmt = KTX_RGBA; + break; + } + case texture_format::cASTC_LDR_5x4: + { + internal_fmt = !astc_srgb_flag ? KTX_COMPRESSED_RGBA_ASTC_5x4_KHR : KTX_COMPRESSED_SRGB8_ALPHA8_ASTC_5x4_KHR; + base_internal_fmt = KTX_RGBA; + break; + } + case texture_format::cASTC_LDR_5x5: + { + internal_fmt = !astc_srgb_flag ? KTX_COMPRESSED_RGBA_ASTC_5x5_KHR : KTX_COMPRESSED_SRGB8_ALPHA8_ASTC_5x5_KHR; + base_internal_fmt = KTX_RGBA; + break; + } + case texture_format::cASTC_LDR_6x5: + { + internal_fmt = !astc_srgb_flag ? KTX_COMPRESSED_RGBA_ASTC_6x5_KHR : KTX_COMPRESSED_SRGB8_ALPHA8_ASTC_6x5_KHR; + base_internal_fmt = KTX_RGBA; + break; + } + case texture_format::cASTC_LDR_6x6: + { + internal_fmt = !astc_srgb_flag ? KTX_COMPRESSED_RGBA_ASTC_6x6_KHR : KTX_COMPRESSED_SRGB8_ALPHA8_ASTC_6x6_KHR; + base_internal_fmt = KTX_RGBA; + break; + } + case texture_format::cASTC_LDR_8x5: + { + internal_fmt = !astc_srgb_flag ? KTX_COMPRESSED_RGBA_ASTC_8x5_KHR : KTX_COMPRESSED_SRGB8_ALPHA8_ASTC_8x5_KHR; + base_internal_fmt = KTX_RGBA; + break; + } + case texture_format::cASTC_LDR_8x6: + { + internal_fmt = !astc_srgb_flag ? KTX_COMPRESSED_RGBA_ASTC_8x6_KHR : KTX_COMPRESSED_SRGB8_ALPHA8_ASTC_8x6_KHR; + base_internal_fmt = KTX_RGBA; + break; + } + case texture_format::cASTC_LDR_10x5: + { + internal_fmt = !astc_srgb_flag ? KTX_COMPRESSED_RGBA_ASTC_10x5_KHR : KTX_COMPRESSED_SRGB8_ALPHA8_ASTC_10x5_KHR; + base_internal_fmt = KTX_RGBA; + break; + } + case texture_format::cASTC_LDR_10x6: + { + internal_fmt = !astc_srgb_flag ? KTX_COMPRESSED_RGBA_ASTC_10x6_KHR : KTX_COMPRESSED_SRGB8_ALPHA8_ASTC_10x6_KHR; + base_internal_fmt = KTX_RGBA; + break; + } + case texture_format::cASTC_LDR_8x8: + { + internal_fmt = !astc_srgb_flag ? KTX_COMPRESSED_RGBA_ASTC_8x8_KHR : KTX_COMPRESSED_SRGB8_ALPHA8_ASTC_8x8_KHR; + base_internal_fmt = KTX_RGBA; + break; + } + case texture_format::cASTC_LDR_10x8: + { + internal_fmt = !astc_srgb_flag ? KTX_COMPRESSED_RGBA_ASTC_10x8_KHR : KTX_COMPRESSED_SRGB8_ALPHA8_ASTC_10x8_KHR; + base_internal_fmt = KTX_RGBA; + break; + } + case texture_format::cASTC_LDR_10x10: + { + internal_fmt = !astc_srgb_flag ? KTX_COMPRESSED_RGBA_ASTC_10x10_KHR : KTX_COMPRESSED_SRGB8_ALPHA8_ASTC_10x10_KHR; + base_internal_fmt = KTX_RGBA; + break; + } + case texture_format::cASTC_LDR_12x10: + { + internal_fmt = !astc_srgb_flag ? KTX_COMPRESSED_RGBA_ASTC_12x10_KHR : KTX_COMPRESSED_SRGB8_ALPHA8_ASTC_12x10_KHR; + base_internal_fmt = KTX_RGBA; + break; + } + case texture_format::cASTC_LDR_12x12: + { + internal_fmt = !astc_srgb_flag ? KTX_COMPRESSED_RGBA_ASTC_12x12_KHR : KTX_COMPRESSED_SRGB8_ALPHA8_ASTC_12x12_KHR; + base_internal_fmt = KTX_RGBA; + break; + } case texture_format::cATC_RGB: { internal_fmt = KTX_ATC_RGB_AMD; @@ -1987,6 +1772,12 @@ namespace basisu append_vector(ktx_data, (uint8_t*)&header, sizeof(header)); + fmt_debug_printf("create_ktx_texture_file: {}x{}, astc_srgb_flag: {}, basis::texture_format: {}, internalFormat: {}, baseInternalFormat: {}, arrayElements: {}, faces: {}, mipLevels: {}\n", + width, height, astc_srgb_flag, (uint32_t)fmt, + (uint32_t)header.m_glInternalFormat, (uint32_t)header.m_glBaseInternalFormat, + (uint32_t)header.m_numberOfArrayElements, (uint32_t)header.m_numberOfFaces, + (uint32_t)header.m_numberOfMipmapLevels); + for (uint32_t level_index = 0; level_index < total_levels; level_index++) { uint32_t img_size = gpu_images[0][level_index].get_size_in_bytes(); @@ -2169,7 +1960,7 @@ namespace basisu } // array_index } #endif - + // Write DDS file using tinydds TinyDDS_WriteCallbacks cbs; cbs.error = [](void* user, char const* msg) { BASISU_NOTE_UNUSED(user); fprintf(stderr, "tinydds: %s\n", msg); }; @@ -2179,7 +1970,7 @@ namespace basisu uint32_t mipmap_sizes[32]; const void* mipmap_ptrs[32]; - + clear_obj(mipmap_sizes); clear_obj(mipmap_ptrs); @@ -2197,7 +1988,7 @@ namespace basisu { case texture_format::cBC1_NV: case texture_format::cBC1_AMD: - case texture_format::cBC1: + case texture_format::cBC1: tinydds_fmt = use_srgb_format ? TDDS_BC1_RGBA_SRGB_BLOCK : TDDS_BC1_RGBA_UNORM_BLOCK; break; case texture_format::cBC3: @@ -2225,10 +2016,10 @@ namespace basisu } } - // DirectXTex's DDSView doesn't handle odd sizes textures correctly. RenderDoc loads them fine, however. - // Trying to work around this here results in invalid mipmaps. - //width = (width + 3) & ~3; - //height = (height + 3) & ~3; + // Note DirectXTex's DDSView doesn't handle odd sizes textures correctly. RenderDoc loads them fine, however. + + fmt_debug_printf("write_dds_file: {}x{}, basis::texture_format: {}, tinydds_fmt: {}, slices: {}, mipLevels: {}, cubemap_flag: {}, use_srgb_format: {}\n", + width, height, (uint32_t)fmt, tinydds_fmt, slices, total_levels, cubemap_flag, use_srgb_format); bool status = TinyDDS_WriteImage(&cbs, &dds_data, @@ -2248,7 +2039,7 @@ namespace basisu fprintf(stderr, "write_dds_file: Failed creating DDS file\n"); return false; } - + return true; } @@ -2267,7 +2058,7 @@ namespace basisu return true; } - + bool read_uncompressed_dds_file(const char* pFilename, basisu::vector &ldr_mips, basisu::vector& hdr_mips) { const uint32_t MAX_IMAGE_DIM = 16384; @@ -2278,7 +2069,7 @@ namespace basisu cbs.allocFn = [](void* user, size_t size) -> void* { BASISU_NOTE_UNUSED(user); return malloc(size); }; cbs.freeFn = [](void* user, void* memory) { BASISU_NOTE_UNUSED(user); free(memory); }; cbs.readFn = [](void* user, void* buffer, size_t byteCount) -> size_t { return (size_t)fread(buffer, 1, byteCount, (FILE*)user); }; - + #ifdef _MSC_VER cbs.seekFn = [](void* user, int64_t ofs) -> bool { return _fseeki64((FILE*)user, ofs, SEEK_SET) == 0; }; cbs.tellFn = [](void* user) -> int64_t { return _ftelli64((FILE*)user); }; @@ -2318,7 +2109,7 @@ namespace basisu error_printf("Failed parsing DDS header in file \"%s\"\n", pFilename); goto failure; } - + if ((!TinyDDS_Is2D(ctx)) || (TinyDDS_ArraySlices(ctx) > 1) || (TinyDDS_IsCubemap(ctx))) { error_printf("Unsupported DDS texture type in file \"%s\"\n", pFilename); @@ -2327,7 +2118,7 @@ namespace basisu width = TinyDDS_Width(ctx); height = TinyDDS_Height(ctx); - + if (!width || !height) { error_printf("DDS texture dimensions invalid in file \"%s\"\n", pFilename); @@ -2339,7 +2130,7 @@ namespace basisu error_printf("DDS texture dimensions too large in file \"%s\"\n", pFilename); goto failure; } - + tfmt = TinyDDS_GetFormat(ctx); switch (tfmt) { @@ -2387,7 +2178,7 @@ namespace basisu } memcpy(ldr_mips[level].get_ptr(), pImage, image_size); - + if ((tfmt == TDDS_B8G8R8A8_SRGB) || (tfmt == TDDS_B8G8R8A8_UNORM)) { // Swap R and B components. @@ -2416,7 +2207,7 @@ namespace basisu else if (fmt == cRGBA_HALF) { hdr_mips[level].resize(level_width, level_height); - + if ((hdr_mips[level].get_total_pixels() * sizeof(basist::half_float) * 4 != image_size)) { assert(0); @@ -2426,7 +2217,7 @@ namespace basisu // Unpack half to float. const basist::half_float* pSrc_comps = static_cast(pImage); vec4F* pDst_texels = hdr_mips[level].get_ptr(); - + for (uint32_t i = 0; i < total_level_texels; i++) { (*pDst_texels)[0] = basist::half_to_float(pSrc_comps[0]); @@ -2462,7 +2253,7 @@ namespace basisu uint8_vec filedata; if (extension == "ktx") { - if (!create_ktx_texture_file(filedata, g, cubemap_flag)) + if (!create_ktx_texture_file(filedata, g, cubemap_flag, use_srgb_format)) return false; } else if (extension == "pvr") @@ -2500,7 +2291,7 @@ namespace basisu } //const uint32_t OUT_FILE_MAGIC = 'TEXC'; - struct out_file_header + struct out_file_header { packed_uint<4> m_magic; packed_uint<4> m_pad; @@ -2532,16 +2323,92 @@ namespace basisu fwrite(&hdr, sizeof(hdr), 1, pFile); fwrite(gi.get_ptr(), gi.get_size_in_bytes(), 1, pFile); - + return fclose(pFile) != EOF; } - // The .astc texture format is readable using ARM's astcenc, AMD Compressonator, and other engines/tools. It oddly doesn't support mipmaps, limiting +#pragma pack(push, 1) + struct astc_file_header + { + uint8_t m_sig[4]; + uint8_t m_block_dim[3]; + uint8_t m_width[3]; + uint8_t m_height[3]; + uint8_t m_depth[3]; + }; +#pragma pack(pop) + + bool read_astc_file(const uint8_t *pImage_data, size_t image_data_size, vector2D& blocks, uint32_t &block_width, uint32_t &block_height, uint32_t &width, uint32_t &height) + { + block_width = 0; + block_height = 0; + width = 0; + height = 0; + blocks.resize(0, 0); + + if (image_data_size < (sizeof(astc_file_header) + sizeof(astc_helpers::astc_block))) + return false; + + const astc_file_header* pHeader = reinterpret_cast(pImage_data); + + if ((pHeader->m_sig[0] != 0x13) || (pHeader->m_sig[1] != 0xAB) || (pHeader->m_sig[2] != 0xA1) || (pHeader->m_sig[3] != 0x5C)) + return false; + + const uint32_t block_depth = pHeader->m_block_dim[2]; + if (block_depth != 1) + return false; + + if ((pHeader->m_depth[0] != 1) || (pHeader->m_depth[1] != 0) || (pHeader->m_depth[2] != 0)) + return false; + + block_width = pHeader->m_block_dim[0]; + block_height = pHeader->m_block_dim[1]; + + if (!astc_helpers::is_valid_block_size(block_width, block_height)) + return false; + + width = pHeader->m_width[0] | ((uint32_t)pHeader->m_width[1] << 8u) | ((uint32_t)pHeader->m_width[2] << 16u); + height = pHeader->m_height[0] | ((uint32_t)pHeader->m_height[1] << 8u) | ((uint32_t)pHeader->m_height[2] << 16u); + + const uint32_t MAX_DIM = 32768; + if ((!width) || (width > MAX_DIM) || (!height) || (height > MAX_DIM)) + return false; + + const uint32_t num_blocks_x = (width + block_width - 1) / block_width; + const uint32_t num_blocks_y = (height + block_height - 1) / block_height; + const uint32_t total_blocks = num_blocks_x * num_blocks_y; + + size_t total_expected_size = sizeof(astc_file_header) + (size_t)total_blocks * sizeof(astc_helpers::astc_block); + if (image_data_size < total_expected_size) + return false; + + if (!blocks.try_resize(num_blocks_x, num_blocks_y)) + return false; + + memcpy(blocks.get_ptr(), pImage_data + sizeof(astc_file_header), (size_t)total_blocks * sizeof(astc_helpers::astc_block)); + + return true; + } + + bool read_astc_file(const char* pFilename, vector2D& blocks, uint32_t& block_width, uint32_t& block_height, uint32_t& width, uint32_t& height) + { + uint8_vec file_data; + if (!read_file_to_vec(pFilename, file_data)) + return false; + + if (!file_data.size()) + return false; + + return read_astc_file(file_data.get_ptr(), file_data.size(), blocks, block_width, block_height, width, height); + } + + // The .astc texture format is readable using ARM's astcenc, AMD Compressonator, and other engines/tools. It oddly doesn't support mipmaps, limiting // its usefulness/relevance. // https://github.com/ARM-software/astc-encoder/blob/main/Docs/FileFormat.md bool write_astc_file(const char* pFilename, const void* pBlocks, uint32_t block_width, uint32_t block_height, uint32_t dim_x, uint32_t dim_y) { - assert(pBlocks && (block_width >= 4) && (block_height >= 4) && (dim_x > 0) && (dim_y > 0)); + assert(pBlocks && (dim_x > 0) && (dim_y > 0)); + assert(astc_helpers::is_valid_block_size(block_width, block_height)); uint8_vec file_data; file_data.push_back(0x13); @@ -2578,5 +2445,6 @@ namespace basisu return write_vec_to_file(pFilename, file_data); } - + } // basisu + diff --git a/external/basis_universal/encoder/basisu_gpu_texture.h b/external/basis_universal/encoder/basisu_gpu_texture.h index 06f2cc09bf..bcfc9cb494 100644 --- a/external/basis_universal/encoder/basisu_gpu_texture.h +++ b/external/basis_universal/encoder/basisu_gpu_texture.h @@ -1,5 +1,5 @@ // basisu_gpu_texture.h -// Copyright (C) 2019-2024 Binomial LLC. All Rights Reserved. +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,6 +14,7 @@ // limitations under the License. #pragma once #include "../transcoder/basisu.h" +#include "../transcoder/basisu_astc_helpers.h" #include "basisu_etc.h" namespace basisu @@ -49,11 +50,12 @@ namespace basisu inline texture_format get_format() const { return m_fmt; } inline bool is_hdr() const { return is_hdr_texture_format(m_fmt); } - + inline bool is_ldr() const { return !is_hdr_texture_format(m_fmt); } + // Width/height in pixels inline uint32_t get_pixel_width() const { return m_width; } inline uint32_t get_pixel_height() const { return m_height; } - + // Width/height in blocks, row pitch is assumed to be m_blocks_x. inline uint32_t get_blocks_x() const { return m_blocks_x; } inline uint32_t get_blocks_y() const { return m_blocks_y; } @@ -68,7 +70,7 @@ namespace basisu inline uint32_t get_row_pitch_in_bytes() const { return get_bytes_per_block() * get_blocks_x(); } inline const uint64_vec &get_blocks() const { return m_blocks; } - + inline const uint64_t *get_ptr() const { return &m_blocks[0]; } inline uint64_t *get_ptr() { return &m_blocks[0]; } @@ -101,12 +103,14 @@ namespace basisu m_blocks.resize(m_blocks_x * m_blocks_y * m_qwords_per_block); } - // Unpacks LDR textures only. - bool unpack(image& img) const; + // Unpacks LDR textures only. Asserts and returns false otherwise. + // astc_srgb: true to use the ASTC sRGB decode profile, false for linear. + // For XUASTC LDR, this should match what was used during encoding. For ETC1S/UASTC LDR 4x4, this should be false. + bool unpack(image& img, bool astc_srgb) const; - // Unpacks HDR textures only. + // Unpacks HDR textures only. Asserts and returns false otherwise. bool unpack_hdr(imagef& img) const; - + inline void override_dimensions(uint32_t w, uint32_t h) { m_width = w; @@ -121,9 +125,10 @@ namespace basisu typedef basisu::vector gpu_image_vec; - // KTX1 file writing - bool create_ktx_texture_file(uint8_vec &ktx_data, const basisu::vector& gpu_images, bool cubemap_flag); - + // KTX1 file writing - compatible with ARM's astcenc tool, and some other tools. + // Note astc_linear_flag used to be always effectively true in older code. It's ignored for ASTC HDR formats. + bool create_ktx_texture_file(uint8_vec &ktx_data, const basisu::vector& gpu_images, bool cubemap_flag, bool astc_srgb_flag); + bool does_dds_support_format(texture_format fmt); bool write_dds_file(uint8_vec& dds_data, const basisu::vector& gpu_images, bool cubemap_flag, bool use_srgb_format); bool write_dds_file(const char* pFilename, const basisu::vector& gpu_images, bool cubemap_flag, bool use_srgb_format); @@ -135,7 +140,7 @@ namespace basisu bool write_compressed_texture_file(const char *pFilename, const basisu::vector& g, bool cubemap_flag, bool use_srgb_format); bool write_compressed_texture_file(const char* pFilename, const gpu_image_vec& g, bool use_srgb_format); bool write_compressed_texture_file(const char *pFilename, const gpu_image &g, bool use_srgb_format); - + bool write_3dfx_out_file(const char* pFilename, const gpu_image& gi); // GPU texture block unpacking @@ -145,8 +150,15 @@ namespace basisu void unpack_bc4(const void *pBlock_bits, uint8_t *pPixels, uint32_t stride); bool unpack_bc3(const void *pBlock_bits, color_rgba *pPixels); void unpack_bc5(const void *pBlock_bits, color_rgba *pPixels); + +#if 0 bool unpack_bc7_mode6(const void *pBlock_bits, color_rgba *pPixels); + int determine_bc7_mode(const void* pBlock); + int determine_bc7_mode_4_index_mode(const void* pBlock); + int determine_bc7_mode_4_or_5_rotation(const void* pBlock); bool unpack_bc7(const void* pBlock_bits, color_rgba* pPixels); // full format +#endif + bool unpack_bc6h(const void* pSrc_block, void* pDst_block, bool is_signed, uint32_t dest_pitch_in_halfs = 4 * 3); // full format, outputs HALF values, RGB texels only (not RGBA) void unpack_atc(const void* pBlock_bits, color_rgba* pPixels); // We only support CC_MIXED non-alpha blocks here because that's the only mode the transcoder uses at the moment. @@ -155,15 +167,18 @@ namespace basisu bool unpack_pvrtc2(const void* p, color_rgba* pPixels); void unpack_etc2_eac_r(const void *p, color_rgba* pPixels, uint32_t c); void unpack_etc2_eac_rg(const void* p, color_rgba* pPixels); - + // unpack_block() is primarily intended to unpack texture data created by the transcoder. // For some texture formats (like ETC2 RGB, PVRTC2, FXT1) it's not yet a complete implementation. // Unpacks LDR texture formats only. - bool unpack_block(texture_format fmt, const void *pBlock, color_rgba *pPixels); + bool unpack_block(texture_format fmt, const void *pBlock, color_rgba *pPixels, bool astc_srgb); // Unpacks HDR texture formats only. bool unpack_block_hdr(texture_format fmt, const void* pBlock, vec4F* pPixels); - + + bool read_astc_file(const uint8_t* pImage_data, size_t image_data_size, vector2D& blocks, uint32_t& block_width, uint32_t& block_height, uint32_t& width, uint32_t& height); + bool read_astc_file(const char* pFilename, vector2D& blocks, uint32_t& block_width, uint32_t& block_height, uint32_t& width, uint32_t& height); bool write_astc_file(const char* pFilename, const void* pBlocks, uint32_t block_width, uint32_t block_height, uint32_t dim_x, uint32_t dim_y); - + } // namespace basisu + diff --git a/external/basis_universal/encoder/basisu_math.h b/external/basis_universal/encoder/basisu_math.h index 66bb749a5b..24b83859ae 100644 --- a/external/basis_universal/encoder/basisu_math.h +++ b/external/basis_universal/encoder/basisu_math.h @@ -8,10 +8,10 @@ namespace bu_math // Would prefer using SSE1 etc. but that would require implementing multiple versions and platform divergence (needing more testing). BASISU_FORCE_INLINE float inv_sqrt(float v) { - union - { - float flt; - uint32_t ui; + union + { + float flt; + uint32_t ui; } un; un.flt = v; @@ -20,6 +20,16 @@ namespace bu_math return 0.703952253f * un.flt * (2.38924456f - v * (un.flt * un.flt)); } + inline float linstep(float edge0, float edge1, float x) + { + assert(edge1 != edge0); + + // Scale, and clamp x to 0..1 range + x = basisu::saturate((x - edge0) / (edge1 - edge0)); + + return x; + } + inline float smoothstep(float edge0, float edge1, float x) { assert(edge1 != edge0); @@ -1130,12 +1140,12 @@ namespace bu_math template Z& matrix_mul_helper(Z& result, const X& lhs, const Y& rhs) { - static_assert((int)Z::num_rows == (int)X::num_rows); - static_assert((int)Z::num_cols == (int)Y::num_cols); - static_assert((int)X::num_cols == (int)Y::num_rows); + static_assert(Z::num_rows == X::num_rows); + static_assert(Z::num_cols == Y::num_cols); + static_assert(X::num_cols == Y::num_rows); assert(((void*)&result != (void*)&lhs) && ((void*)&result != (void*)&rhs)); - for (int r = 0; r < X::num_rows; r++) - for (int c = 0; c < Y::num_cols; c++) + for (uint32_t r = 0; r < X::num_rows; r++) + for (uint32_t c = 0; c < Y::num_cols; c++) { typename Z::scalar_type s = lhs(r, 0) * rhs(0, c); for (uint32_t i = 1; i < X::num_cols; i++) @@ -1148,12 +1158,12 @@ namespace bu_math template Z& matrix_mul_helper_transpose_lhs(Z& result, const X& lhs, const Y& rhs) { - static_assert((int)Z::num_rows == (int)X::num_cols); - static_assert((int)Z::num_cols == (int)Y::num_cols); - static_assert((int)X::num_rows == (int)Y::num_rows); + static_assert(Z::num_rows == X::num_cols); + static_assert(Z::num_cols == Y::num_cols); + static_assert(X::num_rows == Y::num_rows); assert(((void*)&result != (void*)&lhs) && ((void*)&result != (void*)&rhs)); - for (int r = 0; r < X::num_cols; r++) - for (int c = 0; c < Y::num_cols; c++) + for (uint32_t r = 0; r < X::num_cols; r++) + for (uint32_t c = 0; c < Y::num_cols; c++) { typename Z::scalar_type s = lhs(0, r) * rhs(0, c); for (uint32_t i = 1; i < X::num_rows; i++) @@ -1166,12 +1176,12 @@ namespace bu_math template Z& matrix_mul_helper_transpose_rhs(Z& result, const X& lhs, const Y& rhs) { - static_assert((int)Z::num_rows == (int)X::num_rows); - static_assert((int)Z::num_cols == (int)Y::num_rows); - static_assert((int)X::num_cols == (int)Y::num_cols); + static_assert(Z::num_rows == X::num_rows); + static_assert(Z::num_cols == Y::num_rows); + static_assert(X::num_cols == Y::num_cols); assert(((void*)&result != (void*)&lhs) && ((void*)&result != (void*)&rhs)); - for (int r = 0; r < X::num_rows; r++) - for (int c = 0; c < Y::num_rows; c++) + for (uint32_t r = 0; r < X::num_rows; r++) + for (uint32_t c = 0; c < Y::num_rows; c++) { typename Z::scalar_type s = lhs(r, 0) * rhs(c, 0); for (uint32_t i = 1; i < X::num_cols; i++) @@ -1180,17 +1190,21 @@ namespace bu_math } return result; } - + template class matrix { public: typedef T scalar_type; + static const uint32_t num_rows = R; + static const uint32_t num_cols = C; +#if 0 enum { num_rows = R, num_cols = C }; +#endif typedef vec col_vec; typedef vec < (R > 1) ? (R - 1) : 0, T > subcol_vec; @@ -2144,7 +2158,7 @@ namespace bu_math static inline matrix make_tensor_product_matrix(const row_vec& v, const row_vec& w) { matrix ret; - for (int r = 0; r < num_rows; r++) + for (uint32_t r = 0; r < num_rows; r++) ret[r] = row_vec::mul_components(v.broadcast(r), w); return ret; } @@ -2485,6 +2499,31 @@ namespace basisu int64_t m_total2; }; + class tracked_stat_float + { + public: + tracked_stat_float() { clear(); } + + inline void clear() { m_num = 0; m_total = 0; m_total2 = 0; } + + inline void update(float val) { m_num++; m_total += val; m_total2 += val * val; } + + inline tracked_stat_float& operator += (float val) { update(val); return *this; } + + inline uint32_t get_number_of_values() { return m_num; } + inline float get_total() const { return m_total; } + inline float get_total2() const { return m_total2; } + + inline float get_average() const { return m_num ? m_total / (float)m_num : 0.0f; }; + inline float get_std_dev() const { return m_num ? sqrt((float)(m_num * m_total2 - m_total * m_total)) / m_num : 0.0f; } + inline float get_variance() const { float s = get_std_dev(); return s * s; } + + private: + uint32_t m_num; + float m_total; + float m_total2; + }; + class tracked_stat_dbl { public: @@ -2521,14 +2560,14 @@ namespace basisu FloatType m_mad; // mean absolute deviation FloatType m_min, m_max, m_range; // min and max values, and max-min FloatType m_len; // length of values as a vector (Euclidean norm or L2 norm) - FloatType m_coeff_of_var; // coefficient of variation (std_dev/mean), High CV: Indicates greater variability relative to the mean, meaning the data values are more spread out, + FloatType m_coeff_of_var; // coefficient of variation (std_dev/mean), High CV: Indicates greater variability relative to the mean, meaning the data values are more spread out, // Low CV : Indicates less variability relative to the mean, meaning the data values are more consistent. - - FloatType m_skewness; // Skewness = 0: The data is perfectly symmetric around the mean, - // Skewness > 0: The data is positively skewed (right-skewed), + + FloatType m_skewness; // Skewness = 0: The data is perfectly symmetric around the mean, + // Skewness > 0: The data is positively skewed (right-skewed), // Skewness < 0: The data is negatively skewed (left-skewed) // 0-.5 approx. symmetry, .5-1 moderate skew, >= 1 highly skewed - + FloatType m_kurtosis; // Excess Kurtosis: Kurtosis = 0: The distribution has normal kurtosis (mesokurtic) // Kurtosis > 0: The distribution is leptokurtic, with heavy tails and a sharp peak // Kurtosis < 0: The distribution is platykurtic, with light tails and a flatter peak @@ -2538,9 +2577,12 @@ namespace basisu FloatType m_median; uint32_t m_median_index; - stats() - { - clear(); + FloatType m_five_percent_lo; // avg of the lowest 5%, must calc median to be valid + FloatType m_five_percent_hi; // avg of the lowest 5%, must calc median to be valid + + stats() + { + clear(); } void clear() @@ -2557,9 +2599,12 @@ namespace basisu m_skewness = 0; m_kurtosis = 0; m_any_zero = false; - + m_median = 0; m_median_index = 0; + + m_five_percent_lo = 0; + m_five_percent_hi = 0; } template @@ -2588,13 +2633,26 @@ namespace basisu m_median = (m_median + vals[(n / 2) - 1].first) * .5f; m_median_index = vals[n / 2].second; + + // sum and avg low 5% and high 5% + const uint32_t p5_n = clamp((n + 10) / 20, 1u, n); + FloatType lo5_sum = 0, hi5_sum = 0; + + for (uint32_t i = 0; i < p5_n; i++) + { + lo5_sum += vals[i].first; + hi5_sum += vals[n - 1 - i].first; + } + + m_five_percent_lo = lo5_sum / FloatType(p5_n); + m_five_percent_hi = hi5_sum / FloatType(p5_n); } template void calc(uint32_t n, const T* pVals, uint32_t stride = 1, bool calc_median_flag = false) { clear(); - + if (!n) return; @@ -2609,10 +2667,10 @@ namespace basisu if (v == 0.0f) m_any_zero = true; - + m_total += v; m_total_sq += v * v; - + if (!i) { m_min = v; @@ -2634,12 +2692,12 @@ namespace basisu m_avg = m_total / nd; m_avg_sq = m_total_sq / nd; m_rms = sqrt(m_avg_sq); - + for (uint32_t i = 0; i < n; i++) { FloatType v = (FloatType)pVals[i * stride]; FloatType d = v - m_avg; - + const FloatType d2 = d * d; const FloatType d3 = d2 * d; const FloatType d4 = d3 * d; @@ -2680,6 +2738,55 @@ namespace basisu m_total += v; } + + const FloatType nd = (FloatType)n; + + m_avg = m_total / nd; + + for (uint32_t i = 0; i < n; i++) + { + FloatType v = (FloatType)pVals[i * stride]; + FloatType d = v - m_avg; + + const FloatType d2 = d * d; + + m_var += d2; + } + + m_var /= nd; + m_std_dev = sqrt(m_var); + } + + // Only compute average, variance and standard deviation. + template + void calc_simplified_with_range(uint32_t n, const T* pVals, uint32_t stride = 1) + { + clear(); + + if (!n) + return; + + m_n = n; + + for (uint32_t i = 0; i < n; i++) + { + FloatType v = (FloatType)pVals[i * stride]; + + m_total += v; + + if (!i) + { + m_min = v; + m_max = v; + } + else + { + m_min = minimum(m_min, v); + m_max = maximum(m_max, v); + } + } + + m_range = m_max - m_min; const FloatType nd = (FloatType)n; @@ -2712,7 +2819,7 @@ namespace basisu FloatType m_euclidean_dist; // euclidean distance between values as vectors FloatType m_cosine_sim; // normalized dot products of values as vectors FloatType m_min_diff, m_max_diff; // minimum/maximum abs difference between values - + comparative_stats() { clear(); @@ -2738,7 +2845,7 @@ namespace basisu clear(); if (!n) return; - + stats temp_a_stats; if (!pA_stats) { @@ -2757,7 +2864,7 @@ namespace basisu { const FloatType fa = (FloatType)pA[i * a_stride]; const FloatType fb = (FloatType)pB[i * b_stride]; - + if ((pA_stats->m_min >= 0.0f) && (pB_stats->m_min >= 0.0f)) { const FloatType ld = log(fa + 1.0f) - log(fb + 1.0f); @@ -2766,7 +2873,7 @@ namespace basisu const FloatType diff = fa - fb; const FloatType abs_diff = fabs(diff); - + m_mse += diff * diff; m_mae += abs_diff; @@ -2781,7 +2888,7 @@ namespace basisu } const FloatType nd = (FloatType)n; - + m_euclidean_dist = sqrt(m_mse); m_mse /= nd; @@ -2790,7 +2897,7 @@ namespace basisu m_mae /= nd; m_cov /= nd; - + FloatType dv = (pA_stats->m_std_dev * pB_stats->m_std_dev); if (dv != 0.0f) m_pearson = m_cov / dv; @@ -2883,9 +2990,9 @@ namespace basisu const FloatType fb = (FloatType)pB[i * b_stride]; const FloatType diff = fa - fb; - + m_mse += diff * diff; - + const FloatType da = fa - pA_stats->m_avg; const FloatType db = fb - pB_stats->m_avg; m_cov += da * db; @@ -2897,7 +3004,7 @@ namespace basisu m_mse /= nd; m_rmse = sqrt(m_mse); - + m_cov /= nd; } @@ -2938,7 +3045,7 @@ namespace basisu m_cov /= nd; } }; - + class stat_history { public: @@ -3083,12 +3190,12 @@ namespace basisu uint32_t lowerBits = float_union.u & 0xFFFF; // Round to nearest or even - if ((lowerBits & 0x8000) && + if ((lowerBits & 0x8000) && ((lowerBits > 0x8000) || ((lowerBits == 0x8000) && (upperBits & 1))) ) { // Round up - upperBits += 1; + upperBits += 1; // Check for overflow in the exponent after rounding up if (((upperBits & 0x7F80) == 0x7F80) && ((upperBits & 0x007F) == 0)) @@ -3140,6 +3247,7 @@ namespace basisu return res; } - - + + } // namespace basisu + diff --git a/external/basis_universal/encoder/basisu_uastc_enc.cpp b/external/basis_universal/encoder/basisu_uastc_enc.cpp index 701534cc56..88448eef0b 100644 --- a/external/basis_universal/encoder/basisu_uastc_enc.cpp +++ b/external/basis_universal/encoder/basisu_uastc_enc.cpp @@ -1,5 +1,5 @@ // basisu_uastc_enc.cpp -// Copyright (C) 2019-2024 Binomial LLC. All Rights Reserved. +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -223,7 +223,7 @@ namespace basisu default: break; } -#endif +#endif uint32_t total_planes = 1; switch (result.m_uastc_mode) @@ -319,7 +319,15 @@ namespace basisu const uint32_t comp_plane = (total_comps == 2) ? c : ((c == result.m_astc.m_ccs) ? 1 : 0); if (comp_plane == plane_index) - std::swap(endpoints[c * 2 + 0], endpoints[c * 2 + 1]); + { + // shut up a useless gcc warning + assert((c * 2 + 1) < (int)sizeof(endpoints)); + + if ((c * 2 + 1) < (int)sizeof(endpoints)) + { + std::swap(endpoints[c * 2 + 0], endpoints[c * 2 + 1]); + } + } } } else @@ -456,7 +464,7 @@ namespace basisu printf("Total bits: %u, endpoint bits: %u, weight bits: %u\n", block_bit_offset, total_endpoint_bits, total_weight_bits); #endif } - + // MODE 0 // 0. DualPlane: 0, WeightRange: 8 (16), Subsets: 1, CEM: 8 (RGB Direct ), EndpointRange: 19 (192) MODE6 RGB // 18. DualPlane: 0, WeightRange: 11 (32), Subsets: 1, CEM: 8 (RGB Direct ), EndpointRange: 11 (32) MODE6 RGB @@ -507,7 +515,7 @@ namespace basisu astc_results.m_endpoints[3] = ccell_results.m_astc_high_endpoint.m_c[1]; astc_results.m_endpoints[4] = ccell_results.m_astc_low_endpoint.m_c[2]; astc_results.m_endpoints[5] = ccell_results.m_astc_high_endpoint.m_c[2]; - + bool invert = false; if (pForce_selectors == nullptr) @@ -1128,7 +1136,7 @@ namespace basisu } // common_pattern } - // MODE 5 + // MODE 5 // DualPlane: 0, WeightRange: 5 (8), Subsets: 1, CEM: 8 (RGB Direct ), EndpointRange: 20 (256) BC7 MODE 6 (or MODE 1 1-subset) static void astc_mode5(const color_rgba block[4][4], uastc_encode_results* pResults, uint32_t& total_results, bc7enc_compress_block_params& comp_params) { @@ -1259,7 +1267,7 @@ namespace basisu ccell_results_rgb.m_pSelectors_temp = &ccell_result_selectors_temp[0]; uint64_t part_err_rgb = color_cell_compression(255, &ccell_params_rgb, &ccell_results_rgb, &comp_params); - + color_cell_compressor_params ccell_params_a; memset(&ccell_params_a, 0, sizeof(ccell_params_a)); @@ -1416,9 +1424,9 @@ namespace basisu for (uint32_t x = 0; x < 4; x++) { const uint32_t astc_part = bc7_convert_partition_index_3_to_2(g_bc7_partition3[16 * bc7_pattern + x + y * 4], common_pattern_k); -#ifdef _DEBUG +#ifdef _DEBUG assert((int)astc_part == astc_compute_texel_partition(astc_pattern, x, y, 0, 2, true)); -#endif +#endif part_pixel_index[y][x] = num_part_pixels[astc_part]; part_pixels[astc_part][num_part_pixels[astc_part]++] = block[y][x]; @@ -1583,7 +1591,7 @@ namespace basisu } #endif } - + // 9. DualPlane: 0, WeightRange: 2 (4), Subsets: 2, CEM: 12 (RGBA Direct), EndpointRange: 8 (16) - BC7 MODE 7 // 16. DualPlane: 0, WeightRange : 2 (4), Subsets : 2, CEM: 4 (LA Direct), EndpointRange : 20 (256) - BC7 MODE 7 static void astc_mode9_or_16(uint32_t mode, const color_rgba source_block[4][4], uastc_encode_results* pResults, uint32_t& total_results, bc7enc_compress_block_params& comp_params, uint32_t estimate_partition_list_size) @@ -2499,7 +2507,7 @@ namespace basisu total_results++; } } - + static void compute_block_error(const color_rgba block[4][4], const color_rgba decoded_block[4][4], uint64_t &total_rgb_err, uint64_t &total_rgba_err, uint64_t &total_la_err) { uint64_t total_err_r = 0, total_err_g = 0, total_err_b = 0, total_err_a = 0; @@ -2542,18 +2550,18 @@ namespace basisu color_rgba tblock_bc1[4][4]; dxt1_block tbc1_block[8]; basist::encode_bc1(tbc1_block, (const uint8_t*)&decoded_uastc_block[0][0], 0); - unpack_block(texture_format::cBC1, tbc1_block, &tblock_bc1[0][0]); + unpack_block(texture_format::cBC1, tbc1_block, &tblock_bc1[0][0], false); color_rgba tblock_hint0_bc1[4][4]; color_rgba tblock_hint1_bc1[4][4]; - + etc_block etc1_blk; memset(&etc1_blk, 0, sizeof(etc1_blk)); eac_a8_block etc2_blk; memset(&etc2_blk, 0, sizeof(etc2_blk)); etc2_blk.m_multiplier = 1; - + // Pack to UASTC, then unpack, because the endpoints may be swapped. uastc_block temp_ublock; @@ -2561,7 +2569,7 @@ namespace basisu unpacked_uastc_block temp_ublock_unpacked; unpack_uastc(temp_ublock, temp_ublock_unpacked, false); - + unpacked_uastc_block ublock; memset(&ublock, 0, sizeof(ublock)); ublock.m_mode = best_results.m_uastc_mode; @@ -2579,7 +2587,7 @@ namespace basisu { transcode_uastc_to_bc1_hint1(ublock, (color32 (*)[4]) decoded_uastc_block, &b, false); - unpack_block(texture_format::cBC1, &b, &tblock_hint1_bc1[0][0]); + unpack_block(texture_format::cBC1, &b, &tblock_hint1_bc1[0][0], false); } // HINT0 @@ -2590,8 +2598,8 @@ namespace basisu else { transcode_uastc_to_bc1_hint0(ublock, &b); - - unpack_block(texture_format::cBC1, &b, &tblock_hint0_bc1[0][0]); + + unpack_block(texture_format::cBC1, &b, &tblock_hint0_bc1[0][0], false); } // Compute block errors @@ -2612,7 +2620,7 @@ namespace basisu const float err_thresh0 = 1.075f; const float err_thresh1 = 1.075f; - + if ((g_uastc_mode_has_bc1_hint0[best_mode]) && (t_err_hint0 <= t_err * err_thresh0)) bc1_hint0 = true; @@ -2779,7 +2787,7 @@ namespace basisu uint32_t first_flip = 0, last_flip = 2; uint32_t first_individ = 0, last_individ = 2; - + if (flags & cPackUASTCETC1DisableFlipAndIndividual) { last_flip = 1; @@ -2791,7 +2799,7 @@ namespace basisu first_flip = 1; last_flip = first_flip + 1; } - + for (uint32_t flip = first_flip; flip < last_flip; flip++) { trial_block.set_flip_bit(flip != 0); @@ -2799,7 +2807,7 @@ namespace basisu for (uint32_t individ = first_individ; individ < last_individ; individ++) { const uint32_t mul = individ ? 15 : 31; - + trial_block.set_diff_bit(individ == 0); color_rgba unbiased_block_colors[2]; @@ -2815,7 +2823,7 @@ namespace basisu { const etc_coord2 &c = g_etc1_pixel_coords[flip][subset][j]; const color_rgba& p = decoded_uastc_block[c.m_y][c.m_x]; - + avg_color[0] += p.r; avg_color[1] += p.g; avg_color[2] += p.b; @@ -2833,13 +2841,13 @@ namespace basisu unbiased_block_colors[subset][1] = (uint8_t)((avg_color[1] * mul + 1020) / (8 * 255)); unbiased_block_colors[subset][2] = (uint8_t)((avg_color[2] * mul + 1020) / (8 * 255)); unbiased_block_colors[subset][3] = 0; - + } // subset - + for (uint32_t bias_iter = 0; bias_iter < last_bias; bias_iter++) { const uint32_t bias = use_faster_bias_mode_table ? s_sorted_bias_modes[bias_iter] : bias_iter; - + color_rgba block_colors[2]; for (uint32_t subset = 0; subset < 2; subset++) block_colors[subset] = has_bias ? apply_etc1_bias((color32&)unbiased_block_colors[subset], bias, mul, subset) : unbiased_block_colors[subset]; @@ -2873,7 +2881,7 @@ namespace basisu uint64_t best_subset_err = UINT64_MAX; const uint32_t inten_table_limit = (level == cPackUASTCLevelVerySlow) ? 8 : ((range[subset] > 51) ? 8 : (range[subset] >= 7 ? 4 : 2)); - + for (uint32_t inten_table = 0; inten_table < inten_table_limit; inten_table++) { trial_block.set_inten_table(subset, inten_table); @@ -3008,7 +3016,7 @@ namespace basisu uint32_t m_table; uint32_t m_multiplier; }; - + static uint64_t uastc_pack_eac_a8(uastc_pack_eac_a8_results& results, const uint8_t* pPixels, uint32_t num_pixels, uint32_t base_search_rad, uint32_t mul_search_rad, uint32_t table_mask) { assert(num_pixels <= 16); @@ -3152,7 +3160,7 @@ namespace basisu solid_results.m_common_pattern = 0; solid_results.m_solid_color = first_color; memset(&solid_results.m_astc, 0, sizeof(solid_results.m_astc)); - + etc_block etc1_blk; uint32_t etc1_bias = 0; @@ -3168,17 +3176,17 @@ namespace basisu return; } - + int level = flags & 7; const bool favor_uastc_error = (flags & cPackUASTCFavorUASTCError) != 0; const bool favor_bc7_error = !favor_uastc_error && ((flags & cPackUASTCFavorBC7Error) != 0); //const bool etc1_perceptual = true; - + // TODO: This uses 64KB of stack space! uastc_encode_results results[MAX_ENCODE_RESULTS]; - + level = clampi(level, cPackUASTCLevelFastest, cPackUASTCLevelVerySlow); - + // Set all options to slowest, then configure from there depending on the selected level. uint32_t mode_mask = UINT32_MAX; uint32_t uber_level = 6; @@ -3189,12 +3197,12 @@ namespace basisu uint32_t least_squares_passes = 2; bool bc1_hints = true; bool only_use_la_on_transparent_blocks = false; - + switch (level) { case cPackUASTCLevelFastest: { - mode_mask = (1 << 0) | (1 << 8) | + mode_mask = (1 << 0) | (1 << 8) | (1 << 11) | (1 << 12) | (1 << 15); always_try_alpha_modes = false; @@ -3220,7 +3228,7 @@ namespace basisu estimate_partition = true; break; } - case cPackUASTCLevelDefault: + case cPackUASTCLevelDefault: { mode_mask = (1 << 0) | (1 << 1) | (1 << 4) | (1 << 5) | (1 << 6) | (1 << 8) | (1 << 9) | (1 << 10) | (1 << 11) | (1 << 12) | (1 << 13) | @@ -3258,9 +3266,9 @@ namespace basisu // HACK HACK //mode_mask &= ~(1 << 18); //mode_mask = (1 << 18)| (1 << 10); - + uint32_t total_results = 0; - + if (only_use_la_on_transparent_blocks) { if ((is_la) && (!has_alpha)) @@ -3268,7 +3276,7 @@ namespace basisu } const bool try_alpha_modes = has_alpha || always_try_alpha_modes; - + bc7enc_compress_block_params comp_params; memset(&comp_params, 0, sizeof(comp_params)); comp_params.m_max_partitions_mode1 = 64; @@ -3343,7 +3351,7 @@ namespace basisu } assert(total_results); - + // Fix up the errors so we consistently have LA, RGB, or RGBA error. for (uint32_t i = 0; i < total_results; i++) { @@ -3377,7 +3385,7 @@ namespace basisu } } } - + unpacked_uastc_block unpacked_ublock; memset(&unpacked_ublock, 0, sizeof(unpacked_ublock)); @@ -3447,7 +3455,7 @@ namespace basisu encode_bc7_block(&bc7_data, &bc7_results); color_rgba decoded_bc7_block[4][4]; - unpack_block(texture_format::cBC7, &bc7_data, &decoded_bc7_block[0][0]); + unpack_block(texture_format::cBC7, &bc7_data, &decoded_bc7_block[0][0], false); // Compute BC7 error uint64_t total_bc7_la_err, total_bc7_rgb_err, total_bc7_rgba_err; @@ -3544,7 +3552,7 @@ namespace basisu const uastc_encode_results& best_results = results[best_index]; const uint32_t best_mode = best_results.m_uastc_mode; const astc_block_desc& best_astc_results = best_results.m_astc; - + color_rgba decoded_uastc_block[4][4]; bool success = unpack_uastc(best_mode, best_results.m_common_pattern, best_results.m_solid_color.get_color32(), best_astc_results, (basist::color32 *)&decoded_uastc_block[0][0], false); (void)success; @@ -3562,14 +3570,14 @@ namespace basisu basist::uastc_block temp_block; pack_uastc(temp_block, best_results, etc1_blk, 0, etc_eac_a8_blk, false, false); - + basist::color32 temp_block_unpacked[4][4]; success = basist::unpack_uastc(temp_block, (basist::color32 *)temp_block_unpacked, false); VALIDATE(success); // Now round trip to packed ASTC and back, then decode to pixels. uint32_t astc_data[4]; - + if (best_results.m_uastc_mode == UASTC_MODE_INDEX_SOLID_COLOR) pack_astc_solid_block(astc_data, (color32 &)best_results.m_solid_color); else @@ -3587,7 +3595,7 @@ namespace basisu for (uint32_t x = 0; x < 4; x++) { VALIDATE(decoded_astc_block[y][x] == decoded_uastc_block[y][x]); - + VALIDATE(temp_block_unpacked[y][x].c[0] == decoded_uastc_block[y][x].r); VALIDATE(temp_block_unpacked[y][x].c[1] == decoded_uastc_block[y][x].g); VALIDATE(temp_block_unpacked[y][x].c[2] == decoded_uastc_block[y][x].b); @@ -3601,7 +3609,7 @@ namespace basisu bool bc1_hint0 = false, bc1_hint1 = false; if (bc1_hints) compute_bc1_hints(bc1_hint0, bc1_hint1, best_results, block, decoded_uastc_block); - + eac_a8_block eac_a8_blk; if ((g_uastc_mode_has_alpha[best_mode]) && (best_mode != UASTC_MODE_INDEX_SOLID_COLOR)) { @@ -3613,7 +3621,7 @@ namespace basisu uastc_pack_eac_a8_results eac8_a8_results; memset(&eac8_a8_results, 0, sizeof(eac8_a8_results)); uastc_pack_eac_a8(eac8_a8_results, decoded_uastc_block_alpha, 16, 0, eac_a8_mul_search_rad, eac_a8_table_mask); - + // All we care about for hinting is the table and multiplier. eac_a8_blk.m_table = eac8_a8_results.m_table; eac_a8_blk.m_multiplier = eac8_a8_results.m_multiplier; @@ -3810,11 +3818,11 @@ namespace basisu { std::size_t operator()(selector_bitsequence const& s) const noexcept { - return hash_hsieh((const uint8_t*)&s, sizeof(s)); + return basist::hash_hsieh((const uint8_t*)&s, sizeof(s)); } }; - - static bool uastc_rdo_blocks(uint32_t first_index, uint32_t last_index, basist::uastc_block* pBlocks, const color_rgba* pBlock_pixels, const uastc_rdo_params& params, uint32_t flags, + + static bool uastc_rdo_blocks(uint32_t first_index, uint32_t last_index, basist::uastc_block* pBlocks, const color_rgba* pBlock_pixels, const uastc_rdo_params& params, uint32_t flags, uint32_t &total_skipped, uint32_t &total_refined, uint32_t &total_modified, uint32_t &total_smooth) { debug_printf("uastc_rdo_blocks: Processing blocks %u to %u\n", first_index, last_index); @@ -3823,7 +3831,7 @@ namespace basisu const bool perceptual = false; std::unordered_map selector_history; - + for (uint32_t block_index = first_index; block_index < last_index; block_index++) { const basist::uastc_block& blk = pBlocks[block_index]; @@ -3872,8 +3880,8 @@ namespace basisu basist::encode_bc7_block(&b7_block, &b7_results); color_rgba decoded_b7_blk[4][4]; - unpack_block(texture_format::cBC7, &b7_block, &decoded_b7_blk[0][0]); - + unpack_block(texture_format::cBC7, &b7_block, &decoded_b7_blk[0][0], false); + uint64_t bc7_err = 0; for (uint32_t i = 0; i < 16; i++) bc7_err += color_distance(perceptual, pPixels[i], ((color_rgba*)decoded_b7_blk)[i], true); @@ -3928,7 +3936,7 @@ namespace basisu float best_t = cur_ms_err * smooth_block_error_scale + cur_bits * params.m_lambda; - // Now scan through previous blocks, insert their selector bit patterns into the current block, and find + // Now scan through previous blocks, insert their selector bit patterns into the current block, and find // selector bit patterns which don't increase the overall block error too much. for (int prev_block_index = last_block_to_check; prev_block_index >= first_block_to_check; --prev_block_index) { @@ -3981,7 +3989,7 @@ namespace basisu basist::encode_bc7_block(&trial_b7_block, &trial_b7_results); color_rgba decoded_trial_b7_blk[4][4]; - unpack_block(texture_format::cBC7, &trial_b7_block, &decoded_trial_b7_blk[0][0]); + unpack_block(texture_format::cBC7, &trial_b7_block, &decoded_trial_b7_blk[0][0], false); uint64_t trial_bc7_err = 0; for (uint32_t i = 0; i < 16; i++) @@ -4050,7 +4058,7 @@ namespace basisu color_rgba decoded_trial_uastc_block[4][4]; bool success = unpack_uastc(results.m_uastc_mode, results.m_common_pattern, results.m_solid_color.get_color32(), results.m_astc, (basist::color32*) & decoded_trial_uastc_block[0][0], false); assert(success); - + BASISU_NOTE_UNUSED(success); uint64_t trial_uastc_err = 0; @@ -4077,7 +4085,7 @@ namespace basisu // Write the modified block pBlocks[block_index] = best_block; - + } // if (best_block_index != block_index) { @@ -4093,8 +4101,8 @@ namespace basisu return true; } - - // This function implements a basic form of rate distortion optimization (RDO) for UASTC. + + // This function implements a basic form of rate distortion optimization (RDO) for UASTC. // It only changes selectors and then updates the hints. It uses very approximate LZ bitprice estimation. // There's A LOT that can be done better in here, but it's a start. // One nice advantage of the method used here is that it works for any input, no matter which or how many modes it uses. @@ -4133,7 +4141,7 @@ namespace basisu { std::lock_guard lck(stat_mutex); - + all_succeeded = all_succeeded && status; total_skipped += job_skipped; total_modified += job_modified; @@ -4152,7 +4160,12 @@ namespace basisu } debug_printf("uastc_rdo: Total modified: %3.2f%%, total skipped: %3.2f%%, total refined: %3.2f%%, total smooth: %3.2f%%\n", total_modified * 100.0f / num_blocks, total_skipped * 100.0f / num_blocks, total_refined * 100.0f / num_blocks, total_smooth * 100.0f / num_blocks); - + return status; } } // namespace basisu + + + + + diff --git a/external/basis_universal/encoder/basisu_wasm_api.cpp b/external/basis_universal/encoder/basisu_wasm_api.cpp new file mode 100644 index 0000000000..d4db364d20 --- /dev/null +++ b/external/basis_universal/encoder/basisu_wasm_api.cpp @@ -0,0 +1,319 @@ +// File: basisu_wasm_api.cpp - Simplified compression API for WASM WASI modules and Python native support. +// Also useable by plain C callers. +#include "basisu_comp.h" +#include "basisu_wasm_api.h" + +using namespace basisu; + +static inline uint64_t wasm_offset(void* p) +{ + return (uint64_t)(uintptr_t)p; +} + +static inline uint8_t* wasm_ptr(uint64_t offset) +{ + return (uint8_t*)(uintptr_t)offset; +} + +BU_WASM_EXPORT("bu_get_version") +uint32_t bu_get_version() +{ + printf("Hello from basisu_wasm_api.cpp version %u\n", BASISU_LIB_VERSION); + + return BASISU_LIB_VERSION; +} + +BU_WASM_EXPORT("bu_enable_debug_printf") +void bu_enable_debug_printf(uint32_t flag) +{ + enable_debug_printf(flag != 0); +} + +BU_WASM_EXPORT("bu_init") +void bu_init() +{ + basisu_encoder_init(false, false); +} + +// Memory alloc/free — stubs +BU_WASM_EXPORT("bu_alloc") +uint64_t bu_alloc(uint64_t size) +{ + void* p = malloc((size_t)size); + return wasm_offset(p); +} + +BU_WASM_EXPORT("bu_free") +void bu_free(uint64_t ofs) +{ + free(wasm_ptr(ofs)); +} + +const uint32_t COMP_PARAMS_MAGIC = 0x43504D50; // "CPMP" + +struct comp_params +{ + uint32_t m_magic = COMP_PARAMS_MAGIC; + + comp_params() + { + clear(); + } + + void clear() + { + assert(m_magic == COMP_PARAMS_MAGIC); + + m_comp_data.clear(); + m_images.clear(); + m_imagesf.clear(); + + m_stats.clear(); + } + + uint8_vec m_comp_data; + + basisu::vector m_images; + basisu::vector m_imagesf; + + image_stats m_stats; +}; + +BU_WASM_EXPORT("bu_new_comp_params") +uint64_t bu_new_comp_params() +{ + comp_params* p = new comp_params; + return wasm_offset(p); +} + +BU_WASM_EXPORT("bu_delete_comp_params") +wasm_bool_t bu_delete_comp_params(uint64_t params_ofs) +{ + comp_params* p = (comp_params*)wasm_ptr(params_ofs); + if (!p) + return false; + + assert(p->m_magic == COMP_PARAMS_MAGIC); + if (p->m_magic != COMP_PARAMS_MAGIC) + return false; + + delete p; + + return true; +} + +BU_WASM_EXPORT("bu_comp_params_get_comp_data_size") +uint64_t bu_comp_params_get_comp_data_size(uint64_t params_ofs) +{ + comp_params* pParams = (comp_params*)wasm_ptr(params_ofs); + if (!pParams) + return 0; + + assert(pParams->m_magic == COMP_PARAMS_MAGIC); + if (pParams->m_magic != COMP_PARAMS_MAGIC) + return 0; + + return pParams->m_comp_data.size(); +} + +BU_WASM_EXPORT("bu_comp_params_get_comp_data_ofs") +uint64_t bu_comp_params_get_comp_data_ofs(uint64_t params_ofs) +{ + comp_params* pParams = (comp_params*)wasm_ptr(params_ofs); + if (!pParams) + return 0; + + assert(pParams->m_magic == COMP_PARAMS_MAGIC); + if (pParams->m_magic != COMP_PARAMS_MAGIC) + return 0; + + return wasm_offset(pParams->m_comp_data.get_ptr()); +} + +BU_WASM_EXPORT("bu_comp_params_clear") +wasm_bool_t bu_comp_params_clear(uint64_t params_ofs) +{ + comp_params* pParams = (comp_params*)wasm_ptr(params_ofs); + if (!pParams) + return false; + + assert(pParams->m_magic == COMP_PARAMS_MAGIC); + if (pParams->m_magic != COMP_PARAMS_MAGIC) + return false; + + pParams->clear(); + + return true; +} + +// Caller wants to give us a LDR/SDR 32bpp RGBA mipmap level (4 bytes per pixel) +BU_WASM_EXPORT("bu_comp_params_set_image_rgba32") +wasm_bool_t bu_comp_params_set_image_rgba32( + uint64_t params_ofs, + uint32_t image_index, + uint64_t img_data_ofs, + uint32_t width, uint32_t height, + uint32_t pitch_in_bytes) +{ + if ((!width) || (!height) || (!pitch_in_bytes)) + return false; + + comp_params* pParams = (comp_params*)wasm_ptr(params_ofs); + if (!pParams) + return false; + + assert(pParams->m_magic == COMP_PARAMS_MAGIC); + if (pParams->m_magic != COMP_PARAMS_MAGIC) + return false; + + const uint8_t* pImage = wasm_ptr(img_data_ofs); + if (!pImage) + return false; + + const uint32_t bytes_per_pixel = sizeof(color_rgba); + + if (pitch_in_bytes < width * bytes_per_pixel) + return false; + + if (image_index >= pParams->m_images.size()) + { + if (!pParams->m_images.try_resize(image_index + 1)) + return false; + } + + basisu::image& dst_img = pParams->m_images[image_index]; + + dst_img.resize(width, height); + + if (pitch_in_bytes == width * bytes_per_pixel) + { + memcpy(dst_img.get_ptr(), pImage, pitch_in_bytes * height); + } + else + { + for (uint32_t y = 0; y < height; y++) + { + const uint8_t* pSrc_row = pImage + y * pitch_in_bytes; + + uint8_t* pDst_row = (uint8_t *)&dst_img(0, y); + + memcpy(pDst_row, pSrc_row, width * bytes_per_pixel); + } // y + } + + return true; +} + +// Caller wants to give us a float RGBA mipmap level (4*4=16 bytes per pixel) +BU_WASM_EXPORT("bu_comp_params_set_image_float_rgba") +wasm_bool_t bu_comp_params_set_image_float_rgba( + uint64_t params_ofs, + uint32_t image_index, + uint64_t img_data_ofs, + uint32_t width, uint32_t height, + uint32_t pitch_in_bytes) +{ + if ((!width) || (!height) || (!pitch_in_bytes)) + return false; + + comp_params* pParams = (comp_params*)wasm_ptr(params_ofs); + if (!pParams) + return false; + + assert(pParams->m_magic == COMP_PARAMS_MAGIC); + if (pParams->m_magic != COMP_PARAMS_MAGIC) + return false; + + const uint8_t* pImage = wasm_ptr(img_data_ofs); + if (!pImage) + return false; + + const uint32_t bytes_per_pixel = sizeof(float) * 4; + + if (pitch_in_bytes < width * bytes_per_pixel) + return false; + + if (image_index >= pParams->m_images.size()) + { + if (!pParams->m_imagesf.try_resize(image_index + 1)) + return false; + } + + basisu::imagef& dst_img = pParams->m_imagesf[image_index]; + + dst_img.resize(width, height); + + if (pitch_in_bytes == width * bytes_per_pixel) + { + memcpy((void *)dst_img.get_ptr(), (const void *)pImage, pitch_in_bytes * height); + } + else + { + for (uint32_t y = 0; y < height; y++) + { + const uint8_t* pSrc_row = pImage + y * pitch_in_bytes; + + uint8_t* pDst_row = (uint8_t*)&dst_img(0, y); + + memcpy(pDst_row, pSrc_row, width * bytes_per_pixel); + } // y + } + + return true; +} + +BU_WASM_EXPORT("bu_compress_texture") +wasm_bool_t bu_compress_texture( + uint64_t params_ofs, + uint32_t desired_basis_tex_format, // basis_tex_format + int quality_level, int effort_level, + uint64_t flags_and_quality, float low_level_uastc_rdo_or_dct_quality) +{ + //enable_debug_printf((flags_and_quality & cFlagDebug) != 0); + + comp_params* pParams = (comp_params*)wasm_ptr(params_ofs); + if (!pParams) + return false; + + assert(pParams->m_magic == COMP_PARAMS_MAGIC); + if (pParams->m_magic != COMP_PARAMS_MAGIC) + return false; + + pParams->m_comp_data.clear(); + + if (desired_basis_tex_format >= (uint32_t)basist::basis_tex_format::cTotalFormats) + return false; + + if (!pParams->m_images.size() && !pParams->m_imagesf.size()) + return false; + if (pParams->m_images.size() && pParams->m_imagesf.size()) + return false; + + size_t comp_size = 0; + + void* pComp_data = basis_compress_internal( + (basist::basis_tex_format)desired_basis_tex_format, + pParams->m_images.size() ? &pParams->m_images : nullptr, + pParams->m_imagesf.size() ? &pParams->m_imagesf : nullptr, + (uint32_t)flags_and_quality, + low_level_uastc_rdo_or_dct_quality, + &comp_size, + &pParams->m_stats, + quality_level, + effort_level); + + if (!pComp_data) + return false; + + if (!pParams->m_comp_data.try_resize(comp_size)) + { + basis_free_data(pComp_data); + return false; + } + + memcpy(pParams->m_comp_data.get_ptr(), pComp_data, comp_size); + + basis_free_data(pComp_data); + + return true; +} diff --git a/external/basis_universal/encoder/basisu_wasm_api.h b/external/basis_universal/encoder/basisu_wasm_api.h new file mode 100644 index 0000000000..92266bc417 --- /dev/null +++ b/external/basis_universal/encoder/basisu_wasm_api.h @@ -0,0 +1,58 @@ +// File: basisu_wasm_api.h +#pragma once +#include "basisu_wasm_api_common.h" + +BU_WASM_EXPORT("bu_get_version") +uint32_t bu_get_version(); + +BU_WASM_EXPORT("bu_enable_debug_printf") +void bu_enable_debug_printf(uint32_t flag); + +BU_WASM_EXPORT("bu_init") +void bu_init(); + +BU_WASM_EXPORT("bu_alloc") +uint64_t bu_alloc(uint64_t size); + +BU_WASM_EXPORT("bu_free") +void bu_free(uint64_t ofs); + +BU_WASM_EXPORT("bu_new_comp_params") +uint64_t bu_new_comp_params(); + +BU_WASM_EXPORT("bu_delete_comp_params") +wasm_bool_t bu_delete_comp_params(uint64_t params_ofs); + +BU_WASM_EXPORT("bu_comp_params_get_comp_data_size") +uint64_t bu_comp_params_get_comp_data_size(uint64_t params_ofs); + +BU_WASM_EXPORT("bu_comp_params_get_comp_data_ofs") +uint64_t bu_comp_params_get_comp_data_ofs(uint64_t params_ofs); + +BU_WASM_EXPORT("bu_comp_params_clear") +wasm_bool_t bu_comp_params_clear(uint64_t params_ofs); + +BU_WASM_EXPORT("bu_comp_params_set_image_rgba32") +wasm_bool_t bu_comp_params_set_image_rgba32( + uint64_t params_ofs, + uint32_t image_index, + uint64_t img_data_ofs, + uint32_t width, uint32_t height, + uint32_t pitch_in_bytes); + +BU_WASM_EXPORT("bu_comp_params_set_image_float_rgba") +wasm_bool_t bu_comp_params_set_image_float_rgba( + uint64_t params_ofs, + uint32_t image_index, + uint64_t img_data_ofs, + uint32_t width, uint32_t height, + uint32_t pitch_in_bytes); + +BU_WASM_EXPORT("bu_compress_texture") +wasm_bool_t bu_compress_texture( + uint64_t params_ofs, + uint32_t desired_basis_tex_format, + int quality_level, int effort_level, + uint64_t flags_and_quality, + float low_level_uastc_rdo_or_dct_quality); + diff --git a/external/basis_universal/encoder/basisu_wasm_api_common.h b/external/basis_universal/encoder/basisu_wasm_api_common.h new file mode 100644 index 0000000000..d3fe1ae391 --- /dev/null +++ b/external/basis_universal/encoder/basisu_wasm_api_common.h @@ -0,0 +1,156 @@ +// File: basisu_wasm_api_common.h +#pragma once +#include "stdint.h" + +#if defined(__wasm__) + #if defined(__cplusplus) + #define BU_WASM_EXPORT(name) __attribute__((export_name(name))) extern "C" + #else + #define BU_WASM_EXPORT(name) __attribute__((export_name(name))) + #endif +#elif defined(__cplusplus) + #define BU_WASM_EXPORT(name) extern "C" +#else + #define BU_WASM_EXPORT(name) +#endif + +// wasm_bool_t is an alias for uint32_t +typedef uint32_t wasm_bool_t; + +// Compression constants + +#define BU_QUALITY_MIN 0 +#define BU_QUALITY_MAX 100 + +#define BU_EFFORT_MIN 0 +#define BU_EFFORT_MAX 10 +#define BU_EFFORT_SUPER_FAST = 0 +#define BU_EFFORT_FAST = 2 +#define BU_EFFORT_NORMAL = 5 +#define BU_EFFORT_DEFAULT = 2 +#define BU_EFFORT_SLOW = 8 +#define BU_EFFORT_VERY_SLOW = 10 + +#define BU_COMP_FLAGS_NONE (0) +#define BU_COMP_FLAGS_USE_OPENCL (1 << 8 ) +#define BU_COMP_FLAGS_THREADED (1 << 9 ) +#define BU_COMP_FLAGS_DEBUG_OUTPUT (1 << 10) +#define BU_COMP_FLAGS_KTX2_OUTPUT (1 << 11) +#define BU_COMP_FLAGS_KTX2_UASTC_ZSTD (1 << 12) +#define BU_COMP_FLAGS_SRGB (1 << 13) +#define BU_COMP_FLAGS_GEN_MIPS_CLAMP (1 << 14) +#define BU_COMP_FLAGS_GEN_MIPS_WRAP (1 << 15) +#define BU_COMP_FLAGS_Y_FLIP (1 << 16) +#define BU_COMP_FLAGS_PRINT_STATS (1 << 18) +#define BU_COMP_FLAGS_PRINT_STATUS (1 << 19) +#define BU_COMP_FLAGS_DEBUG_IMAGES (1 << 20) +#define BU_COMP_FLAGS_REC2020 (1 << 21) +#define BU_COMP_FLAGS_VALIDATE_OUTPUT (1 << 22) + +#define BU_COMP_FLAGS_XUASTC_LDR_FULL_ARITH (0) +#define BU_COMP_FLAGS_XUASTC_LDR_HYBRID (1 << 23) +#define BU_COMP_FLAGS_XUASTC_LDR_FULL_ZSTD (2 << 23) +#define BU_COMP_FLAGS_XUASTC_LDR_SYNTAX_SHIFT (23) +#define BU_COMP_FLAGS_XUASTC_LDR_SYNTAX_MASK (3) + +#define BU_COMP_FLAGS_TEXTURE_TYPE_2D (0 << 25) +#define BU_COMP_FLAGS_TEXTURE_TYPE_2D_ARRAY (1 << 25) +#define BU_COMP_FLAGS_TEXTURE_TYPE_CUBEMAP_ARRAY (2 << 25) +#define BU_COMP_FLAGS_TEXTURE_TYPE_VIDEO_FRAMES (3 << 25) +#define BU_COMP_FLAGS_TEXTURE_TYPE_SHIFT (25) +#define BU_COMP_FLAGS_TEXTURE_TYPE_MASK (3) + +#define BU_COMP_FLAGS_VERBOSE (BU_COMP_FLAGS_DEBUG_OUTPUT | BU_COMP_FLAGS_PRINT_STATS | BU_COMP_FLAGS_PRINT_STATUS) + +// basist::basis_tex_format: the supported .ktx2 (and .basis) file format types +#define BTF_ETC1S 0 +#define BTF_UASTC_LDR_4X4 1 +#define BTF_UASTC_HDR_4X4 2 +#define BTF_ASTC_HDR_6X6 3 +#define BTF_UASTC_HDR_6X6 4 +#define BTF_XUASTC_LDR_4X4 5 +#define BTF_XUASTC_LDR_5X4 6 +#define BTF_XUASTC_LDR_5X5 7 +#define BTF_XUASTC_LDR_6X5 8 +#define BTF_XUASTC_LDR_6X6 9 +#define BTF_XUASTC_LDR_8X5 10 +#define BTF_XUASTC_LDR_8X6 11 +#define BTF_XUASTC_LDR_10X5 12 +#define BTF_XUASTC_LDR_10X6 13 +#define BTF_XUASTC_LDR_8X8 14 +#define BTF_XUASTC_LDR_10X8 15 +#define BTF_XUASTC_LDR_10X10 16 +#define BTF_XUASTC_LDR_12X10 17 +#define BTF_XUASTC_LDR_12X12 18 +#define BTF_ASTC_LDR_4X4 19 +#define BTF_ASTC_LDR_5X4 20 +#define BTF_ASTC_LDR_5X5 21 +#define BTF_ASTC_LDR_6X5 22 +#define BTF_ASTC_LDR_6X6 23 +#define BTF_ASTC_LDR_8X5 24 +#define BTF_ASTC_LDR_8X6 25 +#define BTF_ASTC_LDR_10X5 26 +#define BTF_ASTC_LDR_10X6 27 +#define BTF_ASTC_LDR_8X8 28 +#define BTF_ASTC_LDR_10X8 29 +#define BTF_ASTC_LDR_10X10 30 +#define BTF_ASTC_LDR_12X10 31 +#define BTF_ASTC_LDR_12X12 32 +#define BTF_TOTAL_FORMATS 33 + +// Transcoding constants + +// basist::transcoder_texture_format: the supported transcode GPU texture formats +#define TF_ETC1_RGB 0 +#define TF_ETC2_RGBA 1 +#define TF_BC1_RGB 2 +#define TF_BC3_RGBA 3 +#define TF_BC4_R 4 +#define TF_BC5_RG 5 +#define TF_BC7_RGBA 6 +#define TF_PVRTC1_4_RGB 8 +#define TF_PVRTC1_4_RGBA 9 +#define TF_ASTC_LDR_4X4_RGBA 10 +#define TF_ATC_RGB 11 +#define TF_ATC_RGBA 12 +#define TF_FXT1_RGB 17 +#define TF_PVRTC2_4_RGB 18 +#define TF_PVRTC2_4_RGBA 19 +#define TF_ETC2_EAC_R11 20 +#define TF_ETC2_EAC_RG11 21 +#define TF_BC6H 22 +#define TF_ASTC_HDR_4X4_RGBA 23 +#define TF_RGBA32 13 +#define TF_RGB565 14 +#define TF_BGR565 15 +#define TF_RGBA4444 16 +#define TF_RGB_HALF 24 +#define TF_RGBA_HALF 25 +#define TF_RGB_9E5 26 +#define TF_ASTC_HDR_6X6_RGBA 27 +#define TF_ASTC_LDR_5X4_RGBA 28 +#define TF_ASTC_LDR_5X5_RGBA 29 +#define TF_ASTC_LDR_6X5_RGBA 30 +#define TF_ASTC_LDR_6X6_RGBA 31 +#define TF_ASTC_LDR_8X5_RGBA 32 +#define TF_ASTC_LDR_8X6_RGBA 33 +#define TF_ASTC_LDR_10X5_RGBA 34 +#define TF_ASTC_LDR_10X6_RGBA 35 +#define TF_ASTC_LDR_8X8_RGBA 36 +#define TF_ASTC_LDR_10X8_RGBA 37 +#define TF_ASTC_LDR_10X10_RGBA 38 +#define TF_ASTC_LDR_12X10_RGBA 39 +#define TF_ASTC_LDR_12X12_RGBA 40 +#define TF_TOTAL_TEXTURE_FORMATS 41 + +// basist::basisu_decode_flags: Transcode decode flags (bt_ktx2_transcode_image_level decode_flags parameter, logically OR'd) +#define DECODE_FLAGS_PVRTC_DECODE_TO_NEXT_POW2 2 +#define DECODE_FLAGS_TRANSCODE_ALPHA_DATA_TO_OPAQUE_FORMATS 4 +#define DECODE_FLAGS_BC1_FORBID_THREE_COLOR_BLOCKS 8 +#define DECODE_FLAGS_OUTPUT_HAS_ALPHA_INDICES 16 +#define DECODE_FLAGS_HIGH_QUALITY 32 +#define DECODE_FLAGS_NO_ETC1S_CHROMA_FILTERING 64 +#define DECODE_FLAGS_NO_DEBLOCK_FILTERING 128 +#define DECODE_FLAGS_STRONGER_DEBLOCK_FILTERING 256 +#define DECODE_FLAGS_FORCE_DEBLOCK_FILTERING 512 +#define DECODE_FLAGS_XUASTC_LDR_DISABLE_FAST_BC7_TRANSCODING 1024 diff --git a/external/basis_universal/encoder/basisu_wasm_transcoder_api.cpp b/external/basis_universal/encoder/basisu_wasm_transcoder_api.cpp new file mode 100644 index 0000000000..ab46525b14 --- /dev/null +++ b/external/basis_universal/encoder/basisu_wasm_transcoder_api.cpp @@ -0,0 +1,1071 @@ +// basisu_wasm_transcoder_api.cpp - Transcoding API support for WASM WASI modules and Python native support. +// Also useable by plain C callers. +#include +#include +#include +#include "../transcoder/basisu_transcoder.h" +#include "basisu_wasm_transcoder_api.h" + +using namespace basisu; +using namespace basist; + +static inline uint64_t wasm_offset(void* p) +{ + return (uint64_t)(uintptr_t)p; +} + +static inline uint8_t* wasm_ptr(uint64_t offset) +{ + return (uint8_t*)(uintptr_t)offset; +} + +// High-level functions + +BU_WASM_EXPORT("bt_get_version") +uint32_t bt_get_version() +{ + printf("Hello from basisu_wasm_transcoder_api.cpp version %u\n", BASISD_LIB_VERSION); + + return BASISD_LIB_VERSION; +} + +BU_WASM_EXPORT("bt_enable_debug_printf") +void bt_enable_debug_printf(uint32_t flag) +{ + enable_debug_printf(flag != 0); +} + +BU_WASM_EXPORT("bt_init") +void bt_init() +{ + basisu_transcoder_init(); +} + +// Memory alloc/free — stubs +BU_WASM_EXPORT("bt_alloc") +uint64_t bt_alloc(uint64_t size) +{ + void* p = malloc((size_t)size); + return wasm_offset(p); +} + +BU_WASM_EXPORT("bt_free") +void bt_free(uint64_t mem_ofs) +{ + free(wasm_ptr(mem_ofs)); +} + +// basis_tex_format helpers + +BU_WASM_EXPORT("bt_basis_tex_format_is_xuastc_ldr") +wasm_bool_t bt_basis_tex_format_is_xuastc_ldr(uint32_t basis_tex_fmt_u32) +{ + assert(basis_tex_fmt_u32 < (uint32_t)basis_tex_format::cTotalFormats); + + basis_tex_format tex_fmt = static_cast(basis_tex_fmt_u32); + + return basis_tex_format_is_xuastc_ldr(tex_fmt); +} + +BU_WASM_EXPORT("bt_basis_tex_format_is_astc_ldr") +wasm_bool_t bt_basis_tex_format_is_astc_ldr(uint32_t basis_tex_fmt_u32) +{ + assert(basis_tex_fmt_u32 < (uint32_t)basis_tex_format::cTotalFormats); + + basis_tex_format tex_fmt = static_cast(basis_tex_fmt_u32); + + return basis_tex_format_is_astc_ldr(tex_fmt); +} + +BU_WASM_EXPORT("bt_basis_tex_format_get_block_width") +uint32_t bt_basis_tex_format_get_block_width(uint32_t basis_tex_fmt_u32) +{ + assert(basis_tex_fmt_u32 < (uint32_t)basis_tex_format::cTotalFormats); + + basis_tex_format tex_fmt = static_cast(basis_tex_fmt_u32); + + return basis_tex_format_get_block_width(tex_fmt); +} + +BU_WASM_EXPORT("bt_basis_tex_format_get_block_height") +uint32_t bt_basis_tex_format_get_block_height(uint32_t basis_tex_fmt_u32) +{ + assert(basis_tex_fmt_u32 < (uint32_t)basis_tex_format::cTotalFormats); + + basis_tex_format tex_fmt = static_cast(basis_tex_fmt_u32); + + return basis_tex_format_get_block_height(tex_fmt); +} + +BU_WASM_EXPORT("bt_basis_tex_format_is_hdr") +wasm_bool_t bt_basis_tex_format_is_hdr(uint32_t basis_tex_fmt_u32) +{ + assert(basis_tex_fmt_u32 < (uint32_t)basis_tex_format::cTotalFormats); + + basis_tex_format tex_fmt = static_cast(basis_tex_fmt_u32); + + return basis_tex_format_is_hdr(tex_fmt); +} + +BU_WASM_EXPORT("bt_basis_tex_format_is_ldr") +wasm_bool_t bt_basis_tex_format_is_ldr(uint32_t basis_tex_fmt_u32) +{ + assert(basis_tex_fmt_u32 < (uint32_t)basis_tex_format::cTotalFormats); + + basis_tex_format tex_fmt = static_cast(basis_tex_fmt_u32); + + return basis_tex_format_is_ldr(tex_fmt); +} + +// transcoder_texture_format helpers + +BU_WASM_EXPORT("bt_basis_get_bytes_per_block_or_pixel") +uint32_t bt_basis_get_bytes_per_block_or_pixel(uint32_t transcoder_texture_format_u32) +{ + assert(transcoder_texture_format_u32 < (uint32_t)transcoder_texture_format::cTFTotalTextureFormats); + + transcoder_texture_format fmt = static_cast(transcoder_texture_format_u32); + + return basis_get_bytes_per_block_or_pixel(fmt); +} + +BU_WASM_EXPORT("bt_basis_transcoder_format_has_alpha") +wasm_bool_t bt_basis_transcoder_format_has_alpha(uint32_t transcoder_texture_format_u32) +{ + assert(transcoder_texture_format_u32 < (uint32_t)transcoder_texture_format::cTFTotalTextureFormats); + + transcoder_texture_format fmt = static_cast(transcoder_texture_format_u32); + + return basis_transcoder_format_has_alpha(fmt); +} + +BU_WASM_EXPORT("bt_basis_transcoder_format_is_hdr") +wasm_bool_t bt_basis_transcoder_format_is_hdr(uint32_t transcoder_texture_format_u32) +{ + assert(transcoder_texture_format_u32 < (uint32_t)transcoder_texture_format::cTFTotalTextureFormats); + + transcoder_texture_format fmt = static_cast(transcoder_texture_format_u32); + + return basis_transcoder_format_is_hdr(fmt); +} + +BU_WASM_EXPORT("bt_basis_transcoder_format_is_ldr") +wasm_bool_t bt_basis_transcoder_format_is_ldr(uint32_t transcoder_texture_format_u32) +{ + assert(transcoder_texture_format_u32 < (uint32_t)transcoder_texture_format::cTFTotalTextureFormats); + + transcoder_texture_format fmt = static_cast(transcoder_texture_format_u32); + + return basis_transcoder_format_is_ldr(fmt); +} + +BU_WASM_EXPORT("bt_basis_transcoder_texture_format_is_astc") +wasm_bool_t bt_basis_transcoder_texture_format_is_astc(uint32_t transcoder_texture_format_u32) +{ + assert(transcoder_texture_format_u32 < (uint32_t)transcoder_texture_format::cTFTotalTextureFormats); + + transcoder_texture_format fmt = static_cast(transcoder_texture_format_u32); + + return basis_is_transcoder_texture_format_astc(fmt); +} + +BU_WASM_EXPORT("bt_basis_transcoder_format_is_uncompressed") +wasm_bool_t bt_basis_transcoder_format_is_uncompressed(uint32_t transcoder_texture_format_u32) +{ + assert(transcoder_texture_format_u32 < (uint32_t)transcoder_texture_format::cTFTotalTextureFormats); + + transcoder_texture_format fmt = static_cast(transcoder_texture_format_u32); + + return basis_transcoder_format_is_uncompressed(fmt); +} + +BU_WASM_EXPORT("bt_basis_get_uncompressed_bytes_per_pixel") +uint32_t bt_basis_get_uncompressed_bytes_per_pixel(uint32_t transcoder_texture_format_u32) +{ + assert(transcoder_texture_format_u32 < (uint32_t)transcoder_texture_format::cTFTotalTextureFormats); + + transcoder_texture_format fmt = static_cast(transcoder_texture_format_u32); + + return basis_get_uncompressed_bytes_per_pixel(fmt); +} + +BU_WASM_EXPORT("bt_basis_get_block_width") +uint32_t bt_basis_get_block_width(uint32_t transcoder_texture_format_u32) +{ + assert(transcoder_texture_format_u32 < (uint32_t)transcoder_texture_format::cTFTotalTextureFormats); + + transcoder_texture_format fmt = static_cast(transcoder_texture_format_u32); + + return basis_get_block_width(fmt); +} + +BU_WASM_EXPORT("bt_basis_get_block_height") +uint32_t bt_basis_get_block_height(uint32_t transcoder_texture_format_u32) +{ + assert(transcoder_texture_format_u32 < (uint32_t)transcoder_texture_format::cTFTotalTextureFormats); + + transcoder_texture_format fmt = static_cast(transcoder_texture_format_u32); + + return basis_get_block_height(fmt); +} + +BU_WASM_EXPORT("bt_basis_get_transcoder_texture_format_from_basis_tex_format") +uint32_t bt_basis_get_transcoder_texture_format_from_basis_tex_format(uint32_t basis_tex_format_u32) +{ + assert(basis_tex_format_u32 < (uint32_t)basis_tex_format::cTotalFormats); + + basis_tex_format fmt = static_cast(basis_tex_format_u32); + + return (uint32_t)basis_get_transcoder_texture_format_from_xuastc_or_astc_ldr_basis_tex_format(fmt); +} + +BU_WASM_EXPORT("bt_basis_is_format_supported") +wasm_bool_t bt_basis_is_format_supported(uint32_t transcoder_texture_format_u32, uint32_t basis_tex_format_u32) +{ + assert(transcoder_texture_format_u32 < (uint32_t)transcoder_texture_format::cTFTotalTextureFormats); + assert(basis_tex_format_u32 < (uint32_t)basis_tex_format::cTotalFormats); + + transcoder_texture_format transcoder_tex_fmt = static_cast(transcoder_texture_format_u32); + basis_tex_format basis_tex_fmt = static_cast(basis_tex_format_u32); + + return basis_is_format_supported(transcoder_tex_fmt, basis_tex_fmt); +} + +BU_WASM_EXPORT("bt_basis_compute_transcoded_image_size_in_bytes") +uint32_t bt_basis_compute_transcoded_image_size_in_bytes(uint32_t transcoder_texture_format_u32, uint32_t orig_width, uint32_t orig_height) +{ + assert(transcoder_texture_format_u32 < (uint32_t)transcoder_texture_format::cTFTotalTextureFormats); + + transcoder_texture_format transcoder_tex_fmt = static_cast(transcoder_texture_format_u32); + + return basis_compute_transcoded_image_size_in_bytes(transcoder_tex_fmt, orig_width, orig_height); +} + +// KTX2 inspection and transcoding helpers + +const uint32_t KTX2_HANDLE_MAGIC = 0xAB21EF20; + +struct ktx2_handle_t +{ + uint32_t m_magic = KTX2_HANDLE_MAGIC; + ktx2_transcoder m_transcoder; +}; + +BU_WASM_EXPORT("bt_ktx2_open") +uint64_t bt_ktx2_open(uint64_t data_mem_ofs, uint32_t data_len) +{ + if (!data_mem_ofs || (data_len < 4)) + return 0; + + ktx2_handle_t* pHandle = new ktx2_handle_t(); + + if (!pHandle->m_transcoder.init(wasm_ptr(data_mem_ofs), data_len)) + { + delete pHandle; + return 0; + } + + return wasm_offset(pHandle); +} + +BU_WASM_EXPORT("bt_ktx2_close") +void bt_ktx2_close(uint64_t handle) +{ + if (!handle) + return; + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return; + + delete pHandle; +} + +BU_WASM_EXPORT("bt_ktx2_get_width") +uint32_t bt_ktx2_get_width(uint64_t handle) +{ + if (!handle) + { + assert(0); + return 0; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return 0; + + return pHandle->m_transcoder.get_width(); +} + +BU_WASM_EXPORT("bt_ktx2_get_height") +uint32_t bt_ktx2_get_height(uint64_t handle) +{ + if (!handle) + { + assert(0); + return 0; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return 0; + + return pHandle->m_transcoder.get_height(); +} + +BU_WASM_EXPORT("bt_ktx2_get_levels") +uint32_t bt_ktx2_get_levels(uint64_t handle) +{ + if (!handle) + { + assert(0); + return 0; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return 0; + + return pHandle->m_transcoder.get_levels(); +} + +BU_WASM_EXPORT("bt_ktx2_get_faces") +uint32_t bt_ktx2_get_faces(uint64_t handle) +{ + if (!handle) + { + assert(0); + return 0; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return 0; + + return pHandle->m_transcoder.get_faces(); +} + +BU_WASM_EXPORT("bt_ktx2_get_layers") +uint32_t bt_ktx2_get_layers(uint64_t handle) +{ + if (!handle) + { + assert(0); + return 0; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return 0; + + return pHandle->m_transcoder.get_layers(); +} + +BU_WASM_EXPORT("bt_ktx2_get_basis_tex_format") +uint32_t bt_ktx2_get_basis_tex_format(uint64_t handle) +{ + if (!handle) + { + assert(0); + return 0; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return 0; + + return (uint32_t)pHandle->m_transcoder.get_basis_tex_format(); +} + +BU_WASM_EXPORT("bt_ktx2_is_etc1s") +wasm_bool_t bt_ktx2_is_etc1s(uint64_t handle) +{ + if (!handle) + { + assert(0); + return false; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return false; + + return pHandle->m_transcoder.is_etc1s(); +} + +BU_WASM_EXPORT("bt_ktx2_is_uastc_ldr_4x4") +wasm_bool_t bt_ktx2_is_uastc_ldr_4x4(uint64_t handle) +{ + if (!handle) + { + assert(0); + return false; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return false; + + return pHandle->m_transcoder.is_uastc(); +} + +BU_WASM_EXPORT("bt_ktx2_is_hdr") +wasm_bool_t bt_ktx2_is_hdr(uint64_t handle) +{ + if (!handle) + { + assert(0); + return false; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return false; + + return pHandle->m_transcoder.is_hdr(); +} + +BU_WASM_EXPORT("bt_ktx2_is_hdr_4x4") +wasm_bool_t bt_ktx2_is_hdr_4x4(uint64_t handle) +{ + if (!handle) + { + assert(0); + return false; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return false; + + return pHandle->m_transcoder.is_hdr_4x4(); +} + +BU_WASM_EXPORT("bt_ktx2_is_hdr_6x6") +wasm_bool_t bt_ktx2_is_hdr_6x6(uint64_t handle) +{ + if (!handle) + { + assert(0); + return false; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return false; + + return pHandle->m_transcoder.is_hdr_6x6(); +} + +BU_WASM_EXPORT("bt_ktx2_is_ldr") +wasm_bool_t bt_ktx2_is_ldr(uint64_t handle) +{ + if (!handle) + { + assert(0); + return false; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return false; + + return pHandle->m_transcoder.is_ldr(); +} + +BU_WASM_EXPORT("bt_ktx2_is_astc_ldr") +wasm_bool_t bt_ktx2_is_astc_ldr(uint64_t handle) +{ + if (!handle) + { + assert(0); + return false; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return false; + + return pHandle->m_transcoder.is_astc_ldr(); +} + +BU_WASM_EXPORT("bt_ktx2_is_xuastc_ldr") +wasm_bool_t bt_ktx2_is_xuastc_ldr(uint64_t handle) +{ + if (!handle) + { + assert(0); + return false; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return false; + + return pHandle->m_transcoder.is_xuastc_ldr(); +} + +BU_WASM_EXPORT("bt_ktx2_get_block_width") +uint32_t bt_ktx2_get_block_width(uint64_t handle) +{ + if (!handle) + { + assert(0); + return 0; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return 0; + + return pHandle->m_transcoder.get_block_width(); +} + +BU_WASM_EXPORT("bt_ktx2_get_block_height") +uint32_t bt_ktx2_get_block_height(uint64_t handle) +{ + if (!handle) + { + assert(0); + return 0; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return 0; + + return pHandle->m_transcoder.get_block_height(); +} + +BU_WASM_EXPORT("bt_ktx2_has_alpha") +wasm_bool_t bt_ktx2_has_alpha(uint64_t handle) +{ + if (!handle) + { + assert(0); + return false; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return false; + + return pHandle->m_transcoder.get_has_alpha(); +} + +BU_WASM_EXPORT("bt_ktx2_get_dfd_color_model") +uint32_t bt_ktx2_get_dfd_color_model(uint64_t handle) +{ + if (!handle) + { + assert(0); + return 0; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return 0; + + return pHandle->m_transcoder.get_dfd_color_model(); +} + +BU_WASM_EXPORT("bt_ktx2_get_dfd_color_primaries") +uint32_t bt_ktx2_get_dfd_color_primaries(uint64_t handle) +{ + if (!handle) + { + assert(0); + return 0; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return 0; + + return pHandle->m_transcoder.get_dfd_color_primaries(); +} + +BU_WASM_EXPORT("bt_ktx2_get_dfd_transfer_func") +uint32_t bt_ktx2_get_dfd_transfer_func(uint64_t handle) +{ + if (!handle) + { + assert(0); + return 0; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return 0; + + return pHandle->m_transcoder.get_dfd_transfer_func(); +} + +BU_WASM_EXPORT("bt_ktx2_is_srgb") +wasm_bool_t bt_ktx2_is_srgb(uint64_t handle) +{ + if (!handle) + { + assert(0); + return false; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return false; + + return pHandle->m_transcoder.is_srgb(); +} + +BU_WASM_EXPORT("bt_ktx2_get_dfd_flags") +uint32_t bt_ktx2_get_dfd_flags(uint64_t handle) +{ + if (!handle) + { + assert(0); + return 0; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return 0; + + return pHandle->m_transcoder.get_dfd_flags(); +} + +BU_WASM_EXPORT("bt_ktx2_get_dfd_total_samples") +uint32_t bt_ktx2_get_dfd_total_samples(uint64_t handle) +{ + if (!handle) + { + assert(0); + return 0; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return 0; + + return pHandle->m_transcoder.get_dfd_total_samples(); +} + +BU_WASM_EXPORT("bt_ktx2_get_dfd_channel_id0") +uint32_t bt_ktx2_get_dfd_channel_id0(uint64_t handle) +{ + if (!handle) + { + assert(0); + return 0; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return 0; + + return pHandle->m_transcoder.get_dfd_channel_id0(); +} + +BU_WASM_EXPORT("bt_ktx2_get_dfd_channel_id1") +uint32_t bt_ktx2_get_dfd_channel_id1(uint64_t handle) +{ + if (!handle) + { + assert(0); + return 0; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return 0; + + return pHandle->m_transcoder.get_dfd_channel_id1(); +} + +BU_WASM_EXPORT("bt_ktx2_is_video") +wasm_bool_t bt_ktx2_is_video(uint64_t handle) +{ + if (!handle) + { + assert(0); + return false; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return false; + + return pHandle->m_transcoder.is_video(); +} + +BU_WASM_EXPORT("bt_ktx2_get_ldr_hdr_upconversion_nit_multiplier") +float bt_ktx2_get_ldr_hdr_upconversion_nit_multiplier(uint64_t handle) +{ + if (!handle) + { + assert(0); + return 0.0f; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return 0.0f; + + return pHandle->m_transcoder.get_ldr_hdr_upconversion_nit_multiplier(); +} + +BU_WASM_EXPORT("bt_ktx2_get_level_orig_width") +uint32_t bt_ktx2_get_level_orig_width(uint64_t handle, uint32_t level_index, uint32_t layer_index, uint32_t face_index) +{ + if (!handle) + { + assert(0); + return 0; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return 0; + + // FIXME slow - most info is thrown away. + ktx2_image_level_info level_info; + if (!pHandle->m_transcoder.get_image_level_info(level_info, level_index, layer_index, face_index)) + return 0; + + return level_info.m_orig_width; +} + +BU_WASM_EXPORT("bt_ktx2_get_level_orig_height") +uint32_t bt_ktx2_get_level_orig_height(uint64_t handle, uint32_t level_index, uint32_t layer_index, uint32_t face_index) +{ + if (!handle) + { + assert(0); + return 0; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return 0; + + // FIXME slow - most info is thrown away. + ktx2_image_level_info level_info; + if (!pHandle->m_transcoder.get_image_level_info(level_info, level_index, layer_index, face_index)) + return 0; + + return level_info.m_orig_height; +} + +BU_WASM_EXPORT("bt_ktx2_get_level_actual_width") +uint32_t bt_ktx2_get_level_actual_width(uint64_t handle, uint32_t level_index, uint32_t layer_index, uint32_t face_index) +{ + if (!handle) + { + assert(0); + return 0; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return 0; + + // FIXME slow - most info is thrown away. + ktx2_image_level_info level_info; + if (!pHandle->m_transcoder.get_image_level_info(level_info, level_index, layer_index, face_index)) + return 0; + + return level_info.m_width; +} + +BU_WASM_EXPORT("bt_ktx2_get_level_actual_height") +uint32_t bt_ktx2_get_level_actual_height(uint64_t handle, uint32_t level_index, uint32_t layer_index, uint32_t face_index) +{ + if (!handle) + { + assert(0); + return 0; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return 0; + + // FIXME slow - most info is thrown away. + ktx2_image_level_info level_info; + if (!pHandle->m_transcoder.get_image_level_info(level_info, level_index, layer_index, face_index)) + return 0; + + return level_info.m_height; +} + +BU_WASM_EXPORT("bt_ktx2_get_level_num_blocks_x") +uint32_t bt_ktx2_get_level_num_blocks_x(uint64_t handle, uint32_t level_index, uint32_t layer_index, uint32_t face_index) +{ + if (!handle) + { + assert(0); + return 0; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return 0; + + // FIXME slow - most info is thrown away. + ktx2_image_level_info level_info; + if (!pHandle->m_transcoder.get_image_level_info(level_info, level_index, layer_index, face_index)) + return 0; + + return level_info.m_num_blocks_x; +} + +BU_WASM_EXPORT("bt_ktx2_get_level_num_blocks_y") +uint32_t bt_ktx2_get_level_num_blocks_y(uint64_t handle, uint32_t level_index, uint32_t layer_index, uint32_t face_index) +{ + if (!handle) + { + assert(0); + return 0; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return 0; + + // FIXME slow - most info is thrown away. + ktx2_image_level_info level_info; + if (!pHandle->m_transcoder.get_image_level_info(level_info, level_index, layer_index, face_index)) + return 0; + + return level_info.m_num_blocks_y; +} + +BU_WASM_EXPORT("bt_ktx2_get_level_total_blocks") +uint32_t bt_ktx2_get_level_total_blocks(uint64_t handle, uint32_t level_index, uint32_t layer_index, uint32_t face_index) +{ + if (!handle) + { + assert(0); + return 0; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return 0; + + // FIXME slow - most info is thrown away. + ktx2_image_level_info level_info; + if (!pHandle->m_transcoder.get_image_level_info(level_info, level_index, layer_index, face_index)) + return 0; + + return level_info.m_total_blocks; +} + +BU_WASM_EXPORT("bt_ktx2_get_level_alpha_flag") +wasm_bool_t bt_ktx2_get_level_alpha_flag(uint64_t handle, uint32_t level_index, uint32_t layer_index, uint32_t face_index) +{ + if (!handle) + { + assert(0); + return false; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return false; + + // FIXME slow - most info is thrown away. + ktx2_image_level_info level_info; + if (!pHandle->m_transcoder.get_image_level_info(level_info, level_index, layer_index, face_index)) + return false; + + return level_info.m_alpha_flag; +} + +BU_WASM_EXPORT("bt_ktx2_get_level_iframe_flag") +wasm_bool_t bt_ktx2_get_level_iframe_flag(uint64_t handle, uint32_t level_index, uint32_t layer_index, uint32_t face_index) +{ + if (!handle) + { + assert(0); + return false; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return false; + + // FIXME slow - most info is thrown away. + ktx2_image_level_info level_info; + if (!pHandle->m_transcoder.get_image_level_info(level_info, level_index, layer_index, face_index)) + return false; + + return level_info.m_iframe_flag; +} + +BU_WASM_EXPORT("bt_ktx2_start_transcoding") +wasm_bool_t bt_ktx2_start_transcoding(uint64_t handle) +{ + if (!handle) + { + assert(0); + return false; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return false; + + return pHandle->m_transcoder.start_transcoding(); +} + +const uint32_t KTX2_TRANSCODE_STATE_MAGIC = 0x2B21CF21; + +struct ktx2_transcode_state_t +{ + uint32_t m_magic = KTX2_TRANSCODE_STATE_MAGIC; + + ktx2_transcoder_state m_state; +}; + +BU_WASM_EXPORT("bt_ktx2_create_transcode_state") +uint64_t bt_ktx2_create_transcode_state() +{ + return wasm_offset(new ktx2_transcode_state_t()); +} + +BU_WASM_EXPORT("bt_ktx2_destroy_transcode_state") +void bt_ktx2_destroy_transcode_state(uint64_t handle) +{ + if (!handle) + return; + + ktx2_transcode_state_t* pState = reinterpret_cast(wasm_ptr(handle)); + + assert(pState->m_magic == KTX2_TRANSCODE_STATE_MAGIC); + if (pState->m_magic != KTX2_TRANSCODE_STATE_MAGIC) + return; + + delete pState; +} + +BU_WASM_EXPORT("bt_ktx2_transcode_image_level") +wasm_bool_t bt_ktx2_transcode_image_level( + uint64_t ktx2_handle, + uint32_t level_index, uint32_t layer_index, uint32_t face_index, + uint64_t output_block_mem_ofs, uint32_t output_blocks_buf_size_in_blocks_or_pixels, + uint32_t transcoder_texture_format_u32, + uint32_t decode_flags, + uint32_t output_row_pitch_in_blocks_or_pixels, + uint32_t output_rows_in_pixels, + int channel0, int channel1, + uint64_t state_handle) +{ + if ((!ktx2_handle) || (!output_block_mem_ofs)) + { + assert(0); + return false; + } + + ktx2_handle_t* pHandle = reinterpret_cast(wasm_ptr(ktx2_handle)); + + assert(pHandle->m_magic == KTX2_HANDLE_MAGIC); + if (pHandle->m_magic != KTX2_HANDLE_MAGIC) + return false; + + assert(transcoder_texture_format_u32 < (uint32_t)transcoder_texture_format::cTFTotalTextureFormats); + transcoder_texture_format tex_fmt = static_cast(transcoder_texture_format_u32); + + ktx2_transcode_state_t* pTranscode_state = nullptr; + + if (state_handle) + { + pTranscode_state = reinterpret_cast(wasm_ptr(state_handle)); + + assert(pTranscode_state->m_magic == KTX2_TRANSCODE_STATE_MAGIC); + if (pTranscode_state->m_magic != KTX2_TRANSCODE_STATE_MAGIC) + return false; + } + + return pHandle->m_transcoder.transcode_image_level( + level_index, layer_index, face_index, + wasm_ptr(output_block_mem_ofs), output_blocks_buf_size_in_blocks_or_pixels, + tex_fmt, + decode_flags, output_row_pitch_in_blocks_or_pixels, output_rows_in_pixels, channel0, channel1, + pTranscode_state ? &pTranscode_state->m_state : nullptr); +} diff --git a/external/basis_universal/encoder/basisu_wasm_transcoder_api.h b/external/basis_universal/encoder/basisu_wasm_transcoder_api.h new file mode 100644 index 0000000000..a7389acee9 --- /dev/null +++ b/external/basis_universal/encoder/basisu_wasm_transcoder_api.h @@ -0,0 +1,216 @@ +// File: basisu_wasm_transcoder_api.h - Transcoding API support for WASM WASI modules and Python native support. +#pragma once +#include "basisu_wasm_api_common.h" + +// High-level functions + +BU_WASM_EXPORT("bt_get_version") +uint32_t bt_get_version(); + +BU_WASM_EXPORT("bt_enable_debug_printf") +void bt_enable_debug_printf(uint32_t flag); + +BU_WASM_EXPORT("bt_init") +void bt_init(); + +BU_WASM_EXPORT("bt_alloc") +uint64_t bt_alloc(uint64_t size); + +BU_WASM_EXPORT("bt_free") +void bt_free(uint64_t ofs); + +// basis_tex_format helpers + +BU_WASM_EXPORT("bt_basis_tex_format_is_xuastc_ldr") +wasm_bool_t bt_basis_tex_format_is_xuastc_ldr(uint32_t basis_tex_fmt_u32); + +BU_WASM_EXPORT("bt_basis_tex_format_is_astc_ldr") +wasm_bool_t bt_basis_tex_format_is_astc_ldr(uint32_t basis_tex_fmt_u32); + +BU_WASM_EXPORT("bt_basis_tex_format_get_block_width") +uint32_t bt_basis_tex_format_get_block_width(uint32_t basis_tex_fmt_u32); + +BU_WASM_EXPORT("bt_basis_tex_format_get_block_height") +uint32_t bt_basis_tex_format_get_block_height(uint32_t basis_tex_fmt_u32); + +BU_WASM_EXPORT("bt_basis_tex_format_is_hdr") +wasm_bool_t bt_basis_tex_format_is_hdr(uint32_t basis_tex_format_u32); + +BU_WASM_EXPORT("bt_basis_tex_format_is_ldr") +wasm_bool_t bt_basis_tex_format_is_ldr(uint32_t basis_tex_format_u32); + +// transcoder_texture_format helpers + +BU_WASM_EXPORT("bt_basis_get_bytes_per_block_or_pixel") +uint32_t bt_basis_get_bytes_per_block_or_pixel(uint32_t transcoder_texture_format_u32); + +BU_WASM_EXPORT("bt_basis_transcoder_format_has_alpha") +wasm_bool_t bt_basis_transcoder_format_has_alpha(uint32_t transcoder_texture_format_u32); + +BU_WASM_EXPORT("bt_basis_transcoder_format_is_hdr") +wasm_bool_t bt_basis_transcoder_format_is_hdr(uint32_t transcoder_texture_format_u32); + +BU_WASM_EXPORT("bt_basis_transcoder_format_is_ldr") +wasm_bool_t bt_basis_transcoder_format_is_ldr(uint32_t transcoder_texture_format_u32); + +BU_WASM_EXPORT("bt_basis_transcoder_texture_format_is_astc") +wasm_bool_t bt_basis_transcoder_texture_format_is_astc(uint32_t transcoder_texture_format_u32); + +BU_WASM_EXPORT("bt_basis_transcoder_format_is_uncompressed") +wasm_bool_t bt_basis_transcoder_format_is_uncompressed(uint32_t transcoder_texture_format_u32); + +BU_WASM_EXPORT("bt_basis_get_uncompressed_bytes_per_pixel") +uint32_t bt_basis_get_uncompressed_bytes_per_pixel(uint32_t transcoder_texture_format_u32); + +BU_WASM_EXPORT("bt_basis_get_block_width") +uint32_t bt_basis_get_block_width(uint32_t transcoder_texture_format_u32); + +BU_WASM_EXPORT("bt_basis_get_block_height") +uint32_t bt_basis_get_block_height(uint32_t transcoder_texture_format_u32); + +BU_WASM_EXPORT("bt_basis_get_transcoder_texture_format_from_basis_tex_format") +uint32_t bt_basis_get_transcoder_texture_format_from_basis_tex_format(uint32_t basis_tex_format_u32); + +BU_WASM_EXPORT("bt_basis_is_format_supported") +wasm_bool_t bt_basis_is_format_supported(uint32_t transcoder_texture_format_u32, uint32_t basis_tex_format_u32); + +BU_WASM_EXPORT("bt_basis_compute_transcoded_image_size_in_bytes") +uint32_t bt_basis_compute_transcoded_image_size_in_bytes(uint32_t transcoder_texture_format_u32, uint32_t orig_width, uint32_t orig_height); + +// Transcoding +BU_WASM_EXPORT("bt_ktx2_open") +uint64_t bt_ktx2_open(uint64_t data_mem_ofs, uint32_t data_len); + +BU_WASM_EXPORT("bt_ktx2_close") +void bt_ktx2_close(uint64_t handle); + +BU_WASM_EXPORT("bt_ktx2_get_width") +uint32_t bt_ktx2_get_width(uint64_t handle); + +BU_WASM_EXPORT("bt_ktx2_get_height") +uint32_t bt_ktx2_get_height(uint64_t handle); + +BU_WASM_EXPORT("bt_ktx2_get_levels") +uint32_t bt_ktx2_get_levels(uint64_t handle); + +BU_WASM_EXPORT("bt_ktx2_get_faces") +uint32_t bt_ktx2_get_faces(uint64_t handle); + +BU_WASM_EXPORT("bt_ktx2_get_layers") +uint32_t bt_ktx2_get_layers(uint64_t handle); + +BU_WASM_EXPORT("bt_ktx2_get_basis_tex_format") +uint32_t bt_ktx2_get_basis_tex_format(uint64_t handle); + +BU_WASM_EXPORT("bt_ktx2_is_etc1s") +wasm_bool_t bt_ktx2_is_etc1s(uint64_t handle); + +BU_WASM_EXPORT("bt_ktx2_is_uastc_ldr_4x4") +wasm_bool_t bt_ktx2_is_uastc_ldr_4x4(uint64_t handle); + +BU_WASM_EXPORT("bt_ktx2_is_hdr") +wasm_bool_t bt_ktx2_is_hdr(uint64_t handle); + +BU_WASM_EXPORT("bt_ktx2_is_hdr_4x4") +wasm_bool_t bt_ktx2_is_hdr_4x4(uint64_t handle); + +BU_WASM_EXPORT("bt_ktx2_is_hdr_6x6") +wasm_bool_t bt_ktx2_is_hdr_6x6(uint64_t handle); + +BU_WASM_EXPORT("bt_ktx2_is_ldr") +wasm_bool_t bt_ktx2_is_ldr(uint64_t handle); + +BU_WASM_EXPORT("bt_ktx2_is_astc_ldr") +wasm_bool_t bt_ktx2_is_astc_ldr(uint64_t handle); + +BU_WASM_EXPORT("bt_ktx2_is_xuastc_ldr") +wasm_bool_t bt_ktx2_is_xuastc_ldr(uint64_t handle); + +BU_WASM_EXPORT("bt_ktx2_get_block_width") +uint32_t bt_ktx2_get_block_width(uint64_t handle); + +BU_WASM_EXPORT("bt_ktx2_get_block_height") +uint32_t bt_ktx2_get_block_height(uint64_t handle); + +BU_WASM_EXPORT("bt_ktx2_has_alpha") +wasm_bool_t bt_ktx2_has_alpha(uint64_t handle); + +BU_WASM_EXPORT("bt_ktx2_get_dfd_color_model") +uint32_t bt_ktx2_get_dfd_color_model(uint64_t handle); + +BU_WASM_EXPORT("bt_ktx2_get_dfd_color_primaries") +uint32_t bt_ktx2_get_dfd_color_primaries(uint64_t handle); + +BU_WASM_EXPORT("bt_ktx2_get_dfd_transfer_func") +uint32_t bt_ktx2_get_dfd_transfer_func(uint64_t handle); + +BU_WASM_EXPORT("bt_ktx2_is_srgb") +wasm_bool_t bt_ktx2_is_srgb(uint64_t handle); + +BU_WASM_EXPORT("bt_ktx2_get_dfd_flags") +uint32_t bt_ktx2_get_dfd_flags(uint64_t handle); + +BU_WASM_EXPORT("bt_ktx2_get_dfd_total_samples") +uint32_t bt_ktx2_get_dfd_total_samples(uint64_t handle); + +BU_WASM_EXPORT("bt_ktx2_get_dfd_channel_id0") +uint32_t bt_ktx2_get_dfd_channel_id0(uint64_t handle); + +BU_WASM_EXPORT("bt_ktx2_get_dfd_channel_id1") +uint32_t bt_ktx2_get_dfd_channel_id1(uint64_t handle); + +BU_WASM_EXPORT("bt_ktx2_is_video") +wasm_bool_t bt_ktx2_is_video(uint64_t handle); + +BU_WASM_EXPORT("bt_ktx2_get_ldr_hdr_upconversion_nit_multiplier") +float bt_ktx2_get_ldr_hdr_upconversion_nit_multiplier(uint64_t handle); + +BU_WASM_EXPORT("bt_ktx2_get_level_orig_width") +uint32_t bt_ktx2_get_level_orig_width(uint64_t handle, uint32_t level_index, uint32_t layer_index, uint32_t face_index); + +BU_WASM_EXPORT("bt_ktx2_get_level_orig_height") +uint32_t bt_ktx2_get_level_orig_height(uint64_t handle, uint32_t level_index, uint32_t layer_index, uint32_t face_index); + +BU_WASM_EXPORT("bt_ktx2_get_level_actual_width") +uint32_t bt_ktx2_get_level_actual_width(uint64_t handle, uint32_t level_index, uint32_t layer_index, uint32_t face_index); + +BU_WASM_EXPORT("bt_ktx2_get_level_actual_height") +uint32_t bt_ktx2_get_level_actual_height(uint64_t handle, uint32_t level_index, uint32_t layer_index, uint32_t face_index); + +BU_WASM_EXPORT("bt_ktx2_get_level_num_blocks_x") +uint32_t bt_ktx2_get_level_num_blocks_x(uint64_t handle, uint32_t level_index, uint32_t layer_index, uint32_t face_index); + +BU_WASM_EXPORT("bt_ktx2_get_level_num_blocks_y") +uint32_t bt_ktx2_get_level_num_blocks_y(uint64_t handle, uint32_t level_index, uint32_t layer_index, uint32_t face_index); + +BU_WASM_EXPORT("bt_ktx2_get_level_total_blocks") +uint32_t bt_ktx2_get_level_total_blocks(uint64_t handle, uint32_t level_index, uint32_t layer_index, uint32_t face_index); + +BU_WASM_EXPORT("bt_ktx2_get_level_alpha_flag") +wasm_bool_t bt_ktx2_get_level_alpha_flag(uint64_t handle, uint32_t level_index, uint32_t layer_index, uint32_t face_index); + +BU_WASM_EXPORT("bt_ktx2_get_level_iframe_flag") +wasm_bool_t bt_ktx2_get_level_iframe_flag(uint64_t handle, uint32_t level_index, uint32_t layer_index, uint32_t face_index); + +BU_WASM_EXPORT("bt_ktx2_start_transcoding") +wasm_bool_t bt_ktx2_start_transcoding(uint64_t handle); + +BU_WASM_EXPORT("bt_ktx2_create_transcode_state") +uint64_t bt_ktx2_create_transcode_state(); + +BU_WASM_EXPORT("bt_ktx2_destroy_transcode_state") +void bt_ktx2_destroy_transcode_state(uint64_t handle); + +BU_WASM_EXPORT("bt_ktx2_transcode_image_level") +wasm_bool_t bt_ktx2_transcode_image_level( + uint64_t ktx2_handle, // handle to KTX2 file, see bt_ktx2_open() + uint32_t level_index, uint32_t layer_index, uint32_t face_index, // KTX2 level/layer/face to transcode + uint64_t output_block_mem_ofs, // allocate using bt_alloc() + uint32_t output_blocks_buf_size_in_blocks_or_pixels, + uint32_t transcoder_texture_format_u32, // target format, TF_ETC1_RGB etc. + uint32_t decode_flags, // DECODE_FLAGS_ + uint32_t output_row_pitch_in_blocks_or_pixels, // can be 0 + uint32_t output_rows_in_pixels, // can be 0 + int channel0, int channel1, // both default to -1 + uint64_t state_handle); // thread local state: can be 0, or bt_ktx2_create_transcode_state() + diff --git a/external/basis_universal/encoder/cppspmd_flow.h b/external/basis_universal/encoder/cppspmd_flow.h index 07b592455d..3e83e9eda0 100644 --- a/external/basis_universal/encoder/cppspmd_flow.h +++ b/external/basis_universal/encoder/cppspmd_flow.h @@ -48,7 +48,7 @@ CPPSPMD_FORCE_INLINE void spmd_kernel::spmd_return() m_kernel_exec = andnot(m_exec, m_kernel_exec); m_exec = exec_mask::all_off(); } - + template CPPSPMD_FORCE_INLINE void spmd_kernel::spmd_unmasked(const UnmaskedBody& unmaskedBody) { @@ -61,7 +61,7 @@ CPPSPMD_FORCE_INLINE void spmd_kernel::spmd_unmasked(const UnmaskedBody& unmaske m_kernel_exec = m_kernel_exec & orig_kernel_exec; m_exec = m_exec & orig_exec; - + check_masks(); } @@ -69,9 +69,9 @@ struct scoped_unmasked_restorer { spmd_kernel *m_pKernel; exec_mask m_orig_exec, m_orig_kernel_exec; - - CPPSPMD_FORCE_INLINE scoped_unmasked_restorer(spmd_kernel *pKernel) : - m_pKernel(pKernel), + + CPPSPMD_FORCE_INLINE scoped_unmasked_restorer(spmd_kernel *pKernel) : + m_pKernel(pKernel), m_orig_exec(pKernel->m_exec), m_orig_kernel_exec(pKernel->m_kernel_exec) { @@ -79,15 +79,15 @@ struct scoped_unmasked_restorer pKernel->m_exec = exec_mask::all_on(); } - CPPSPMD_FORCE_INLINE ~scoped_unmasked_restorer() - { + CPPSPMD_FORCE_INLINE ~scoped_unmasked_restorer() + { m_pKernel->m_kernel_exec = m_pKernel->m_kernel_exec & m_orig_kernel_exec; m_pKernel->m_exec = m_pKernel->m_exec & m_orig_exec; m_pKernel->check_masks(); } }; -#define SPMD_UNMASKED_BEGIN { scoped_unmasked_restorer _unmasked_restorer(this); +#define SPMD_UNMASKED_BEGIN { scoped_unmasked_restorer _unmasked_restorer(this); #define SPMD_UNMASKED_END } #if 0 @@ -113,9 +113,9 @@ CPPSPMD_FORCE_INLINE void spmd_kernel::spmd_if_break(const vbool& cond) #ifdef _DEBUG assert(m_in_loop); #endif - + exec_mask cond_exec(cond); - + m_exec = andnot(m_exec & cond_exec, m_exec); check_masks(); @@ -157,7 +157,7 @@ CPPSPMD_FORCE_INLINE void spmd_kernel::spmd_sifelse(const vbool& cond, const IfB m_exec = em; elseBody(); } - + m_exec = orig_exec; } @@ -165,7 +165,7 @@ template CPPSPMD_FORCE_INLINE void spmd_kernel::spmd_if(const vbool& cond, const IfBody& ifBody) { exec_mask cond_exec(cond); - + exec_mask pre_if_exec = cond_exec & m_exec; if (any(pre_if_exec)) @@ -188,7 +188,7 @@ CPPSPMD_FORCE_INLINE void spmd_kernel::spmd_ifelse(const vbool& cond, const IfBo bool all_flag = false; exec_mask cond_exec(cond); - + { exec_mask pre_if_exec = cond_exec & m_exec; @@ -218,9 +218,10 @@ CPPSPMD_FORCE_INLINE void spmd_kernel::spmd_ifelse(const vbool& cond, const IfBo exec_mask unexecuted_lanes = cond_exec & m_exec; m_exec = pre_if_exec; - ifBody(); + // 11/22/2025: changed to elseBody() here, simple bug, we use the macro variants of ifelse anyway + elseBody(); - // Propagate any lanes that got disabled inside the if body into the exec mask outside the if body, but turn on any lanes that didn't execute inside the if body. + // Propagate any lanes that got disabled inside the else body into the exec mask outside the else body, but turn on any lanes that didn't execute inside the else body. m_exec = m_exec | unexecuted_lanes; check_masks(); @@ -290,17 +291,17 @@ struct scoped_exec_restorer2 { spmd_kernel *m_pKernel; exec_mask m_unexecuted_lanes; - - CPPSPMD_FORCE_INLINE scoped_exec_restorer2(spmd_kernel *pKernel, const vbool &cond) : + + CPPSPMD_FORCE_INLINE scoped_exec_restorer2(spmd_kernel *pKernel, const vbool &cond) : m_pKernel(pKernel) - { + { exec_mask cond_exec(cond); m_unexecuted_lanes = andnot(cond_exec, pKernel->m_exec); pKernel->m_exec = cond_exec & pKernel->m_exec; } - CPPSPMD_FORCE_INLINE ~scoped_exec_restorer2() - { + CPPSPMD_FORCE_INLINE ~scoped_exec_restorer2() + { m_pKernel->m_exec = m_pKernel->m_exec | m_unexecuted_lanes; m_pKernel->check_masks(); } @@ -327,17 +328,17 @@ class scoped_exec_saver inline scoped_exec_saver(spmd_kernel *pKernel) : m_exec(pKernel->m_exec), m_kernel_exec(pKernel->m_kernel_exec), m_continue_mask(pKernel->m_continue_mask), m_pKernel(pKernel) - { + { #ifdef _DEBUG m_in_loop = pKernel->m_in_loop; #endif } - + inline ~scoped_exec_saver() - { - m_pKernel->m_exec = m_exec; - m_pKernel->m_continue_mask = m_continue_mask; - m_pKernel->m_kernel_exec = m_kernel_exec; + { + m_pKernel->m_exec = m_exec; + m_pKernel->m_continue_mask = m_continue_mask; + m_pKernel->m_kernel_exec = m_kernel_exec; #ifdef _DEBUG m_pKernel->m_in_loop = m_in_loop; m_pKernel->check_masks(); @@ -353,7 +354,7 @@ CPPSPMD_FORCE_INLINE void spmd_kernel::spmd_foreach(int begin, int end, const Fo { if (begin == end) return; - + if (!any(m_exec)) return; @@ -362,12 +363,12 @@ CPPSPMD_FORCE_INLINE void spmd_kernel::spmd_foreach(int begin, int end, const Fo std::swap(begin, end); exec_mask prev_continue_mask = m_continue_mask, prev_exec = m_exec; - + int total_full = (end - begin) / PROGRAM_COUNT; int total_partial = (end - begin) % PROGRAM_COUNT; lint_t loop_index = begin + program_index; - + const int total_loops = total_full + (total_partial ? 1 : 0); m_continue_mask = exec_mask::all_off(); @@ -390,7 +391,7 @@ CPPSPMD_FORCE_INLINE void spmd_kernel::spmd_foreach(int begin, int end, const Fo m_continue_mask = exec_mask::all_off(); check_masks(); - + store_all(loop_index, loop_index + PROGRAM_COUNT); } @@ -443,9 +444,9 @@ struct scoped_while_restorer #ifdef _DEBUG bool m_prev_in_loop; #endif - - CPPSPMD_FORCE_INLINE scoped_while_restorer(spmd_kernel *pKernel) : - m_pKernel(pKernel), + + CPPSPMD_FORCE_INLINE scoped_while_restorer(spmd_kernel *pKernel) : + m_pKernel(pKernel), m_orig_exec(pKernel->m_exec), m_orig_continue_mask(pKernel->m_continue_mask) { @@ -457,8 +458,8 @@ struct scoped_while_restorer #endif } - CPPSPMD_FORCE_INLINE ~scoped_while_restorer() - { + CPPSPMD_FORCE_INLINE ~scoped_while_restorer() + { m_pKernel->m_exec = m_orig_exec & m_pKernel->m_kernel_exec; m_pKernel->m_continue_mask = m_orig_continue_mask; #ifdef _DEBUG @@ -514,7 +515,7 @@ struct scoped_simple_while_restorer m_pKernel(pKernel), m_orig_exec(pKernel->m_exec) { - + #ifdef _DEBUG m_prev_in_loop = pKernel->m_in_loop; pKernel->m_in_loop = true; @@ -536,18 +537,18 @@ struct scoped_simple_while_restorer #define SPMD_SWHILE(cond) { scoped_simple_while_restorer CPPSPMD_GLUER2(_while_restore_, __LINE__)(this); \ while(true) { \ exec_mask CPPSPMD_GLUER2(cond_exec, __LINE__) = exec_mask(vbool(cond)); m_exec = m_exec & CPPSPMD_GLUER2(cond_exec, __LINE__); if (!any(m_exec)) break; -#define SPMD_SWEND } } +#define SPMD_SWEND } } // Cannot use SPMD break, continue, or return inside simple do #define SPMD_SDO { scoped_simple_while_restorer CPPSPMD_GLUER2(_while_restore_, __LINE__)(this); while(true) { -#define SPMD_SEND_DO(cond) exec_mask CPPSPMD_GLUER2(cond_exec, __LINE__) = exec_mask(vbool(cond)); m_exec = m_exec & CPPSPMD_GLUER2(cond_exec, __LINE__); if (!any(m_exec)) break; } } +#define SPMD_SEND_DO(cond) exec_mask CPPSPMD_GLUER2(cond_exec, __LINE__) = exec_mask(vbool(cond)); m_exec = m_exec & CPPSPMD_GLUER2(cond_exec, __LINE__); if (!any(m_exec)) break; } } #undef SPMD_FOR #undef SPMD_END_FOR #define SPMD_FOR(for_init, for_cond) { for_init; scoped_while_restorer CPPSPMD_GLUER2(_while_restore_, __LINE__)(this); while(true) { exec_mask CPPSPMD_GLUER2(cond_exec, __LINE__) = exec_mask(vbool(for_cond)); \ m_exec = m_exec & CPPSPMD_GLUER2(cond_exec, __LINE__); if (!any(m_exec)) break; #define SPMD_END_FOR(for_inc) m_exec = m_exec | m_continue_mask; m_continue_mask = exec_mask::all_off(); check_masks(); for_inc; } } - + template CPPSPMD_FORCE_INLINE void spmd_kernel::spmd_for(const ForInitBody& forInitBody, const ForCondBody& forCondBody, const ForIncrBody& forIncrBody, const ForBody& forBody) { @@ -576,7 +577,7 @@ CPPSPMD_FORCE_INLINE void spmd_kernel::spmd_for(const ForInitBody& forInitBody, m_exec = m_exec | m_continue_mask; m_continue_mask = exec_mask::all_off(); check_masks(); - + forIncrBody(); } diff --git a/external/basis_universal/encoder/jpgd.cpp b/external/basis_universal/encoder/jpgd.cpp index 57c7ec7b68..9a534b3ee1 100644 --- a/external/basis_universal/encoder/jpgd.cpp +++ b/external/basis_universal/encoder/jpgd.cpp @@ -3,10 +3,11 @@ // Supports box and linear chroma upsampling. // // Released under two licenses. You are free to choose which license you want: -// License 1: +// License 1: // Public Domain // // License 2: +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -24,6 +25,10 @@ // v2.00, March 20, 2020: Fuzzed with zzuf and afl. Fixed several issues, converted most assert()'s to run-time checks. Added chroma upsampling. Removed freq. domain upsampling. gcc/clang warnings. // +#if defined(__wasi__) +#pragma message("__wasi__ defined in jpgd.cpp: note if a decode error occurs, the app will exit because wasi doesn't support longjmp yet.") +#endif + #include "jpgd.h" #include #include @@ -138,7 +143,7 @@ namespace jpgd { { static void idct(int* pTemp, const jpgd_block_t* pSrc) { - (void)pTemp; + (void)pTemp; (void)pSrc; } }; @@ -253,10 +258,10 @@ namespace jpgd { 8,8,8,8,8,7,6,4, 8,8,8,8,8,7,6,5, 8,8,8,8,8,7,6,6, 8,8,8,8,8,7,7,6, 8,8,8,8,8,8,7,6, 8,8,8,8,8,8,8,6, 8,8,8,8,8,8,8,7, 8,8,8,8,8,8,8,8, }; - static const uint8 s_idct_col_table[] = - { - 1, 1, 2, 3, 3, 3, 3, 3, 3, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, - 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8 + static const uint8 s_idct_col_table[] = + { + 1, 1, 2, 3, 3, 3, 3, 3, 3, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8 }; // Scalar "fast pathing" IDCT. @@ -608,7 +613,14 @@ namespace jpgd { { m_error_code = status; free_all_blocks(); + +#ifdef __wasi__ + // HACK HACK for wasi's lack of longjmp support + fprintf(stderr, "jpeg_decoder::stop_decoding: JPEG decode failed with status: %i\n", (int)status); + exit(EXIT_FAILURE); +#else longjmp(m_jmp_state, status); +#endif } void* jpeg_decoder::alloc(size_t nSize, bool zero) @@ -2071,8 +2083,10 @@ namespace jpgd { int jpeg_decoder::decode_next_mcu_row() { +#ifndef __wasi__ if (setjmp(m_jmp_state)) return JPGD_FAILED; +#endif const bool chroma_y_filtering = (m_flags & cFlagLinearChromaFiltering) && ((m_scan_type == JPGD_YH2V2) || (m_scan_type == JPGD_YH1V2)) && (m_image_x_size >= 2) && (m_image_y_size >= 2); if (chroma_y_filtering) @@ -2987,8 +3001,10 @@ namespace jpgd { jpeg_decoder::jpeg_decoder(jpeg_decoder_stream* pStream, uint32_t flags) { +#ifndef __wasi__ if (setjmp(m_jmp_state)) return; +#endif decode_init(pStream, flags); } @@ -3000,8 +3016,10 @@ namespace jpgd { if (m_error_code) return JPGD_FAILED; +#ifndef __wasi__ if (setjmp(m_jmp_state)) return JPGD_FAILED; +#endif decode_start(); diff --git a/external/basis_universal/encoder/jpgd.h b/external/basis_universal/encoder/jpgd.h index 92e53335c6..bd1ad808c0 100644 --- a/external/basis_universal/encoder/jpgd.h +++ b/external/basis_universal/encoder/jpgd.h @@ -1,16 +1,18 @@ // jpgd.h - C++ class for JPEG decompression. -// Public domain, Rich Geldreich +// Dual licensed: Public domain, Rich Geldreich , or Apache 2.0 (see jpgd.cpp) #ifndef JPEG_DECODER_H #define JPEG_DECODER_H #include #include +#ifndef __wasi__ #include +#endif #include #include #ifdef _MSC_VER -#define JPGD_NORETURN __declspec(noreturn) +#define JPGD_NORETURN __declspec(noreturn) #elif defined(__GNUC__) #define JPGD_NORETURN __attribute__ ((noreturn)) #else @@ -140,7 +142,7 @@ namespace jpgd int begin_decoding(); // Returns the next scan line. - // For grayscale images, pScan_line will point to a buffer containing 8-bit pixels (get_bytes_per_pixel() will return 1). + // For grayscale images, pScan_line will point to a buffer containing 8-bit pixels (get_bytes_per_pixel() will return 1). // Otherwise, it will always point to a buffer containing 32-bit RGBA pixels (A will always be 255, and get_bytes_per_pixel() will return 4). // Returns JPGD_SUCCESS if a scan line has been returned. // Returns JPGD_DONE if all scan lines have been returned. @@ -191,7 +193,10 @@ namespace jpgd char m_data[1]; }; + // TODO: we can get rid of longjmp entirely +#ifndef __wasi__ jmp_buf m_jmp_state; +#endif uint32_t m_flags; mem_block* m_pMem_blocks; int m_image_x_size; diff --git a/external/basis_universal/encoder_lib/encoder_lib.vcxproj b/external/basis_universal/encoder_lib/encoder_lib.vcxproj index 3ffa09af26..90daddeb04 100644 --- a/external/basis_universal/encoder_lib/encoder_lib.vcxproj +++ b/external/basis_universal/encoder_lib/encoder_lib.vcxproj @@ -31,6 +31,8 @@ + + @@ -59,6 +61,8 @@ + + @@ -92,12 +96,14 @@ + + @@ -119,40 +125,40 @@ StaticLibrary true Unicode - v143 + v145 StaticLibrary false true Unicode - v143 + v145 StaticLibrary true Unicode - v143 + v145 StaticLibrary true Unicode - v143 + v145 StaticLibrary false true Unicode - v143 + v145 StaticLibrary false true Unicode - v143 + v145 @@ -188,7 +194,7 @@ pch.h ..\OpenCL stdcpp17 - StreamingSIMDExtensions2 + AdvancedVectorExtensions @@ -207,7 +213,7 @@ NotUsing pch.h ..\OpenCL - StreamingSIMDExtensions2 + AdvancedVectorExtensions false false stdcpp17 @@ -227,7 +233,7 @@ true NotUsing pch.h - StreamingSIMDExtensions2 + AdvancedVectorExtensions ..\OpenCL Level4 stdcpp17 @@ -267,7 +273,7 @@ pch.h false false - StreamingSIMDExtensions2 + AdvancedVectorExtensions ..\OpenCL Full AnySuitable @@ -313,4 +319,4 @@ - + \ No newline at end of file diff --git a/external/basis_universal/encoder_lib/encoder_lib.vcxproj.filters b/external/basis_universal/encoder_lib/encoder_lib.vcxproj.filters index aaee9d98ac..093809c2df 100644 --- a/external/basis_universal/encoder_lib/encoder_lib.vcxproj.filters +++ b/external/basis_universal/encoder_lib/encoder_lib.vcxproj.filters @@ -1,4 +1,4 @@ - + @@ -99,6 +99,12 @@ Source Files\encoder + + Source Files\encoder + + + Source Files\encoder + @@ -227,6 +233,15 @@ Source Files\encoder + + Source Files\encoder + + + Source Files\encoder + + + Source Files\transcoder + @@ -256,5 +271,8 @@ Source Files\encoder + + Source Files\transcoder + - + \ No newline at end of file diff --git a/external/basis_universal/example/example.cpp b/external/basis_universal/example/example.cpp index 5062d3d209..d0090169f2 100644 --- a/external/basis_universal/example/example.cpp +++ b/external/basis_universal/example/example.cpp @@ -1,15 +1,18 @@ // File: example.cpp // This minimal LDR/HDR encoding/transcoder example relies on encoder_lib. It shows how to use the encoder in a few different ways, and the transcoder. -// -// It should be compiled with the preprocessor macros BASISU_SUPPORT_SSE (typically 1) and BASISU_SUPPORT_OPENCL (typically 1). +// +// It should be compiled with the preprocessor macros BASISU_SUPPORT_SSE (typically 1) and BASISU_SUPPORT_OPENCL (typically 1). // They should be set to the same preprocesor options as the encoder. // If OpenCL is enabled, the "..\OpenCL" directory should be in your compiler's include path. Additionally, link against "..\OpenCL\lib\opencl64.lib". #include "../encoder/basisu_comp.h" #include "../transcoder/basisu_transcoder.h" #include "../encoder/basisu_gpu_texture.h" +#include "../encoder/basisu_astc_ldr_encode.h" #define USE_ENCODER (1) +//#define FORCE_SAN_FAILURE + const bool USE_OPENCL = false; // The encoder lives in the "basisu" namespace. @@ -17,7 +20,7 @@ const bool USE_OPENCL = false; using namespace basisu; // Quick function to create a visualization of the Mandelbrot set as an float HDR image. -static void create_mandelbrot(imagef& img) +static void create_mandelbrot(imagef& img) { const int width = 256; const int height = 256; @@ -25,30 +28,30 @@ static void create_mandelbrot(imagef& img) // Create a more interesting color palette uint8_t palette[256][3]; - for (int i = 0; i < 256; i++) + for (int i = 0; i < 256; i++) { - if (i < 64) + if (i < 64) { // Blue to cyan transition palette[i][0] = static_cast(0); // Red component palette[i][1] = static_cast(i * 4); // Green component palette[i][2] = static_cast(255); // Blue component } - else if (i < 128) + else if (i < 128) { // Cyan to green transition palette[i][0] = static_cast(0); // Red component palette[i][1] = static_cast(255); // Green component palette[i][2] = static_cast(255 - (i - 64) * 4); // Blue component } - else if (i < 192) + else if (i < 192) { // Green to yellow transition palette[i][0] = static_cast((i - 128) * 4); // Red component palette[i][1] = static_cast(255); // Green component palette[i][2] = static_cast(0); // Blue component } - else + else { // Yellow to red transition palette[i][0] = static_cast(255); // Red component @@ -58,9 +61,9 @@ static void create_mandelbrot(imagef& img) } // Iterate over each pixel in the image - for (int px = 0; px < width; px++) + for (int px = 0; px < width; px++) { - for (int py = 0; py < height; py++) + for (int py = 0; py < height; py++) { double x0 = (px - width / 2.0) * 4.0 / width; double y0 = (py - height / 2.0) * 4.0 / height; @@ -71,7 +74,7 @@ static void create_mandelbrot(imagef& img) double x_temp; int iter; - for (iter = 0; iter < max_iter; iter++) + for (iter = 0; iter < max_iter; iter++) { zx_squared = zx * zx; zy_squared = zy * zy; @@ -148,7 +151,7 @@ static bool encode_uastc_ldr() // basis_compress() is a simple wrapper around the basis_compressor_params and basis_compressor classes. void* pKTX2_data = basis_compress( - basist::basis_tex_format::cUASTC4x4, + basist::basis_tex_format::cUASTC_LDR_4x4, source_images, cFlagThreaded | cFlagPrintStats | cFlagDebug | cFlagPrintStatus, 0.0f, &file_size, @@ -173,7 +176,7 @@ static bool encode_uastc_ldr() static bool encode_uastc_hdr() { const uint32_t W = 256, H = 256; - + imagef img(W, H); #if 1 @@ -196,7 +199,7 @@ static bool encode_uastc_hdr() params.m_write_output_basis_or_ktx2_files = true; params.m_out_filename = "test_uastc_hdr.ktx2"; params.m_perceptual = true; - + #if 1 // Create a job pool containing 7 total threads (the calling thread plus 6 additional threads). // A job pool must be created, even if threading is disabled. It's fine to pass in 0 for NUM_THREADS. @@ -219,13 +222,13 @@ static bool encode_uastc_hdr() basisu::basis_compressor::error_code ec = comp.process(); if (ec != basisu::basis_compressor::cECSuccess) return false; - + return true; } // This example function loads a .KTX2 file and then transcodes it to various compressed/uncompressed texture formats. -// It writes .DDS and .ASTC files. -// ARM's astcenc tool can be used to unpack the .ASTC file: +// It writes .DDS and .ASTC files. +// ARM's astcenc tool can be used to unpack the .ASTC file: // astcenc-avx2.exe -dh test_uastc_hdr_astc.astc out.exr static bool transcode_hdr() { @@ -252,10 +255,10 @@ static bool transcode_hdr() // This example only transcodes UASTC HDR textures. if (!transcoder.is_hdr()) return false; - + // Begin transcoding (this will be a no-op with UASTC HDR textures, but you still need to do it. For ETC1S it'll unpack the global codebooks.) transcoder.start_transcoding(); - + // Transcode to BC6H and write a BC6H .DDS file. { gpu_image tex(texture_format::cBC6HUnsigned, width, height); @@ -268,7 +271,7 @@ static bool transcode_hdr() gpu_image_vec tex_vec; tex_vec.push_back(tex); - if (!write_compressed_texture_file("test_uastc_hdr_bc6h.dds", tex_vec, true)) + if (!write_compressed_texture_file("test_uastc_hdr_bc6h.dds", tex_vec, false)) return false; } @@ -423,7 +426,7 @@ const uint32_t NUM_TEST_BLOCKS = (sizeof(g_test_blocks) / sizeof(g_test_blocks[0 static bool block_unpack_and_transcode_example(void) { printf("block_unpack_and_transcode_example:\n"); - + for (uint32_t test_block_iter = 0; test_block_iter < NUM_TEST_BLOCKS; test_block_iter++) { printf("-- Test block %u:\n", test_block_iter); @@ -431,7 +434,7 @@ static bool block_unpack_and_transcode_example(void) const uint8_t* pASTC_blk = &g_test_blocks[test_block_iter * 2 + 0][0]; const uint8_t* pBC6H_blk = &g_test_blocks[test_block_iter * 2 + 1][0]; - // Unpack the physical ASTC block to logical. + // Unpack the physical ASTC block to logical. // Note this is a full ASTC block unpack, and is not specific to UASTC. It does not verify that the block follows the UASTC HDR spec, only ASTC. astc_helpers::log_astc_block log_blk; bool status = astc_helpers::unpack_block(pASTC_blk, log_blk, 4, 4); @@ -470,7 +473,7 @@ static bool block_unpack_and_transcode_example(void) return false; } } // test_block_iter - + printf("Transcode test OK\n"); return true; @@ -492,7 +495,7 @@ static void fuzz_uastc_hdr_transcoder_test() for (uint32_t t = 0; t < NUM_TRIES; t++) { basist::astc_blk astc_blk; - + if (rg.frand(0.0f, 1.0f) < .3f) { // Fully random block @@ -564,6 +567,85 @@ static void fuzz_uastc_hdr_transcoder_test() printf("OK\n"); } +void wrap_image(const image& src, image& dst, int gridX, int gridY, float maxOffset, bool randomize, basisu::rand &rnd) +{ + if (gridX < 1) gridX = 1; + if (gridY < 1) gridY = 1; + + const int vxCountX = gridX + 1; + const int vxCountY = gridY + 1; + const int stride = vxCountX; + + const int w = src.get_width(); + const int h = src.get_height(); + + dst.resize(w, h); + + dst.set_all(g_black_color); + + basisu::vector verts(vxCountX * vxCountY); + basisu::vector uvs(vxCountX * vxCountY); + basisu::vector cols(vxCountX * vxCountY); + + for (int gy = 0; gy <= gridY; ++gy) + { + for (int gx = 0; gx <= gridX; ++gx) + { + float x = (gx / float(gridX)) * (w - 1); + float y = (gy / float(gridY)) * (h - 1); + + float rx = x; + float ry = y; + + if (randomize) + { + rx += rnd.frand(-maxOffset, maxOffset); + ry += rnd.frand(-maxOffset, maxOffset); + } + + verts[gy * stride + gx] = { rx, ry }; + + float u = gx / float(gridX); + float v = gy / float(gridY); + + u = std::max(0.0f, std::min(1.0f, u)); + v = std::max(0.0f, std::min(1.0f, v)); + + uvs[gy * stride + gx] = { u, v }; + + color_rgba c(g_white_color); + + cols[gy * stride + gx] = c; + } + } + + for (int gy = 0; gy < gridY; ++gy) + { + for (int gx = 0; gx < gridX; ++gx) + { + int i0 = gy * stride + gx; + int i1 = i0 + 1; + int i2 = i0 + stride; + int i3 = i2 + 1; + + tri2 tA; + tA.p0 = verts[i0]; tA.p1 = verts[i1]; tA.p2 = verts[i3]; + tA.t0 = uvs[i0]; tA.t1 = uvs[i1]; tA.t2 = uvs[i3]; + tA.c0 = cols[i0]; tA.c1 = cols[i1]; tA.c2 = cols[i3]; + + draw_tri2(dst, &src, tA, randomize); + + tri2 tB; + tB.p0 = verts[i0]; tB.p1 = verts[i3]; tB.p2 = verts[i2]; + tB.t0 = uvs[i0]; tB.t1 = uvs[i3]; tB.t2 = uvs[i2]; + tB.c0 = cols[i0]; tB.c1 = cols[i3]; tB.c2 = cols[i2]; + + draw_tri2(dst, &src, tB, randomize); + } // gx + } // by + +} + enum class codec_class { cETC1S = 0, @@ -571,20 +653,28 @@ enum class codec_class cUASTC_HDR_4x4 = 2, cASTC_HDR_6x6 = 3, cUASTC_HDR_6x6 = 4, + cASTC_LDR = 5, + cXUASTC_LDR = 6, cTOTAL }; -// The main point of this test is to exercise lots of internal compressor code paths, and transcoder code paths. -bool random_compression_fuzz_test() +// The main point of this test is to exercise lots of internal code paths. +bool random_compress_test() { printf("Random XUASTC/ASTC LDR 4x4-12x12 compression test:\n"); - //const uint32_t N = 256; - const uint32_t N = 64; - const uint32_t MAX_WIDTH = 1024, MAX_HEIGHT = 1024; + const uint32_t num_images = 18; + image test_images[num_images + 1]; + for (uint32_t i = 0; i < num_images; i++) + load_png(fmt_string("../test_files/kodim{02}.png", 1 + i).c_str(), test_images[i]); + + const uint32_t N = 16; + //const uint32_t N = 5000; + const uint32_t MAX_WIDTH = 1024, MAX_HEIGHT = 1024; + basisu::rand rnd; - + float lowest_psnr1 = BIG_FLOAT_VAL, lowest_psnr2 = BIG_FLOAT_VAL; struct result @@ -598,9 +688,11 @@ bool random_compression_fuzz_test() for (uint32_t i = 0; i < N; i++) { - uint32_t seed = 0x2603455 + i; + uint32_t seed = 166136844 + i; - //seed = 23082246; // ETC1S perceptual colorspace error overflow test + //seed = 23082246; // etc1s 1-bit SSE overflow + //seed = 56636601; // UASTC HDR 4x4 assert tol + //seed = 56636744; // HDR 6x6 float overflow fmt_printf("------------------------------ Seed: {}\n", seed); rnd.seed(seed); @@ -609,7 +701,7 @@ bool random_compression_fuzz_test() const uint32_t h = rnd.irand(1, MAX_HEIGHT); const bool mips = rnd.bit(); const bool use_a = rnd.bit(); - + fmt_printf("Trying {}x{}, mips: {}, use_a: {}\n", w, h, mips, use_a); // Chose a random codec/block size to test @@ -618,7 +710,10 @@ bool random_compression_fuzz_test() bool is_hdr = false; uint32_t rnd_codec_class = rnd.irand(0, (uint32_t)codec_class::cTOTAL - 1); - + + // TODO - make this a command line + //rnd_codec_class = rnd.bit() ? (uint32_t)codec_class::cXUASTC_LDR : (uint32_t)codec_class::cASTC_LDR; + //rnd_codec_class = (uint32_t)codec_class::cXUASTC_LDR; //rnd_codec_class = (uint32_t)codec_class::cETC1S; switch (rnd_codec_class) @@ -630,7 +725,7 @@ bool random_compression_fuzz_test() } case (uint32_t)codec_class::cUASTC_LDR_4x4: { - tex_mode = basist::basis_tex_format::cUASTC4x4; + tex_mode = basist::basis_tex_format::cUASTC_LDR_4x4; break; } case (uint32_t)codec_class::cUASTC_HDR_4x4: @@ -647,10 +742,24 @@ bool random_compression_fuzz_test() } case (uint32_t)codec_class::cUASTC_HDR_6x6: { - tex_mode = basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE; + tex_mode = basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE; is_hdr = true; break; } + case (uint32_t)codec_class::cASTC_LDR: + { + // ASTC LDR 4x4-12x12 + const uint32_t block_variant = rnd.irand(0, astc_helpers::NUM_ASTC_BLOCK_SIZES - 1); + tex_mode = (basist::basis_tex_format)((uint32_t)basist::basis_tex_format::cASTC_LDR_4x4 + block_variant); + break; + } + case (uint32_t)codec_class::cXUASTC_LDR: + { + // XUASTC LDR 4x4-12x12 + const uint32_t block_variant = rnd.irand(0, astc_helpers::NUM_ASTC_BLOCK_SIZES - 1); + tex_mode = (basist::basis_tex_format)((uint32_t)basist::basis_tex_format::cXUASTC_LDR_4x4 + block_variant); + break; + } default: assert(0); tex_mode = basist::basis_tex_format::cETC1S; @@ -658,9 +767,9 @@ bool random_compression_fuzz_test() } fmt_printf("Testing basis_tex_format={}\n", (uint32_t)tex_mode); - + size_t comp_size = 0; - + // Create random LDR source image to compress image src_img; src_img.resize(w, h, w, color_rgba(rnd.byte(), rnd.byte(), rnd.byte(), use_a ? rnd.byte() : 255)); @@ -668,7 +777,7 @@ bool random_compression_fuzz_test() if (rnd.irand(0, 7) >= 1) { const uint32_t nt = rnd.irand(0, 1000); - + for (uint32_t k = 0; k < nt; k++) { color_rgba c(rnd.byte(), rnd.byte(), rnd.byte(), use_a ? rnd.byte() : 255); @@ -680,7 +789,7 @@ bool random_compression_fuzz_test() uint32_t xe = rnd.irand(0, w - 1); if (xs > xe) std::swap(xs, xe); - + uint32_t ys = rnd.irand(0, h - 1); uint32_t ye = rnd.irand(0, h - 1); if (ys > ye) @@ -695,7 +804,7 @@ bool random_compression_fuzz_test() uint32_t ys = rnd.irand(0, h - 1); uint32_t ye = rnd.irand(0, h - 1); - + basisu::draw_line(src_img, xs, ys, xe, ye, c); } else if (r == 6) @@ -712,9 +821,9 @@ bool random_compression_fuzz_test() uint32_t y = rnd.irand(0, h - 1); uint32_t sx = rnd.irand(1, 3); uint32_t sy = rnd.irand(1, 3); - + uint32_t l = rnd.irand(1, 10); - + char buf[32] = {}; for (uint32_t j = 0; j < l; j++) buf[j] = (char)rnd.irand(32, 127); @@ -725,13 +834,13 @@ bool random_compression_fuzz_test() { uint32_t xs = rnd.irand(0, w - 1); uint32_t ys = rnd.irand(0, h - 1); - + uint32_t xl = rnd.irand(1, 100); uint32_t yl = rnd.irand(1, 100); uint32_t xe = minimum(xs + xl - 1, w - 1); uint32_t ye = minimum(ys + yl - 1, h - 1); - + color_rgba cols[4]; cols[0] = c; for (uint32_t j = 1; j < 4; j++) @@ -778,24 +887,101 @@ bool random_compression_fuzz_test() } else src_img(x, y) = q; - } // x + } // x } // y } + else if ((r < 20) && (num_images)) + { + uint32_t image_index = rnd.irand(0, num_images - 1); + + const image& img = test_images[image_index]; + if (img.get_width()) + { + float tw = (float)rnd.irand(1, minimum(128, img.get_width())); + float th = (float)rnd.irand(1, minimum(128, img.get_height())); + + float u = (float)rnd.irand(0, img.get_width() - (int)tw); + float v = (float)rnd.irand(0, img.get_height() - (int)th); + + u /= (float)img.get_width(); + v /= (float)img.get_height(); + + tw /= (float)img.get_width(); + th /= (float)img.get_height(); + + float dx = (float)rnd.irand(0, src_img.get_width() - 1); + float dy = (float)rnd.irand(0, src_img.get_height() - 1); + + float dw = (float)rnd.irand(1, minimum(256, img.get_width())); + float dh = (float)rnd.irand(1, minimum(256, img.get_height())); + + tri2 tri; + tri.p0.set(dx, dy); + tri.t0.set(u, v); + + tri.p1.set(dx + dw, dy); + tri.t1.set(u + tw, v); + + tri.p2.set(dx + dw, dy + dh); + tri.t2.set(u + tw, v + th); + + bool alpha_blend = rnd.bit(); + + if (alpha_blend) + { + tri.c0.set(rnd.irand(100, 255), rnd.irand(100, 255), rnd.irand(100, 255), rnd.irand(1, 255)); + tri.c1.set(rnd.irand(100, 255), rnd.irand(100, 255), rnd.irand(100, 255), rnd.irand(1, 255)); + tri.c2.set(rnd.irand(100, 255), rnd.irand(100, 255), rnd.irand(100, 255), rnd.irand(1, 255)); + } + else + { + tri.c0 = g_white_color; + tri.c1 = g_white_color; + tri.c2 = g_white_color; + } + + draw_tri2(src_img, &img, tri, alpha_blend); + + tri.p0.set(dx, dy); + tri.t0.set(u, v); + + tri.p1.set(dx + dw, dy + dh); + tri.t1.set(u + tw, v + th); + tri.c1 = tri.c2; + + tri.p2.set(dx, dy + dh); + tri.t2.set(u, v + th); + tri.c2.set(rnd.irand(100, 255), rnd.irand(100, 255), rnd.irand(100, 255), rnd.irand(1, 255)); + + draw_tri2(src_img, &img, tri, alpha_blend); + } + } else { src_img(rnd.irand(0, w - 1), rnd.irand(0, h - 1)) = c; } } } - + if ((use_a) && (rnd.irand(0, 3) >= 2)) { const uint32_t nt = rnd.irand(0, 1000); - + for (uint32_t k = 0; k < nt; k++) src_img(rnd.irand(0, w - 1), rnd.irand(0, h - 1)).a = rnd.byte(); } + if (rnd.bit()) + { + int gridX = rnd.irand(8, 24); + int gridY = rnd.irand(8, 24); + float maxOffset = rnd.frand(0.0f, (float)maximum(gridX, gridY)); + + image tmp_img; + wrap_image(src_img, tmp_img, gridX, gridY, maxOffset, true, rnd); + src_img.swap(tmp_img); + } + if (!use_a) { for (uint32_t y = 0; y < h; y++) @@ -805,12 +991,12 @@ bool random_compression_fuzz_test() //save_png("test.png", src_img); //fmt_printf("Has alpha: {}\n", src_img.has_alpha()); - + // Choose randomized codec parameters uint32_t flags = cFlagPrintStats | cFlagValidateOutput | cFlagPrintStatus; - - //flags |= cFlagDebug; - + + flags |= cFlagDebug; + flags |= cFlagThreaded; if (rnd.bit()) @@ -826,16 +1012,16 @@ bool random_compression_fuzz_test() flags |= cFlagREC2020; float quality = 0.0f; - + switch (rnd_codec_class) { case (uint32_t)codec_class::cETC1S: { // ETC1S - + // Choose random ETC1S quality level flags |= rnd.irand(1, 255); - + break; } case (uint32_t)codec_class::cUASTC_LDR_4x4: @@ -846,7 +1032,6 @@ bool random_compression_fuzz_test() { // Choose random RDO lambda quality = rnd.frand(0.0, 10.0f); - flags |= cFlagUASTCRDO; } // Choose random effort level @@ -857,7 +1042,7 @@ bool random_compression_fuzz_test() case (uint32_t)codec_class::cUASTC_HDR_4x4: { // UASTC HDR 4x4 - + // Choose random effort level. flags |= rnd.irand(uastc_hdr_4x4_codec_options::cMinLevel, uastc_hdr_4x4_codec_options::cMaxLevel); @@ -867,7 +1052,7 @@ bool random_compression_fuzz_test() case (uint32_t)codec_class::cUASTC_HDR_6x6: { // RDO ASTC HDR 6x6 or UASTC HDR 6x6 - + // Chose random effort level flags |= rnd.irand(0, astc_6x6_hdr::ASTC_HDR_6X6_MAX_USER_COMP_LEVEL); @@ -879,12 +1064,33 @@ bool random_compression_fuzz_test() break; } + case (uint32_t)codec_class::cASTC_LDR: + case (uint32_t)codec_class::cXUASTC_LDR: + { + // ASTC/XUASTC LDR 4x4-12x12 + + // Choose random profile + uint32_t xuastc_ldr_syntax = rnd.irand(0, (uint32_t)basist::astc_ldr_t::xuastc_ldr_syntax::cTotal - 1); + flags |= (xuastc_ldr_syntax << cFlagXUASTCLDRSyntaxShift); + + // Choose random effort + uint32_t effort = rnd.irand(basisu::astc_ldr::EFFORT_LEVEL_MIN, basisu::astc_ldr::EFFORT_LEVEL_MAX); + flags |= effort; + + // Choose random weight grid DCT quality + quality = (float)rnd.frand(1.0f, 100.0f); + + if (rnd.irand(0, 7) == 0) + quality = 0.0f; // sometimes disable DCT + + break; + } default: { assert(0); } } - + void* pComp_data = nullptr; image_stats stats; @@ -892,7 +1098,7 @@ bool random_compression_fuzz_test() { basisu::vector hdr_source_images; imagef hdr_src_img(src_img.get_width(), src_img.get_height()); - + const float max_y = rnd.frand(.000125f, 30000.0f) / 255.0f; for (uint32_t y = 0; y < src_img.get_height(); y++) @@ -907,7 +1113,7 @@ bool random_compression_fuzz_test() } //write_exr("test.exr", hdr_src_img, 3, 0); - + hdr_source_images.push_back(hdr_src_img); pComp_data = basisu::basis_compress(tex_mode, hdr_source_images, flags, quality, &comp_size, &stats); } @@ -917,7 +1123,8 @@ bool random_compression_fuzz_test() ldr_source_images.push_back(src_img); //save_png("test.png", src_img); - + //save_png(fmt_string("test_{}.png", seed), src_img); + pComp_data = basisu::basis_compress(tex_mode, ldr_source_images, flags, quality, &comp_size, &stats); } @@ -941,39 +1148,66 @@ bool random_compression_fuzz_test() psnr2 }); } // i - + printf("PSNR Results:\n"); - + for (uint32_t i = 0; i < results.size(); i++) fmt_printf("{},{},{},{}\n", results[i].m_seed, (uint32_t)results[i].m_fmt, results[i].m_psnr1, results[i].m_psnr2); - + printf("\n"); for (uint32_t i = 0; i < results.size(); i++) fmt_printf("seed={} tex_mode={}, psnr1={}, psnr2={}\n", results[i].m_seed, (uint32_t)results[i].m_fmt, results[i].m_psnr1, results[i].m_psnr2); - + // Success here is essentially not crashing or asserting or SAN'ing earlier printf("Success\n"); - + return true; } +#ifdef FORCE_SAN_FAILURE +static void force_san_failure() +{ + // Purposely do things that should trigger the address sanitizer + int arr[5] = { 0, 1, 2, 3, 4 }; + printf("Out of bounds element: %d\n", arr[10]); + + //uint8_t* p = (uint8_t *)malloc(10); + //p[10] = 99; + + //uint8_t* p = (uint8_t *)malloc(10); + //free(p); + //p[0] = 99; +} +#endif // FORCE_SAN_FAILURE + int main(int arg_c, char* arg_v[]) { BASISU_NOTE_UNUSED(arg_c); BASISU_NOTE_UNUSED(arg_v); +#if defined(DEBUG) | defined(_DEBUG) + printf("DEBUG\n"); +#endif +#ifdef __SANITIZE_ADDRESS__ + printf("__SANITIZE_ADDRESS__\n"); +#endif + +#ifdef FORCE_SAN_FAILURE + force_san_failure(); +#endif + #if USE_ENCODER basisu_encoder_init(USE_OPENCL, false); - if (!random_compression_fuzz_test()) + if (!random_compress_test()) return EXIT_FAILURE; - + if (!block_unpack_and_transcode_example()) return EXIT_FAILURE; fuzz_uastc_hdr_transcoder_test(); - + if (!encode_etc1s()) { fprintf(stderr, "encode_etc1s() failed!\n"); diff --git a/external/basis_universal/example/example.vcxproj b/external/basis_universal/example/example.vcxproj index bd10772210..d33702aa81 100644 --- a/external/basis_universal/example/example.vcxproj +++ b/external/basis_universal/example/example.vcxproj @@ -38,40 +38,40 @@ Application true Unicode - v143 + v145 Application false true Unicode - v143 + v145 Application true Unicode - v143 + v145 Application true Unicode - v143 + v145 Application false true Unicode - v143 + v145 Application false true Unicode - v143 + v145 @@ -100,10 +100,21 @@ $(SolutionDir)\bin\ - + + $(SolutionDir)\bin\ + $(SolutionDir)\bin\ + + $(SolutionDir)\bin\ + + + $(SolutionDir)\bin\ + + + $(SolutionDir)\bin\ + Level4 @@ -112,7 +123,7 @@ true ..\OpenCL stdcpp17 - StreamingSIMDExtensions2 + AdvancedVectorExtensions Console @@ -130,7 +141,7 @@ WIN32;NDEBUG;_HAS_EXCEPTIONS=0;_CONSOLE;%(PreprocessorDefinitions);BASISU_SUPPORT_SSE=1;BASISU_SUPPORT_OPENCL=1 true ..\OpenCL - StreamingSIMDExtensions2 + AdvancedVectorExtensions false false stdcpp17 @@ -152,8 +163,8 @@ StreamingSIMDExtensions2 false ..\OpenCL - Level4 stdcpp17 + Level4 Console @@ -243,4 +254,4 @@ - + \ No newline at end of file diff --git a/external/basis_universal/example_capi/example_capi.c b/external/basis_universal/example_capi/example_capi.c new file mode 100644 index 0000000000..f2c24392a4 --- /dev/null +++ b/external/basis_universal/example_capi/example_capi.c @@ -0,0 +1,707 @@ +// example_capi.c - Plain C API examples +// Compresses a procedurally generated 32bpp 512x512 test image to a XUASTC LDR 8x5 .ktx2 file with mipmaps and writes a .ktx2 file. +// The .ktx2 file is then opened by the transcoder module, examined and unpacked to RGBA 32bpp and ASTC textures which are saved to disk as .tga and .astc files. +// The .tga image files can be viewed by many common image editors/viewers. +// The standard .astc texture files can be unpacked to .PNG using ARM's astcenc tool, using a command line like this: astcenc-avx2.exe -ds transcoded_0_0_0.astc 0.png + +#include +#include +#include +#include +#include +#include + +typedef int BOOL; +#define TRUE (1) +#define FALSE (0) + +// Include compressor and transcoder C API definitions +#include "../encoder/basisu_wasm_api.h" +#include "../encoder/basisu_wasm_transcoder_api.h" + +// Write a blob of data in memory to a file +int write_blob_to_file(const char* pFilename, const void* pData, size_t len) +{ + assert(pFilename != NULL); + assert(pData != NULL); + + if (!pFilename || !pData) + return FALSE; + + FILE* f = fopen(pFilename, "wb"); + if (!f) + return FALSE; + + /* Write the data */ + size_t written = fwrite(pData, 1, len, f); + if (written != len) + { + fclose(f); + return FALSE; + } + + if (fclose(f) != 0) + return FALSE; + + return TRUE; /* success */ +} + +// Writes 24/32bpp .TGA image files +int write_tga_image(const char* pFilename, int w, int h, int has_alpha, const uint8_t* pPixelsRGBA) +{ + assert(pFilename != NULL); + assert(pPixelsRGBA != NULL); + assert(w > 0); + assert(h > 0); + assert((has_alpha == 0) || (has_alpha == 1)); + + /* Runtime argument validation */ + if ((!pFilename) || (!pPixelsRGBA) || (w <= 0) || (h <= 0)) + return -1; // invalid argument + + FILE* pFile = fopen(pFilename, "wb"); + if (!pFile) + return -2; // cannot open file + + uint8_t header[18] = { 0 }; + header[2] = 2; // uncompressed true-color + header[12] = (uint8_t)(w & 0xFF); + header[13] = (uint8_t)((w >> 8) & 0xFF); + header[14] = (uint8_t)(h & 0xFF); + header[15] = (uint8_t)((h >> 8) & 0xFF); + header[16] = has_alpha ? 32 : 24; + + /* Classic TGA: bottom-left origin */ + header[17] = has_alpha ? 8 : 0; + + if (fwrite(header, 1, 18, pFile) != 18) + { + fclose(pFile); + return -3; // header write failed + } + + uint64_t bytes_per_pixel = has_alpha ? 4ULL : 3ULL; + uint64_t pixel_bytes_u64 = (uint64_t)w * (uint64_t)h * bytes_per_pixel; + size_t pixel_bytes = (size_t)pixel_bytes_u64; + + if ((uint64_t)pixel_bytes != pixel_bytes_u64) + return -6; // overflow bogus dimensions + + /* allocate one scanline for BGRA/BGR output */ + size_t row_bytes = (size_t)w * bytes_per_pixel; + uint8_t* pRow = (uint8_t*)malloc(row_bytes); + if (!pRow) + { + fclose(pFile); + return -7; // out of memory + } + + /* TGA expects rows in bottom-to-top order */ + for (int y = 0; y < h; y++) + { + const uint8_t* pSrcRow = pPixelsRGBA + (size_t)(h - 1 - y) * w * bytes_per_pixel; + + /* Convert RGBA->BGRA or RGB->BGR for this row */ + if (has_alpha) + { + /* 4 bytes per pixel */ + for (int x = 0; x < w; x++) + { + const uint8_t* s = &pSrcRow[x * 4]; + uint8_t* d = &pRow[x * 4]; + + d[0] = s[2]; // B + d[1] = s[1]; // G + d[2] = s[0]; // R + d[3] = s[3]; // A + } + } + else + { + /* 3 bytes per pixel */ + for (int x = 0; x < w; x++) + { + const uint8_t* s = &pSrcRow[x * 3]; + uint8_t* d = &pRow[x * 3]; + + d[0] = s[2]; // B + d[1] = s[1]; // G + d[2] = s[0]; // R + } + } + + if (fwrite(pRow, 1, row_bytes, pFile) != row_bytes) + { + free(pRow); + fclose(pFile); + return -4; // pixel write failed + } + } + + free(pRow); + + if (fclose(pFile) != 0) + return -5; // close failed + + return 0; // success +} + +// Write standard ARM .ASTC format texture files +int write_astc_file(const char* pFilename, + const void* pBlocks, // pointer to ASTC blocks + uint32_t block_width, // in texels [4,12] + uint32_t block_height, // in texels [4,12] + uint32_t dim_x, // image actual dimension in texels + uint32_t dim_y) // image actual dimension in texels +{ + assert(pFilename != NULL); + assert(pBlocks != NULL); + assert(dim_x > 0); + assert(dim_y > 0); + assert((block_width >= 4) && (block_width <= 12)); + assert((block_height >= 4) && (block_height <= 12)); + + FILE* f = fopen(pFilename, "wb"); + if (!f) + return 0; + + /* Helper macro for writing single bytes with error check */ +#define PUTB(v) do { if (fputc((int)(v), f) == EOF) { fclose(f); return 0; } } while (0) + + /* Magic */ + PUTB(0x13); + PUTB(0xAB); + PUTB(0xA1); + PUTB(0x5C); + + /* Block dimensions: x, y, z = 1 */ + PUTB((uint8_t)block_width); + PUTB((uint8_t)block_height); + PUTB(1); /* block depth */ + + /* dim_x (24-bit little endian) */ + PUTB((uint8_t)(dim_x & 0xFF)); + PUTB((uint8_t)((dim_x >> 8) & 0xFF)); + PUTB((uint8_t)((dim_x >> 16) & 0xFF)); + + /* dim_y (24-bit little endian) */ + PUTB((uint8_t)(dim_y & 0xFF)); + PUTB((uint8_t)((dim_y >> 8) & 0xFF)); + PUTB((uint8_t)((dim_y >> 16) & 0xFF)); + + /* dim_z = 1 (24-bit LE) */ + PUTB(1); + PUTB(0); + PUTB(0); + + /* Compute block count and total bytes */ + uint32_t num_blocks_x = (dim_x + block_width - 1) / block_width; + uint32_t num_blocks_y = (dim_y + block_height - 1) / block_height; + + uint64_t total_bytes_u64 = + (uint64_t)num_blocks_x * (uint64_t)num_blocks_y * 16ULL; + + size_t total_bytes = (size_t)total_bytes_u64; + + if ((uint64_t)total_bytes != total_bytes_u64) + { + fclose(f); + return 0; /* overflow → fail */ + } + + /* Write block data directly */ + size_t written = fwrite(pBlocks, 1, total_bytes, f); + if (written != total_bytes) + { + fclose(f); /* still close even if error */ + return 0; + } + + if (fclose(f) != 0) + return 0; + + return 1; /* success */ + +#undef PUTB +} + +// Procedurally create a simple test image in memory +uint8_t* create_pretty_rgba_pattern(int w, int h, float q) +{ + if (w <= 0 || h <= 0) + return NULL; + + uint8_t* pImage = (uint8_t*)malloc((size_t)w * h * 4); + if (!pImage) + return NULL; + + for (int y = 0; y < h; y++) + { + for (int x = 0; x < w; x++) + { + /* normalized coordinates 0..1 */ + float fx = (float)x / (float)w; + float fy = (float)y / (float)h; + + /* --- Extra coordinate warping when q != 0 --- */ + if (q != 0.0f) { + float warp = sinf((fx + fy) * 10.0f * q); + fx += 0.15f * q * warp; + fy += 0.15f * q * sinf((fx - fy) * 8.0f * q); + } + + /* Original plasma formula */ + float v = sinf(fx * 12.0f + fy * 4.0f); + v += sinf(fy * 9.0f - fx * 6.0f); + v += sinf((fx + fy) * 7.0f); + + /* Extra variation term — contributes only when q != 0 */ + if (q != 0.0f) + { + v += q * 0.7f * sinf((fx * fx + fy) * 20.0f); + v += q * 0.4f * cosf((fx - fy) * 18.0f); + } + + /* scale to 0..1 */ + v = v * 0.25f + 0.5f; + + float L = 1.5f; + + /* Convert to RGB colors */ + int r = (int)roundf(255.0f * sinf(v * 6.28f) * L); + int g = (int)roundf(255.0f * (1.0f - v) * L); + int b = (int)roundf(255.0f * v * L); + + /* clamp */ + if (r < 0) r = 0; else if (r > 255) r = 255; + if (g < 0) g = 0; else if (g > 255) g = 255; + if (b < 0) b = 0; else if (b > 255) b = 255; + + /* write RGBA */ + uint8_t* p = &pImage[(y * w + x) * 4]; + p[0] = (uint8_t)r; + p[1] = (uint8_t)g; + p[2] = (uint8_t)b; + p[3] = 255; + } + } + + return pImage; +} + +// Takes a KTX2 file in memory and displays info about it, then transcodes it to RGBA32 and ASTC, writing .tga/.astc files to disk +int transcode_ktx2_file(const void* pKTX2_data, size_t ktx2_data_size, const char *pDesc) +{ + printf("------ transcode_ktx2_file(): ktx2 size: %zu, desc: %s\n", ktx2_data_size, pDesc); + + if (!pKTX2_data || !ktx2_data_size) + return FALSE; + + if ((uint32_t)ktx2_data_size != ktx2_data_size) + return FALSE; + + uint64_t ktx2_data_ofs = bt_alloc(ktx2_data_size); + if (!ktx2_data_ofs) + return FALSE; + + memcpy((void*)ktx2_data_ofs, pKTX2_data, ktx2_data_size); + + uint64_t ktx2_handle = bt_ktx2_open(ktx2_data_ofs, (uint32_t)ktx2_data_size); + if (!ktx2_handle) + { + bt_free(ktx2_data_ofs); + return FALSE; + } + + // Just testing LDR here for now + if (!bt_ktx2_is_ldr(ktx2_handle)) + { + bt_ktx2_close(ktx2_handle); + bt_free(ktx2_data_ofs); + return FALSE; + } + + if (!bt_ktx2_start_transcoding(ktx2_handle)) + { + bt_ktx2_close(ktx2_handle); + bt_free(ktx2_data_ofs); + return FALSE; + } + + uint32_t width = bt_ktx2_get_width(ktx2_handle), height = bt_ktx2_get_height(ktx2_handle); + uint32_t levels = bt_ktx2_get_levels(ktx2_handle); // number of mipmap levels, must be >= 1 + uint32_t faces = bt_ktx2_get_faces(ktx2_handle); // 1 or 6 + uint32_t layers = bt_ktx2_get_layers(ktx2_handle); // 0 or array size + + uint32_t basis_tex_format = bt_ktx2_get_basis_tex_format(ktx2_handle); + uint32_t block_width = bt_ktx2_get_block_width(ktx2_handle); + uint32_t block_height = bt_ktx2_get_block_height(ktx2_handle); + uint32_t is_srgb = bt_ktx2_is_srgb(ktx2_handle); + uint32_t is_video = bt_ktx2_is_video(ktx2_handle); // only reliably set after calling bt_ktx2_start_transcoding() + + printf("KTX2 Dimensions: %ux%u, Levels: %u, Faces: %u, Layers: %u\n", width, height, levels, faces, layers); + printf("basis_tex_format: %u\n", basis_tex_format); + printf("Block dimensions: %ux%u\n", block_width, block_height); + printf("is sRGB: %u\n", is_srgb); + printf("is video: %u\n", is_video); + + assert((width >= 1) && (height >= 1)); + assert(levels >= 1); + assert((faces == 6) || (faces == 1)); + + // If layers==0 it's not a texture array + if (layers < 1) + layers = 1; + + // Create our transcoding state handle (which contains thread-local state) + // This is actually optional, and only needed for thread-safe transcoding, but we'll test it here. + uint64_t transcode_state_handle = bt_ktx2_create_transcode_state(); + + for (uint32_t level_index = 0; level_index < levels; level_index++) + { + for (uint32_t layer_index = 0; layer_index < layers; layer_index++) + { + for (uint32_t face_index = 0; face_index < faces; face_index++) + { + printf("- Level: %u, layer: %u, face: %u\n", level_index, layer_index, face_index); + + uint32_t orig_width = bt_ktx2_get_level_orig_width(ktx2_handle, level_index, layer_index, face_index); + uint32_t orig_height = bt_ktx2_get_level_orig_height(ktx2_handle, level_index, layer_index, face_index); + + printf(" Orig dimensions: %ux%u, actual: %ux%u\n", + orig_width, orig_height, + bt_ktx2_get_level_actual_width(ktx2_handle, level_index, layer_index, face_index), bt_ktx2_get_level_actual_height(ktx2_handle, level_index, layer_index, face_index)); + + printf(" Block dimensions: %ux%u, total blocks: %u\n", + bt_ktx2_get_level_num_blocks_x(ktx2_handle, level_index, layer_index, face_index), + bt_ktx2_get_level_num_blocks_y(ktx2_handle, level_index, layer_index, face_index), + bt_ktx2_get_level_total_blocks(ktx2_handle, level_index, layer_index, face_index)); + + printf(" Alpha flag: %u, iframe flag: %u\n", + bt_ktx2_get_level_alpha_flag(ktx2_handle, level_index, layer_index, face_index), + bt_ktx2_get_level_iframe_flag(ktx2_handle, level_index, layer_index, face_index)); + + // First transcode level to uncompressed RGBA32 and write a .tga file + { + char tga_filename[256]; + snprintf(tga_filename, sizeof(tga_filename), "transcoded_%s_L%u_Y%u_F%u.tga", pDesc, level_index, layer_index, face_index); + + uint32_t transcode_buf_size = bt_basis_compute_transcoded_image_size_in_bytes(TF_RGBA32, orig_width, orig_height); + assert(transcode_buf_size); + + uint64_t transcode_buf_ofs = bt_alloc(transcode_buf_size); + + uint32_t decode_flags = 0; + + if (!bt_ktx2_transcode_image_level(ktx2_handle, level_index, layer_index, face_index, + transcode_buf_ofs, transcode_buf_size, + TF_RGBA32, + decode_flags, + 0, 0, -1, -1, transcode_state_handle)) + { + bt_free(transcode_buf_ofs); + bt_ktx2_destroy_transcode_state(transcode_state_handle); + bt_ktx2_close(ktx2_handle); + bt_free(ktx2_data_ofs); + return FALSE; + } + + write_tga_image(tga_filename, orig_width, orig_height, TRUE, (uint8_t*)transcode_buf_ofs); + printf("Wrote file %s\n", tga_filename); + + bt_free(transcode_buf_ofs); + transcode_buf_ofs = 0; + } + + // Now transcode to ASTC and write a .astc file + { + char astc_filename[256]; + snprintf(astc_filename, sizeof(astc_filename), "transcoded_%s_L%u_Y%u_F%u.astc", pDesc, level_index, layer_index, face_index); + + // Determine the correct ASTC transcode texture format from the ktx2 format + uint32_t target_transcode_fmt = bt_basis_get_transcoder_texture_format_from_basis_tex_format(basis_tex_format); + + uint32_t transcode_buf_size = bt_basis_compute_transcoded_image_size_in_bytes(target_transcode_fmt, orig_width, orig_height); + assert(transcode_buf_size); + + uint64_t transcode_buf_ofs = bt_alloc(transcode_buf_size); + + uint32_t decode_flags = 0; + + if (!bt_ktx2_transcode_image_level(ktx2_handle, level_index, layer_index, face_index, + transcode_buf_ofs, transcode_buf_size, + target_transcode_fmt, + decode_flags, + 0, 0, -1, -1, transcode_state_handle)) + { + bt_free(transcode_buf_ofs); + bt_ktx2_destroy_transcode_state(transcode_state_handle); + bt_ktx2_close(ktx2_handle); + bt_free(ktx2_data_ofs); + return FALSE; + } + + write_astc_file(astc_filename, (void*)transcode_buf_ofs, block_width, block_height, orig_width, orig_height); + printf("Wrote .astc file %s\n", astc_filename); + + bt_free(transcode_buf_ofs); + transcode_buf_ofs = 0; + } + + } // face_index + + } // layer_index + + } // level_index + + bt_ktx2_destroy_transcode_state(transcode_state_handle); + transcode_state_handle = 0; + + bt_ktx2_close(ktx2_handle); + ktx2_handle = 0; + + bt_free(ktx2_data_ofs); + ktx2_data_ofs = 0; + + return TRUE; +} + +// Simple 2D test +int test_2D() +{ + printf("------ test_2D():\n"); + + // Generate a test image + int W = 512, H = 512; + + uint8_t* pSrc_image = create_pretty_rgba_pattern(W, H, 0.0f); + + // Save the test image to a .tga file + write_tga_image("test_image.tga", W, H, TRUE, pSrc_image); + printf("Wrote file test_image.tga\n"); + + // Compress it to .ktx2 + uint64_t comp_params = bu_new_comp_params(); + + // Allocate memory + uint64_t img_ofs = bu_alloc(W * H * 4); + if (!img_ofs) + { + fprintf(stderr, "bu_alloc() failed\n"); + return EXIT_FAILURE; + } + + // Copy the test image into the allocated memory + memcpy((void*)img_ofs, pSrc_image, W * H * 4); + + // Supply the image to the compressor - it'll immediately make a copy of the data + if (!bu_comp_params_set_image_rgba32(comp_params, 0, img_ofs, W, H, W * 4)) + { + fprintf(stderr, "bu_comp_params_set_image_rgba32() failed\n"); + return EXIT_FAILURE; + } + + bu_free(img_ofs); + img_ofs = 0; + + // Now compress it to XUASTC LDR 8x5 with weight grid DCT + uint32_t basis_tex_format = BTF_XUASTC_LDR_8X5; + //uint32_t basis_tex_format = BTF_ASTC_LDR_8X5; + //uint32_t basis_tex_format = BTF_ETC1S; + //uint32_t basis_tex_format = BTF_UASTC_LDR_4X4; + + uint32_t quality_level = 85; + uint32_t effort_level = 2; + + uint32_t flags = BU_COMP_FLAGS_KTX2_OUTPUT | BU_COMP_FLAGS_SRGB | + BU_COMP_FLAGS_THREADED | BU_COMP_FLAGS_GEN_MIPS_CLAMP | + BU_COMP_FLAGS_PRINT_STATS | BU_COMP_FLAGS_PRINT_STATUS; + + if (!bu_compress_texture(comp_params, basis_tex_format, quality_level, effort_level, flags, 0.0f)) + { + fprintf(stderr, "bu_compress_texture() failed\n"); + return EXIT_FAILURE; + } + + // Retrieve the compressed .KTX2 file data + uint64_t comp_size = bu_comp_params_get_comp_data_size(comp_params); + if (!comp_size) + { + fprintf(stderr, "bu_comp_params_get_comp_data_size() failed\n"); + return EXIT_FAILURE; + } + + void* pComp_data = (void*)bu_comp_params_get_comp_data_ofs(comp_params); + if (!pComp_data) + { + fprintf(stderr, "bu_comp_params_get_comp_data_ofs() failed\n"); + return EXIT_FAILURE; + } + + // Write the data to disk + write_blob_to_file("test.ktx2", pComp_data, comp_size); + printf("Wrote file test.ktx2\n"); + + // Now inspect and transcode the .KTX2 data to png/astc files + if (!transcode_ktx2_file(pComp_data, comp_size, "2D")) + { + fprintf(stderr, "transcode_ktx2_file() failed\n"); + return EXIT_FAILURE; + } + + bu_delete_comp_params(comp_params); + + free(pSrc_image); + return EXIT_SUCCESS; +} + +// 2D array/texture video test +int test_2D_array(BOOL tex_video_flag, int L, BOOL mipmap_flag) +{ + printf("------ test_2D_array() %i %i %i:\n", tex_video_flag, L, mipmap_flag); + + // Generate a test image + int W = 256, H = 256; + + // Compress it to .ktx2 + uint64_t comp_params = bu_new_comp_params(); + + const char* pDesc = tex_video_flag ? "video" : "array"; + + char filename_buf[256]; + + for (int layer = 0; layer < L; layer++) + { + uint8_t* pSrc_image = create_pretty_rgba_pattern(W, H, (float)layer * .05f); + + // Save the test image to a .tga file + snprintf(filename_buf, sizeof(filename_buf), "test_%s_layer_%u.tga", pDesc, layer); + + write_tga_image(filename_buf, W, H, TRUE, pSrc_image); + printf("Wrote file %s\n", filename_buf); + + // Allocate memory + uint64_t img_ofs = bu_alloc(W * H * 4); + if (!img_ofs) + { + fprintf(stderr, "bu_alloc() failed\n"); + return EXIT_FAILURE; + } + + // Copy the test image into the allocated memory + memcpy((void*)img_ofs, pSrc_image, W * H * 4); + + // Supply the image to the compressor - it'll immediately make a copy of the data + if (!bu_comp_params_set_image_rgba32(comp_params, layer, img_ofs, W, H, W * 4)) + { + fprintf(stderr, "bu_comp_params_set_image_rgba32() failed\n"); + return EXIT_FAILURE; + } + + bu_free(img_ofs); + img_ofs = 0; + + free(pSrc_image); + + } // layer + + // ETC1S has special optimizations for texture video (basic p-frames with skip blocks). + uint32_t basis_tex_format = tex_video_flag ? BTF_ETC1S : BTF_XUASTC_LDR_4X4; + + uint32_t quality_level = 100; + uint32_t effort_level = 4; + + uint32_t flags = BU_COMP_FLAGS_KTX2_OUTPUT | BU_COMP_FLAGS_SRGB | + BU_COMP_FLAGS_THREADED | + BU_COMP_FLAGS_PRINT_STATS | BU_COMP_FLAGS_PRINT_STATUS; + + if (tex_video_flag) + flags |= BU_COMP_FLAGS_TEXTURE_TYPE_VIDEO_FRAMES; + else + flags |= BU_COMP_FLAGS_TEXTURE_TYPE_2D_ARRAY; + + if (mipmap_flag) + flags |= BU_COMP_FLAGS_GEN_MIPS_CLAMP; + + if (!bu_compress_texture(comp_params, basis_tex_format, quality_level, effort_level, flags, 0.0f)) + { + fprintf(stderr, "bu_compress_texture() failed\n"); + return EXIT_FAILURE; + } + + // Retrieve the compressed .KTX2 file data + uint64_t comp_size = bu_comp_params_get_comp_data_size(comp_params); + if (!comp_size) + { + fprintf(stderr, "bu_comp_params_get_comp_data_size() failed\n"); + return EXIT_FAILURE; + } + + void* pComp_data = (void*)bu_comp_params_get_comp_data_ofs(comp_params); + if (!pComp_data) + { + fprintf(stderr, "bu_comp_params_get_comp_data_ofs() failed\n"); + return EXIT_FAILURE; + } + + // Write the data to disk + snprintf(filename_buf, sizeof(filename_buf), "test_%s.ktx2", pDesc); + write_blob_to_file(filename_buf, pComp_data, comp_size); + printf("Wrote file %s\n", filename_buf); + + // Now inspect and transcode the .KTX2 data to png/astc files + if (!transcode_ktx2_file(pComp_data, comp_size, pDesc)) + { + fprintf(stderr, "transcode_ktx2_file() failed\n"); + return EXIT_FAILURE; + } + + bu_delete_comp_params(comp_params); + + return EXIT_SUCCESS; +} + +int main(int argc, char **argv) +{ + (void)argc; + (void)argv; + printf("example_capi.c:\n"); + + // Initialize the encoder (which initializers the transcoder for us) + printf("bu_init:\n"); + bu_init(); + + // bu_init() already does this for us, but it's harmless to call again. + printf("bt_init:\n"); + bt_init(); + + // Control debug output from the compressor + bu_enable_debug_printf(FALSE); + + // simple 2D + if (test_2D() != EXIT_SUCCESS) + { + fprintf(stderr, "test_2D() failed!\n"); + return EXIT_FAILURE; + } + + // 2D array + if (test_2D_array(FALSE, 8, FALSE) != EXIT_SUCCESS) + { + fprintf(stderr, "test_2D_array() (array mode) failed!\n"); + return EXIT_FAILURE; + } + + // texture video + if (test_2D_array(TRUE, 8, TRUE) != EXIT_SUCCESS) + { + fprintf(stderr, "test_2D_array() (texture video mode) failed!\n"); + return EXIT_FAILURE; + } + + printf("Success\n"); + + return EXIT_SUCCESS; +} + + diff --git a/external/basis_universal/example_capi/example_capi.vcxproj b/external/basis_universal/example_capi/example_capi.vcxproj new file mode 100644 index 0000000000..67b4a24e5e --- /dev/null +++ b/external/basis_universal/example_capi/example_capi.vcxproj @@ -0,0 +1,238 @@ + + + + + Debug + ARM64EC + + + Debug + Win32 + + + Release + ARM64EC + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + 18.0 + Win32Proj + {be889347-e4fd-47dd-bbf4-81f98faa8ba9} + examplecapi + 10.0 + + + + Application + true + v145 + Unicode + + + Application + false + v145 + true + Unicode + + + Application + true + v145 + Unicode + + + Application + true + v145 + Unicode + + + Application + false + v145 + true + Unicode + + + Application + false + v145 + true + Unicode + + + + + + + + + + + + + + + + + + + + + + + + + + + $(SolutionDir)\bin\ + + + $(SolutionDir)\bin\ + + + $(SolutionDir)\bin\ + + + $(SolutionDir)\bin\ + + + $(SolutionDir)\bin\ + + + $(SolutionDir)\bin\ + + + + Level3 + true + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + stdcpp20 + ..\OpenCL + + + Console + true + ..\OpenCL\lib + opencl64.lib;$(CoreLibraryDependencies);%(AdditionalDependencies) + + + + + Level3 + true + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + stdcpp20 + ..\OpenCL + + + Console + true + ..\OpenCL\lib + opencl64.lib;$(CoreLibraryDependencies);%(AdditionalDependencies) + + + + + Level3 + true + _DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + stdcpp20 + ..\OpenCL + + + Console + true + ..\OpenCL\lib + opencl64.lib;$(CoreLibraryDependencies);%(AdditionalDependencies) + + + + + Level3 + true + _DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + stdcpp20 + ..\OpenCL + + + Console + true + ..\OpenCL\lib + opencl64.lib;softintrin.lib;%(AdditionalDependencies) + + + + + Level3 + true + true + true + NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + stdcpp20 + ..\OpenCL + + + Console + true + ..\OpenCL\lib + opencl64.lib;$(CoreLibraryDependencies);%(AdditionalDependencies) + + + + + Level3 + true + true + true + NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + stdcpp20 + ..\OpenCL + + + Console + true + ..\OpenCL\lib + opencl64.lib;softintrin.lib;%(AdditionalDependencies) + + + + + + + + + + + + + + + {97c34996-f458-4030-a402-b32c581872f1} + + + + + + \ No newline at end of file diff --git a/external/basis_universal/example_capi/example_capi.vcxproj.filters b/external/basis_universal/example_capi/example_capi.vcxproj.filters new file mode 100644 index 0000000000..aa9303e4e8 --- /dev/null +++ b/external/basis_universal/example_capi/example_capi.vcxproj.filters @@ -0,0 +1,39 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;c++;cppm;ixx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hh;hpp;hxx;h++;hm;inl;inc;ipp;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Source Files + + + Source Files + + + Source Files + + + + + Source Files + + + Source Files + + + Source Files + + + \ No newline at end of file diff --git a/external/basis_universal/example_transcoding/dds_defs.h b/external/basis_universal/example_transcoding/dds_defs.h new file mode 100644 index 0000000000..cbca0a5a21 --- /dev/null +++ b/external/basis_universal/example_transcoding/dds_defs.h @@ -0,0 +1,286 @@ +// File: dds_defs.h +// DX9/10 .DDS file header definitions. +#pragma once + +#define PIXEL_FMT_FOURCC(a, b, c, d) ((a) | ((b) << 8U) | ((c) << 16U) | ((d) << 24U)) + +enum pixel_format +{ + PIXEL_FMT_INVALID = 0, + + PIXEL_FMT_DXT1 = PIXEL_FMT_FOURCC('D', 'X', 'T', '1'), + PIXEL_FMT_DXT2 = PIXEL_FMT_FOURCC('D', 'X', 'T', '2'), + PIXEL_FMT_DXT3 = PIXEL_FMT_FOURCC('D', 'X', 'T', '3'), + PIXEL_FMT_DXT4 = PIXEL_FMT_FOURCC('D', 'X', 'T', '4'), + PIXEL_FMT_DXT5 = PIXEL_FMT_FOURCC('D', 'X', 'T', '5'), + PIXEL_FMT_3DC = PIXEL_FMT_FOURCC('A', 'T', 'I', '2'), // DXN_YX + PIXEL_FMT_DXN = PIXEL_FMT_FOURCC('A', '2', 'X', 'Y'), // DXN_XY + PIXEL_FMT_DXT5A = PIXEL_FMT_FOURCC('A', 'T', 'I', '1'), // ATI1N, http://developer.amd.com/media/gpu_assets/Radeon_X1x00_Programming_Guide.pdf + + // Non-standard formats (some of these are supported by ATI's Compressonator) + PIXEL_FMT_DXT5_CCxY = PIXEL_FMT_FOURCC('C', 'C', 'x', 'Y'), + PIXEL_FMT_DXT5_xGxR = PIXEL_FMT_FOURCC('x', 'G', 'x', 'R'), + PIXEL_FMT_DXT5_xGBR = PIXEL_FMT_FOURCC('x', 'G', 'B', 'R'), + PIXEL_FMT_DXT5_AGBR = PIXEL_FMT_FOURCC('A', 'G', 'B', 'R'), + + PIXEL_FMT_DXT1A = PIXEL_FMT_FOURCC('D', 'X', '1', 'A'), + PIXEL_FMT_ETC1 = PIXEL_FMT_FOURCC('E', 'T', 'C', '1'), + + PIXEL_FMT_R8G8B8 = PIXEL_FMT_FOURCC('R', 'G', 'B', 'x'), + PIXEL_FMT_L8 = PIXEL_FMT_FOURCC('L', 'x', 'x', 'x'), + PIXEL_FMT_A8 = PIXEL_FMT_FOURCC('x', 'x', 'x', 'A'), + PIXEL_FMT_A8L8 = PIXEL_FMT_FOURCC('L', 'x', 'x', 'A'), + PIXEL_FMT_A8R8G8B8 = PIXEL_FMT_FOURCC('R', 'G', 'B', 'A') +}; + +const uint32_t cDDSMaxImageDimensions = 8192U; + +// Total size of header is sizeof(uint32)+cDDSSizeofDDSurfaceDesc2; +const uint32_t cDDSSizeofDDSurfaceDesc2 = 124; + +// "DDS " +const uint32_t cDDSFileSignature = 0x20534444; + +struct DDCOLORKEY +{ + uint32_t dwUnused0; + uint32_t dwUnused1; +}; + +struct DDPIXELFORMAT +{ + uint32_t dwSize; + uint32_t dwFlags; + uint32_t dwFourCC; + uint32_t dwRGBBitCount; // ATI compressonator will place a FOURCC code here for swizzled/cooked DXTn formats + uint32_t dwRBitMask; + uint32_t dwGBitMask; + uint32_t dwBBitMask; + uint32_t dwRGBAlphaBitMask; +}; + +struct DDSCAPS2 +{ + uint32_t dwCaps; + uint32_t dwCaps2; + uint32_t dwCaps3; + uint32_t dwCaps4; +}; + +struct DDSURFACEDESC2 +{ + uint32_t dwSize; + uint32_t dwFlags; + uint32_t dwHeight; + uint32_t dwWidth; + union + { + int32_t lPitch; + uint32_t dwLinearSize; + }; + uint32_t dwBackBufferCount; + uint32_t dwMipMapCount; + uint32_t dwAlphaBitDepth; + uint32_t dwUnused0; + uint32_t lpSurface; + DDCOLORKEY unused0; + DDCOLORKEY unused1; + DDCOLORKEY unused2; + DDCOLORKEY unused3; + DDPIXELFORMAT ddpfPixelFormat; + DDSCAPS2 ddsCaps; + uint32_t dwUnused1; +}; + +const uint32_t DDSD_CAPS = 0x00000001; +const uint32_t DDSD_HEIGHT = 0x00000002; +const uint32_t DDSD_WIDTH = 0x00000004; +const uint32_t DDSD_PITCH = 0x00000008; + +const uint32_t DDSD_BACKBUFFERCOUNT = 0x00000020; +const uint32_t DDSD_ZBUFFERBITDEPTH = 0x00000040; +const uint32_t DDSD_ALPHABITDEPTH = 0x00000080; + +const uint32_t DDSD_LPSURFACE = 0x00000800; + +const uint32_t DDSD_PIXELFORMAT = 0x00001000; +const uint32_t DDSD_CKDESTOVERLAY = 0x00002000; +const uint32_t DDSD_CKDESTBLT = 0x00004000; +const uint32_t DDSD_CKSRCOVERLAY = 0x00008000; + +const uint32_t DDSD_CKSRCBLT = 0x00010000; +const uint32_t DDSD_MIPMAPCOUNT = 0x00020000; +const uint32_t DDSD_REFRESHRATE = 0x00040000; +const uint32_t DDSD_LINEARSIZE = 0x00080000; + +const uint32_t DDSD_TEXTURESTAGE = 0x00100000; +const uint32_t DDSD_FVF = 0x00200000; +const uint32_t DDSD_SRCVBHANDLE = 0x00400000; +const uint32_t DDSD_DEPTH = 0x00800000; + +const uint32_t DDSD_ALL = 0x00fff9ee; + +const uint32_t DDPF_ALPHAPIXELS = 0x00000001; +const uint32_t DDPF_ALPHA = 0x00000002; +const uint32_t DDPF_FOURCC = 0x00000004; +const uint32_t DDPF_PALETTEINDEXED8 = 0x00000020; +const uint32_t DDPF_RGB = 0x00000040; +const uint32_t DDPF_LUMINANCE = 0x00020000; + +const uint32_t DDSCAPS_COMPLEX = 0x00000008; +const uint32_t DDSCAPS_TEXTURE = 0x00001000; +const uint32_t DDSCAPS_MIPMAP = 0x00400000; + +const uint32_t DDSCAPS2_CUBEMAP = 0x00000200; +const uint32_t DDSCAPS2_CUBEMAP_POSITIVEX = 0x00000400; +const uint32_t DDSCAPS2_CUBEMAP_NEGATIVEX = 0x00000800; + +const uint32_t DDSCAPS2_CUBEMAP_POSITIVEY = 0x00001000; +const uint32_t DDSCAPS2_CUBEMAP_NEGATIVEY = 0x00002000; +const uint32_t DDSCAPS2_CUBEMAP_POSITIVEZ = 0x00004000; +const uint32_t DDSCAPS2_CUBEMAP_NEGATIVEZ = 0x00008000; + +const uint32_t DDSCAPS2_VOLUME = 0x00200000; + +typedef enum DXGI_FORMAT +{ + DXGI_FORMAT_UNKNOWN = 0, + DXGI_FORMAT_R32G32B32A32_TYPELESS = 1, + DXGI_FORMAT_R32G32B32A32_FLOAT = 2, + DXGI_FORMAT_R32G32B32A32_UINT = 3, + DXGI_FORMAT_R32G32B32A32_SINT = 4, + DXGI_FORMAT_R32G32B32_TYPELESS = 5, + DXGI_FORMAT_R32G32B32_FLOAT = 6, + DXGI_FORMAT_R32G32B32_UINT = 7, + DXGI_FORMAT_R32G32B32_SINT = 8, + DXGI_FORMAT_R16G16B16A16_TYPELESS = 9, + DXGI_FORMAT_R16G16B16A16_FLOAT = 10, + DXGI_FORMAT_R16G16B16A16_UNORM = 11, + DXGI_FORMAT_R16G16B16A16_UINT = 12, + DXGI_FORMAT_R16G16B16A16_SNORM = 13, + DXGI_FORMAT_R16G16B16A16_SINT = 14, + DXGI_FORMAT_R32G32_TYPELESS = 15, + DXGI_FORMAT_R32G32_FLOAT = 16, + DXGI_FORMAT_R32G32_UINT = 17, + DXGI_FORMAT_R32G32_SINT = 18, + DXGI_FORMAT_R32G8X24_TYPELESS = 19, + DXGI_FORMAT_D32_FLOAT_S8X24_UINT = 20, + DXGI_FORMAT_R32_FLOAT_X8X24_TYPELESS = 21, + DXGI_FORMAT_X32_TYPELESS_G8X24_UINT = 22, + DXGI_FORMAT_R10G10B10A2_TYPELESS = 23, + DXGI_FORMAT_R10G10B10A2_UNORM = 24, + DXGI_FORMAT_R10G10B10A2_UINT = 25, + DXGI_FORMAT_R11G11B10_FLOAT = 26, + DXGI_FORMAT_R8G8B8A8_TYPELESS = 27, + DXGI_FORMAT_R8G8B8A8_UNORM = 28, + DXGI_FORMAT_R8G8B8A8_UNORM_SRGB = 29, + DXGI_FORMAT_R8G8B8A8_UINT = 30, + DXGI_FORMAT_R8G8B8A8_SNORM = 31, + DXGI_FORMAT_R8G8B8A8_SINT = 32, + DXGI_FORMAT_R16G16_TYPELESS = 33, + DXGI_FORMAT_R16G16_FLOAT = 34, + DXGI_FORMAT_R16G16_UNORM = 35, + DXGI_FORMAT_R16G16_UINT = 36, + DXGI_FORMAT_R16G16_SNORM = 37, + DXGI_FORMAT_R16G16_SINT = 38, + DXGI_FORMAT_R32_TYPELESS = 39, + DXGI_FORMAT_D32_FLOAT = 40, + DXGI_FORMAT_R32_FLOAT = 41, + DXGI_FORMAT_R32_UINT = 42, + DXGI_FORMAT_R32_SINT = 43, + DXGI_FORMAT_R24G8_TYPELESS = 44, + DXGI_FORMAT_D24_UNORM_S8_UINT = 45, + DXGI_FORMAT_R24_UNORM_X8_TYPELESS = 46, + DXGI_FORMAT_X24_TYPELESS_G8_UINT = 47, + DXGI_FORMAT_R8G8_TYPELESS = 48, + DXGI_FORMAT_R8G8_UNORM = 49, + DXGI_FORMAT_R8G8_UINT = 50, + DXGI_FORMAT_R8G8_SNORM = 51, + DXGI_FORMAT_R8G8_SINT = 52, + DXGI_FORMAT_R16_TYPELESS = 53, + DXGI_FORMAT_R16_FLOAT = 54, + DXGI_FORMAT_D16_UNORM = 55, + DXGI_FORMAT_R16_UNORM = 56, + DXGI_FORMAT_R16_UINT = 57, + DXGI_FORMAT_R16_SNORM = 58, + DXGI_FORMAT_R16_SINT = 59, + DXGI_FORMAT_R8_TYPELESS = 60, + DXGI_FORMAT_R8_UNORM = 61, + DXGI_FORMAT_R8_UINT = 62, + DXGI_FORMAT_R8_SNORM = 63, + DXGI_FORMAT_R8_SINT = 64, + DXGI_FORMAT_A8_UNORM = 65, + DXGI_FORMAT_R1_UNORM = 66, + DXGI_FORMAT_R9G9B9E5_SHAREDEXP = 67, + DXGI_FORMAT_R8G8_B8G8_UNORM = 68, + DXGI_FORMAT_G8R8_G8B8_UNORM = 69, + DXGI_FORMAT_BC1_TYPELESS = 70, + DXGI_FORMAT_BC1_UNORM = 71, + DXGI_FORMAT_BC1_UNORM_SRGB = 72, + DXGI_FORMAT_BC2_TYPELESS = 73, + DXGI_FORMAT_BC2_UNORM = 74, + DXGI_FORMAT_BC2_UNORM_SRGB = 75, + DXGI_FORMAT_BC3_TYPELESS = 76, + DXGI_FORMAT_BC3_UNORM = 77, + DXGI_FORMAT_BC3_UNORM_SRGB = 78, + DXGI_FORMAT_BC4_TYPELESS = 79, + DXGI_FORMAT_BC4_UNORM = 80, + DXGI_FORMAT_BC4_SNORM = 81, + DXGI_FORMAT_BC5_TYPELESS = 82, + DXGI_FORMAT_BC5_UNORM = 83, + DXGI_FORMAT_BC5_SNORM = 84, + DXGI_FORMAT_B5G6R5_UNORM = 85, + DXGI_FORMAT_B5G5R5A1_UNORM = 86, + DXGI_FORMAT_B8G8R8A8_UNORM = 87, + DXGI_FORMAT_B8G8R8X8_UNORM = 88, + DXGI_FORMAT_R10G10B10_XR_BIAS_A2_UNORM = 89, + DXGI_FORMAT_B8G8R8A8_TYPELESS = 90, + DXGI_FORMAT_B8G8R8A8_UNORM_SRGB = 91, + DXGI_FORMAT_B8G8R8X8_TYPELESS = 92, + DXGI_FORMAT_B8G8R8X8_UNORM_SRGB = 93, + DXGI_FORMAT_BC6H_TYPELESS = 94, + DXGI_FORMAT_BC6H_UF16 = 95, + DXGI_FORMAT_BC6H_SF16 = 96, + DXGI_FORMAT_BC7_TYPELESS = 97, + DXGI_FORMAT_BC7_UNORM = 98, + DXGI_FORMAT_BC7_UNORM_SRGB = 99, + DXGI_FORMAT_AYUV = 100, + DXGI_FORMAT_Y410 = 101, + DXGI_FORMAT_Y416 = 102, + DXGI_FORMAT_NV12 = 103, + DXGI_FORMAT_P010 = 104, + DXGI_FORMAT_P016 = 105, + DXGI_FORMAT_420_OPAQUE = 106, + DXGI_FORMAT_YUY2 = 107, + DXGI_FORMAT_Y210 = 108, + DXGI_FORMAT_Y216 = 109, + DXGI_FORMAT_NV11 = 110, + DXGI_FORMAT_AI44 = 111, + DXGI_FORMAT_IA44 = 112, + DXGI_FORMAT_P8 = 113, + DXGI_FORMAT_A8P8 = 114, + DXGI_FORMAT_B4G4R4A4_UNORM = 115, + DXGI_FORMAT_P208 = 130, + DXGI_FORMAT_V208 = 131, + DXGI_FORMAT_V408 = 132, + DXGI_FORMAT_FORCE_UINT = 0xffffffff +} DXGI_FORMAT; + +enum D3D10_RESOURCE_DIMENSION +{ + D3D10_RESOURCE_DIMENSION_UNKNOWN = 0, + D3D10_RESOURCE_DIMENSION_BUFFER = 1, + D3D10_RESOURCE_DIMENSION_TEXTURE1D = 2, + D3D10_RESOURCE_DIMENSION_TEXTURE2D = 3, + D3D10_RESOURCE_DIMENSION_TEXTURE3D = 4 +}; + +struct DDS_HEADER_DXT10 +{ + DXGI_FORMAT dxgiFormat; + D3D10_RESOURCE_DIMENSION resourceDimension; + uint32_t miscFlag; + uint32_t arraySize; + uint32_t miscFlags2; +}; + diff --git a/external/basis_universal/example_transcoding/example_transcoding.cpp b/external/basis_universal/example_transcoding/example_transcoding.cpp new file mode 100644 index 0000000000..32335e4345 --- /dev/null +++ b/external/basis_universal/example_transcoding/example_transcoding.cpp @@ -0,0 +1,100 @@ +// example_transcoding.cpp: Very simple transcoding-only example. Does not depend on the basisu encoder library at all, just basisu_transcoder.cpp. +// You can use AMD Compressonator or Microsoft's DirectXTex tools on github to view the written DX10 .DDS file. +#include +#include + +// for testing +//#define BASISD_SUPPORT_XUASTC (0) +//#define BASISD_SUPPORT_KTX2_ZSTD (0) + +#include "../transcoder/basisu_transcoder.h" +#include "utils.h" + +int main() +{ + basist::basisu_transcoder_init(); + + // Read the .KTX2 file's data into memory. + utils::uint8_vec ktx2_file_data; + if (!utils::read_file("../test_files/base_xuastc_arith.ktx2", ktx2_file_data)) + { + if (!utils::read_file("base_xuastc_arith.ktx2", ktx2_file_data)) + { + fprintf(stderr, "Can't read file ../test_files/base_xuastc_arith.ktx2 or base_xuastc_arith.ktx2\n"); + return EXIT_FAILURE; + } + } + + printf("Read file base_xuastc_arith.ktx2\n"); + + if (ktx2_file_data.size() > UINT32_MAX) + { + fprintf(stderr, "KTX2 file too large\n"); + return EXIT_FAILURE; + } + + basist::ktx2_transcoder transcoder; + + // Initialize the transcoder. + if (!transcoder.init(ktx2_file_data.data(), (uint32_t)ktx2_file_data.size())) + return EXIT_FAILURE; + + const uint32_t width = transcoder.get_width(); + const uint32_t height = transcoder.get_height(); + const uint32_t num_levels = transcoder.get_levels(); + const bool is_srgb = transcoder.is_srgb(); + + printf("KTX2 dimensions: %ux%u, num mip levels: %u, sRGB: %u\n", width, height, num_levels, is_srgb); + + // Can't transcode HDR to LDR formats. + if (transcoder.is_hdr()) + { + fprintf(stderr, "Expected LDR KTX2 file\n"); + return EXIT_FAILURE; + } + + // Ensure BC7 support was enabled at compilation time (it will be enabled by default). + const basist::transcoder_texture_format tex_fmt = basist::transcoder_texture_format::cTFBC7_RGBA; + if (!basist::basis_is_format_supported(tex_fmt, transcoder.get_basis_tex_format())) + { + printf("BC7 was disabled in the transcoder at compilation\n"); + return EXIT_FAILURE; + } + + // Begin transcoding (this will be a no-op with UASTC HDR textures, but you still need to do it. For ETC1S it'll unpack the global codebooks). + transcoder.start_transcoding(); + + // Transcode to BC7 and write a BC7 .DDS file. + + // Bytes per block (8 or 16 for BC1-7) + const uint32_t bytes_per_block = basist::basis_get_bytes_per_block_or_pixel(tex_fmt); + // Compute total bytes needed to transcode the slice + const uint32_t total_bytes = basist::basis_compute_transcoded_image_size_in_bytes(tex_fmt, width, height); + // Derive the total number of blocks the output buffer can hold. The transcoder will use this to verify the buffer is large enough. + const uint32_t total_blocks = total_bytes / bytes_per_block; + + // Allocate the buffer to hold the blocks + utils::uint8_vec tex_buffer(total_bytes); + + // Transcode the level + bool status = transcoder.transcode_image_level(0, 0, 0, + tex_buffer.data(), total_blocks, + tex_fmt, 0); + + if (!status) + { + fprintf(stderr, "transcoder.transcode_image_level() failed\n"); + return EXIT_FAILURE; + } + + // Write an sRGB DX10-style .DDS file. + if (!utils::save_dds("out.dds", width, height, tex_buffer.data(), 8, DXGI_FORMAT_BC7_UNORM_SRGB, true, true)) + { + fprintf(stderr, "save_dds() failed\n"); + return EXIT_FAILURE; + } + + printf("Wrote out.dds\n"); + + return EXIT_SUCCESS; +} diff --git a/external/basis_universal/example_transcoding/example_transcoding.manifest b/external/basis_universal/example_transcoding/example_transcoding.manifest new file mode 100644 index 0000000000..b4baf6b96d --- /dev/null +++ b/external/basis_universal/example_transcoding/example_transcoding.manifest @@ -0,0 +1,10 @@ + + + + + + UTF-8 + + + + diff --git a/external/basis_universal/example_transcoding/example_transcoding.vcxproj b/external/basis_universal/example_transcoding/example_transcoding.vcxproj new file mode 100644 index 0000000000..2cd44e1412 --- /dev/null +++ b/external/basis_universal/example_transcoding/example_transcoding.vcxproj @@ -0,0 +1,202 @@ + + + + + Debug + ARM64EC + + + Debug + Win32 + + + Release + ARM64EC + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + 17.0 + Win32Proj + {13333092-fcfe-4d74-8e76-f10c6037593c} + exampletranscoding + 10.0 + + + + Application + true + Unicode + v145 + + + Application + false + true + Unicode + v145 + + + Application + true + v145 + Unicode + + + Application + true + v145 + Unicode + + + Application + false + v145 + true + Unicode + + + Application + false + v145 + true + Unicode + + + + + + + + + + + + + + + + + + + + + + + + + + + + Level4 + true + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + stdcpp17 + AdvancedVectorExtensions + + + Console + true + + + + + Level4 + true + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + stdcpp17 + AdvancedVectorExtensions + + + Console + true + + + + + true + _DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + stdcpp17 + Level4 + + + Console + true + + + + + true + _DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + stdcpp17 + Level4 + + + Console + true + + + + + Level4 + true + true + true + NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + stdcpp17 + + + Console + true + + + + + Level4 + true + true + true + NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + stdcpp17 + + + Console + true + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/external/basis_universal/example_transcoding/example_transcoding.vcxproj.filters b/external/basis_universal/example_transcoding/example_transcoding.vcxproj.filters new file mode 100644 index 0000000000..563e6b9ffa --- /dev/null +++ b/external/basis_universal/example_transcoding/example_transcoding.vcxproj.filters @@ -0,0 +1,47 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;c++;cppm;ixx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hh;hpp;hxx;h++;hm;inl;inc;ipp;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + {db43163f-6d1b-46cf-90ad-24650d502e6a} + + + + + Source Files + + + Source Files + + + Source Files\utils + + + Source Files\utils + + + + + Source Files + + + + + Source Files\utils + + + Source Files\utils + + + \ No newline at end of file diff --git a/external/basis_universal/example_transcoding/utils.cpp b/external/basis_universal/example_transcoding/utils.cpp new file mode 100644 index 0000000000..e42e71feeb --- /dev/null +++ b/external/basis_universal/example_transcoding/utils.cpp @@ -0,0 +1,948 @@ +// File: utils.cpp +#include "utils.h" +//#include "lodepng.h" +//#include "miniz.h" + +namespace utils +{ + +#define FLOOD_PUSH(y, xl, xr, dy) if (((y + (dy)) >= 0) && ((y + (dy)) < (int)m_height)) { stack.push_back(fill_segment(y, xl, xr, dy)); } + +// See http://www.realtimerendering.com/resources/GraphicsGems/gems/SeedFill.c +uint32_t image_u8::flood_fill(int x, int y, const color_quad_u8& c, const color_quad_u8& b, std::vector* pSet_pixels) +{ + uint32_t total_set = 0; + + if (!flood_fill_is_inside(x, y, b)) + return 0; + + std::vector stack; + stack.reserve(64); + + FLOOD_PUSH(y, x, x, 1); + FLOOD_PUSH(y + 1, x, x, -1); + + while (stack.size()) + { + fill_segment s = stack.back(); + stack.pop_back(); + + int x1 = s.m_xl, x2 = s.m_xr, dy = s.m_dy; + y = s.m_y + s.m_dy; + + for (x = x1; (x >= 0) && flood_fill_is_inside(x, y, b); x--) + { + (*this)(x, y) = c; + total_set++; + if (pSet_pixels) + pSet_pixels->push_back(pixel_coord(x, y)); + } + + int l; + + if (x >= x1) + goto skip; + + l = x + 1; + if (l < x1) + FLOOD_PUSH(y, l, x1 - 1, -dy); + + x = x1 + 1; + + do + { + for (; x <= ((int)m_width - 1) && flood_fill_is_inside(x, y, b); x++) + { + (*this)(x, y) = c; + total_set++; + if (pSet_pixels) + pSet_pixels->push_back(pixel_coord(x, y)); + } + FLOOD_PUSH(y, l, x - 1, dy); + + if (x > (x2 + 1)) + FLOOD_PUSH(y, x2 + 1, x - 1, -dy); + + skip: + for (x++; x <= x2 && !flood_fill_is_inside(x, y, b); x++) + ; + + l = x; + } while (x <= x2); + } + + return total_set; +} + +void image_u8::draw_line(int xs, int ys, int xe, int ye, const color_quad_u8& color) +{ + if (xs > xe) + { + std::swap(xs, xe); + std::swap(ys, ye); + } + + int dx = xe - xs, dy = ye - ys; + if (!dx) + { + if (ys > ye) + std::swap(ys, ye); + for (int i = ys; i <= ye; i++) + set_pixel_clipped(xs, i, color); + } + else if (!dy) + { + for (int i = xs; i < xe; i++) + set_pixel_clipped(i, ys, color); + } + else if (dy > 0) + { + if (dy <= dx) + { + int e = 2 * dy - dx, e_no_inc = 2 * dy, e_inc = 2 * (dy - dx); + rasterize_line(xs, ys, xe, ye, 0, 1, e, e_inc, e_no_inc, color); + } + else + { + int e = 2 * dx - dy, e_no_inc = 2 * dx, e_inc = 2 * (dx - dy); + rasterize_line(xs, ys, xe, ye, 1, 1, e, e_inc, e_no_inc, color); + } + } + else + { + dy = -dy; + if (dy <= dx) + { + int e = 2 * dy - dx, e_no_inc = 2 * dy, e_inc = 2 * (dy - dx); + rasterize_line(xs, ys, xe, ye, 0, -1, e, e_inc, e_no_inc, color); + } + else + { + int e = 2 * dx - dy, e_no_inc = (2 * dx), e_inc = 2 * (dx - dy); + rasterize_line(xe, ye, xs, ys, 1, -1, e, e_inc, e_no_inc, color); + } + } +} + +void image_u8::rasterize_line(int xs, int ys, int xe, int ye, int pred, int inc_dec, int e, int e_inc, int e_no_inc, const color_quad_u8& color) +{ + int start, end, var; + + if (pred) + { + start = ys; + end = ye; + var = xs; + for (int i = start; i <= end; i++) + { + set_pixel_clipped(var, i, color); + if (e < 0) + e += e_no_inc; + else + { + var += inc_dec; + e += e_inc; + } + } + } + else + { + start = xs; + end = xe; + var = ys; + for (int i = start; i <= end; i++) + { + set_pixel_clipped(i, var, color); + if (e < 0) + e += e_no_inc; + else + { + var += inc_dec; + e += e_inc; + } + } + } +} + +#if 0 +bool load_png(const char* pFilename, image_u8& img) +{ + img.clear(); + + std::vector pixels; + unsigned int w = 0, h = 0; + unsigned int e = lodepng::decode(pixels, w, h, pFilename); + if (e != 0) + { + fprintf(stderr, "Failed loading PNG file %s\n", pFilename); + return false; + } + + img.init(w, h); + memcpy(&img.get_pixels()[0], &pixels[0], w * h * sizeof(uint32_t)); + + return true; +} + +bool save_png(const char* pFilename, const image_u8& img, bool save_alpha) +{ + const uint32_t w = img.width(); + const uint32_t h = img.height(); + + std::vector pixels; + if (save_alpha) + { + pixels.resize(w * h * sizeof(color_quad_u8)); + memcpy(&pixels[0], &img.get_pixels()[0], w * h * sizeof(color_quad_u8)); + } + else + { + pixels.resize(w * h * 3); + unsigned char* pDst = &pixels[0]; + for (uint32_t y = 0; y < h; y++) + for (uint32_t x = 0; x < w; x++, pDst += 3) + pDst[0] = img(x, y)[0], pDst[1] = img(x, y)[1], pDst[2] = img(x, y)[2]; + } + + return lodepng::encode(pFilename, pixels, w, h, save_alpha ? LCT_RGBA : LCT_RGB) == 0; +} +#endif + +static float gauss(int x, int y, float sigma_sqr) +{ + float pow = expf(-((x * x + y * y) / (2.0f * sigma_sqr))); + float g = (1.0f / (sqrtf((float)(2.0f * M_PI * sigma_sqr)))) * pow; + return g; +} + +// size_x/y should be odd +void compute_gaussian_kernel(float* pDst, int size_x, int size_y, float sigma_sqr, uint32_t flags) +{ + assert(size_x & size_y & 1); + + if (!(size_x | size_y)) + return; + + int mid_x = size_x / 2; + int mid_y = size_y / 2; + + double sum = 0; + for (int x = 0; x < size_x; x++) + { + for (int y = 0; y < size_y; y++) + { + float g; + if ((x > mid_x) && (y < mid_y)) + g = pDst[(size_x - x - 1) + y * size_x]; + else if ((x < mid_x) && (y > mid_y)) + g = pDst[x + (size_y - y - 1) * size_x]; + else if ((x > mid_x) && (y > mid_y)) + g = pDst[(size_x - x - 1) + (size_y - y - 1) * size_x]; + else + g = gauss(x - mid_x, y - mid_y, sigma_sqr); + + pDst[x + y * size_x] = g; + sum += g; + } + } + + if (flags & cComputeGaussianFlagNormalizeCenterToOne) + { + sum = pDst[mid_x + mid_y * size_x]; + } + + if (flags & (cComputeGaussianFlagNormalizeCenterToOne | cComputeGaussianFlagNormalize)) + { + double one_over_sum = 1.0f / sum; + for (int i = 0; i < size_x * size_y; i++) + pDst[i] = static_cast(pDst[i] * one_over_sum); + + if (flags & cComputeGaussianFlagNormalizeCenterToOne) + pDst[mid_x + mid_y * size_x] = 1.0f; + } + + if (flags & cComputeGaussianFlagPrint) + { + printf("{\n"); + for (int y = 0; y < size_y; y++) + { + printf(" "); + for (int x = 0; x < size_x; x++) + { + printf("%f, ", pDst[x + y * size_x]); + } + printf("\n"); + } + printf("}"); + } +} + +void gaussian_filter(imagef& dst, const imagef& orig_img, uint32_t odd_filter_width, float sigma_sqr, bool wrapping, uint32_t width_divisor, uint32_t height_divisor) +{ + assert(odd_filter_width && (odd_filter_width & 1)); + odd_filter_width |= 1; + + std::vector kernel(odd_filter_width * odd_filter_width); + compute_gaussian_kernel(&kernel[0], odd_filter_width, odd_filter_width, sigma_sqr, cComputeGaussianFlagNormalize); + + const int dst_width = orig_img.get_width() / width_divisor; + const int dst_height = orig_img.get_height() / height_divisor; + + const int H = odd_filter_width / 2; + const int L = -H; + + dst.crop(dst_width, dst_height); + +//#pragma omp parallel for + for (int oy = 0; oy < dst_height; oy++) + { + for (int ox = 0; ox < dst_width; ox++) + { + vec4F c(0.0f); + + for (int yd = L; yd <= H; yd++) + { + int y = oy * height_divisor + (height_divisor >> 1) + yd; + + for (int xd = L; xd <= H; xd++) + { + int x = ox * width_divisor + (width_divisor >> 1) + xd; + + const vec4F& p = orig_img.get_clamped_or_wrapped(x, y, wrapping, wrapping); + + float w = kernel[(xd + H) + (yd + H) * odd_filter_width]; + c[0] += p[0] * w; + c[1] += p[1] * w; + c[2] += p[2] * w; + c[3] += p[3] * w; + } + } + + dst(ox, oy).set(c[0], c[1], c[2], c[3]); + } + } +} + +static void pow_image(const imagef& src, imagef& dst, const vec4F& power) +{ + dst.resize(src); + +//#pragma omp parallel for + for (int y = 0; y < (int)dst.get_height(); y++) + { + for (uint32_t x = 0; x < dst.get_width(); x++) + { + const vec4F& p = src(x, y); + + if ((power[0] == 2.0f) && (power[1] == 2.0f) && (power[2] == 2.0f) && (power[3] == 2.0f)) + dst(x, y).set(p[0] * p[0], p[1] * p[1], p[2] * p[2], p[3] * p[3]); + else + dst(x, y).set(powf(p[0], power[0]), powf(p[1], power[1]), powf(p[2], power[2]), powf(p[3], power[3])); + } + } +} + +#if 0 +static void mul_image(const imagef& src, imagef& dst, const vec4F& mul) +{ + dst.resize(src); + +//#pragma omp parallel for + for (int y = 0; y < (int)dst.get_height(); y++) + { + for (uint32_t x = 0; x < dst.get_width(); x++) + { + const vec4F& p = src(x, y); + dst(x, y).set(p[0] * mul[0], p[1] * mul[1], p[2] * mul[2], p[3] * mul[3]); + } + } +} +#endif + +static void scale_image(const imagef& src, imagef& dst, const vec4F& scale, const vec4F& shift) +{ + dst.resize(src); + +//#pragma omp parallel for + for (int y = 0; y < (int)dst.get_height(); y++) + { + for (uint32_t x = 0; x < dst.get_width(); x++) + { + const vec4F& p = src(x, y); + + vec4F d; + + for (uint32_t c = 0; c < 4; c++) + d[c] = scale[c] * p[c] + shift[c]; + + dst(x, y).set(d[0], d[1], d[2], d[3]); + } + } +} + +static void add_weighted_image(const imagef& src1, const vec4F& alpha, const imagef& src2, const vec4F& beta, const vec4F& gamma, imagef& dst) +{ + dst.resize(src1); + +//#pragma omp parallel for + for (int y = 0; y < (int)dst.get_height(); y++) + { + for (uint32_t x = 0; x < dst.get_width(); x++) + { + const vec4F& s1 = src1(x, y); + const vec4F& s2 = src2(x, y); + + dst(x, y).set( + s1[0] * alpha[0] + s2[0] * beta[0] + gamma[0], + s1[1] * alpha[1] + s2[1] * beta[1] + gamma[1], + s1[2] * alpha[2] + s2[2] * beta[2] + gamma[2], + s1[3] * alpha[3] + s2[3] * beta[3] + gamma[3]); + } + } +} + +static void add_image(const imagef& src1, const imagef& src2, imagef& dst) +{ + dst.resize(src1); + +//#pragma omp parallel for + for (int y = 0; y < (int)dst.get_height(); y++) + { + for (uint32_t x = 0; x < dst.get_width(); x++) + { + const vec4F& s1 = src1(x, y); + const vec4F& s2 = src2(x, y); + + dst(x, y).set(s1[0] + s2[0], s1[1] + s2[1], s1[2] + s2[2], s1[3] + s2[3]); + } + } +} + +static void adds_image(const imagef& src, const vec4F& value, imagef& dst) +{ + dst.resize(src); + +//#pragma omp parallel for + for (int y = 0; y < (int)dst.get_height(); y++) + { + for (uint32_t x = 0; x < dst.get_width(); x++) + { + const vec4F& p = src(x, y); + + dst(x, y).set(p[0] + value[0], p[1] + value[1], p[2] + value[2], p[3] + value[3]); + } + } +} + +static void mul_image(const imagef& src1, const imagef& src2, imagef& dst, const vec4F& scale) +{ + dst.resize(src1); + +//#pragma omp parallel for + for (int y = 0; y < (int)dst.get_height(); y++) + { + for (uint32_t x = 0; x < dst.get_width(); x++) + { + const vec4F& s1 = src1(x, y); + const vec4F& s2 = src2(x, y); + + vec4F d; + + for (uint32_t c = 0; c < 4; c++) + { + float v1 = s1[c]; + float v2 = s2[c]; + d[c] = v1 * v2 * scale[c]; + } + + dst(x, y) = d; + } + } +} + +static void div_image(const imagef& src1, const imagef& src2, imagef& dst, const vec4F& scale) +{ + dst.resize(src1); + +//#pragma omp parallel for + for (int y = 0; y < (int)dst.get_height(); y++) + { + for (uint32_t x = 0; x < dst.get_width(); x++) + { + const vec4F& s1 = src1(x, y); + const vec4F& s2 = src2(x, y); + + vec4F d; + + for (uint32_t c = 0; c < 4; c++) + { + float v = s2[c]; + if (v == 0.0f) + d[c] = 0.0f; + else + d[c] = (s1[c] * scale[c]) / v; + } + + dst(x, y) = d; + } + } +} + +static vec4F avg_image(const imagef& src) +{ + vec4F avg(0.0f); + + for (uint32_t y = 0; y < src.get_height(); y++) + { + for (uint32_t x = 0; x < src.get_width(); x++) + { + const vec4F& s = src(x, y); + + avg += vec4F(s[0], s[1], s[2], s[3]); + } + } + + avg /= static_cast(src.get_total_pixels()); + + return avg; +} + +// Reference: https://ece.uwaterloo.ca/~z70wang/research/ssim/index.html +vec4F compute_ssim(const imagef& a, const imagef& b) +{ + imagef axb, a_sq, b_sq, mu1, mu2, mu1_sq, mu2_sq, mu1_mu2, s1_sq, s2_sq, s12, smap, t1, t2, t3; + + const float C1 = 6.50250f, C2 = 58.52250f; + + pow_image(a, a_sq, vec4F(2)); + pow_image(b, b_sq, vec4F(2)); + mul_image(a, b, axb, vec4F(1.0f)); + + gaussian_filter(mu1, a, 11, 1.5f * 1.5f); + gaussian_filter(mu2, b, 11, 1.5f * 1.5f); + + pow_image(mu1, mu1_sq, vec4F(2)); + pow_image(mu2, mu2_sq, vec4F(2)); + mul_image(mu1, mu2, mu1_mu2, vec4F(1.0f)); + + gaussian_filter(s1_sq, a_sq, 11, 1.5f * 1.5f); + add_weighted_image(s1_sq, vec4F(1), mu1_sq, vec4F(-1), vec4F(0), s1_sq); + + gaussian_filter(s2_sq, b_sq, 11, 1.5f * 1.5f); + add_weighted_image(s2_sq, vec4F(1), mu2_sq, vec4F(-1), vec4F(0), s2_sq); + + gaussian_filter(s12, axb, 11, 1.5f * 1.5f); + add_weighted_image(s12, vec4F(1), mu1_mu2, vec4F(-1), vec4F(0), s12); + + scale_image(mu1_mu2, t1, vec4F(2), vec4F(0)); + adds_image(t1, vec4F(C1), t1); + + scale_image(s12, t2, vec4F(2), vec4F(0)); + adds_image(t2, vec4F(C2), t2); + + mul_image(t1, t2, t3, vec4F(1)); + + add_image(mu1_sq, mu2_sq, t1); + adds_image(t1, vec4F(C1), t1); + + add_image(s1_sq, s2_sq, t2); + adds_image(t2, vec4F(C2), t2); + + mul_image(t1, t2, t1, vec4F(1)); + + div_image(t3, t1, smap, vec4F(1)); + + return avg_image(smap); +} + +vec4F compute_ssim(const image_u8& a, const image_u8& b, bool luma) +{ + image_u8 ta(a), tb(b); + + if ((ta.width() != tb.width()) || (ta.height() != tb.height())) + { + fprintf(stderr, "compute_ssim: Cropping input images to equal dimensions\n"); + + const uint32_t w = std::min(a.width(), b.width()); + const uint32_t h = std::min(a.height(), b.height()); + ta.crop(w, h); + tb.crop(w, h); + } + + if (!ta.width() || !ta.height()) + { + assert(0); + return vec4F(0); + } + + if (luma) + { + for (uint32_t y = 0; y < ta.height(); y++) + { + for (uint32_t x = 0; x < ta.width(); x++) + { + ta(x, y).set((uint8_t)ta(x, y).get_luma(), ta(x, y).a); + tb(x, y).set((uint8_t)tb(x, y).get_luma(), tb(x, y).a); + } + } + } + + imagef fta, ftb; + + fta.set(ta); + ftb.set(tb); + + return compute_ssim(fta, ftb); +} + +bool save_dds(const char* pFilename, uint32_t width, uint32_t height, const void* pBlocks, uint32_t pixel_format_bpp, DXGI_FORMAT dxgi_format, bool srgb, bool force_dx10_header) +{ + (void)srgb; + + FILE* pFile = NULL; +#ifdef _MSC_VER + fopen_s(&pFile, pFilename, "wb"); +#else + pFile = fopen(pFilename, "wb"); +#endif + if (!pFile) + { + fprintf(stderr, "Failed creating file %s!\n", pFilename); + return false; + } + + fwrite("DDS ", 4, 1, pFile); + + DDSURFACEDESC2 desc; + memset(&desc, 0, sizeof(desc)); + + desc.dwSize = sizeof(desc); + desc.dwFlags = DDSD_WIDTH | DDSD_HEIGHT | DDSD_PIXELFORMAT | DDSD_CAPS; + + desc.dwWidth = width; + desc.dwHeight = height; + + desc.ddsCaps.dwCaps = DDSCAPS_TEXTURE; + desc.ddpfPixelFormat.dwSize = sizeof(desc.ddpfPixelFormat); + + desc.ddpfPixelFormat.dwFlags |= DDPF_FOURCC; + + desc.lPitch = (((desc.dwWidth + 3) & ~3) * ((desc.dwHeight + 3) & ~3) * pixel_format_bpp) >> 3; + desc.dwFlags |= DDSD_LINEARSIZE; + + desc.ddpfPixelFormat.dwRGBBitCount = 0; + + if ((!force_dx10_header) && + ((dxgi_format == DXGI_FORMAT_BC1_UNORM) || + (dxgi_format == DXGI_FORMAT_BC3_UNORM) || + (dxgi_format == DXGI_FORMAT_BC4_UNORM) || + (dxgi_format == DXGI_FORMAT_BC5_UNORM))) + { + if (dxgi_format == DXGI_FORMAT_BC1_UNORM) + desc.ddpfPixelFormat.dwFourCC = (uint32_t)PIXEL_FMT_FOURCC('D', 'X', 'T', '1'); + else if (dxgi_format == DXGI_FORMAT_BC3_UNORM) + desc.ddpfPixelFormat.dwFourCC = (uint32_t)PIXEL_FMT_FOURCC('D', 'X', 'T', '5'); + else if (dxgi_format == DXGI_FORMAT_BC4_UNORM) + desc.ddpfPixelFormat.dwFourCC = (uint32_t)PIXEL_FMT_FOURCC('A', 'T', 'I', '1'); + else if (dxgi_format == DXGI_FORMAT_BC5_UNORM) + desc.ddpfPixelFormat.dwFourCC = (uint32_t)PIXEL_FMT_FOURCC('A', 'T', 'I', '2'); + + fwrite(&desc, sizeof(desc), 1, pFile); + } + else + { + desc.ddpfPixelFormat.dwFourCC = (uint32_t)PIXEL_FMT_FOURCC('D', 'X', '1', '0'); + + fwrite(&desc, sizeof(desc), 1, pFile); + + DDS_HEADER_DXT10 hdr10; + memset(&hdr10, 0, sizeof(hdr10)); + + // Not all tools support DXGI_FORMAT_BC7_UNORM_SRGB (like NVTT), but ddsview in DirectXTex pays attention to it. So not sure what to do here. + // For best compatibility just write DXGI_FORMAT_BC7_UNORM. + //hdr10.dxgiFormat = srgb ? DXGI_FORMAT_BC7_UNORM_SRGB : DXGI_FORMAT_BC7_UNORM; + hdr10.dxgiFormat = dxgi_format; // DXGI_FORMAT_BC7_UNORM; + hdr10.resourceDimension = D3D10_RESOURCE_DIMENSION_TEXTURE2D; + hdr10.arraySize = 1; + + fwrite(&hdr10, sizeof(hdr10), 1, pFile); + } + + fwrite(pBlocks, desc.lPitch, 1, pFile); + + if (fclose(pFile) == EOF) + { + fprintf(stderr, "Failed writing to DDS file %s!\n", pFilename); + return false; + } + + return true; +} + +void strip_extension(std::string& s) +{ + for (int32_t i = (int32_t)s.size() - 1; i >= 0; i--) + { + if (s[i] == '.') + { + s.resize(i); + break; + } + } +} + +void strip_path(std::string& s) +{ + for (int32_t i = (int32_t)s.size() - 1; i >= 0; i--) + { + if ((s[i] == '/') || (s[i] == ':') || (s[i] == '\\')) + { + s.erase(0, i + 1); + break; + } + } +} + +uint32_t hash_hsieh(const uint8_t* pBuf, size_t len) +{ + if (!pBuf || !len) + return 0; + + uint32_t h = static_cast(len); + + const uint32_t bytes_left = len & 3; + len >>= 2; + + while (len--) + { + const uint16_t* pWords = reinterpret_cast(pBuf); + + h += pWords[0]; + + const uint32_t t = (pWords[1] << 11) ^ h; + h = (h << 16) ^ t; + + pBuf += sizeof(uint32_t); + + h += h >> 11; + } + + switch (bytes_left) + { + case 1: + h += *reinterpret_cast(pBuf); + h ^= h << 10; + h += h >> 1; + break; + case 2: + h += *reinterpret_cast(pBuf); + h ^= h << 11; + h += h >> 17; + break; + case 3: + h += *reinterpret_cast(pBuf); + h ^= h << 16; + h ^= (static_cast(pBuf[sizeof(uint16_t)])) << 18; + h += h >> 11; + break; + default: + break; + } + + h ^= h << 3; + h += h >> 5; + h ^= h << 4; + h += h >> 17; + h ^= h << 25; + h += h >> 6; + + return h; +} + +float compute_block_max_std_dev(const color_quad_u8* pPixels, uint32_t block_width, uint32_t block_height, uint32_t num_comps) +{ + tracked_stat comp_stats[4]; + + for (uint32_t y = 0; y < block_height; y++) + { + for (uint32_t x = 0; x < block_width; x++) + { + const color_quad_u8* pPixel = pPixels + x + y * block_width; + + for (uint32_t c = 0; c < num_comps; c++) + comp_stats[c].update(pPixel->m_c[c]); + } + } + + float max_std_dev = 0.0f; + for (uint32_t i = 0; i < num_comps; i++) + max_std_dev = std::max(max_std_dev, comp_stats[i].get_std_dev()); + return max_std_dev; +} + +const uint32_t ASTC_SIG = 0x5CA1AB13; + +#pragma pack(push, 1) +struct astc_header +{ + uint32_t m_sig; + uint8_t m_block_x; + uint8_t m_block_y; + uint8_t m_block_z; + uint8_t m_width[3]; + uint8_t m_height[3]; + uint8_t m_depth[3]; +}; +#pragma pack(pop) + +bool save_astc_file(const char* pFilename, block16_vec& blocks, uint32_t width, uint32_t height, uint32_t block_width, uint32_t block_height) +{ + FILE* pFile = nullptr; + +#ifdef _MSC_VER + fopen_s(&pFile, pFilename, "wb"); +#else + pFile = fopen(pFilename, "wb"); +#endif + + if (!pFile) + return false; + + astc_header hdr; + memset(&hdr, 0, sizeof(hdr)); + + hdr.m_sig = ASTC_SIG; + hdr.m_block_x = (uint8_t)block_width; + hdr.m_block_y = (uint8_t)block_height; + hdr.m_block_z = 1; + hdr.m_width[0] = (uint8_t)(width); + hdr.m_width[1] = (uint8_t)(width >> 8); + hdr.m_width[2] = (uint8_t)(width >> 16); + hdr.m_height[0] = (uint8_t)(height); + hdr.m_height[1] = (uint8_t)(height >> 8); + hdr.m_height[2] = (uint8_t)(height >> 16); + hdr.m_depth[0] = 1; + fwrite(&hdr, sizeof(hdr), 1, pFile); + + fwrite(blocks.data(), 16, blocks.size(), pFile); + if (fclose(pFile) == EOF) + return false; + + return true; +} + +bool load_astc_file(const char* pFilename, block16_vec& blocks, uint32_t& width, uint32_t& height, uint32_t& block_width, uint32_t& block_height) +{ + FILE* pFile = nullptr; + +#ifdef _MSC_VER + fopen_s(&pFile, pFilename, "rb"); +#else + pFile = fopen(pFilename, "rb"); +#endif + + if (!pFile) + return false; + + astc_header hdr; + if (fread(&hdr, sizeof(hdr), 1, pFile) != 1) + { + fclose(pFile); + return false; + } + + if (hdr.m_sig != ASTC_SIG) + { + fclose(pFile); + return false; + } + + width = hdr.m_width[0] + (hdr.m_width[1] << 8) + (hdr.m_width[2] << 16); + height = hdr.m_height[0] + (hdr.m_height[1] << 8) + (hdr.m_height[2] << 16); + uint32_t depth = hdr.m_depth[0] + (hdr.m_depth[1] << 8) + (hdr.m_depth[2] << 16); + + if ((width < 1) || (width > 32768) || (height < 1) || (height > 32768)) + return false; + if ((hdr.m_block_z != 1) || (depth != 1)) + return false; + + block_width = hdr.m_block_x; + block_height = hdr.m_block_y; + + if ((block_width < 4) || (block_width > 12) || (block_height < 4) || (block_height > 12)) + return false; + + uint32_t blocks_x = (width + block_width - 1) / block_width; + uint32_t blocks_y = (height + block_height - 1) / block_height; + uint32_t total_blocks = blocks_x * blocks_y; + + blocks.resize(total_blocks); + + if (fread(blocks.data(), 16, total_blocks, pFile) != total_blocks) + { + fclose(pFile); + return false; + } + + fclose(pFile); + return true; +} + +#if 0 +uint32_t get_deflate_size(const void* pData, size_t data_size) +{ + size_t comp_size = 0; + void* pPre_RDO_Comp_data = tdefl_compress_mem_to_heap(pData, data_size, &comp_size, TDEFL_MAX_PROBES_MASK);// TDEFL_DEFAULT_MAX_PROBES); + mz_free(pPre_RDO_Comp_data); + + if (comp_size > UINT32_MAX) + return UINT32_MAX; + + return (uint32_t)comp_size; +} +#endif + +bool read_file(const char* pFilename, uint8_vec& buf) +{ + buf.resize(0); + + FILE* pFile = nullptr; +#if _MSC_VER + fopen_s(&pFile, pFilename, "rb"); +#else + pFile = fopen(pFilename, "rb"); +#endif + if (!pFile) + return false; + + fseek(pFile, 0, SEEK_END); + + long file_end_ofs = ftell(pFile); + if (file_end_ofs <= 0) + { + fclose(pFile); + return false; + } + + size_t sz = static_cast(file_end_ofs); + if (sz != (unsigned long)file_end_ofs) + { + fclose(pFile); + return false; + } + + fseek(pFile, 0, SEEK_SET); + + buf.resize(sz); + + if (fread(buf.data(), sizeof(uint8_t), sz, pFile) != sz) + { + fclose(pFile); + return false; + } + + fclose(pFile); + return true; +} + +} // namespace utils diff --git a/external/basis_universal/example_transcoding/utils.h b/external/basis_universal/example_transcoding/utils.h new file mode 100644 index 0000000000..d161e5ff27 --- /dev/null +++ b/external/basis_universal/example_transcoding/utils.h @@ -0,0 +1,2621 @@ +// File: utils.h +#pragma once +#ifdef _MSC_VER +#pragma warning (push) +#pragma warning (disable:4127) // conditional expression is constant +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "dds_defs.h" + +#ifndef M_PI +#define M_PI 3.14159265358979323846 +#endif + +#define ASSUME(c) static_assert(c, #c) +#define ARRAY_SIZE(a) (sizeof(a)/sizeof(a[0])) + +#define VECTOR_TEXT_LINE_SIZE (30.0f) +#define VECTOR_TEXT_CORE_LINE_SIZE (21.0f) + +#define UNUSED(x) (void)x + +namespace utils +{ +typedef std::vector uint8_vec; + +extern const uint32_t g_pretty_colors[]; +extern const uint32_t g_num_pretty_colors; + +const float cDegToRad = 0.01745329252f; +const float cRadToDeg = 57.29577951f; + +enum eClear { cClear }; +enum eZero { cZero }; +enum eInitExpand { cInitExpand }; + +inline int iabs(int i) { if (i < 0) i = -i; return i; } +inline uint8_t clamp255(int32_t i) { return (uint8_t)((i & 0xFFFFFF00U) ? (~(i >> 31)) : i); } +template inline S clamp(S value, S low, S high) { return (value < low) ? low : ((value > high) ? high : value); } +template inline F lerp(F a, F b, F s) { return a + (b - a) * s; } +template inline F square(F a) { return a * a; } + +template +inline T prev_wrap(T i, T n) +{ + T temp = i - 1; + if (temp < 0) + temp = n - 1; + return temp; +} + +template +inline T next_wrap(T i, T n) +{ + T temp = i + 1; + if (temp >= n) + temp = 0; + return temp; +} + +inline int posmod(int x, int y) +{ + if (x >= 0) + return (x < y) ? x : (x % y); + int m = (-x) % y; + return (m != 0) ? (y - m) : m; +} + +inline float deg_to_rad(float f) +{ + return f * cDegToRad; +}; + +inline float rad_to_deg(float f) +{ + return f * cRadToDeg; +}; + +template +struct rel_ops +{ + friend bool operator!=(const T& x, const T& y) + { + return (!(x == y)); + } + friend bool operator>(const T& x, const T& y) + { + return (y < x); + } + friend bool operator<=(const T& x, const T& y) + { + return (!(y < x)); + } + friend bool operator>=(const T& x, const T& y) + { + return (!(x < y)); + } +}; + +template +class vec : public rel_ops > +{ +public: + typedef T scalar_type; + enum + { + num_elements = N + }; + + inline vec() + { + } + + inline vec(eClear) + { + clear(); + } + + inline vec(const vec& other) + { + for (uint32_t i = 0; i < N; i++) + m_s[i] = other.m_s[i]; + } + + template + inline vec(const vec& other) + { + set(other); + } + + template + inline vec(const vec& other, T w) + { + *this = other; + m_s[N - 1] = w; + } + + explicit inline vec(T val) + { + set(val); + } + + inline vec(T val0, T val1) + { + set(val0, val1); + } + + inline vec(T val0, T val1, T val2) + { + set(val0, val1, val2); + } + + inline vec(T val0, T val1, T val2, T val3) + { + set(val0, val1, val2, val3); + } + + inline vec(T val0, T val1, T val2, T val3, T val4, T val5) + { + set(val0, val1, val2, val3, val4, val5); + } + + inline vec( + T val0, T val1, T val2, T val3, + T val4, T val5, T val6, T val7, + T val8, T val9, T val10, T val11, + T val12, T val13, T val14, T val15) + { + set(val0, val1, val2, val3, + val4, val5, val6, val7, + val8, val9, val10, val11, + val12, val13, val14, val15); + } + + inline vec( + T val0, T val1, T val2, T val3, + T val4, T val5, T val6, T val7, + T val8, T val9, T val10, T val11, + T val12, T val13, T val14, T val15, + T val16, T val17, T val18, T val19) + { + set(val0, val1, val2, val3, + val4, val5, val6, val7, + val8, val9, val10, val11, + val12, val13, val14, val15, + val16, val17, val18, val19); + } + + inline vec( + T val0, T val1, T val2, T val3, + T val4, T val5, T val6, T val7, + T val8, T val9, T val10, T val11, + T val12, T val13, T val14, T val15, + T val16, T val17, T val18, T val19, + T val20, T val21, T val22, T val23, + T val24) + { + set(val0, val1, val2, val3, + val4, val5, val6, val7, + val8, val9, val10, val11, + val12, val13, val14, val15, + val16, val17, val18, val19, + val20, val21, val22, val23, + val24); + } + + inline void clear() + { + if (N > 4) + memset(m_s, 0, sizeof(m_s)); + else + { + for (uint32_t i = 0; i < N; i++) + m_s[i] = 0; + } + } + + template + inline vec& set(const vec& other) + { + if ((void*)this == (void*)&other) + return *this; + const uint32_t m = std::min(N, ON); + uint32_t i; + for (i = 0; i < m; i++) + m_s[i] = static_cast(other[i]); + for (; i < N; i++) + m_s[i] = 0; + return *this; + } + + inline vec& set_component(uint32_t index, T val) + { + assert(index < N); + m_s[index] = val; + return *this; + } + + inline vec& set(T val) + { + for (uint32_t i = 0; i < N; i++) + m_s[i] = val; + return *this; + } + + inline vec& set(T val0, T val1) + { + m_s[0] = val0; + if (N >= 2) + { + m_s[1] = val1; + + for (uint32_t i = 2; i < N; i++) + m_s[i] = 0; + } + return *this; + } + + inline vec& set(T val0, T val1, T val2) + { + m_s[0] = val0; + if (N >= 2) + { + m_s[1] = val1; + + if (N >= 3) + { + m_s[2] = val2; + + for (uint32_t i = 3; i < N; i++) + m_s[i] = 0; + } + } + return *this; + } + + inline vec& set(T val0, T val1, T val2, T val3) + { + m_s[0] = val0; + if (N >= 2) + { + m_s[1] = val1; + + if (N >= 3) + { + m_s[2] = val2; + + if (N >= 4) + { + m_s[3] = val3; + + for (uint32_t i = 4; i < N; i++) + m_s[i] = 0; + } + } + } + return *this; + } + + inline vec& set(T val0, T val1, T val2, T val3, T val4, T val5) + { + m_s[0] = val0; + if (N >= 2) + { + m_s[1] = val1; + + if (N >= 3) + { + m_s[2] = val2; + + if (N >= 4) + { + m_s[3] = val3; + + if (N >= 5) + { + m_s[4] = val4; + + if (N >= 6) + { + m_s[5] = val5; + + for (uint32_t i = 6; i < N; i++) + m_s[i] = 0; + } + } + } + } + } + return *this; + } + + inline vec& set( + T val0, T val1, T val2, T val3, + T val4, T val5, T val6, T val7, + T val8, T val9, T val10, T val11, + T val12, T val13, T val14, T val15) + { + m_s[0] = val0; + if (N >= 2) + m_s[1] = val1; + if (N >= 3) + m_s[2] = val2; + if (N >= 4) + m_s[3] = val3; + + if (N >= 5) + m_s[4] = val4; + if (N >= 6) + m_s[5] = val5; + if (N >= 7) + m_s[6] = val6; + if (N >= 8) + m_s[7] = val7; + + if (N >= 9) + m_s[8] = val8; + if (N >= 10) + m_s[9] = val9; + if (N >= 11) + m_s[10] = val10; + if (N >= 12) + m_s[11] = val11; + + if (N >= 13) + m_s[12] = val12; + if (N >= 14) + m_s[13] = val13; + if (N >= 15) + m_s[14] = val14; + if (N >= 16) + m_s[15] = val15; + + for (uint32_t i = 16; i < N; i++) + m_s[i] = 0; + + return *this; + } + + inline vec& set( + T val0, T val1, T val2, T val3, + T val4, T val5, T val6, T val7, + T val8, T val9, T val10, T val11, + T val12, T val13, T val14, T val15, + T val16, T val17, T val18, T val19) + { + m_s[0] = val0; + if (N >= 2) + m_s[1] = val1; + if (N >= 3) + m_s[2] = val2; + if (N >= 4) + m_s[3] = val3; + + if (N >= 5) + m_s[4] = val4; + if (N >= 6) + m_s[5] = val5; + if (N >= 7) + m_s[6] = val6; + if (N >= 8) + m_s[7] = val7; + + if (N >= 9) + m_s[8] = val8; + if (N >= 10) + m_s[9] = val9; + if (N >= 11) + m_s[10] = val10; + if (N >= 12) + m_s[11] = val11; + + if (N >= 13) + m_s[12] = val12; + if (N >= 14) + m_s[13] = val13; + if (N >= 15) + m_s[14] = val14; + if (N >= 16) + m_s[15] = val15; + + if (N >= 17) + m_s[16] = val16; + if (N >= 18) + m_s[17] = val17; + if (N >= 19) + m_s[18] = val18; + if (N >= 20) + m_s[19] = val19; + + for (uint32_t i = 20; i < N; i++) + m_s[i] = 0; + + return *this; + } + + inline vec& set( + T val0, T val1, T val2, T val3, + T val4, T val5, T val6, T val7, + T val8, T val9, T val10, T val11, + T val12, T val13, T val14, T val15, + T val16, T val17, T val18, T val19, + T val20, T val21, T val22, T val23, + T val24) + { + m_s[0] = val0; + if (N >= 2) + m_s[1] = val1; + if (N >= 3) + m_s[2] = val2; + if (N >= 4) + m_s[3] = val3; + + if (N >= 5) + m_s[4] = val4; + if (N >= 6) + m_s[5] = val5; + if (N >= 7) + m_s[6] = val6; + if (N >= 8) + m_s[7] = val7; + + if (N >= 9) + m_s[8] = val8; + if (N >= 10) + m_s[9] = val9; + if (N >= 11) + m_s[10] = val10; + if (N >= 12) + m_s[11] = val11; + + if (N >= 13) + m_s[12] = val12; + if (N >= 14) + m_s[13] = val13; + if (N >= 15) + m_s[14] = val14; + if (N >= 16) + m_s[15] = val15; + + if (N >= 17) + m_s[16] = val16; + if (N >= 18) + m_s[17] = val17; + if (N >= 19) + m_s[18] = val18; + if (N >= 20) + m_s[19] = val19; + + if (N >= 21) + m_s[20] = val20; + if (N >= 22) + m_s[21] = val21; + if (N >= 23) + m_s[22] = val22; + if (N >= 24) + m_s[23] = val23; + + if (N >= 25) + m_s[24] = val24; + + for (uint32_t i = 25; i < N; i++) + m_s[i] = 0; + + return *this; + } + + inline vec& set(const T* pValues) + { + for (uint32_t i = 0; i < N; i++) + m_s[i] = pValues[i]; + return *this; + } + + template + inline vec& swizzle_set(const vec& other, uint32_t i) + { + return set(static_cast(other[i])); + } + + template + inline vec& swizzle_set(const vec& other, uint32_t i, uint32_t j) + { + return set(static_cast(other[i]), static_cast(other[j])); + } + + template + inline vec& swizzle_set(const vec& other, uint32_t i, uint32_t j, uint32_t k) + { + return set(static_cast(other[i]), static_cast(other[j]), static_cast(other[k])); + } + + template + inline vec& swizzle_set(const vec& other, uint32_t i, uint32_t j, uint32_t k, uint32_t l) + { + return set(static_cast(other[i]), static_cast(other[j]), static_cast(other[k]), static_cast(other[l])); + } + + inline vec& operator=(const vec& rhs) + { + if (this != &rhs) + { + for (uint32_t i = 0; i < N; i++) + m_s[i] = rhs.m_s[i]; + } + return *this; + } + + template + inline vec& operator=(const vec& other) + { + if ((void*)this == (void*)&other) + return *this; + + uint32_t s = std::min(N, O); + + uint32_t i; + for (i = 0; i < s; i++) + m_s[i] = static_cast(other[i]); + + for (; i < N; i++) + m_s[i] = 0; + + return *this; + } + + inline bool operator==(const vec& rhs) const + { + for (uint32_t i = 0; i < N; i++) + if (!(m_s[i] == rhs.m_s[i])) + return false; + return true; + } + + inline bool operator<(const vec& rhs) const + { + for (uint32_t i = 0; i < N; i++) + { + if (m_s[i] < rhs.m_s[i]) + return true; + else if (!(m_s[i] == rhs.m_s[i])) + return false; + } + + return false; + } + + inline T operator[](uint32_t i) const + { + assert(i < N); + return m_s[i]; + } + + inline T& operator[](uint32_t i) + { + assert(i < N); + return m_s[i]; + } + + template + inline uint64_t get_component_as_uint() const + { + ASSUME(index < N); + if (sizeof(T) == sizeof(float)) + return *reinterpret_cast(&m_s[index]); + else + return *reinterpret_cast(&m_s[index]); + } + + inline T get_x(void) const + { + return m_s[0]; + } + inline T get_y(void) const + { + ASSUME(N >= 2); + return m_s[1]; + } + inline T get_z(void) const + { + ASSUME(N >= 3); + return m_s[2]; + } + inline T get_w(void) const + { + ASSUME(N >= 4); + return m_s[3]; + } + + inline vec get_x_vector() const + { + return broadcast<0>(); + } + inline vec get_y_vector() const + { + return broadcast<1>(); + } + inline vec get_z_vector() const + { + return broadcast<2>(); + } + inline vec get_w_vector() const + { + return broadcast<3>(); + } + + inline T get_component(uint32_t i) const + { + return (*this)[i]; + } + + inline vec& set_x(T v) + { + m_s[0] = v; + return *this; + } + inline vec& set_y(T v) + { + ASSUME(N >= 2); + m_s[1] = v; + return *this; + } + inline vec& set_z(T v) + { + ASSUME(N >= 3); + m_s[2] = v; + return *this; + } + inline vec& set_w(T v) + { + ASSUME(N >= 4); + m_s[3] = v; + return *this; + } + + inline const T* get_ptr() const + { + return reinterpret_cast(&m_s[0]); + } + inline T* get_ptr() + { + return reinterpret_cast(&m_s[0]); + } + + inline vec as_point() const + { + vec result(*this); + result[N - 1] = 1; + return result; + } + + inline vec as_dir() const + { + vec result(*this); + result[N - 1] = 0; + return result; + } + + inline vec<2, T> select2(uint32_t i, uint32_t j) const + { + assert((i < N) && (j < N)); + return vec<2, T>(m_s[i], m_s[j]); + } + + inline vec<3, T> select3(uint32_t i, uint32_t j, uint32_t k) const + { + assert((i < N) && (j < N) && (k < N)); + return vec<3, T>(m_s[i], m_s[j], m_s[k]); + } + + inline vec<4, T> select4(uint32_t i, uint32_t j, uint32_t k, uint32_t l) const + { + assert((i < N) && (j < N) && (k < N) && (l < N)); + return vec<4, T>(m_s[i], m_s[j], m_s[k], m_s[l]); + } + + inline bool is_dir() const + { + return m_s[N - 1] == 0; + } + inline bool is_vector() const + { + return is_dir(); + } + inline bool is_point() const + { + return m_s[N - 1] == 1; + } + + inline vec project() const + { + vec result(*this); + if (result[N - 1]) + result /= result[N - 1]; + return result; + } + + inline vec broadcast(unsigned i) const + { + return vec((*this)[i]); + } + + template + inline vec broadcast() const + { + return vec((*this)[i]); + } + + inline vec swizzle(uint32_t i, uint32_t j) const + { + return vec((*this)[i], (*this)[j]); + } + + inline vec swizzle(uint32_t i, uint32_t j, uint32_t k) const + { + return vec((*this)[i], (*this)[j], (*this)[k]); + } + + inline vec swizzle(uint32_t i, uint32_t j, uint32_t k, uint32_t l) const + { + return vec((*this)[i], (*this)[j], (*this)[k], (*this)[l]); + } + + inline vec operator-() const + { + vec result; + for (uint32_t i = 0; i < N; i++) + result.m_s[i] = -m_s[i]; + return result; + } + + inline vec operator+() const + { + return *this; + } + + inline vec& operator+=(const vec& other) + { + for (uint32_t i = 0; i < N; i++) + m_s[i] += other.m_s[i]; + return *this; + } + + inline vec& operator-=(const vec& other) + { + for (uint32_t i = 0; i < N; i++) + m_s[i] -= other.m_s[i]; + return *this; + } + + inline vec& operator*=(const vec& other) + { + for (uint32_t i = 0; i < N; i++) + m_s[i] *= other.m_s[i]; + return *this; + } + + inline vec& operator/=(const vec& other) + { + for (uint32_t i = 0; i < N; i++) + m_s[i] /= other.m_s[i]; + return *this; + } + + inline vec& operator*=(T s) + { + for (uint32_t i = 0; i < N; i++) + m_s[i] *= s; + return *this; + } + + inline vec& operator/=(T s) + { + for (uint32_t i = 0; i < N; i++) + m_s[i] /= s; + return *this; + } + + // component-wise multiply (not a dot product like in previous versions) + // just remarking it out because it's too ambiguous, use dot() or mul_components() instead +#if 0 + friend inline vec operator*(const vec& lhs, const vec& rhs) + { + return vec::mul_components(lhs, rhs); + } +#endif + + friend inline vec operator*(const vec& lhs, T val) + { + vec result; + for (uint32_t i = 0; i < N; i++) + result.m_s[i] = lhs.m_s[i] * val; + return result; + } + + friend inline vec operator*(T val, const vec& rhs) + { + vec result; + for (uint32_t i = 0; i < N; i++) + result.m_s[i] = val * rhs.m_s[i]; + return result; + } + + friend inline vec operator/(const vec& lhs, const vec& rhs) + { + vec result; + for (uint32_t i = 0; i < N; i++) + result.m_s[i] = lhs.m_s[i] / rhs.m_s[i]; + return result; + } + + friend inline vec operator/(const vec& lhs, T val) + { + vec result; + for (uint32_t i = 0; i < N; i++) + result.m_s[i] = lhs.m_s[i] / val; + return result; + } + + friend inline vec operator+(const vec& lhs, const vec& rhs) + { + vec result; + for (uint32_t i = 0; i < N; i++) + result.m_s[i] = lhs.m_s[i] + rhs.m_s[i]; + return result; + } + + friend inline vec operator-(const vec& lhs, const vec& rhs) + { + vec result; + for (uint32_t i = 0; i < N; i++) + result.m_s[i] = lhs.m_s[i] - rhs.m_s[i]; + return result; + } + + static inline vec<3, T> cross2(const vec& a, const vec& b) + { + ASSUME(N >= 2); + return vec<3, T>(0, 0, a[0] * b[1] - a[1] * b[0]); + } + + inline vec<3, T> cross2(const vec& b) const + { + return cross2(*this, b); + } + + static inline vec<3, T> cross3(const vec& a, const vec& b) + { + ASSUME(N >= 3); + return vec<3, T>(a[1] * b[2] - a[2] * b[1], a[2] * b[0] - a[0] * b[2], a[0] * b[1] - a[1] * b[0]); + } + + inline vec<3, T> cross3(const vec& b) const + { + return cross3(*this, b); + } + + static inline vec<3, T> cross(const vec& a, const vec& b) + { + ASSUME(N >= 2); + + if (N == 2) + return cross2(a, b); + else + return cross3(a, b); + } + + inline vec<3, T> cross(const vec& b) const + { + ASSUME(N >= 2); + return cross(*this, b); + } + + inline T dot(const vec& rhs) const + { + return dot(*this, rhs); + } + + inline vec dot_vector(const vec& rhs) const + { + return vec(dot(*this, rhs)); + } + + static inline T dot(const vec& lhs, const vec& rhs) + { + T result = lhs.m_s[0] * rhs.m_s[0]; + for (uint32_t i = 1; i < N; i++) + result += lhs.m_s[i] * rhs.m_s[i]; + return result; + } + + inline T dot2(const vec& rhs) const + { + ASSUME(N >= 2); + return m_s[0] * rhs.m_s[0] + m_s[1] * rhs.m_s[1]; + } + + inline T dot3(const vec& rhs) const + { + ASSUME(N >= 3); + return m_s[0] * rhs.m_s[0] + m_s[1] * rhs.m_s[1] + m_s[2] * rhs.m_s[2]; + } + + inline T dot4(const vec& rhs) const + { + ASSUME(N >= 4); + return m_s[0] * rhs.m_s[0] + m_s[1] * rhs.m_s[1] + m_s[2] * rhs.m_s[2] + m_s[3] * rhs.m_s[3]; + } + + inline T norm(void) const + { + T sum = m_s[0] * m_s[0]; + for (uint32_t i = 1; i < N; i++) + sum += m_s[i] * m_s[i]; + return sum; + } + + inline T length(void) const + { + return sqrt(norm()); + } + + inline T squared_distance(const vec& rhs) const + { + T dist2 = 0; + for (uint32_t i = 0; i < N; i++) + { + T d = m_s[i] - rhs.m_s[i]; + dist2 += d * d; + } + return dist2; + } + + inline T squared_distance(const vec& rhs, T early_out) const + { + T dist2 = 0; + for (uint32_t i = 0; i < N; i++) + { + T d = m_s[i] - rhs.m_s[i]; + dist2 += d * d; + if (dist2 > early_out) + break; + } + return dist2; + } + + inline T distance(const vec& rhs) const + { + T dist2 = 0; + for (uint32_t i = 0; i < N; i++) + { + T d = m_s[i] - rhs.m_s[i]; + dist2 += d * d; + } + return sqrt(dist2); + } + + inline vec inverse() const + { + vec result; + for (uint32_t i = 0; i < N; i++) + result[i] = m_s[i] ? (1.0f / m_s[i]) : 0; + return result; + } + + // returns squared length (norm) + inline double normalize(const vec* pDefaultVec = NULL) + { + double n = m_s[0] * m_s[0]; + for (uint32_t i = 1; i < N; i++) + n += m_s[i] * m_s[i]; + + if (n != 0) + *this *= static_cast(1.0f / sqrt(n)); + else if (pDefaultVec) + *this = *pDefaultVec; + return n; + } + + inline double normalize3(const vec* pDefaultVec = NULL) + { + ASSUME(N >= 3); + + double n = m_s[0] * m_s[0] + m_s[1] * m_s[1] + m_s[2] * m_s[2]; + + if (n != 0) + *this *= static_cast((1.0f / sqrt(n))); + else if (pDefaultVec) + *this = *pDefaultVec; + return n; + } + + inline vec& normalize_in_place(const vec* pDefaultVec = NULL) + { + normalize(pDefaultVec); + return *this; + } + + inline vec& normalize3_in_place(const vec* pDefaultVec = NULL) + { + normalize3(pDefaultVec); + return *this; + } + + inline vec get_normalized(const vec* pDefaultVec = NULL) const + { + vec result(*this); + result.normalize(pDefaultVec); + return result; + } + + inline vec get_normalized3(const vec* pDefaultVec = NULL) const + { + vec result(*this); + result.normalize3(pDefaultVec); + return result; + } + + inline vec& clamp(T l, T h) + { + for (uint32_t i = 0; i < N; i++) + m_s[i] = static_cast(clamp(m_s[i], l, h)); + return *this; + } + + inline vec& saturate() + { + return clamp(0.0f, 1.0f); + } + + inline vec& clamp(const vec& l, const vec& h) + { + for (uint32_t i = 0; i < N; i++) + m_s[i] = static_cast(clamp(m_s[i], l[i], h[i])); + return *this; + } + + inline bool is_within_bounds(const vec& l, const vec& h) const + { + for (uint32_t i = 0; i < N; i++) + if ((m_s[i] < l[i]) || (m_s[i] > h[i])) + return false; + + return true; + } + + inline bool is_within_bounds(T l, T h) const + { + for (uint32_t i = 0; i < N; i++) + if ((m_s[i] < l) || (m_s[i] > h)) + return false; + + return true; + } + + inline uint32_t get_major_axis(void) const + { + T m = fabs(m_s[0]); + uint32_t r = 0; + for (uint32_t i = 1; i < N; i++) + { + const T c = fabs(m_s[i]); + if (c > m) + { + m = c; + r = i; + } + } + return r; + } + + inline uint32_t get_minor_axis(void) const + { + T m = fabs(m_s[0]); + uint32_t r = 0; + for (uint32_t i = 1; i < N; i++) + { + const T c = fabs(m_s[i]); + if (c < m) + { + m = c; + r = i; + } + } + return r; + } + + inline void get_projection_axes(uint32_t& u, uint32_t& v) const + { + const int axis = get_major_axis(); + if (m_s[axis] < 0.0f) + { + v = next_wrap(axis, N); + u = next_wrap(v, N); + } + else + { + u = next_wrap(axis, N); + v = next_wrap(u, N); + } + } + + inline T get_absolute_minimum(void) const + { + T result = fabs(m_s[0]); + for (uint32_t i = 1; i < N; i++) + result = std::min(result, fabs(m_s[i])); + return result; + } + + inline T get_absolute_maximum(void) const + { + T result = fabs(m_s[0]); + for (uint32_t i = 1; i < N; i++) + result = std::max(result, fabs(m_s[i])); + return result; + } + + inline T get_minimum(void) const + { + T result = m_s[0]; + for (uint32_t i = 1; i < N; i++) + result = std::min(result, m_s[i]); + return result; + } + + inline T get_maximum(void) const + { + T result = m_s[0]; + for (uint32_t i = 1; i < N; i++) + result = std::max(result, m_s[i]); + return result; + } + + inline vec& remove_unit_direction(const vec& dir) + { + *this -= (dot(dir) * dir); + return *this; + } + + inline vec get_remove_unit_direction(const vec& dir) const + { + return *this - (dot(dir) * dir); + } + + inline bool all_less(const vec& b) const + { + for (uint32_t i = 0; i < N; i++) + if (m_s[i] >= b.m_s[i]) + return false; + return true; + } + + inline bool all_less_equal(const vec& b) const + { + for (uint32_t i = 0; i < N; i++) + if (m_s[i] > b.m_s[i]) + return false; + return true; + } + + inline bool all_greater(const vec& b) const + { + for (uint32_t i = 0; i < N; i++) + if (m_s[i] <= b.m_s[i]) + return false; + return true; + } + + inline bool all_greater_equal(const vec& b) const + { + for (uint32_t i = 0; i < N; i++) + if (m_s[i] < b.m_s[i]) + return false; + return true; + } + + inline vec negate_xyz() const + { + vec ret; + + ret[0] = -m_s[0]; + if (N >= 2) + ret[1] = -m_s[1]; + if (N >= 3) + ret[2] = -m_s[2]; + + for (uint32_t i = 3; i < N; i++) + ret[i] = m_s[i]; + + return ret; + } + + inline vec& invert() + { + for (uint32_t i = 0; i < N; i++) + if (m_s[i] != 0.0f) + m_s[i] = 1.0f / m_s[i]; + return *this; + } + + inline scalar_type perp_dot(const vec& b) const + { + ASSUME(N == 2); + return m_s[0] * b.m_s[1] - m_s[1] * b.m_s[0]; + } + + inline vec perp() const + { + ASSUME(N == 2); + return vec(-m_s[1], m_s[0]); + } + + inline vec get_floor() const + { + vec result; + for (uint32_t i = 0; i < N; i++) + result[i] = floor(m_s[i]); + return result; + } + + inline vec get_ceil() const + { + vec result; + for (uint32_t i = 0; i < N; i++) + result[i] = ceil(m_s[i]); + return result; + } + + // static helper methods + + static inline vec mul_components(const vec& lhs, const vec& rhs) + { + vec result; + for (uint32_t i = 0; i < N; i++) + result[i] = lhs.m_s[i] * rhs.m_s[i]; + return result; + } + + static inline vec mul_add_components(const vec& a, const vec& b, const vec& c) + { + vec result; + for (uint32_t i = 0; i < N; i++) + result[i] = a.m_s[i] * b.m_s[i] + c.m_s[i]; + return result; + } + + static inline vec make_axis(uint32_t i) + { + vec result; + result.clear(); + result[i] = 1; + return result; + } + + static inline vec equals_mask(const vec& a, const vec& b) + { + vec ret; + for (uint32_t i = 0; i < N; i++) + ret[i] = (a[i] == b[i]); + return ret; + } + + static inline vec not_equals_mask(const vec& a, const vec& b) + { + vec ret; + for (uint32_t i = 0; i < N; i++) + ret[i] = (a[i] != b[i]); + return ret; + } + + static inline vec less_mask(const vec& a, const vec& b) + { + vec ret; + for (uint32_t i = 0; i < N; i++) + ret[i] = (a[i] < b[i]); + return ret; + } + + static inline vec less_equals_mask(const vec& a, const vec& b) + { + vec ret; + for (uint32_t i = 0; i < N; i++) + ret[i] = (a[i] <= b[i]); + return ret; + } + + static inline vec greater_equals_mask(const vec& a, const vec& b) + { + vec ret; + for (uint32_t i = 0; i < N; i++) + ret[i] = (a[i] >= b[i]); + return ret; + } + + static inline vec greater_mask(const vec& a, const vec& b) + { + vec ret; + for (uint32_t i = 0; i < N; i++) + ret[i] = (a[i] > b[i]); + return ret; + } + + static inline vec component_max(const vec& a, const vec& b) + { + vec ret; + for (uint32_t i = 0; i < N; i++) + ret.m_s[i] = std::max(a.m_s[i], b.m_s[i]); + return ret; + } + + static inline vec component_min(const vec& a, const vec& b) + { + vec ret; + for (uint32_t i = 0; i < N; i++) + ret.m_s[i] = std::min(a.m_s[i], b.m_s[i]); + return ret; + } + + static inline vec lerp(const vec& a, const vec& b, float t) + { + vec ret; + for (uint32_t i = 0; i < N; i++) + ret.m_s[i] = a.m_s[i] + (b.m_s[i] - a.m_s[i]) * t; + return ret; + } + + static inline bool equal_tol(const vec& a, const vec& b, float t) + { + for (uint32_t i = 0; i < N; i++) + if (!equal_tol(a.m_s[i], b.m_s[i], t)) + return false; + return true; + } + + inline bool equal_tol(const vec& b, float t) const + { + return equal_tol(*this, b, t); + } + +protected: + T m_s[N]; +}; + +typedef vec<1, double> vec1D; +typedef vec<2, double> vec2D; +typedef vec<3, double> vec3D; +typedef vec<4, double> vec4D; + +typedef vec<1, float> vec1F; + +typedef vec<2, float> vec2F; +typedef std::vector vec2F_array; + +typedef vec<3, float> vec3F; +typedef std::vector vec3F_array; + +typedef vec<4, float> vec4F; +typedef std::vector vec4F_array; + +typedef vec<2, uint32_t> vec2U; +typedef vec<3, uint32_t> vec3U; +typedef vec<2, int> vec2I; +typedef vec<3, int> vec3I; +typedef vec<4, int> vec4I; + +typedef vec<2, int16_t> vec2I16; +typedef vec<3, int16_t> vec3I16; + +inline vec2F rotate_point(const vec2F& p, float rad) +{ + float c = cos(rad); + float s = sin(rad); + + float x = p[0]; + float y = p[1]; + + return vec2F(x * c - y * s, x * s + y * c); +} + +class rect +{ +public: + inline rect() + { + } + + inline rect(eClear) + { + clear(); + } + + inline rect(eInitExpand) + { + init_expand(); + } + + // up to, but not including right/bottom + inline rect(int left, int top, int right, int bottom) + { + set(left, top, right, bottom); + } + + inline rect(const vec2I& lo, const vec2I& hi) + { + m_corner[0] = lo; + m_corner[1] = hi; + } + + inline rect(const vec2I& point) + { + m_corner[0] = point; + m_corner[1].set(point[0] + 1, point[1] + 1); + } + + inline bool operator==(const rect& r) const + { + return (m_corner[0] == r.m_corner[0]) && (m_corner[1] == r.m_corner[1]); + } + + inline bool operator<(const rect& r) const + { + for (uint32_t i = 0; i < 2; i++) + { + if (m_corner[i] < r.m_corner[i]) + return true; + else if (!(m_corner[i] == r.m_corner[i])) + return false; + } + + return false; + } + + inline void clear() + { + m_corner[0].clear(); + m_corner[1].clear(); + } + + inline void set(int left, int top, int right, int bottom) + { + m_corner[0].set(left, top); + m_corner[1].set(right, bottom); + } + + inline void set(const vec2I& lo, const vec2I& hi) + { + m_corner[0] = lo; + m_corner[1] = hi; + } + + inline void set(const vec2I& point) + { + m_corner[0] = point; + m_corner[1].set(point[0] + 1, point[1] + 1); + } + + inline uint32_t get_width() const + { + return m_corner[1][0] - m_corner[0][0]; + } + inline uint32_t get_height() const + { + return m_corner[1][1] - m_corner[0][1]; + } + + inline int get_left() const + { + return m_corner[0][0]; + } + inline int get_top() const + { + return m_corner[0][1]; + } + inline int get_right() const + { + return m_corner[1][0]; + } + inline int get_bottom() const + { + return m_corner[1][1]; + } + + inline bool is_empty() const + { + return (m_corner[1][0] <= m_corner[0][0]) || (m_corner[1][1] <= m_corner[0][1]); + } + + inline uint32_t get_dimension(uint32_t axis) const + { + return m_corner[1][axis] - m_corner[0][axis]; + } + inline uint32_t get_area() const + { + return get_dimension(0) * get_dimension(1); + } + + inline const vec2I& operator[](uint32_t i) const + { + assert(i < 2); + return m_corner[i]; + } + inline vec2I& operator[](uint32_t i) + { + assert(i < 2); + return m_corner[i]; + } + + inline rect& translate(int x_ofs, int y_ofs) + { + m_corner[0][0] += x_ofs; + m_corner[0][1] += y_ofs; + m_corner[1][0] += x_ofs; + m_corner[1][1] += y_ofs; + return *this; + } + + inline rect& init_expand() + { + m_corner[0].set(INT_MAX); + m_corner[1].set(INT_MIN); + return *this; + } + + inline rect& expand(int x, int y) + { + m_corner[0][0] = std::min(m_corner[0][0], x); + m_corner[0][1] = std::min(m_corner[0][1], y); + m_corner[1][0] = std::max(m_corner[1][0], x + 1); + m_corner[1][1] = std::max(m_corner[1][1], y + 1); + return *this; + } + + inline rect& expand(const rect& r) + { + m_corner[0][0] = std::min(m_corner[0][0], r[0][0]); + m_corner[0][1] = std::min(m_corner[0][1], r[0][1]); + m_corner[1][0] = std::max(m_corner[1][0], r[1][0]); + m_corner[1][1] = std::max(m_corner[1][1], r[1][1]); + return *this; + } + + inline bool touches(const rect& r) const + { + for (uint32_t i = 0; i < 2; i++) + { + if (r[1][i] <= m_corner[0][i]) + return false; + else if (r[0][i] >= m_corner[1][i]) + return false; + } + + return true; + } + + inline bool fully_within(const rect& r) const + { + for (uint32_t i = 0; i < 2; i++) + { + if (m_corner[0][i] < r[0][i]) + return false; + else if (m_corner[1][i] > r[1][i]) + return false; + } + + return true; + } + + inline bool intersect(const rect& r) + { + if (!touches(r)) + { + clear(); + return false; + } + + for (uint32_t i = 0; i < 2; i++) + { + m_corner[0][i] = std::max(m_corner[0][i], r[0][i]); + m_corner[1][i] = std::min(m_corner[1][i], r[1][i]); + } + + return true; + } + + inline bool contains(int x, int y) const + { + return (x >= m_corner[0][0]) && (x < m_corner[1][0]) && + (y >= m_corner[0][1]) && (y < m_corner[1][1]); + } + + inline bool contains(const vec2I& p) const + { + return contains(p[0], p[1]); + } + +private: + vec2I m_corner[2]; +}; + +inline rect make_rect(uint32_t width, uint32_t height) +{ + return rect(0, 0, width, height); +} + +struct color_quad_u8 +{ +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable:4201) +#endif + union + { + uint8_t m_c[4]; + struct + { + uint8_t r; + uint8_t g; + uint8_t b; + uint8_t a; + }; + }; +#ifdef _MSC_VER +#pragma warning(pop) +#endif + + inline color_quad_u8(eClear) : color_quad_u8(0, 0, 0, 0) { } + + inline color_quad_u8(uint8_t cr, uint8_t cg, uint8_t cb, uint8_t ca) + { + set(cr, cg, cb, ca); + } + + inline color_quad_u8(uint8_t cy = 0, uint8_t ca = 255) + { + set(cy, ca); + } + + inline void clear() + { + set(0, 0, 0, 0); + } + + inline color_quad_u8& set(uint8_t cy, uint8_t ca = 255) + { + m_c[0] = cy; + m_c[1] = cy; + m_c[2] = cy; + m_c[3] = ca; + return *this; + } + + inline color_quad_u8& set(uint8_t cr, uint8_t cg, uint8_t cb, uint8_t ca) + { + m_c[0] = cr; + m_c[1] = cg; + m_c[2] = cb; + m_c[3] = ca; + return *this; + } + + inline color_quad_u8& set_clamped(int cr, int cg, int cb, int ca) + { + m_c[0] = (uint8_t)clamp(cr, 0, 255); + m_c[1] = (uint8_t)clamp(cg, 0, 255); + m_c[2] = (uint8_t)clamp(cb, 0, 255); + m_c[3] = (uint8_t)clamp(ca, 0, 255); + return *this; + } + + color_quad_u8& set_alpha(int ca) { a = (uint8_t)clamp(ca, 0, 255); return *this; } + + inline uint8_t& operator[] (uint32_t i) { assert(i < 4); return m_c[i]; } + inline uint8_t operator[] (uint32_t i) const { assert(i < 4); return m_c[i]; } + + inline int get_luma() const { return (13938U * m_c[0] + 46869U * m_c[1] + 4729U * m_c[2] + 32768U) >> 16U; } // REC709 weightings + + inline bool operator== (const color_quad_u8& other) const + { + return (m_c[0] == other.m_c[0]) && (m_c[1] == other.m_c[1]) && (m_c[2] == other.m_c[2]) && (m_c[3] == other.m_c[3]); + } + + inline bool operator!= (const color_quad_u8& other) const + { + return !(*this == other); + } + + inline uint32_t squared_distance(const color_quad_u8& c, bool alpha = true) const + { + return square(r - c.r) + square(g - c.g) + square(b - c.b) + (alpha ? square(a - c.a) : 0); + } + + inline bool rgb_equals(const color_quad_u8& rhs) const + { + return (r == rhs.r) && (g == rhs.g) && (b == rhs.b); + } +}; +typedef std::vector color_quad_u8_vec; + +inline uint32_t color_distance(bool perceptual, const color_quad_u8& e1, const color_quad_u8& e2, bool alpha) +{ + if (perceptual) + { + const float l1 = e1.r * .2126f + e1.g * .715f + e1.b * .0722f; + const float cr1 = e1.r - l1; + const float cb1 = e1.b - l1; + + const float l2 = e2.r * .2126f + e2.g * .715f + e2.b * .0722f; + const float cr2 = e2.r - l2; + const float cb2 = e2.b - l2; + + const float dl = l1 - l2; + const float dcr = cr1 - cr2; + const float dcb = cb1 - cb2; + + uint32_t d = static_cast( + 32.0f * 4.0f * dl * dl + + 32.0f * 2.0f * (.5f / (1.0f - .2126f)) * (.5f / (1.0f - .2126f)) * dcr * dcr + + 32.0f * .25f * (.5f / (1.0f - .0722f)) * (.5f / (1.0f - .0722f)) * dcb * dcb); + + if (alpha) + { + int da = (int)e1.a - (int)e2.a; + + d += static_cast(128.0f * da * da); + } + + return d; + } + else + return e1.squared_distance(e2, alpha); +} + +extern color_quad_u8 g_white_color_u8, g_black_color_u8, g_red_color_u8, g_green_color_u8, g_blue_color_u8, g_yellow_color_u8, g_purple_color_u8, g_magenta_color_u8, g_cyan_color_u8; + +class image_u8 +{ +public: + image_u8() : + m_width(0), m_height(0), + m_clip_rect(cClear) + { + } + + image_u8(uint32_t width, uint32_t height) : + m_width(width), m_height(height), + m_clip_rect(0, 0, width, height) + { + m_pixels.resize(width * height); + } + + inline const color_quad_u8_vec& get_pixels() const { return m_pixels; } + inline color_quad_u8_vec& get_pixels() { return m_pixels; } + + inline uint32_t width() const { return m_width; } + inline uint32_t height() const { return m_height; } + inline uint32_t total_pixels() const { return m_width * m_height; } + + inline const rect& get_clip_rect() const { return m_clip_rect; } + + inline void set_clip_rect(const rect& r) + { + assert((r.get_left() >= 0) && (r.get_top() >= 0) && (r.get_right() <= (int)m_width) && (r.get_bottom() <= (int)m_height)); + + m_clip_rect = r; + } + + inline void clear_clip_rect() { m_clip_rect.set(0, 0, m_width, m_height); } + + inline bool is_clipped(int x, int y) const { return !m_clip_rect.contains(x, y); } + + inline rect get_bounds() const { return rect(0, 0, m_width, m_height); } + + inline color_quad_u8& operator()(uint32_t x, uint32_t y) { assert((x < m_width) && (y < m_height)); return m_pixels[x + m_width * y]; } + inline const color_quad_u8& operator()(uint32_t x, uint32_t y) const { assert((x < m_width) && (y < m_height)); return m_pixels[x + m_width * y]; } + + image_u8& clear() + { + m_width = m_height = 0; + m_clip_rect.clear(); + m_pixels.clear(); + return *this; + } + + image_u8& init(uint32_t width, uint32_t height) + { + clear(); + + m_width = width; + m_height = height; + m_clip_rect.set(0, 0, width, height); + m_pixels.resize(width * height); + return *this; + } + + image_u8& set_all(const color_quad_u8& p) + { + for (uint32_t i = 0; i < m_pixels.size(); i++) + m_pixels[i] = p; + return *this; + } + + inline const color_quad_u8& get_clamped(int x, int y) const { return (*this)(clamp(x, 0, m_width - 1), clamp(y, 0, m_height - 1)); } + inline color_quad_u8& get_clamped(int x, int y) { return (*this)(clamp(x, 0, m_width - 1), clamp(y, 0, m_height - 1)); } + + inline image_u8& set_pixel_clipped(int x, int y, const color_quad_u8& c) + { + if (!is_clipped(x, y)) + (*this)(x, y) = c; + return *this; + } + + inline image_u8& fill_box(int x, int y, int w, int h, const color_quad_u8& c) + { + for (int y_ofs = 0; y_ofs < h; y_ofs++) + for (int x_ofs = 0; x_ofs < w; x_ofs++) + set_pixel_clipped(x + x_ofs, y + y_ofs, c); + return *this; + } + + void invert_box(int inX, int inY, int inW, int inH) + { + for (int y = 0; y < inH; y++) + { + const uint32_t yy = inY + y; + + for (int x = 0; x < inW; x++) + { + const uint32_t xx = inX + x; + + if (is_clipped(xx, yy)) + continue; + + color_quad_u8 c((*this)(xx, yy)); + + c.r = 255 - c.r; + c.g = 255 - c.g; + c.b = 255 - c.b; + + set_pixel_clipped(xx, yy, c); + } + } + } + + image_u8& crop_dup_borders(uint32_t w, uint32_t h) + { + const uint32_t orig_w = m_width, orig_h = m_height; + + crop(w, h); + + if (orig_w && orig_h) + { + if (m_width > orig_w) + { + for (uint32_t x = orig_w; x < m_width; x++) + for (uint32_t y = 0; y < m_height; y++) + set_pixel_clipped(x, y, get_clamped(std::min(x, orig_w - 1U), std::min(y, orig_h - 1U))); + } + + if (m_height > orig_h) + { + for (uint32_t y = orig_h; y < m_height; y++) + for (uint32_t x = 0; x < m_width; x++) + set_pixel_clipped(x, y, get_clamped(std::min(x, orig_w - 1U), std::min(y, orig_h - 1U))); + } + } + return *this; + } + + image_u8& crop(uint32_t new_width, uint32_t new_height) + { + if ((m_width == new_width) && (m_height == new_height)) + return *this; + + image_u8 new_image(new_width, new_height); + + const uint32_t w = std::min(m_width, new_width); + const uint32_t h = std::min(m_height, new_height); + + for (uint32_t y = 0; y < h; y++) + for (uint32_t x = 0; x < w; x++) + new_image(x, y) = (*this)(x, y); + + return swap(new_image); + } + + image_u8& swap(image_u8& other) + { + std::swap(m_width, other.m_width); + std::swap(m_height, other.m_height); + std::swap(m_pixels, other.m_pixels); + std::swap(m_clip_rect, other.m_clip_rect); + return *this; + } + + // No clipping + inline void get_block(uint32_t bx, uint32_t by, uint32_t width, uint32_t height, color_quad_u8* pPixels) const + { + assert((bx * width + width) <= m_width); + assert((by * height + height) <= m_height); + + for (uint32_t y = 0; y < height; y++) + memcpy(pPixels + y * width, &(*this)(bx * width, by * height + y), width * sizeof(color_quad_u8)); + } + + inline void get_block_clamped(uint32_t bx, uint32_t by, uint32_t width, uint32_t height, color_quad_u8* pPixels) const + { + for (uint32_t y = 0; y < height; y++) + for (uint32_t x = 0; x < width; x++) + pPixels[x + y * width] = get_clamped(bx * width + x, by * height + y); + } + + // No clipping + inline void set_block(uint32_t bx, uint32_t by, uint32_t width, uint32_t height, const color_quad_u8* pPixels) + { + assert((bx * width + width) <= m_width); + assert((by * height + height) <= m_height); + + for (uint32_t y = 0; y < height; y++) + memcpy(&(*this)(bx * width, by * height + y), pPixels + y * width, width * sizeof(color_quad_u8)); + } + + image_u8& swizzle(uint32_t r, uint32_t g, uint32_t b, uint32_t a) + { + assert((r | g | b | a) <= 3); + for (uint32_t y = 0; y < m_height; y++) + { + for (uint32_t x = 0; x < m_width; x++) + { + color_quad_u8 tmp((*this)(x, y)); + (*this)(x, y).set(tmp[r], tmp[g], tmp[b], tmp[a]); + } + } + + return *this; + } + + struct pixel_coord + { + uint16_t m_x, m_y; + pixel_coord() { } + pixel_coord(uint32_t x, uint32_t y) : m_x((uint16_t)x), m_y((uint16_t)y) { } + }; + + uint32_t flood_fill(int x, int y, const color_quad_u8& c, const color_quad_u8& b, std::vector* pSet_pixels = nullptr); + + void draw_line(int xs, int ys, int xe, int ye, const color_quad_u8& color); + + inline void set_pixel_clipped_alphablend(int x, int y, const color_quad_u8& c) + { + if (is_clipped(x, y)) + return; + + color_quad_u8 ct(m_pixels[x + y * m_width]); + + ct.r = static_cast(ct.r + ((c.r - ct.r) * c.a) / 255); + ct.g = static_cast(ct.g + ((c.g - ct.g) * c.a) / 255); + ct.b = static_cast(ct.b + ((c.b - ct.b) * c.a) / 255); + + m_pixels[x + y * m_width] = ct; + } + +private: + color_quad_u8_vec m_pixels; + uint32_t m_width, m_height; + rect m_clip_rect; + + struct fill_segment + { + int16_t m_y, m_xl, m_xr, m_dy; + + fill_segment(int y, int xl, int xr, int dy) : + m_y((int16_t)y), m_xl((int16_t)xl), m_xr((int16_t)xr), m_dy((int16_t)dy) + { + } + }; + + inline bool flood_fill_is_inside(int x, int y, const color_quad_u8& b) const + { + if (is_clipped(x, y)) + return false; + + return (*this)(x, y) == b; + } + + void rasterize_line(int xs, int ys, int xe, int ye, int pred, int inc_dec, int e, int e_inc, int e_no_inc, const color_quad_u8& color); + + void draw_aaline_pixel(int x, int y, int a, color_quad_u8 color) + { + color.a = static_cast(255 - a); + set_pixel_clipped_alphablend(x, y, color); + } +}; + +//bool load_png(const char* pFilename, image_u8& img); + +//bool save_png(const char* pFilename, const image_u8& img, bool save_alpha); + +class image_metrics +{ +public: + double m_max, m_mean, m_mean_squared, m_root_mean_squared, m_peak_snr; + + image_metrics() + { + clear(); + } + + void clear() + { + memset(this, 0, sizeof(*this)); + } + + void compute(const image_u8& a, const image_u8& b, uint32_t first_channel, uint32_t num_channels) + { + const bool average_component_error = true; + + const uint32_t width = std::min(a.width(), b.width()); + const uint32_t height = std::min(a.height(), b.height()); + + assert((first_channel < 4U) && (first_channel + num_channels <= 4U)); + + // Histogram approach originally due to Charles Bloom. + double hist[256]; + memset(hist, 0, sizeof(hist)); + + for (uint32_t y = 0; y < height; y++) + { + for (uint32_t x = 0; x < width; x++) + { + const color_quad_u8& ca = a(x, y); + const color_quad_u8& cb = b(x, y); + + if (!num_channels) + hist[iabs(ca.get_luma() - cb.get_luma())]++; + else + { + for (uint32_t c = 0; c < num_channels; c++) + hist[iabs(ca[first_channel + c] - cb[first_channel + c])]++; + } + } + } + + m_max = 0; + double sum = 0.0f, sum2 = 0.0f; + for (uint32_t i = 0; i < 256; i++) + { + if (!hist[i]) + continue; + + m_max = std::max(m_max, i); + + double x = i * hist[i]; + + sum += x; + sum2 += i * x; + } + + // See http://richg42.blogspot.com/2016/09/how-to-compute-psnr-from-old-berkeley.html + double total_values = width * height; + + if (average_component_error) + total_values *= clamp(num_channels, 1, 4); + + m_mean = clamp(sum / total_values, 0.0f, 255.0f); + m_mean_squared = clamp(sum2 / total_values, 0.0f, 255.0f * 255.0f); + + m_root_mean_squared = sqrt(m_mean_squared); + + if (!m_root_mean_squared) + m_peak_snr = 100.0f; + else + m_peak_snr = clamp(log10(255.0f / m_root_mean_squared) * 20.0f, 0.0f, 100.0f); + } +}; + +class imagef +{ +public: + imagef() : + m_width(0), m_height(0), m_pitch(0) + { + } + + imagef(uint32_t w, uint32_t h, uint32_t p = UINT32_MAX) : + m_width(0), m_height(0), m_pitch(0) + { + resize(w, h, p); + } + + imagef(const imagef& other) : + m_width(0), m_height(0), m_pitch(0) + { + *this = other; + } + + imagef& swap(imagef& other) + { + std::swap(m_width, other.m_width); + std::swap(m_height, other.m_height); + std::swap(m_pitch, other.m_pitch); + m_pixels.swap(other.m_pixels); + return *this; + } + + imagef& operator= (const imagef& rhs) + { + if (this != &rhs) + { + m_width = rhs.m_width; + m_height = rhs.m_height; + m_pitch = rhs.m_pitch; + m_pixels = rhs.m_pixels; + } + return *this; + } + + imagef& clear() + { + m_width = 0; + m_height = 0; + m_pitch = 0; + m_pixels.resize(0); + return *this; + } + + imagef& set(const image_u8& src, const vec4F& scale = vec4F(1), const vec4F& bias = vec4F(0)) + { + const uint32_t width = src.width(); + const uint32_t height = src.height(); + + resize(width, height); + + for (int y = 0; y < (int)height; y++) + { + for (uint32_t x = 0; x < width; x++) + { + const color_quad_u8& src_pixel = src(x, y); + (*this)(x, y).set((float)src_pixel.r * scale[0] + bias[0], (float)src_pixel.g * scale[1] + bias[1], (float)src_pixel.b * scale[2] + bias[2], (float)src_pixel.a * scale[3] + bias[3]); + } + } + + return *this; + } + + imagef& resize(const imagef& other, uint32_t p = UINT32_MAX, const vec4F& background = vec4F(0, 0, 0, 1)) + { + return resize(other.get_width(), other.get_height(), p, background); + } + + imagef& resize(uint32_t w, uint32_t h, uint32_t p = UINT32_MAX, const vec4F& background = vec4F(0, 0, 0, 1)) + { + return crop(w, h, p, background); + } + + imagef& set_all(const vec4F& c) + { + for (uint32_t i = 0; i < m_pixels.size(); i++) + m_pixels[i] = c; + return *this; + } + + imagef& fill_box(uint32_t x, uint32_t y, uint32_t w, uint32_t h, const vec4F& c) + { + for (uint32_t iy = 0; iy < h; iy++) + for (uint32_t ix = 0; ix < w; ix++) + set_pixel_clipped(x + ix, y + iy, c); + return *this; + } + + imagef& crop(uint32_t w, uint32_t h, uint32_t p = UINT32_MAX, const vec4F& background = vec4F(0, 0, 0, 1)) + { + if (p == UINT32_MAX) + p = w; + + if ((w == m_width) && (m_height == h) && (m_pitch == p)) + return *this; + + if ((!w) || (!h) || (!p)) + { + clear(); + return *this; + } + + vec4F_array cur_state; + cur_state.swap(m_pixels); + + m_pixels.resize(p * h); + + for (uint32_t y = 0; y < h; y++) + { + for (uint32_t x = 0; x < w; x++) + { + if ((x < m_width) && (y < m_height)) + m_pixels[x + y * p] = cur_state[x + y * m_pitch]; + else + m_pixels[x + y * p] = background; + } + } + + m_width = w; + m_height = h; + m_pitch = p; + + return *this; + } + + inline const vec4F& operator() (uint32_t x, uint32_t y) const { assert(x < m_width&& y < m_height); return m_pixels[x + y * m_pitch]; } + inline vec4F& operator() (uint32_t x, uint32_t y) { assert(x < m_width&& y < m_height); return m_pixels[x + y * m_pitch]; } + + inline const vec4F& get_clamped(int x, int y) const { return (*this)(clamp(x, 0, m_width - 1), clamp(y, 0, m_height - 1)); } + inline vec4F& get_clamped(int x, int y) { return (*this)(clamp(x, 0, m_width - 1), clamp(y, 0, m_height - 1)); } + + inline const vec4F& get_clamped_or_wrapped(int x, int y, bool wrap_u, bool wrap_v) const + { + x = wrap_u ? posmod(x, m_width) : clamp(x, 0, m_width - 1); + y = wrap_v ? posmod(y, m_height) : clamp(y, 0, m_height - 1); + return m_pixels[x + y * m_pitch]; + } + + inline vec4F& get_clamped_or_wrapped(int x, int y, bool wrap_u, bool wrap_v) + { + x = wrap_u ? posmod(x, m_width) : clamp(x, 0, m_width - 1); + y = wrap_v ? posmod(y, m_height) : clamp(y, 0, m_height - 1); + return m_pixels[x + y * m_pitch]; + } + + inline imagef& set_pixel_clipped(int x, int y, const vec4F& c) + { + if ((static_cast(x) < m_width) && (static_cast(y) < m_height)) + (*this)(x, y) = c; + return *this; + } + + // Very straightforward blit with full clipping. Not fast, but it works. + imagef& blit(const imagef& src, int src_x, int src_y, int src_w, int src_h, int dst_x, int dst_y) + { + for (int y = 0; y < src_h; y++) + { + const int sy = src_y + y; + if (sy < 0) + continue; + else if (sy >= (int)src.get_height()) + break; + + for (int x = 0; x < src_w; x++) + { + const int sx = src_x + x; + if (sx < 0) + continue; + else if (sx >= (int)src.get_height()) + break; + + set_pixel_clipped(dst_x + x, dst_y + y, src(sx, sy)); + } + } + + return *this; + } + + const imagef& extract_block_clamped(vec4F* pDst, uint32_t src_x, uint32_t src_y, uint32_t w, uint32_t h) const + { + for (uint32_t y = 0; y < h; y++) + for (uint32_t x = 0; x < w; x++) + *pDst++ = get_clamped(src_x + x, src_y + y); + return *this; + } + + imagef& set_block_clipped(const vec4F* pSrc, uint32_t dst_x, uint32_t dst_y, uint32_t w, uint32_t h) + { + for (uint32_t y = 0; y < h; y++) + for (uint32_t x = 0; x < w; x++) + set_pixel_clipped(dst_x + x, dst_y + y, *pSrc++); + return *this; + } + + inline uint32_t get_width() const { return m_width; } + inline uint32_t get_height() const { return m_height; } + inline uint32_t get_pitch() const { return m_pitch; } + inline uint32_t get_total_pixels() const { return m_width * m_height; } + + inline uint32_t get_block_width(uint32_t w) const { return (m_width + (w - 1)) / w; } + inline uint32_t get_block_height(uint32_t h) const { return (m_height + (h - 1)) / h; } + inline uint32_t get_total_blocks(uint32_t w, uint32_t h) const { return get_block_width(w) * get_block_height(h); } + + inline const vec4F_array& get_pixels() const { return m_pixels; } + inline vec4F_array& get_pixels() { return m_pixels; } + + inline const vec4F* get_ptr() const { return &m_pixels[0]; } + inline vec4F* get_ptr() { return &m_pixels[0]; } + +private: + uint32_t m_width, m_height, m_pitch; // all in pixels + vec4F_array m_pixels; +}; + +enum +{ + cComputeGaussianFlagNormalize = 1, + cComputeGaussianFlagPrint = 2, + cComputeGaussianFlagNormalizeCenterToOne = 4 +}; + +// size_x/y should be odd +void compute_gaussian_kernel(float* pDst, int size_x, int size_y, float sigma_sqr, uint32_t flags); + +void gaussian_filter(imagef& dst, const imagef& orig_img, uint32_t odd_filter_width, float sigma_sqr, bool wrapping = false, uint32_t width_divisor = 1, uint32_t height_divisor = 1); + +vec4F compute_ssim(const imagef& a, const imagef& b); + +vec4F compute_ssim(const image_u8& a, const image_u8& b, bool luma); + +struct block8 +{ + uint64_t m_vals[1]; +}; + +typedef std::vector block8_vec; + +struct block16 +{ + uint64_t m_vals[2]; +}; + +typedef std::vector block16_vec; + +bool save_dds(const char* pFilename, uint32_t width, uint32_t height, const void* pBlocks, uint32_t pixel_format_bpp, DXGI_FORMAT dxgi_format, bool srgb, bool force_dx10_header); + +void strip_extension(std::string& s); +void strip_path(std::string& s); + +uint32_t hash_hsieh(const uint8_t* pBuf, size_t len); + +// https://www.johndcook.com/blog/standard_deviation/ +// This class is for small numbers of integers, so precision shouldn't be an issue. +class tracked_stat +{ +public: + tracked_stat() { clear(); } + + void clear() { m_num = 0; m_total = 0; m_total2 = 0; } + + void update(uint32_t val) { m_num++; m_total += val; m_total2 += val * val; } + + tracked_stat& operator += (uint32_t val) { update(val); return *this; } + + uint32_t get_number_of_values() const { return m_num; } + uint64_t get_total() const { return m_total; } + uint64_t get_total2() const { return m_total2; } + + float get_mean() const { return m_num ? (float)m_total / m_num : 0.0f; }; + + float get_variance() const { return m_num ? ((float)(m_num * m_total2 - m_total * m_total)) / (m_num * m_num) : 0.0f; } + float get_std_dev() const { return m_num ? sqrtf((float)(m_num * m_total2 - m_total * m_total)) / m_num : 0.0f; } + + float get_sample_variance() const { return (m_num > 1) ? ((float)(m_num * m_total2 - m_total * m_total)) / (m_num * (m_num - 1)) : 0.0f; } + float get_sample_std_dev() const { return (m_num > 1) ? sqrtf(get_sample_variance()) : 0.0f; } + +private: + uint32_t m_num; + uint64_t m_total; + uint64_t m_total2; +}; + +inline float compute_covariance(const float* pA, const float* pB, const tracked_stat& a, const tracked_stat& b, bool sample) +{ + const uint32_t n = a.get_number_of_values(); + assert(n == b.get_number_of_values()); + + if (!n) + { + assert(0); + return 0.0f; + } + if ((sample) && (n == 1)) + { + assert(0); + return 0; + } + + const float mean_a = a.get_mean(); + const float mean_b = b.get_mean(); + + float total = 0.0f; + for (uint32_t i = 0; i < n; i++) + total += (pA[i] - mean_a) * (pB[i] - mean_b); + + return total / (sample ? (n - 1) : n); +} + +inline float compute_correlation_coefficient(const float* pA, const float* pB, const tracked_stat& a, const tracked_stat& b, float c, bool sample) +{ + if (!a.get_number_of_values()) + return 1.0f; + + float covar = compute_covariance(pA, pB, a, b, sample); + float std_dev_a = sample ? a.get_sample_std_dev() : a.get_std_dev(); + float std_dev_b = sample ? b.get_sample_std_dev() : b.get_std_dev(); + float denom = std_dev_a * std_dev_b + c; + + if (denom < .0000125f) + return 1.0f; + + float result = (covar + c) / denom; + + return clamp(result, -1.0f, 1.0f); +} + +float compute_block_max_std_dev(const color_quad_u8* pPixels, uint32_t block_width, uint32_t block_height, uint32_t num_comps); + +class rand +{ + std::mt19937 m_mt; + +public: + rand() { } + + rand(uint32_t s) { seed(s); } + void seed(uint32_t s) { m_mt.seed(s); } + + // between [l,h] + int irand(int l, int h) { std::uniform_int_distribution d(l, h); return d(m_mt); } + + uint32_t urand32() { return static_cast(irand(INT32_MIN, INT32_MAX)); } + + bool bit() { return irand(0, 1) == 1; } + + uint8_t byte() { return static_cast(urand32()); } + + // between [l,h) + float frand(float l, float h) { std::uniform_real_distribution d(l, h); return d(m_mt); } + + float gaussian(float mean, float stddev) { std::normal_distribution d(mean, stddev); return d(m_mt); } +}; + +bool save_astc_file(const char* pFilename, block16_vec& blocks, uint32_t width, uint32_t height, uint32_t block_width, uint32_t block_height); +bool load_astc_file(const char* pFilename, block16_vec& blocks, uint32_t& width, uint32_t& height, uint32_t& block_width, uint32_t& block_height); + +class value_stats +{ +public: + value_stats() + { + clear(); + } + + void clear() + { + m_sum = 0; + m_sum2 = 0; + m_num = 0; + m_min = 1e+39; + m_max = -1e+39; + m_vals.clear(); + } + + void add(double val) + { + m_sum += val; + m_sum2 += val * val; + + m_num++; + + m_min = std::min(m_min, val); + m_max = std::max(m_max, val); + + m_vals.push_back(val); + } + + void add(int val) + { + add(static_cast(val)); + } + + void add(uint32_t val) + { + add(static_cast(val)); + } + + void add(int64_t val) + { + add(static_cast(val)); + } + + void add(uint64_t val) + { + add(static_cast(val)); + } + + void print(const char* pPrefix = "") + { + if (!m_vals.size()) + printf("%s: Empty\n", pPrefix); + else + printf("%s: Samples: %llu, Total: %f, Avg: %f, Std Dev: %f, Min: %f, Max: %f, Mean: %f\n", + pPrefix, (unsigned long long)get_num(), get_total(), get_average(), get_std_dev(), get_min(), get_max(), get_mean()); + } + + double get_total() const + { + return m_sum; + } + + double get_average() const + { + return m_num ? (m_sum / m_num) : 0.0f; + } + + double get_min() const + { + return m_min; + } + + double get_max() const + { + return m_max; + } + + uint64_t get_num() const + { + return m_num; + } + + double get_val(uint32_t index) const + { + return m_vals[index]; + } + + // Returns population standard deviation + double get_std_dev() const + { + if (!m_num) + return 0.0f; + + // TODO: FP precision + return sqrt((m_sum2 - ((m_sum * m_sum) / m_num)) / m_num); + } + + double get_mean() const + { + if (!m_num) + return 0.0f; + + std::vector sorted_vals(m_vals); + std::sort(sorted_vals.begin(), sorted_vals.end()); + + return sorted_vals[sorted_vals.size() / 2]; + } + +private: + double m_sum; + double m_sum2; + + uint64_t m_num; + + double m_min; + double m_max; + + mutable std::vector m_vals; +}; + +//uint32_t get_deflate_size(const void* pData, size_t data_size); + +bool read_file(const char* pFilename, uint8_vec& buf); + +} // namespace utils + +#ifdef _MSC_VER +#pragma warning (pop) +#endif \ No newline at end of file diff --git a/external/basis_universal/python/README.md b/external/basis_universal/python/README.md new file mode 100644 index 0000000000..1b07677833 --- /dev/null +++ b/external/basis_universal/python/README.md @@ -0,0 +1,90 @@ +Python support is still new and coming online, but is entirely functional. +The library's pure C (WASM friendly) API's are completely exposed to Python. Our next goal is to work on official [Wheels](https://pythonwheels.com/), once the API is settled and more examples are written. + +The Python integration first tries to use native .so's in the basisu_py +directory. If they don't exist, it tries the slower and single threaded WASM +fallbacks under basisu_py/wasm, which requires wasmtime for Python to be +installed. Some tests require an input.ktx2 or test.ktx2 to be in the current +directory. + +Building: + +Under the repo's root directory - build the native SO's: + +``` +mkdir build_python +cd build_python +cmake -DBASISU_BUILD_PYTHON=ON .. +make +``` + +Build the WASM modules (see README_WASI.md file for instructions on how to +install the WASI SDK, which is required): + +``` +mkdir build_wasm_st +cd build_wasm_st +cmake .. -DCMAKE_TOOLCHAIN_FILE=$WASI_SDK_PATH/share/cmake/wasi-sdk.cmake -DCMAKE_BUILD_TYPE=Release -DBASISU_WASM_THREADING=OFF +make +``` + +--- + +Running Tests +------------- + +The tests assume the current directory is "python". Under Windows we've tested with Python v3.12.10, and under Linux v3.12.13. + +Higher-level tests: + +- python3 -m tests.test_backend_loading +- python3 -m tests.test_basic_wasm_selection +- python3 -m tests.test_basic_backend_selection +- python3 -m tests.test_basic_decode +- python3 -m tests.test_basic_transcode +- python3 -m tests.test_compress_swirl +- python3 -m tests.test_compress_swirl_hdr +- python3 -m tests.test_transcoder_astc +- python3 -m tests.test_transcoder_backend_loading +- python3 -m tests.test_transcoder_end_to_end +- python3 -m tests.test_transcoder_end_to_end_hdr +- python3 -m tests.test_transcoder_helpers + +Low-level tests (used while bringing up the codec): + +- python3 -m lowlevel_test_native.basic_test +- python3 -m lowlevel_test_native.test_transcoder_basic +- python3 -m lowlevel_test_native.example_capi_python +- python3 -m lowlevel_test_wasm.basic_test +- python3 -m lowlevel_test_wasm.compress_test +- python3 -m lowlevel_test_wasm.compress_test_float + +Example output: + +``` +richg@ryzen9:/mnt/c/dev/bu_1_22_snapshot2/basis_universal-master/python$ python3 -m tests.test_backend_loading +========== BACKEND LOADING TEST ========== + +Testing native backend... +[Encoder] Using native backend + [OK] Native backend loaded +Hello from basisu_wasm_api.cpp version 200 + Native get_version() ? 200 + Native alloc() returned ptr = 685784256 + Native free() OK + [OK] Native basic operations working. + + +Testing WASM backend... +[WASM Encoder] Loaded: /mnt/c/dev/bu_1_22_snapshot2/basis_universal-master/python/basisu_py/wasm/basisu_module_st.wasm +[Encoder] Using WASM backend + [OK] WASM backend loaded +Hello from basisu_wasm_api.cpp version 200 + WASM get_version() ? 200 + WASM alloc() returned ptr = 26920160 + WASM free() OK + [OK] WASM basic operations working. + + +========== DONE ========== +``` diff --git a/external/basis_universal/python/README_win.md b/external/basis_universal/python/README_win.md new file mode 100644 index 0000000000..74a312b1fa --- /dev/null +++ b/external/basis_universal/python/README_win.md @@ -0,0 +1,85 @@ +Windows Native Python Build Instructions +======================================== + +This project uses pybind11 to build Python .pyd extension modules on Windows. +Because Windows installs multiple Python versions, and pybind11 currently only +supports up to Python 3.12, you must follow these steps exactly. + +Requirements +------------ +- Visual Studio Developer Command Prompt (VS C++ Build Tools installed) +- Python 3.12 (pybind11 does NOT support 3.13+ at the time of writing) +- pybind11 installed into Python 3.12 + +Check installed Python versions: + py -0 + +If Python 3.12 is missing: + winget install Python.Python.3.12 + +Install pybind11 for Python 3.12: + py -3.12 -m pip install pybind11 + +IMPORTANT: +You must build AND run with the same Python interpreter version (3.12). + +Building the .pyd Modules +------------------------- +Open the "Developer Command Prompt for Visual Studio". + +From the project root: + + mkdir build_python_win + cd build_python_win + +Run CMake using the exact path to python.exe for Python 3.12: + + cmake -G "Visual Studio 17 2022" -A x64 -DBASISU_BUILD_PYTHON=ON -DBASISU_BUILD_WASM=OFF -DPYTHON_EXECUTABLE="C:\Users\\AppData\Local\Programs\Python\Python312\python.exe" .. + +Build: + + cmake --build . --config Release + +Output files will be created in: + + python/basisu_py/basisu_python.pyd + python/basisu_py/basisu_transcoder_python.pyd + +Running the Modules +------------------- +Always run using Python 3.12: + + py -3.12 + +Inside Python: + + import basisu_py + print("Modules loaded OK.") + +While in the "python" directory: + + py -m tests.test_backend_loading + +WASM Backend (Optional) +----------------------- +Install wasmtime: + + py -3.12 -m pip install wasmtime + +Ensure these files exist: + + python/basisu_py/wasm/*.wasm + +Common Problems +--------------- +1. "pybind11 not found" + -> Installed into wrong Python version. Use: + py -3.12 -m pip install pybind11 + +2. "Python config failure" + -> You are using Python 3.13 or 3.14. Must use Python 3.12. + +3. Modules not loading + -> You must run them with the same interpreter used to build them: + py -3.12 + diff --git a/external/basis_universal/python/astc_writer.py b/external/basis_universal/python/astc_writer.py new file mode 100644 index 0000000000..5752620d72 --- /dev/null +++ b/external/basis_universal/python/astc_writer.py @@ -0,0 +1,83 @@ +# astc_writer.py +# +# Minimal ASTC writer that mirrors the C/C++ write_astc_file() logic from example_capi.c. +# Writes a valid single-slice 2D ASTC texture file (no array slices, no 3D, no mips). +# +# Usage: +# from astc_writer import write_astc_file +# write_astc_file("output.astc", blocks, block_width, block_height, width, height) +# +# "blocks" must be a bytes-like object containing the full ASTC block data +# using 16 bytes per block (standard ASTC block size). + + +def write_astc_file( + filename: str, + blocks: bytes, + block_width: int, + block_height: int, + width: int, + height: int +) -> None: + """ + Write an ASTC file to disk. + + Parameters: + filename : Output filename ("something.astc") + blocks : Bytes-like object containing ASTC blocks (16 bytes per block) + block_width : ASTC block width (e.g. 4-12) + block_height : ASTC block height (e.g. 4-12) + width : Original image width in pixels + height : Original image height in pixels + + Notes: + - ASTC files use 2D blocks; depth is always 1. + - Block layout goes row-major: (num_blocks_y num_blocks_x) blocks. + - No mipmaps are stored in this format. + """ + + # Validate block dimensions + if block_width < 4 or block_width > 12: + raise ValueError(f"ASTC block_width {block_width} out of range (412)") + if block_height < 4 or block_height > 12: + raise ValueError(f"ASTC block_height {block_height} out of range (412)") + + # Compute block grid + num_blocks_x = (width + block_width - 1) // block_width + num_blocks_y = (height + block_height - 1) // block_height + total_blocks = num_blocks_x * num_blocks_y + expected_size = total_blocks * 16 # 16 bytes per ASTC block (always) + + if len(blocks) != expected_size: + raise ValueError( + f"ASTC block buffer incorrect size: expected {expected_size}, got {len(blocks)}" + ) + + # Write file + with open(filename, "wb") as f: + # ASTC magic number (0x13AB A15C) + f.write(bytes([0x13, 0xAB, 0xA1, 0x5C])) + + # Block dims: x, y, z (z=1) + f.write(bytes([ + block_width & 0xFF, + block_height & 0xFF, + 1 + ])) + + # ASTC stores width/height/depth as 24-bit LE + def write_24bit_le(v: int): + f.write(bytes([ + v & 0xFF, + (v >> 8) & 0xFF, + (v >> 16) & 0xFF + ])) + + write_24bit_le(width) + write_24bit_le(height) + write_24bit_le(1) # depth + + # Write actual block payload + f.write(blocks) + + print(f"[ASTC Writer] Wrote: {filename} ({width}x{height}, {block_width}x{block_height} blocks)") diff --git a/external/basis_universal/python/basisu_encoder_pybind11.cpp b/external/basis_universal/python/basisu_encoder_pybind11.cpp new file mode 100644 index 0000000000..c6a698892b --- /dev/null +++ b/external/basis_universal/python/basisu_encoder_pybind11.cpp @@ -0,0 +1,109 @@ +// File: basisu_encoder_pybind11.cpp +// pybind11 native bindings for the compressor's pure C API basisu_wasm_api.h +#include +#include +#include + +// include the basisu compression plain C API +#include "../encoder/basisu_wasm_api.h" + +namespace py = pybind11; + +// Convert wasm_bool_t (uint32_t) ? Python bool +static inline bool to_bool(uint32_t v) { return v != 0; } + +PYBIND11_MODULE(basisu_python, m) { + m.doc() = "Native Basis Universal encoder (pybind11 binding over basisu_wasm_api)"; + + // + // Initialization / Version + // + m.def("init", &bu_init, "Initialize the BasisU codec library"); + m.def("get_version", &bu_get_version, "Return BASISU_LIB_VERSION"); + + // + // Memory allocation helpers + // + m.def("alloc", &bu_alloc, + "Allocate memory inside native heap and return pointer as uint64"); + m.def("free", &bu_free, + "Free previously allocated pointer"); + + // + // Compression params handles + // + m.def("new_params", &bu_new_comp_params, + "Create a new comp_params struct inside native heap"); + m.def("delete_params", + [](uint64_t h) { return to_bool(bu_delete_comp_params(h)); }, + "Destroy a comp_params struct"); + + m.def("params_clear", + [](uint64_t h) { return to_bool(bu_comp_params_clear(h)); }, + "Clear comp_params struct"); + + // + // Image upload API + // + m.def("set_image_rgba32", + [](uint64_t params, uint32_t index, + uint64_t img_ptr, uint32_t w, uint32_t h, uint32_t pitch) { + return to_bool(bu_comp_params_set_image_rgba32( + params, index, img_ptr, w, h, pitch)); + }, + "Set 8-bit RGBA32 image into parameters"); + + m.def("set_image_float_rgba", + [](uint64_t params, uint32_t index, + uint64_t img_ptr, uint32_t w, uint32_t h, uint32_t pitch) { + return to_bool(bu_comp_params_set_image_float_rgba( + params, index, img_ptr, w, h, pitch)); + }, + "Set float32 RGBA image into parameters"); + + // + // Compression + // + m.def("compress", + [](uint64_t params, + int tex_format, + int quality, + int effort, + uint64_t flags, + float rdo_quality) + { + return to_bool(bu_compress_texture( + params, tex_format, quality, effort, flags, rdo_quality)); + }, + py::arg("params"), + py::arg("tex_format"), + py::arg("quality"), + py::arg("effort"), + py::arg("flags"), + py::arg("rdo_quality") = 0.0f + ); + + // + // Output blob access + // + m.def("get_comp_data_size", + &bu_comp_params_get_comp_data_size, + "Return size (bytes) of compressed output"); + m.def("get_comp_data_ofs", + &bu_comp_params_get_comp_data_ofs, + "Return pointer (uint64) to compressed output buffer"); + + // Memory read/write + m.def("read_memory", + [](uint64_t ptr, uint32_t size) { + return py::bytes((const char*)ptr, size); + }, + "Read `size` bytes starting at native memory address `ptr`"); + + m.def("write_memory", + [](uint64_t dest_ptr, py::buffer src) { + py::buffer_info info = src.request(); + memcpy((void*)dest_ptr, info.ptr, info.size * info.itemsize); + }, + "Write bytes/buffer-like object into native memory at address `ptr`"); +} diff --git a/external/basis_universal/python/basisu_py/MANIFEST.in b/external/basis_universal/python/basisu_py/MANIFEST.in new file mode 100644 index 0000000000..993a897b05 --- /dev/null +++ b/external/basis_universal/python/basisu_py/MANIFEST.in @@ -0,0 +1,2 @@ +recursive-include basisu_py *.py *.so *.wasm +include README.md diff --git a/external/basis_universal/python/basisu_py/READMD.md b/external/basis_universal/python/basisu_py/READMD.md new file mode 100644 index 0000000000..ca5adf873d --- /dev/null +++ b/external/basis_universal/python/basisu_py/READMD.md @@ -0,0 +1,5 @@ +This is the Python support directory for the Basis Universal KTX2 compressor +and transcoder modules. + +License: Apache 2.0 + diff --git a/external/basis_universal/python/basisu_py/__init__.py b/external/basis_universal/python/basisu_py/__init__.py new file mode 100644 index 0000000000..2186700a29 --- /dev/null +++ b/external/basis_universal/python/basisu_py/__init__.py @@ -0,0 +1,35 @@ +""" +basisu_py +========= +Python bindings for the Basis Universal encoder and transcoder, with +automatic fallback between native C++ extensions and WASM modules. + +Main entry points: + - Transcoder : basisu_py.transcoder.Transcoder + - Encoder : basisu_py.codec.Encoder + - constants : basisu_py.constants +""" + +from .codec import Encoder +from .transcoder import Transcoder, KTX2Handle +from .constants import ( + BasisTexFormat, + BasisQuality, + BasisEffort, + BasisFlags, + TranscoderTextureFormat, + TranscodeDecodeFlags, +) + +# What the package publicly exposes +__all__ = [ + "Encoder", + "Transcoder", + "KTX2Handle", + "BasisTexFormat", + "BasisQuality", + "BasisEffort", + "BasisFlags", + "TranscoderTextureFormat", + "TranscodeDecodeFlags", +] diff --git a/external/basis_universal/python/basisu_py/basisu_python.cpython-312-x86_64-linux-gnu.so b/external/basis_universal/python/basisu_py/basisu_python.cpython-312-x86_64-linux-gnu.so new file mode 100644 index 0000000000..efe98fb32a Binary files /dev/null and b/external/basis_universal/python/basisu_py/basisu_python.cpython-312-x86_64-linux-gnu.so differ diff --git a/external/basis_universal/python/basisu_py/basisu_python.pyd b/external/basis_universal/python/basisu_py/basisu_python.pyd new file mode 100644 index 0000000000..b140cb6544 Binary files /dev/null and b/external/basis_universal/python/basisu_py/basisu_python.pyd differ diff --git a/external/basis_universal/python/basisu_py/basisu_transcoder_python.cpython-312-x86_64-linux-gnu.so b/external/basis_universal/python/basisu_py/basisu_transcoder_python.cpython-312-x86_64-linux-gnu.so new file mode 100644 index 0000000000..0a695e9f4d Binary files /dev/null and b/external/basis_universal/python/basisu_py/basisu_transcoder_python.cpython-312-x86_64-linux-gnu.so differ diff --git a/external/basis_universal/python/basisu_py/basisu_transcoder_python.pyd b/external/basis_universal/python/basisu_py/basisu_transcoder_python.pyd new file mode 100644 index 0000000000..8bafa76db4 Binary files /dev/null and b/external/basis_universal/python/basisu_py/basisu_transcoder_python.pyd differ diff --git a/external/basis_universal/python/basisu_py/codec.py b/external/basis_universal/python/basisu_py/codec.py new file mode 100644 index 0000000000..e12b22cf0d --- /dev/null +++ b/external/basis_universal/python/basisu_py/codec.py @@ -0,0 +1,222 @@ +# basisu_py/codec.py + +import importlib +import numpy as np +from PIL import Image +import ctypes + +from .constants import BasisTexFormat, BasisQuality, BasisEffort, BasisFlags +from pathlib import Path + +class EncoderBackend: + NATIVE = "native" + WASM = "wasm" + AUTO = "auto" + +class Encoder: + + def __init__(self, backend=EncoderBackend.AUTO): + self.backend = backend + self._native = None + self._wasm = None + self.backend_name = None + + # ------------------------------------------------------------------ + # Try native first (AUTO or NATIVE modes) + # ------------------------------------------------------------------ + if backend in (EncoderBackend.AUTO, EncoderBackend.NATIVE): + try: + import basisu_py.basisu_python as native_encoder + native_encoder.init() + + self._native = native_encoder + self._wasm = None + self.backend_name = "NATIVE" + + print("[Encoder] Using native backend") + return + + except Exception as e: + if backend == EncoderBackend.NATIVE: + raise RuntimeError( + f"[Encoder] Native backend requested but unavailable: {e}" + ) + print("[Encoder] Native unavailable; falling back to WASM:", e) + + # ------------------------------------------------------------------ + # Fallback to WASM (AUTO or explicitly WASM) + # ------------------------------------------------------------------ + try: + from basisu_py.wasm.wasm_encoder import BasisuWasmEncoder + except Exception as e: + raise RuntimeError( + f"[Encoder] WASM backend cannot be imported: {e}\n" + "Make sure wasmtime is installed and basisu_py/wasm/*.wasm exist." + ) + + wasm_path = Path(__file__).parent / "wasm" / "basisu_module_st.wasm" + self._wasm = BasisuWasmEncoder(str(wasm_path)) + self._wasm.load() + self._native = None + self.backend_name = "WASM" + + print("[Encoder] Using WASM backend") + + + # ------------------------------------------------------ + # Public API + # ------------------------------------------------------ + def compress(self, + image, + format=-1, + quality=BasisQuality.MAX, + effort=BasisEffort.DEFAULT, + flags=BasisFlags.KTX2_OUTPUT | BasisFlags.SRGB | BasisFlags.THREADED | BasisFlags.XUASTC_LDR_FULL_ZSTD): + + rgba_bytes, w, h, is_hdr = self._convert_input_to_rgba_bytes(image) + + # Auto-select format if user passed -1 + if format == -1: + if is_hdr: + format = BasisTexFormat.cUASTC_HDR_6x6 + else: + format = BasisTexFormat.cXUASTC_LDR_6x6 + + if self._native: + return self._compress_native(rgba_bytes, w, h, format, quality, effort, flags, is_hdr) + else: + return self._compress_wasm(rgba_bytes, w, h, format, quality, effort, flags, is_hdr) + + def compress_float32(self, arr, **kwargs): + if not isinstance(arr, np.ndarray) or arr.dtype != np.float32: + raise ValueError("compress_float32 requires float32 NumPy HxWx4 array") + + return self.compress(arr, **kwargs) + + # ------------------------------------------------------ + # Native backend + # ------------------------------------------------------ + def _compress_native(self, bytes_data, w, h, fmt, quality, effort, flags, is_hdr=False): + enc = self._native + + params = enc.new_params() + + try: + buf_ptr = enc.alloc(len(bytes_data)) + + # Write raw bytes (uint8 or float32) + ctypes.memmove(buf_ptr, bytes_data, len(bytes_data)) + + if is_hdr: + ok = enc.set_image_float_rgba(params, 0, buf_ptr, w, h, w * 16) # 4 floats = 16 bytes per pixel + else: + ok = enc.set_image_rgba32(params, 0, buf_ptr, w, h, w * 4) + + if not ok: + raise RuntimeError("Native encoder: set_image failed (HDR or LDR)") + + ok = enc.compress(params, fmt, quality, effort, flags, 0.0) + if not ok: + raise RuntimeError("Native encoder: compress() failed") + + size = enc.get_comp_data_size(params) + ofs = enc.get_comp_data_ofs(params) + blob = enc.read_memory(ofs, size) + return blob + + finally: + enc.delete_params(params) + if buf_ptr: + enc.free(buf_ptr) + + # ------------------------------------------------------ + # WASM backend + # ------------------------------------------------------ + def _compress_wasm(self, bytes_data, w, h, fmt, quality, effort, flags, is_hdr=False): + enc = self._wasm + + params = enc.new_params() + + try: + buf_ptr = enc.alloc(len(bytes_data)) + enc.write_bytes(buf_ptr, bytes_data) + + if is_hdr: + ok = enc.set_image_float_rgba(params, 0, buf_ptr, w, h, w * 16) + else: + ok = enc.set_image_rgba32(params, 0, buf_ptr, w, h, w * 4) + + if not ok: + raise RuntimeError("WASM encoder: set_image failed (HDR or LDR)") + + ok = enc.compress(params, fmt, quality, effort, flags, 0.0) + if not ok: + raise RuntimeError("WASM encoder: compress() failed") + + size = enc.get_comp_data_size(params) + ofs = enc.get_comp_data_ofs(params) + blob = enc.read_bytes(ofs, size) + return blob + + finally: + enc.delete_params(params) + if buf_ptr: + enc.free(buf_ptr) + + # ------------------------------------------------------ + # Image conversion + # ------------------------------------------------------ + def _convert_input_to_rgba_bytes(self, image): + """ + Accept: + - Pillow Image (LDR) -> returns uint8 bytes + - NumPy uint8 LDR -> returns uint8 bytes + - NumPy float32 HDR -> returns float32 bytes + Returns (bytes, width, height, is_hdr) + """ + + # Pillow image -> LDR + if isinstance(image, Image.Image): + image = image.convert("RGBA") + arr = np.array(image, dtype=np.uint8) + h, w = arr.shape[:2] + return arr.tobytes(), w, h, False + + # NumPy array + elif isinstance(image, np.ndarray): + + # HDR float32 image + if image.dtype == np.float32: + if image.ndim != 3 or image.shape[2] not in (3,4): + raise ValueError("HDR NumPy image must be HxWx3 or HxWx4 float32") + + h, w, c = image.shape + + # Expand RGB -> RGBA if needed + if c == 3: + alpha = np.ones((h, w, 1), dtype=np.float32) + arr = np.concatenate([image, alpha], axis=2) + else: + arr = image + + return arr.tobytes(), w, h, True + + # LDR uint8 image + if image.dtype == np.uint8: + if image.ndim != 3 or image.shape[2] not in (3,4): + raise ValueError("LDR NumPy image must be HxWx3 or HxWx4 uint8") + + h, w, c = image.shape + + if c == 3: + alpha = np.full((h, w, 1), 255, dtype=np.uint8) + arr = np.concatenate([image, alpha], axis=2) + else: + arr = image + + return arr.tobytes(), w, h, False + + raise ValueError("NumPy image must be uint8 (LDR) or float32 (HDR)") + + else: + raise TypeError("compress() expects Pillow Image or NumPy array") diff --git a/external/basis_universal/python/basisu_py/constants.py b/external/basis_universal/python/basisu_py/constants.py new file mode 100644 index 0000000000..04be0b0137 --- /dev/null +++ b/external/basis_universal/python/basisu_py/constants.py @@ -0,0 +1,183 @@ +# basisu_constants.py + +# ============================================================ +# .KTX2/.basis file types +# basist::basis_tex_format +# ============================================================ +class BasisTexFormat: + # Original LDR formats + cETC1S = 0 + cUASTC_LDR_4x4 = 1 + + # HDR + cUASTC_HDR_4x4 = 2 + cASTC_HDR_6x6 = 3 + cUASTC_HDR_6x6 = 4 + + # XUASTC supercompressed LDR formats + cXUASTC_LDR_4x4 = 5 + cXUASTC_LDR_5x4 = 6 + cXUASTC_LDR_5x5 = 7 + cXUASTC_LDR_6x5 = 8 + cXUASTC_LDR_6x6 = 9 + cXUASTC_LDR_8x5 = 10 + cXUASTC_LDR_8x6 = 11 + cXUASTC_LDR_10x5 = 12 + cXUASTC_LDR_10x6 = 13 + cXUASTC_LDR_8x8 = 14 + cXUASTC_LDR_10x8 = 15 + cXUASTC_LDR_10x10= 16 + cXUASTC_LDR_12x10= 17 + cXUASTC_LDR_12x12= 18 + + # Standard ASTC LDR + cASTC_LDR_4x4 = 19 + cASTC_LDR_5x4 = 20 + cASTC_LDR_5x5 = 21 + cASTC_LDR_6x5 = 22 + cASTC_LDR_6x6 = 23 + cASTC_LDR_8x5 = 24 + cASTC_LDR_8x6 = 25 + cASTC_LDR_10x5 = 26 + cASTC_LDR_10x6 = 27 + cASTC_LDR_8x8 = 28 + cASTC_LDR_10x8 = 29 + cASTC_LDR_10x10= 30 + cASTC_LDR_12x10= 31 + cASTC_LDR_12x12= 32 + +# ============================================================ +# Unified quality level: 1-100 (higher=better quality, 100 disables some codec options) +# ============================================================ +class BasisQuality: + MIN = 1 + MAX = 100 + +# ============================================================ +# Unified effort level: 0-10 (0=fastest, 10=very slow, higher=slower but higher potential quality/more features utilized) +# ============================================================ +class BasisEffort: + MIN = 0 + MAX = 10 + + SUPER_FAST = 0 + FAST = 2 + NORMAL = 5 + DEFAULT = 2 + SLOW = 8 + VERY_SLOW = 10 + +# ============================================================ +# C-style API flags +# ============================================================ +class BasisFlags: + NONE = 0 + USE_OPENCL = 1 << 8 + THREADED = 1 << 9 + DEBUG_OUTPUT = 1 << 10 + + KTX2_OUTPUT = 1 << 11 + KTX2_UASTC_ZSTD = 1 << 12 + + SRGB = 1 << 13 + GEN_MIPS_CLAMP = 1 << 14 + GEN_MIPS_WRAP = 1 << 15 + + Y_FLIP = 1 << 16 + + PRINT_STATS = 1 << 18 + PRINT_STATUS = 1 << 19 + + DEBUG_IMAGES = 1 << 20 + + REC2020 = 1 << 21 + VALIDATE_OUTPUT = 1 << 22 + + XUASTC_LDR_FULL_ARITH = 0 + XUASTC_LDR_HYBRID = 1 << 23 + XUASTC_LDR_FULL_ZSTD = 2 << 23 + XUASTC_LDR_SYNTAX_SHIFT = 23 + XUASTC_LDR_SYNTAX_MASK = 3 + + TEXTURE_TYPE_2D = 0 << 25 + TEXTURE_TYPE_2D_ARRAY = 1 << 25 + TEXTURE_TYPE_CUBEMAP_ARRAY = 2 << 25 + TEXTURE_TYPE_VIDEO_FRAMES = 3 << 25 + TEXTURE_TYPE_SHIFT = 25 + TEXTURE_TYPE_MASK = 3 + + VERBOSE = PRINT_STATS | PRINT_STATUS + MIPMAP_CLAMP = GEN_MIPS_CLAMP + MIPMAP_WRAP = GEN_MIPS_WRAP + +# ============================================================ +# Transcoder Texture Formats (GPU block formats) +# basist::transcoder_texture_format +# ============================================================ +class TranscoderTextureFormat: + TF_ETC1_RGB = 0 + TF_ETC2_RGBA = 1 + TF_BC1_RGB = 2 + TF_BC3_RGBA = 3 + TF_BC4_R = 4 + TF_BC5_RG = 5 + TF_BC7_RGBA = 6 + + TF_PVRTC1_4_RGB = 8 + TF_PVRTC1_4_RGBA = 9 + + TF_ASTC_LDR_4X4_RGBA = 10 + TF_ATC_RGB = 11 + TF_ATC_RGBA = 12 + + # Uncompressed + TF_RGBA32 = 13 + TF_RGB565 = 14 + TF_BGR565 = 15 + TF_RGBA4444 = 16 + + TF_FXT1_RGB = 17 + TF_PVRTC2_4_RGB = 18 + TF_PVRTC2_4_RGBA = 19 + + TF_ETC2_EAC_R11 = 20 + TF_ETC2_EAC_RG11 = 21 + TF_BC6H = 22 + + TF_ASTC_HDR_4X4_RGBA = 23 + + TF_RGB_HALF = 24 + TF_RGBA_HALF = 25 + TF_RGB_9E5 = 26 + TF_ASTC_HDR_6X6_RGBA = 27 + + TF_ASTC_LDR_5X4_RGBA = 28 + TF_ASTC_LDR_5X5_RGBA = 29 + TF_ASTC_LDR_6X5_RGBA = 30 + TF_ASTC_LDR_6X6_RGBA = 31 + TF_ASTC_LDR_8X5_RGBA = 32 + TF_ASTC_LDR_8X6_RGBA = 33 + TF_ASTC_LDR_10X5_RGBA = 34 + TF_ASTC_LDR_10X6_RGBA = 35 + TF_ASTC_LDR_8X8_RGBA = 36 + TF_ASTC_LDR_10X8_RGBA = 37 + TF_ASTC_LDR_10X10_RGBA= 38 + TF_ASTC_LDR_12X10_RGBA= 39 + TF_ASTC_LDR_12X12_RGBA= 40 + + TOTAL = 41 + +# ============================================================ +# Transcoder Decode Flags +# ============================================================ +class TranscodeDecodeFlags: + PVRTC_DECODE_TO_NEXT_POW2 = 2 + TRANSCODE_ALPHA_TO_OPAQUE = 4 + BC1_FORBID_THREE_COLOR_BLOCKS = 8 + OUTPUT_HAS_ALPHA_INDICES = 16 + HIGH_QUALITY = 32 + NO_ETC1S_CHROMA_FILTERING = 64 + NO_DEBLOCK_FILTERING = 128 + STRONGER_DEBLOCK_FILTERING = 256 + FORCE_DEBLOCK_FILTERING = 512 + XUASTC_LDR_DISABLE_FAST_BC7_TRANSCODING = 1024 diff --git a/external/basis_universal/python/basisu_py/transcoder.py b/external/basis_universal/python/basisu_py/transcoder.py new file mode 100644 index 0000000000..868e69e63a --- /dev/null +++ b/external/basis_universal/python/basisu_py/transcoder.py @@ -0,0 +1,735 @@ +# basisu_py/transcoder.py +import numpy as np +from dataclasses import dataclass +from pathlib import Path + +from basisu_py.constants import ( + TranscoderTextureFormat, +) + +import importlib +import ctypes + + +# --------------------------------------------------------------------------- +# Enum to select backend +# --------------------------------------------------------------------------- +class TranscoderBackend: + NATIVE = "native" + WASM = "wasm" + AUTO = "auto" + + +# --------------------------------------------------------------------------- +# Wrapper class storing pointer+handle +# --------------------------------------------------------------------------- +@dataclass +class KTX2Handle: + ptr: int + handle: int + + +# --------------------------------------------------------------------------- +# Main Transcoder class +# --------------------------------------------------------------------------- +class Transcoder: + def __init__(self, backend=TranscoderBackend.AUTO): + self._native = None + self._wasm = None + self.backend_name = None + self.backend = None + + use_native = False + + # ------------------------------------------------------------------ + # Try native backend first if AUTO or NATIVE + # ------------------------------------------------------------------ + if backend in (TranscoderBackend.AUTO, TranscoderBackend.NATIVE): + try: + native_mod = importlib.import_module("basisu_py.basisu_transcoder_python") + native_mod.init() + self._native = native_mod + self.backend = native_mod + self.backend_name = "NATIVE" + use_native = True + print("[Transcoder] Using native backend") + except Exception as e: + if backend == TranscoderBackend.NATIVE: + # Caller explicitly requested native - fail hard + raise RuntimeError(f"Native transcoder backend failed: {e}") + print("[Transcoder] Native backend unavailable, reason:", e) + self._native = None + + # ------------------------------------------------------------------ + # Fallback to WASM if native is not being used + # ------------------------------------------------------------------ + if not use_native: + try: + from basisu_py.wasm.wasm_transcoder import BasisuWasmTranscoder + except Exception as e: + raise RuntimeError( + f"WASM backend cannot be imported: {e}\n" + "Ensure that:\n" + " - 'wasmtime' is installed\n" + " - basisu_py/wasm/*.wasm files are present in the install\n" + ) + + wasm_path = Path(__file__).parent / "wasm" / "basisu_transcoder_module_st.wasm" + self._wasm = BasisuWasmTranscoder(str(wasm_path)) + self._wasm.load() + self.backend = self._wasm + self.backend_name = "WASM" + print("[Transcoder] Using WASM backend") + + # Finally, bind the unified API to whichever backend we chose + self._bind_backend(self.backend) + + # ----------------------------------------------------------------------- + # Unified backend binding (native or wasm) + # ----------------------------------------------------------------------- + def _bind_backend(self, b): + self.backend = b + + # ------------------ memory operations ------------------ + memory_mapping = [ + ("_alloc", "alloc"), + ("_free", "free"), + ("_write", "write_memory"), + ("_read", "read_memory"), + ] + + # ------------------ KTX2 core ------------------ + basis_mapping = [ + # basis_tex_format helpers + ("basis_tex_format_is_xuastc_ldr", "basis_tex_format_is_xuastc_ldr"), + ("basis_tex_format_is_astc_ldr", "basis_tex_format_is_astc_ldr"), + ("basis_tex_format_get_block_width", "basis_tex_format_get_block_width"), + ("basis_tex_format_get_block_height", "basis_tex_format_get_block_height"), + ("basis_tex_format_is_hdr", "basis_tex_format_is_hdr"), + ("basis_tex_format_is_ldr", "basis_tex_format_is_ldr"), + + # transcoder_texture_format helpers + ("basis_get_bytes_per_block_or_pixel", "basis_get_bytes_per_block_or_pixel"), + ("basis_transcoder_format_has_alpha", "basis_transcoder_format_has_alpha"), + ("basis_transcoder_format_is_hdr", "basis_transcoder_format_is_hdr"), + ("basis_transcoder_format_is_ldr", "basis_transcoder_format_is_ldr"), + ("basis_transcoder_texture_format_is_astc", "basis_transcoder_texture_format_is_astc"), + ("basis_transcoder_format_is_uncompressed", "basis_transcoder_format_is_uncompressed"), + ("basis_get_uncompressed_bytes_per_pixel", "basis_get_uncompressed_bytes_per_pixel"), + ("basis_get_block_width", "basis_get_block_width"), + ("basis_get_block_height", "basis_get_block_height"), + ("basis_get_transcoder_texture_format_from_basis_tex_format","basis_get_transcoder_texture_format_from_basis_tex_format"), + ("basis_is_format_supported", "basis_is_format_supported"), + ("basis_compute_transcoded_image_size_in_bytes","basis_compute_transcoded_image_size_in_bytes"), + ] + + ktx2_mapping = [ + + ("ktx2_open", "ktx2_open"), + ("ktx2_close", "ktx2_close"), + + ("ktx2_get_width", "ktx2_get_width"), + ("ktx2_get_height", "ktx2_get_height"), + ("ktx2_get_levels", "ktx2_get_levels"), + ("ktx2_get_faces", "ktx2_get_faces"), + ("ktx2_get_layers", "ktx2_get_layers"), + + ("ktx2_get_basis_tex_format", "ktx2_get_basis_tex_format"), + + ("ktx2_get_block_width", "ktx2_get_block_width"), + ("ktx2_get_block_height", "ktx2_get_block_height"), + + ("ktx2_has_alpha", "ktx2_has_alpha"), + + # flags + ("ktx2_is_hdr", "ktx2_is_hdr"), + ("ktx2_is_hdr_4x4", "ktx2_is_hdr_4x4"), + ("ktx2_is_hdr_6x6", "ktx2_is_hdr_6x6"), + ("ktx2_is_ldr", "ktx2_is_ldr"), + ("ktx2_is_srgb", "ktx2_is_srgb"), + ("ktx2_is_etc1s", "ktx2_is_etc1s"), + ("ktx2_is_uastc_ldr_4x4", "ktx2_is_uastc_ldr_4x4"), + ("ktx2_is_xuastc_ldr", "ktx2_is_xuastc_ldr"), + ("ktx2_is_astc_ldr", "ktx2_is_astc_ldr"), + ("ktx2_is_video", "ktx2_is_video"), + ("ktx2_get_ldr_hdr_upconversion_nit_multiplier", "ktx2_get_ldr_hdr_upconversion_nit_multiplier"), + + # DFD access + ("ktx2_get_dfd_flags", "ktx2_get_dfd_flags"), + ("ktx2_get_dfd_total_samples", "ktx2_get_dfd_total_samples"), + ("ktx2_get_dfd_channel_id0", "ktx2_get_dfd_channel_id0"), + ("ktx2_get_dfd_channel_id1", "ktx2_get_dfd_channel_id1"), + ("ktx2_get_dfd_color_model", "ktx2_get_dfd_color_model"), + ("ktx2_get_dfd_color_primaries", "ktx2_get_dfd_color_primaries"), + ("ktx2_get_dfd_transfer_func", "ktx2_get_dfd_transfer_func"), + + # per-level info + ("ktx2_get_level_orig_width", "ktx2_get_level_orig_width"), + ("ktx2_get_level_orig_height", "ktx2_get_level_orig_height"), + ("ktx2_get_level_actual_width", "ktx2_get_level_actual_width"), + ("ktx2_get_level_actual_height", "ktx2_get_level_actual_height"), + + ("ktx2_get_level_num_blocks_x", "ktx2_get_level_num_blocks_x"), + ("ktx2_get_level_num_blocks_y", "ktx2_get_level_num_blocks_y"), + ("ktx2_get_level_total_blocks", "ktx2_get_level_total_blocks"), + + ("ktx2_get_level_alpha_flag", "ktx2_get_level_alpha_flag"), + ("ktx2_get_level_iframe_flag", "ktx2_get_level_iframe_flag"), + + # transcoding + ("ktx2_start_transcoding", "ktx2_start_transcoding"), + ("ktx2_transcode_image_level", "ktx2_transcode_image_level"), + + # version + ("get_version_fn", "get_version"), + ] + + # Apply all mappings + for public_name, backend_name in (memory_mapping + ktx2_mapping + basis_mapping): + setattr(self, public_name, getattr(b, backend_name)) + + # ----------------------------------------------------------------------- + # Public version query + # ----------------------------------------------------------------------- + def get_version(self): + return self.get_version_fn() + + # ----------------------------------------------------------------------- + # Enable library debug printing to stdout (also set BASISU_FORCE_DEVEL_MESSAGES to 1 in transcoder/basisu.h) + # ----------------------------------------------------------------------- + def enable_debug_printf(self, flag: bool = True): + return self.backend.enable_debug_printf(flag) + + # ----------------------------------------------------------------------- + # KTX2 Handle API: open/close + all queries + # ----------------------------------------------------------------------- + def open(self, ktx2_bytes: bytes) -> KTX2Handle: + ptr = self._alloc(len(ktx2_bytes)) + self._write(ptr, ktx2_bytes) + handle = self.ktx2_open(ptr, len(ktx2_bytes)) + return KTX2Handle(ptr, handle) + + def close(self, ktx2_handle: KTX2Handle): + self.ktx2_close(ktx2_handle.handle) + self._free(ktx2_handle.ptr) + + # ---- Basic queries ---- + def get_width(self, ktx2_handle: KTX2Handle): + return self.ktx2_get_width(ktx2_handle.handle) + + def get_height(self, ktx2_handle: KTX2Handle): + return self.ktx2_get_height(ktx2_handle.handle) + + def get_levels(self, ktx2_handle: KTX2Handle): + return self.ktx2_get_levels(ktx2_handle.handle) + + def get_faces(self, ktx2_handle: KTX2Handle): + return self.ktx2_get_faces(ktx2_handle.handle) + + def get_layers(self, ktx2_handle: KTX2Handle): + return self.ktx2_get_layers(ktx2_handle.handle) + + def get_basis_tex_format(self, ktx2_handle: KTX2Handle): + return self.ktx2_get_basis_tex_format(ktx2_handle.handle) + + def has_alpha(self, ktx2_handle: KTX2Handle) -> bool: + """ + Return true if the KTX2 container has alpha. + """ + return bool(self.ktx2_has_alpha(ktx2_handle.handle)) + + # ---- Format flags ---- + def is_hdr(self, ktx2_handle): return bool(self.ktx2_is_hdr(ktx2_handle.handle)) + def is_hdr_4x4(self, ktx2_handle): return bool(self.ktx2_is_hdr_4x4(ktx2_handle.handle)) + def is_hdr_6x6(self, ktx2_handle): return bool(self.ktx2_is_hdr_6x6(ktx2_handle.handle)) + def is_ldr(self, ktx2_handle): return bool(self.ktx2_is_ldr(ktx2_handle.handle)) + def is_srgb(self, ktx2_handle): return bool(self.ktx2_is_srgb(ktx2_handle.handle)) + def is_video(self, ktx2_handle): return bool(self.ktx2_is_video(ktx2_handle.handle)) + def get_ldr_hdr_upconversion_nit_multiplier(self, ktx2_handle): return self.ktx2_get_ldr_hdr_upconversion_nit_multiplier(ktx2_handle.handle) + def is_etc1s(self, ktx2_handle): return bool(self.ktx2_is_etc1s(ktx2_handle.handle)) + def is_uastc_ldr_4x4(self, ktx2_handle): return bool(self.ktx2_is_uastc_ldr_4x4(ktx2_handle.handle)) + def is_xuastc_ldr(self, ktx2_handle): return bool(self.ktx2_is_xuastc_ldr(ktx2_handle.handle)) + def is_astc_ldr(self, ktx2_handle): return bool(self.ktx2_is_astc_ldr(ktx2_handle.handle)) + + # ---- DFD access + def get_dfd_flags(self, ktx2_handle): return self.ktx2_get_dfd_flags(ktx2_handle.handle) + def get_dfd_total_samples(self, ktx2_handle): return self.ktx2_get_dfd_total_samples(ktx2_handle.handle) + def get_dfd_color_model(self, ktx2_handle): return self.ktx2_get_dfd_color_model(ktx2_handle.handle) + def get_dfd_color_primaries(self, ktx2_handle): return self.ktx2_get_dfd_color_primaries(ktx2_handle.handle) + def get_dfd_transfer_func(self, ktx2_handle): return self.ktx2_get_dfd_transfer_func(ktx2_handle.handle) + def get_dfd_channel_id0(self, ktx2_handle): return self.ktx2_get_dfd_channel_id0(ktx2_handle.handle) + def get_dfd_channel_id1(self, ktx2_handle): return self.ktx2_get_dfd_channel_id1(ktx2_handle.handle) + + # ---- Block dimensions ---- + def get_block_width(self, ktx2_handle): return self.ktx2_get_block_width(ktx2_handle.handle) + def get_block_height(self, ktx2_handle): return self.ktx2_get_block_height(ktx2_handle.handle) + + # ----------------------------------------------------------------------- + # Explicit: start transcoding on an already-open KTX2 file + # ----------------------------------------------------------------------- + def start_transcoding(self, ktx2_handle: KTX2Handle): + """ + Must be called before per-level iframe flags become valid. + """ + ok = self.ktx2_start_transcoding(ktx2_handle.handle) + if not ok: + raise RuntimeError("start_transcoding() failed") + return True + + # ---- Level info ---- + def get_level_orig_width(self, ktx2_handle, level, layer=0, face=0): + return self.ktx2_get_level_orig_width(ktx2_handle.handle, level, layer, face) + + def get_level_orig_height(self, ktx2_handle, level, layer=0, face=0): + return self.ktx2_get_level_orig_height(ktx2_handle.handle, level, layer, face) + + def get_level_actual_width(self, ktx2_handle, level, layer=0, face=0): + return self.ktx2_get_level_actual_width(ktx2_handle.handle, level, layer, face) + + def get_level_actual_height(self, ktx2_handle, level, layer=0, face=0): + return self.ktx2_get_level_actual_height(ktx2_handle.handle, level, layer, face) + + def get_level_num_blocks_x(self, ktx2_handle, level, layer=0, face=0): + return self.ktx2_get_level_num_blocks_x(ktx2_handle.handle, level, layer, face) + + def get_level_num_blocks_y(self, ktx2_handle, level, layer=0, face=0): + return self.ktx2_get_level_num_blocks_y(ktx2_handle.handle, level, layer, face) + + def get_level_total_blocks(self, ktx2_handle, level, layer=0, face=0): + return self.ktx2_get_level_total_blocks(ktx2_handle.handle, level, layer, face) + + def get_level_alpha_flag(self, ktx2_handle, level, layer=0, face=0): + return bool(self.ktx2_get_level_alpha_flag(ktx2_handle.handle, level, layer, face)) + + def get_level_iframe_flag(self, ktx2_handle, level, layer=0, face=0): + return bool(self.ktx2_get_level_iframe_flag(ktx2_handle.handle, level, layer, face)) + + # ----------------------------------------------------------------------- + # Low-level: Decode RGBA8 from an already-open KTX2 handle + # ----------------------------------------------------------------------- + def decode_rgba_handle(self, ktx2_handle: KTX2Handle, level=0, layer=0, face=0): + """ + Low-level fast decode. Requires an already-open KTX2Handle. + Returns HxWx4 uint8 NumPy array. + """ + w = self.ktx2_get_level_orig_width(ktx2_handle.handle, level, layer, face) + h = self.ktx2_get_level_orig_height(ktx2_handle.handle, level, layer, face) + + out_size = w * h * 4 + out_ptr = self._alloc(out_size) + + # MUST start transcoding before ANY decode + ok = self.ktx2_start_transcoding(ktx2_handle.handle) + if not ok: + self._free(out_ptr) + raise RuntimeError("start_transcoding failed") + + ok = self.ktx2_transcode_image_level( + ktx2_handle.handle, + level, layer, face, + out_ptr, + out_size, + TranscoderTextureFormat.TF_RGBA32, + 0, 0, 0, -1, -1, 0 + ) + if not ok: + self._free(out_ptr) + raise RuntimeError("transcode_image_level failed") + + raw_bytes = self._read(out_ptr, out_size) + self._free(out_ptr) + + arr = np.frombuffer(raw_bytes, dtype=np.uint8) + return arr.reshape((h, w, 4)) + + # ----------------------------------------------------------------------- + # High-level: Decode RGBA8 directly from KTX2 file data + # ----------------------------------------------------------------------- + def decode_rgba(self, ktx2_bytes: bytes, level=0, layer=0, face=0): + """ + High-level convenience decode. Opens the KTX2 file bytes for you. + """ + ktx2_handle = self.open(ktx2_bytes) + try: + return self.decode_rgba_handle(ktx2_handle, level, layer, face) + finally: + self.close(ktx2_handle) + + # ----------------------------------------------------------------------- + # Low-level: Decode HDR (RGBA float32) from open KTX2 + # ----------------------------------------------------------------------- + def decode_rgba_hdr_handle(self, ktx2_handle: KTX2Handle, level=0, layer=0, face=0): + """ + Low-level HDR decode. Returns HxWx4 float32 NumPy array. + """ + w = self.ktx2_get_level_orig_width(ktx2_handle.handle, level, layer, face) + h = self.ktx2_get_level_orig_height(ktx2_handle.handle, level, layer, face) + + bytes_per_pixel = 8 # 4 * half-float + out_size = w * h * bytes_per_pixel + out_ptr = self._alloc(out_size) + + ok = self.ktx2_start_transcoding(ktx2_handle.handle) + if not ok: + self._free(out_ptr) + raise RuntimeError("start_transcoding failed") + + ok = self.ktx2_transcode_image_level( + ktx2_handle.handle, + level, layer, face, + out_ptr, + out_size, + TranscoderTextureFormat.TF_RGBA_HALF, + 0, 0, 0, -1, -1, 0 + ) + if not ok: + self._free(out_ptr) + raise RuntimeError("transcode_image_level failed") + + raw_bytes = self._read(out_ptr, out_size) + self._free(out_ptr) + + arr = np.frombuffer(raw_bytes, dtype=np.float16).astype(np.float32) + return arr.reshape((h, w, 4)) + + # ----------------------------------------------------------------------- + # High-level: Decode HDR (RGBA float32) from KTX2 file data + # ----------------------------------------------------------------------- + def decode_rgba_hdr(self, ktx2_bytes: bytes, level=0, layer=0, face=0): + """ + High-level convenience HDR decode. Opens the KTX2 file bytes for you. + """ + ktx2_handle = self.open(ktx2_bytes) + try: + return self.decode_rgba_hdr_handle(ktx2_handle, level, layer, face) + finally: + self.close(ktx2_handle) + + # ----------------------------------------------------------------------- + # Low-level: General-purpose transcode using a chosen TranscoderTextureFormat format + # ----------------------------------------------------------------------- + def transcode_tfmt_handle(self, ktx2_handle: KTX2Handle, tfmt: int, + level=0, layer=0, face=0, decode_flags=0, + channel0=-1, channel1=-1): + """ + Low-level direct transcoding from an already-open KTX2 handle. + + Parameters: + ktx2_handle: KTX2Handle -> already-open KTX2 + tfmt: int -> TranscoderTextureFormat to transcode to (for ASTC: block size and LDR/HDR MUST match the KTX2 file, for HDR: must be a HDR texture format) + level/layer/face: int -> which image slice to decode + decode_flags: int -> basist::decode_flags + row_pitch, rows_in_pixels, channel0, channel1 -> advanced options + + Returns: bytes (transcoded GPU texture data or uncompressed image) + """ + + # Determine actual output size in bytes + ow = self.ktx2_get_level_orig_width(ktx2_handle.handle, level, layer, face) + oh = self.ktx2_get_level_orig_height(ktx2_handle.handle, level, layer, face) + + out_size = self.basis_compute_transcoded_image_size_in_bytes(tfmt, ow, oh) + if out_size == 0: + raise RuntimeError("basis_compute_transcoded_image_size_in_bytes returned 0") + + # print(f"*** ow={ow}, oh={oh}, out_size={out_size}") + + out_ptr = self._alloc(out_size) + + # Call transcoder + ok = self.ktx2_start_transcoding(ktx2_handle.handle) + if not ok: + self._free(out_ptr) + raise RuntimeError("start_transcoding failed") + + ok = self.ktx2_transcode_image_level( + ktx2_handle.handle, + level, layer, face, + out_ptr, + out_size, + tfmt, + decode_flags, + 0, + 0, + channel0, channel1, + 0 # no per-thread state object + ) + if not ok: + self._free(out_ptr) + raise RuntimeError("ktx2_transcode_image_level failed") + + # Extract bytes + raw_bytes = self._read(out_ptr, out_size) + + self._free(out_ptr) + return raw_bytes + + # ----------------------------------------------------------------------- + # High-level: General-purpose transcode (opens the KTX2 for you) + # tfmt: the TranscoderTextureFormat to transcode too + # ----------------------------------------------------------------------- + def transcode_tfmt(self, ktx2_bytes: bytes, tfmt: int, + level=0, layer=0, face=0, decode_flags=0, + channel0=-1, channel1=-1): + """ + High-level convenience wrapper for transcode_tfmt_handle(). + Automatically opens/closes the KTX2 file. + """ + ktx2_handle = self.open(ktx2_bytes) + try: + return self.transcode_tfmt_handle( + ktx2_handle, tfmt, + level=level, + layer=layer, + face=face, + decode_flags=decode_flags, + channel0=channel0, + channel1=channel1 + ) + finally: + self.close(ktx2_handle) + + # ----------------------------------------------------------------------- + # Low-level: choose a specific transcoder_texture_format from a family string + # ----------------------------------------------------------------------- + def choose_transcoder_format(self, ktx2_handle: KTX2Handle, family: str) -> int: + """ + Given an already-opened KTX2 and a desired family string, choose a concrete + TranscoderTextureFormat enum. + + family: one of: + "ASTC", "BC1", "BC3", "BC4", "BC5", "BC6H", "BC7", + "PVRTC1", "PVRTC2", + "ETC1", "ETC2", "ETC2_EAC_R11", "ETC2_EAC_RG11", + "ATC", "FXT1", + "RGBA32", "RGB_HALF", "RGBA_HALF", "RGB_FLOAT", "RGBA_FLOAT", + "RGB_9E5" + + Returns: + int: TranscoderTextureFormat value + """ + + s = family.strip().upper().replace(" ", "") + hdr_tex = self.is_hdr(ktx2_handle) + has_alpha = self.has_alpha(ktx2_handle) + basis_fmt = self.get_basis_tex_format(ktx2_handle) + + tfmt = None + + # ------------------------------------------------------------------- + # Uncompressed families + # ------------------------------------------------------------------- + if s in ("RGBA32", "RGBA8", "UNCOMPRESSED"): + tfmt = TranscoderTextureFormat.TF_RGBA32 + + elif s in ("RGBHALF", "RGB16F", "RGB_FLOAT", "RGBFLOAT"): + tfmt = TranscoderTextureFormat.TF_RGB_HALF + + elif s in ("RGBAHALF", "RGBA16F", "RGBA_FLOAT", "RGBAFLOAT"): + tfmt = TranscoderTextureFormat.TF_RGBA_HALF + + elif s in ("RGB9E5", "RGB_9E5"): + tfmt = TranscoderTextureFormat.TF_RGB_9E5 + + # ------------------------------------------------------------------- + # BC families + # ------------------------------------------------------------------- + elif s == "BC1": + tfmt = TranscoderTextureFormat.TF_BC1_RGB + elif s == "BC3": + tfmt = TranscoderTextureFormat.TF_BC3_RGBA + elif s == "BC4": + tfmt = TranscoderTextureFormat.TF_BC4_R + elif s == "BC5": + tfmt = TranscoderTextureFormat.TF_BC5_RG + elif s == "BC6H": + tfmt = TranscoderTextureFormat.TF_BC6H + elif s == "BC7": + tfmt = TranscoderTextureFormat.TF_BC7_RGBA + + # ------------------------------------------------------------------- + # PVRTC families + # ------------------------------------------------------------------- + elif s == "PVRTC1": + tfmt = (TranscoderTextureFormat.TF_PVRTC1_4_RGBA + if has_alpha else TranscoderTextureFormat.TF_PVRTC1_4_RGB) + + elif s == "PVRTC2": + tfmt = (TranscoderTextureFormat.TF_PVRTC2_4_RGBA + if has_alpha else TranscoderTextureFormat.TF_PVRTC2_4_RGB) + + # ------------------------------------------------------------------- + # ETC / EAC families + # ------------------------------------------------------------------- + elif s == "ETC1": + tfmt = TranscoderTextureFormat.TF_ETC1_RGB + + elif s == "ETC2": + tfmt = TranscoderTextureFormat.TF_ETC2_RGBA + + elif s in ("ETC2_EAC_R11", "EAC_R11"): + tfmt = TranscoderTextureFormat.TF_ETC2_EAC_R11 + + elif s in ("ETC2_EAC_RG11", "EAC_RG11"): + tfmt = TranscoderTextureFormat.TF_ETC2_EAC_RG11 + + # ------------------------------------------------------------------- + # ATC / FXT + # ------------------------------------------------------------------- + elif s == "ATC": + tfmt = (TranscoderTextureFormat.TF_ATC_RGBA + if has_alpha else TranscoderTextureFormat.TF_ATC_RGB) + + elif s == "FXT1": + tfmt = TranscoderTextureFormat.TF_FXT1_RGB + + # ------------------------------------------------------------------- + # ASTC family + # ------------------------------------------------------------------- + elif s == "ASTC": + # Let BasisU decide correct ASTC format (block size + LDR/HDR) + tfmt = self.basis_get_transcoder_texture_format_from_basis_tex_format(basis_fmt) + + else: + # Unknown family: choose a safe uncompressed default + if hdr_tex: + tfmt = TranscoderTextureFormat.TF_RGBA_HALF + else: + tfmt = TranscoderTextureFormat.TF_RGBA32 + + # ------------------------------------------------------------------- + # Validate HDR/LDR compatibility (optional but recommended) + # ------------------------------------------------------------------- + # Use helpers to ensure we don't do HDR->LDR or LDR->HDR accidentally. + is_tfmt_hdr = self.basis_transcoder_format_is_hdr(tfmt) + if hdr_tex and not is_tfmt_hdr: + raise ValueError(f"Requested {family} (LDR transcoder format) for HDR KTX2.") + if not hdr_tex and is_tfmt_hdr: + raise ValueError(f"Requested {family} (HDR transcoder format) for LDR KTX2.") + + return tfmt + + # ----------------------------------------------------------------------- + # Low-level: General-purpose transcode using a family string + # from an already opened ktx2 file. + # Returns: + # (data_bytes, chosen_tfmt, block_width, block_height) + # ----------------------------------------------------------------------- + def transcode_handle( + self, + ktx2_handle: KTX2Handle, + family: str, + level=0, + layer=0, + face=0, + decode_flags=0, + channel0=-1, + channel1=-1 + ): + """ + Low-level direct transcoding from an already-open KTX2 handle, + using a high-level family string such as: + "BC7", "BC3", "BC1", "ETC1", "ETC2", "ASTC", "PVRTC1", + "RGBA32", "RGB_HALF", "RGBA_HALF", "RGB_9E5", etc. + See choose_transcoder_format(). + Returns: + (data_bytes, tfmt, block_width, block_height) + """ + + # Decide the exact transcoder format (BC1/BC7/etc.) + tfmt = self.choose_transcoder_format(ktx2_handle, family) + + # Get original dims of the requested slice + ow = self.get_level_orig_width(ktx2_handle, level, layer, face) + oh = self.get_level_orig_height(ktx2_handle, level, layer, face) + + # Compute correct output size for the chosen format + out_size = self.basis_compute_transcoded_image_size_in_bytes(tfmt, ow, oh) + if out_size == 0: + raise RuntimeError( + f"Computed output size is 0 for tfmt={tfmt}, dims={ow}x{oh}" + ) + + # Allocate output buffer + out_ptr = self._alloc(out_size) + + # Ensure transcoding tables are ready + ok = self.ktx2_start_transcoding(ktx2_handle.handle) + if not ok: + self._free(out_ptr) + raise RuntimeError("start_transcoding failed") + + # Perform the transcode + ok = self.ktx2_transcode_image_level( + ktx2_handle.handle, + level, layer, face, + out_ptr, + out_size, + tfmt, + decode_flags, + 0, # row_pitch_in_blocks_or_pixels + 0, # rows_in_pixels + channel0, + channel1, + 0 # no thread-local state + ) + if not ok: + self._free(out_ptr) + raise RuntimeError("ktx2_transcode_image_level failed") + + # Extract bytes from native/WASM memory + data_bytes = self._read(out_ptr, out_size) + + # Free the output buffer + self._free(out_ptr) + + # Determine block dims for this texture format + if self.basis_transcoder_format_is_uncompressed(tfmt): + bw = None + bh = None + else: + bw = self.basis_get_block_width(tfmt) + bh = self.basis_get_block_height(tfmt) + + return data_bytes, tfmt, bw, bh + + # ----------------------------------------------------------------------- + # High-level: one-shot transcode using a family string + # directly from ktx2 file data. (Slower if you're transcoding multiple + # levels/faces/layers.) + # ----------------------------------------------------------------------- + def transcode( + self, + ktx2_bytes: bytes, + family: str, + level=0, + layer=0, + face=0, + decode_flags=0, + channel0=-1, + channel1=-1 + ): + """ + High-level version of transcode_handle(). + Calls transcode_handle() internally. + + Returns: + (data_bytes, tfmt, block_width, block_height) + """ + ktx2_handle = self.open(ktx2_bytes) + try: + return self.transcode_handle( + ktx2_handle, + family, + level=level, + layer=layer, + face=face, + decode_flags=decode_flags, + channel0=channel0, + channel1=channel1 + ) + finally: + self.close(ktx2_handle) + + def tfmt_name(self, tfmt: int): + return TranscoderTextureFormat(tfmt).name diff --git a/external/basis_universal/python/basisu_py/wasm/__init__.py b/external/basis_universal/python/basisu_py/wasm/__init__.py new file mode 100644 index 0000000000..76d8f38a1b --- /dev/null +++ b/external/basis_universal/python/basisu_py/wasm/__init__.py @@ -0,0 +1 @@ +# Purposely empty diff --git a/external/basis_universal/python/basisu_py/wasm/basisu_module_mt.wasm b/external/basis_universal/python/basisu_py/wasm/basisu_module_mt.wasm new file mode 100644 index 0000000000..b9045c1361 Binary files /dev/null and b/external/basis_universal/python/basisu_py/wasm/basisu_module_mt.wasm differ diff --git a/external/basis_universal/python/basisu_py/wasm/basisu_module_st.wasm b/external/basis_universal/python/basisu_py/wasm/basisu_module_st.wasm new file mode 100644 index 0000000000..13b61f7fcd Binary files /dev/null and b/external/basis_universal/python/basisu_py/wasm/basisu_module_st.wasm differ diff --git a/external/basis_universal/python/basisu_py/wasm/basisu_transcoder_module_mt.wasm b/external/basis_universal/python/basisu_py/wasm/basisu_transcoder_module_mt.wasm new file mode 100644 index 0000000000..b9c90be47e Binary files /dev/null and b/external/basis_universal/python/basisu_py/wasm/basisu_transcoder_module_mt.wasm differ diff --git a/external/basis_universal/python/basisu_py/wasm/basisu_transcoder_module_st.wasm b/external/basis_universal/python/basisu_py/wasm/basisu_transcoder_module_st.wasm new file mode 100644 index 0000000000..0b717d4b9e Binary files /dev/null and b/external/basis_universal/python/basisu_py/wasm/basisu_transcoder_module_st.wasm differ diff --git a/external/basis_universal/python/basisu_py/wasm/wasm_encoder.py b/external/basis_universal/python/basisu_py/wasm/wasm_encoder.py new file mode 100644 index 0000000000..e6d3516ab5 --- /dev/null +++ b/external/basis_universal/python/basisu_py/wasm/wasm_encoder.py @@ -0,0 +1,126 @@ +# basisu_py/wasm/wasm_encoder.py + +import wasmtime +import ctypes + +from ..constants import BasisTexFormat, BasisQuality, BasisEffort, BasisFlags + + +class BasisuWasmEncoder: + def __init__(self, wasm_path): + self.wasm_path = wasm_path + self.engine = None + self.store = None + self.memory = None + self.exports = None + + # ------------------------------------------------------ + # Initialize WASM + WASI + # ------------------------------------------------------ + def _init_engine(self): + self.engine = wasmtime.Engine() + self.store = wasmtime.Store(self.engine) + + wasi = wasmtime.WasiConfig() + wasi.argv = ["basisu-wasm"] + wasi.inherit_stdout() + wasi.inherit_stderr() + self.store.set_wasi(wasi) + + def load(self): + self._init_engine() + + module = wasmtime.Module.from_file(self.engine, self.wasm_path) + linker = wasmtime.Linker(self.engine) + linker.define_wasi() + + instance = linker.instantiate(self.store, module) + self.exports = instance.exports(self.store) + self.memory = self.exports["memory"] + + # Initialize if present + if "bu_init" in self.exports: + self.exports["bu_init"](self.store) + + print("[WASM Encoder] Loaded:", self.wasm_path) + + # ------------------------------------------------------ + # Access raw linear memory buffer + # ------------------------------------------------------ + def _buf(self): + raw_ptr = self.memory.data_ptr(self.store) + size = self.memory.data_len(self.store) + addr = ctypes.addressof(raw_ptr.contents) + return (ctypes.c_ubyte * size).from_address(addr) + + # ------------------------------------------------------ + # Version + # ------------------------------------------------------ + def get_version(self): + return self.exports["bu_get_version"](self.store) + + # ------------------------------------------------------ + # Memory alloc/free + # ------------------------------------------------------ + def alloc(self, size): + return self.exports["bu_alloc"](self.store, size) + + def free(self, ptr): + self.exports["bu_free"](self.store, ptr) + + # ------------------------------------------------------ + # Params + # ------------------------------------------------------ + def new_params(self): + return self.exports["bu_new_comp_params"](self.store) + + def delete_params(self, params): + return self.exports["bu_delete_comp_params"](self.store, params) + + # ------------------------------------------------------ + # Image input + # ------------------------------------------------------ + def set_image_rgba32(self, params, index, ptr, w, h, pitch): + return self.exports["bu_comp_params_set_image_rgba32"]( + self.store, params, index, ptr, w, h, pitch + ) + + def set_image_float_rgba(self, params, index, ptr, w, h, pitch): + return self.exports["bu_comp_params_set_image_float_rgba"]( + self.store, params, index, ptr, w, h, pitch + ) + + # ------------------------------------------------------ + # Compression + # ------------------------------------------------------ + def compress(self, params, fmt, quality, effort, flags, rdo): + return bool(self.exports["bu_compress_texture"]( + self.store, params, fmt, quality, effort, flags, rdo + )) + + # ------------------------------------------------------ + # Output blob + # ------------------------------------------------------ + def get_comp_data_size(self, params): + return self.exports["bu_comp_params_get_comp_data_size"](self.store, params) + + def get_comp_data_ofs(self, params): + return self.exports["bu_comp_params_get_comp_data_ofs"](self.store, params) + + # ------------------------------------------------------ + # Raw memory I/O + # ------------------------------------------------------ + def write_bytes(self, ptr, data): + buf = self._buf() + buf[ptr:ptr + len(data)] = data + + def read_bytes(self, ptr, size): + buf = self._buf() + return bytes(buf[ptr:ptr + size]) + + # NEW unified names: + def write_memory(self, ptr, data): + self.write_bytes(ptr, data) + + def read_memory(self, ptr, size): + return self.read_bytes(ptr, size) diff --git a/external/basis_universal/python/basisu_py/wasm/wasm_transcoder.py b/external/basis_universal/python/basisu_py/wasm/wasm_transcoder.py new file mode 100644 index 0000000000..01e96ee604 --- /dev/null +++ b/external/basis_universal/python/basisu_py/wasm/wasm_transcoder.py @@ -0,0 +1,326 @@ +# basisu_py/wasm/wasm_transcoder.py + +import wasmtime +import ctypes + + +class BasisuWasmTranscoder: + """ + Lowest-level WASM transcoder wrapper. + Direct mapping to basisu_wasm_transcoder_api.h/.cpp + + NOTE: + - This layer does NOT interpret formats or block sizes. + - It only wraps the raw C API (bt_* and basis_* exports). + - Higher-level logic (TranscoderCore, Transcoder) will build on top. + """ + + def __init__(self, wasm_path: str): + self.wasm_path = wasm_path + self.engine = None + self.store = None + self.memory = None + self.exports = None + + # ------------------------------------------------------ + # Internal: initialize WASM + WASI + # ------------------------------------------------------ + def _init_engine(self): + self.engine = wasmtime.Engine() + self.store = wasmtime.Store(self.engine) + + wasi = wasmtime.WasiConfig() + wasi.argv = ["basisu-transcoder"] + wasi.inherit_stdout() + wasi.inherit_stderr() + self.store.set_wasi(wasi) + + def load(self): + self._init_engine() + + module = wasmtime.Module.from_file(self.engine, self.wasm_path) + linker = wasmtime.Linker(self.engine) + linker.define_wasi() + + instance = linker.instantiate(self.store, module) + self.exports = instance.exports(self.store) + self.memory = self.exports["memory"] + + # Mandatory transcoder init + if "bt_init" in self.exports: + self.exports["bt_init"](self.store) + + print("[WASM Transcoder] Loaded:", self.wasm_path) + + # ------------------------------------------------------ + # Linear memory access helpers + # ------------------------------------------------------ + def _buf(self): + raw_ptr = self.memory.data_ptr(self.store) + size = self.memory.data_len(self.store) + addr = ctypes.addressof(raw_ptr.contents) + return (ctypes.c_ubyte * size).from_address(addr) + + def write_bytes(self, ptr: int, data: bytes): + buf = self._buf() + buf[ptr:ptr + len(data)] = data + + def read_bytes(self, ptr: int, num: int) -> bytes: + buf = self._buf() + return bytes(buf[ptr:ptr + num]) + + # NEW unified names: + def write_memory(self, ptr, data): + self.write_bytes(ptr, data) + + def read_memory(self, ptr, size): + return self.read_bytes(ptr, size) + + # ------------------------------------------------------ + # Memory alloc/free + # ------------------------------------------------------ + def alloc(self, size: int) -> int: + return self.exports["bt_alloc"](self.store, size) + + def free(self, ptr: int): + return self.exports["bt_free"](self.store, ptr) + + # ------------------------------------------------------ + # High-level functions: version, init, debug + # ------------------------------------------------------ + def get_version(self) -> int: + return self.exports["bt_get_version"](self.store) + + def enable_debug_printf(self, flag: bool = True): + return self.exports["bt_enable_debug_printf"](self.store, 1 if flag else 0) + + # ------------------------------------------------------ + # basis_tex_format helpers + # ------------------------------------------------------ + def basis_tex_format_is_xuastc_ldr(self, basis_tex_fmt_u32: int) -> bool: + return bool(self.exports["bt_basis_tex_format_is_xuastc_ldr"](self.store, basis_tex_fmt_u32)) + + def basis_tex_format_is_astc_ldr(self, basis_tex_fmt_u32: int) -> bool: + return bool(self.exports["bt_basis_tex_format_is_astc_ldr"](self.store, basis_tex_fmt_u32)) + + def basis_tex_format_get_block_width(self, basis_tex_fmt_u32: int) -> int: + return self.exports["bt_basis_tex_format_get_block_width"](self.store, basis_tex_fmt_u32) + + def basis_tex_format_get_block_height(self, basis_tex_fmt_u32: int) -> int: + return self.exports["bt_basis_tex_format_get_block_height"](self.store, basis_tex_fmt_u32) + + def basis_tex_format_is_hdr(self, basis_tex_fmt_u32: int) -> bool: + return bool(self.exports["bt_basis_tex_format_is_hdr"](self.store, basis_tex_fmt_u32)) + + def basis_tex_format_is_ldr(self, basis_tex_fmt_u32: int) -> bool: + return bool(self.exports["bt_basis_tex_format_is_ldr"](self.store, basis_tex_fmt_u32)) + + # ------------------------------------------------------ + # transcoder_texture_format helpers + # ------------------------------------------------------ + def basis_get_bytes_per_block_or_pixel(self, tfmt_u32: int) -> int: + return self.exports["bt_basis_get_bytes_per_block_or_pixel"](self.store, tfmt_u32) + + def basis_transcoder_format_has_alpha(self, tfmt_u32: int) -> bool: + return bool(self.exports["bt_basis_transcoder_format_has_alpha"](self.store, tfmt_u32)) + + def basis_transcoder_format_is_hdr(self, tfmt_u32: int) -> bool: + return bool(self.exports["bt_basis_transcoder_format_is_hdr"](self.store, tfmt_u32)) + + def basis_transcoder_format_is_ldr(self, tfmt_u32: int) -> bool: + return bool(self.exports["bt_basis_transcoder_format_is_ldr"](self.store, tfmt_u32)) + + def basis_transcoder_texture_format_is_astc(self, tfmt_u32: int) -> bool: + return bool(self.exports["bt_basis_transcoder_texture_format_is_astc"](self.store, tfmt_u32)) + + def basis_transcoder_format_is_uncompressed(self, tfmt_u32: int) -> bool: + return bool(self.exports["bt_basis_transcoder_format_is_uncompressed"](self.store, tfmt_u32)) + + def basis_get_uncompressed_bytes_per_pixel(self, tfmt_u32: int) -> int: + return self.exports["bt_basis_get_uncompressed_bytes_per_pixel"](self.store, tfmt_u32) + + def basis_get_block_width(self, tfmt_u32: int) -> int: + return self.exports["bt_basis_get_block_width"](self.store, tfmt_u32) + + def basis_get_block_height(self, tfmt_u32: int) -> int: + return self.exports["bt_basis_get_block_height"](self.store, tfmt_u32) + + def basis_get_transcoder_texture_format_from_basis_tex_format(self, basis_tex_fmt_u32: int) -> int: + return self.exports["bt_basis_get_transcoder_texture_format_from_basis_tex_format"](self.store, basis_tex_fmt_u32) + + def basis_is_format_supported(self, tfmt_u32: int, basis_tex_fmt_u32: int) -> bool: + return bool(self.exports["bt_basis_is_format_supported"](self.store, tfmt_u32, basis_tex_fmt_u32)) + + def basis_compute_transcoded_image_size_in_bytes(self, tfmt_u32: int, orig_width: int, orig_height: int) -> int: + return self.exports["bt_basis_compute_transcoded_image_size_in_bytes"]( + self.store, tfmt_u32, orig_width, orig_height + ) + + # ------------------------------------------------------ + # KTX2 handle management + # ------------------------------------------------------ + def ktx2_open(self, data_ptr: int, data_len: int) -> int: + return self.exports["bt_ktx2_open"](self.store, data_ptr, data_len) + + def ktx2_close(self, handle: int): + return self.exports["bt_ktx2_close"](self.store, handle) + + # ------------------------------------------------------ + # Basic KTX2 metadata + # ------------------------------------------------------ + def ktx2_get_width(self, handle: int) -> int: + return self.exports["bt_ktx2_get_width"](self.store, handle) + + def ktx2_get_height(self, handle: int) -> int: + return self.exports["bt_ktx2_get_height"](self.store, handle) + + def ktx2_get_levels(self, handle: int) -> int: + return self.exports["bt_ktx2_get_levels"](self.store, handle) + + def ktx2_get_faces(self, handle: int) -> int: + return self.exports["bt_ktx2_get_faces"](self.store, handle) + + def ktx2_get_layers(self, handle: int) -> int: + return self.exports["bt_ktx2_get_layers"](self.store, handle) + + def ktx2_get_basis_tex_format(self, handle: int) -> int: + return self.exports["bt_ktx2_get_basis_tex_format"](self.store, handle) + + # KTX2 format checks + def ktx2_is_etc1s(self, handle: int) -> bool: + return bool(self.exports["bt_ktx2_is_etc1s"](self.store, handle)) + + def ktx2_is_uastc_ldr_4x4(self, handle: int) -> bool: + return bool(self.exports["bt_ktx2_is_uastc_ldr_4x4"](self.store, handle)) + + def ktx2_is_hdr(self, handle: int) -> bool: + return bool(self.exports["bt_ktx2_is_hdr"](self.store, handle)) + + def ktx2_is_hdr_4x4(self, handle: int) -> bool: + return bool(self.exports["bt_ktx2_is_hdr_4x4"](self.store, handle)) + + def ktx2_is_hdr_6x6(self, handle: int) -> bool: + return bool(self.exports["bt_ktx2_is_hdr_6x6"](self.store, handle)) + + def ktx2_is_ldr(self, handle: int) -> bool: + return bool(self.exports["bt_ktx2_is_ldr"](self.store, handle)) + + def ktx2_is_astc_ldr(self, handle: int) -> bool: + return bool(self.exports["bt_ktx2_is_astc_ldr"](self.store, handle)) + + def ktx2_is_xuastc_ldr(self, handle: int) -> bool: + return bool(self.exports["bt_ktx2_is_xuastc_ldr"](self.store, handle)) + + def ktx2_get_block_width(self, handle: int) -> int: + return self.exports["bt_ktx2_get_block_width"](self.store, handle) + + def ktx2_get_block_height(self, handle: int) -> int: + return self.exports["bt_ktx2_get_block_height"](self.store, handle) + + def ktx2_has_alpha(self, handle: int) -> bool: + return bool(self.exports["bt_ktx2_has_alpha"](self.store, handle)) + + def ktx2_get_dfd_color_model(self, handle: int) -> int: + return self.exports["bt_ktx2_get_dfd_color_model"](self.store, handle) + + def ktx2_get_dfd_color_primaries(self, handle: int) -> int: + return self.exports["bt_ktx2_get_dfd_color_primaries"](self.store, handle) + + def ktx2_get_dfd_transfer_func(self, handle: int) -> int: + return self.exports["bt_ktx2_get_dfd_transfer_func"](self.store, handle) + + def ktx2_is_srgb(self, handle: int) -> bool: + return bool(self.exports["bt_ktx2_is_srgb"](self.store, handle)) + + def ktx2_get_dfd_flags(self, handle: int) -> int: + return self.exports["bt_ktx2_get_dfd_flags"](self.store, handle) + + def ktx2_get_dfd_total_samples(self, handle: int) -> int: + return self.exports["bt_ktx2_get_dfd_total_samples"](self.store, handle) + + def ktx2_get_dfd_channel_id0(self, handle: int) -> int: + return self.exports["bt_ktx2_get_dfd_channel_id0"](self.store, handle) + + def ktx2_get_dfd_channel_id1(self, handle: int) -> int: + return self.exports["bt_ktx2_get_dfd_channel_id1"](self.store, handle) + + def ktx2_is_video(self, handle: int) -> bool: + return bool(self.exports["bt_ktx2_is_video"](self.store, handle)) + + def ktx2_get_ldr_hdr_upconversion_nit_multiplier(self, handle: int) -> float: + return self.exports["bt_ktx2_get_ldr_hdr_upconversion_nit_multiplier"](self.store, handle) + + # ------------------------------------------------------ + # Per-level metadata + # ------------------------------------------------------ + def ktx2_get_level_orig_width(self, h, lvl, layer, face) -> int: + return self.exports["bt_ktx2_get_level_orig_width"](self.store, h, lvl, layer, face) + + def ktx2_get_level_orig_height(self, h, lvl, layer, face) -> int: + return self.exports["bt_ktx2_get_level_orig_height"](self.store, h, lvl, layer, face) + + def ktx2_get_level_actual_width(self, h, lvl, layer, face) -> int: + return self.exports["bt_ktx2_get_level_actual_width"](self.store, h, lvl, layer, face) + + def ktx2_get_level_actual_height(self, h, lvl, layer, face) -> int: + return self.exports["bt_ktx2_get_level_actual_height"](self.store, h, lvl, layer, face) + + def ktx2_get_level_num_blocks_x(self, h, lvl, layer, face) -> int: + return self.exports["bt_ktx2_get_level_num_blocks_x"](self.store, h, lvl, layer, face) + + def ktx2_get_level_num_blocks_y(self, h, lvl, layer, face) -> int: + return self.exports["bt_ktx2_get_level_num_blocks_y"](self.store, h, lvl, layer, face) + + def ktx2_get_level_total_blocks(self, h, lvl, layer, face) -> int: + return self.exports["bt_ktx2_get_level_total_blocks"](self.store, h, lvl, layer, face) + + def ktx2_get_level_alpha_flag(self, h, lvl, layer, face) -> bool: + return bool(self.exports["bt_ktx2_get_level_alpha_flag"](self.store, h, lvl, layer, face)) + + def ktx2_get_level_iframe_flag(self, h, lvl, layer, face) -> bool: + return bool(self.exports["bt_ktx2_get_level_iframe_flag"](self.store, h, lvl, layer, face)) + + # ------------------------------------------------------ + # Transcoding control + # ------------------------------------------------------ + def ktx2_start_transcoding(self, handle: int) -> bool: + return bool(self.exports["bt_ktx2_start_transcoding"](self.store, handle)) + + def ktx2_create_transcode_state(self) -> int: + return self.exports["bt_ktx2_create_transcode_state"](self.store) + + def ktx2_destroy_transcode_state(self, handle: int): + return self.exports["bt_ktx2_destroy_transcode_state"](self.store, handle) + + # ------------------------------------------------------ + # Actual transcoding call + # ------------------------------------------------------ + def ktx2_transcode_image_level( + self, + ktx2_handle: int, + level_index: int, + layer_index: int, + face_index: int, + output_block_mem_ofs: int, + output_blocks_buf_size_in_blocks_or_pixels: int, + transcoder_texture_format_u32: int, + decode_flags: int, + output_row_pitch_in_blocks_or_pixels: int, + output_rows_in_pixels: int, + channel0: int, + channel1: int, + state_handle: int, + ) -> bool: + return bool(self.exports["bt_ktx2_transcode_image_level"]( + self.store, + ktx2_handle, + level_index, layer_index, face_index, + output_block_mem_ofs, + output_blocks_buf_size_in_blocks_or_pixels, + transcoder_texture_format_u32, + decode_flags, + output_row_pitch_in_blocks_or_pixels, + output_rows_in_pixels, + channel0, channel1, + state_handle + )) diff --git a/external/basis_universal/python/basisu_transcoder_pybind11.cpp b/external/basis_universal/python/basisu_transcoder_pybind11.cpp new file mode 100644 index 0000000000..4d4aa0f1b2 --- /dev/null +++ b/external/basis_universal/python/basisu_transcoder_pybind11.cpp @@ -0,0 +1,264 @@ +// File: basisu_transcoder_pybind11.cpp +// pybind11 native bindings for the transcoder's pure C API basisu_wasm_transcoder_api.h + +#include +#include + +#include "../encoder/basisu_wasm_transcoder_api.h" + +namespace py = pybind11; + +// wasm_bool_t is uint32_t — convert to Python bool +static inline bool to_bool(wasm_bool_t v) { return v != 0; } + +PYBIND11_MODULE(basisu_transcoder_python, m) { + m.doc() = "Native Basis Universal transcoder (pybind11 binding over basisu_wasm_transcoder_api)"; + + // ------------------------------------------------------------------------ + // High-level functions + // ------------------------------------------------------------------------ + m.def("get_version", &bt_get_version, + "Get BasisU transcoder version"); + + m.def("enable_debug_printf", + [](bool flag) { bt_enable_debug_printf(flag ? 1u : 0u); }, + "Enable or disable debug printf output"); + + m.def("init", &bt_init, + "Initialize transcoder library"); + + m.def("alloc", &bt_alloc, + "Allocate a buffer, returns uint64 offset/pointer"); + m.def("free", &bt_free, + "Free a buffer allocated by bt_alloc"); + + + // ------------------------------------------------------------------------ + // basis_tex_format helpers + // ------------------------------------------------------------------------ + m.def("basis_tex_format_is_xuastc_ldr", + [](uint32_t fmt) { return to_bool(bt_basis_tex_format_is_xuastc_ldr(fmt)); }); + + m.def("basis_tex_format_is_astc_ldr", + [](uint32_t fmt) { return to_bool(bt_basis_tex_format_is_astc_ldr(fmt)); }); + + m.def("basis_tex_format_get_block_width", + &bt_basis_tex_format_get_block_width); + + m.def("basis_tex_format_get_block_height", + &bt_basis_tex_format_get_block_height); + + m.def("basis_tex_format_is_hdr", + [](uint32_t fmt) { return to_bool(bt_basis_tex_format_is_hdr(fmt)); }); + + m.def("basis_tex_format_is_ldr", + [](uint32_t fmt) { return to_bool(bt_basis_tex_format_is_ldr(fmt)); }); + + + // ------------------------------------------------------------------------ + // transcoder_texture_format helpers + // ------------------------------------------------------------------------ + m.def("basis_get_bytes_per_block_or_pixel", + &bt_basis_get_bytes_per_block_or_pixel); + + m.def("basis_transcoder_format_has_alpha", + [](uint32_t tfmt) { return to_bool(bt_basis_transcoder_format_has_alpha(tfmt)); }); + + m.def("basis_transcoder_format_is_hdr", + [](uint32_t tfmt) { return to_bool(bt_basis_transcoder_format_is_hdr(tfmt)); }); + + m.def("basis_transcoder_format_is_ldr", + [](uint32_t tfmt) { return to_bool(bt_basis_transcoder_format_is_ldr(tfmt)); }); + + m.def("basis_transcoder_texture_format_is_astc", + [](uint32_t tfmt) { return to_bool(bt_basis_transcoder_texture_format_is_astc(tfmt)); }); + + m.def("basis_transcoder_format_is_uncompressed", + [](uint32_t tfmt) { return to_bool(bt_basis_transcoder_format_is_uncompressed(tfmt)); }); + + m.def("basis_get_uncompressed_bytes_per_pixel", + &bt_basis_get_uncompressed_bytes_per_pixel); + + m.def("basis_get_block_width", + &bt_basis_get_block_width); + + m.def("basis_get_block_height", + &bt_basis_get_block_height); + + m.def("basis_get_transcoder_texture_format_from_basis_tex_format", + &bt_basis_get_transcoder_texture_format_from_basis_tex_format); + + m.def("basis_is_format_supported", + [](uint32_t tfmt, uint32_t basis_fmt) { + return to_bool(bt_basis_is_format_supported(tfmt, basis_fmt)); + }); + + m.def("basis_compute_transcoded_image_size_in_bytes", + &bt_basis_compute_transcoded_image_size_in_bytes); + + + // ------------------------------------------------------------------------ + // KTX2 open/close & basic info + // ------------------------------------------------------------------------ + m.def("ktx2_open", &bt_ktx2_open, + "Open a KTX2 image from memory; returns handle"); + + m.def("ktx2_close", &bt_ktx2_close, + "Close a previously opened KTX2 handle"); + + m.def("ktx2_get_width", &bt_ktx2_get_width); + m.def("ktx2_get_height", &bt_ktx2_get_height); + m.def("ktx2_get_levels", &bt_ktx2_get_levels); + m.def("ktx2_get_faces", &bt_ktx2_get_faces); + m.def("ktx2_get_layers", &bt_ktx2_get_layers); + + m.def("ktx2_get_basis_tex_format", &bt_ktx2_get_basis_tex_format); + + m.def("ktx2_is_etc1s", + [](uint64_t h) { return to_bool(bt_ktx2_is_etc1s(h)); }); + + m.def("ktx2_is_uastc_ldr_4x4", + [](uint64_t h) { return to_bool(bt_ktx2_is_uastc_ldr_4x4(h)); }); + + m.def("ktx2_is_hdr", + [](uint64_t h) { return to_bool(bt_ktx2_is_hdr(h)); }); + + m.def("ktx2_is_hdr_4x4", + [](uint64_t h) { return to_bool(bt_ktx2_is_hdr_4x4(h)); }); + + m.def("ktx2_is_hdr_6x6", + [](uint64_t h) { return to_bool(bt_ktx2_is_hdr_6x6(h)); }); + + m.def("ktx2_is_ldr", + [](uint64_t h) { return to_bool(bt_ktx2_is_ldr(h)); }); + + m.def("ktx2_is_astc_ldr", + [](uint64_t h) { return to_bool(bt_ktx2_is_astc_ldr(h)); }); + + m.def("ktx2_is_xuastc_ldr", + [](uint64_t h) { return to_bool(bt_ktx2_is_xuastc_ldr(h)); }); + + m.def("ktx2_get_block_width", &bt_ktx2_get_block_width); + + m.def("ktx2_get_block_height", &bt_ktx2_get_block_height); + + m.def("ktx2_has_alpha", + [](uint64_t h) { return to_bool(bt_ktx2_has_alpha(h)); }); + + m.def("ktx2_get_dfd_color_model", &bt_ktx2_get_dfd_color_model); + m.def("ktx2_get_dfd_color_primaries", &bt_ktx2_get_dfd_color_primaries); + m.def("ktx2_get_dfd_transfer_func", &bt_ktx2_get_dfd_transfer_func); + + m.def("ktx2_is_srgb", + [](uint64_t h) { return to_bool(bt_ktx2_is_srgb(h)); }); + + m.def("ktx2_get_dfd_flags", &bt_ktx2_get_dfd_flags); + m.def("ktx2_get_dfd_total_samples", &bt_ktx2_get_dfd_total_samples); + m.def("ktx2_get_dfd_channel_id0", &bt_ktx2_get_dfd_channel_id0); + m.def("ktx2_get_dfd_channel_id1", &bt_ktx2_get_dfd_channel_id1); + + m.def("ktx2_is_video", + [](uint64_t h) { return to_bool(bt_ktx2_is_video(h)); }); + + m.def("ktx2_get_ldr_hdr_upconversion_nit_multiplier", + &bt_ktx2_get_ldr_hdr_upconversion_nit_multiplier); + + + // ------------------------------------------------------------------------ + // KTX2 per-level info + // ------------------------------------------------------------------------ + m.def("ktx2_get_level_orig_width", + &bt_ktx2_get_level_orig_width); + + m.def("ktx2_get_level_orig_height", + &bt_ktx2_get_level_orig_height); + + m.def("ktx2_get_level_actual_width", + &bt_ktx2_get_level_actual_width); + + m.def("ktx2_get_level_actual_height", + &bt_ktx2_get_level_actual_height); + + m.def("ktx2_get_level_num_blocks_x", + &bt_ktx2_get_level_num_blocks_x); + + m.def("ktx2_get_level_num_blocks_y", + &bt_ktx2_get_level_num_blocks_y); + + m.def("ktx2_get_level_total_blocks", + &bt_ktx2_get_level_total_blocks); + + m.def("ktx2_get_level_alpha_flag", + [](uint64_t h, uint32_t level, uint32_t layer, uint32_t face) { + return to_bool(bt_ktx2_get_level_alpha_flag(h, level, layer, face)); + }); + + m.def("ktx2_get_level_iframe_flag", + [](uint64_t h, uint32_t level, uint32_t layer, uint32_t face) { + return to_bool(bt_ktx2_get_level_iframe_flag(h, level, layer, face)); + }); + + + // ------------------------------------------------------------------------ + // Transcoding state and operations + // ------------------------------------------------------------------------ + m.def("ktx2_start_transcoding", + [](uint64_t h) { return to_bool(bt_ktx2_start_transcoding(h)); }); + + m.def("ktx2_create_transcode_state", + &bt_ktx2_create_transcode_state); + + m.def("ktx2_destroy_transcode_state", + &bt_ktx2_destroy_transcode_state); + + m.def("ktx2_transcode_image_level", + [](uint64_t ktx2_handle, + uint32_t level_index, uint32_t layer_index, uint32_t face_index, + uint64_t out_mem_ofs, + uint32_t out_blocks_or_pixels, + uint32_t transcoder_texture_format_u32, + uint32_t decode_flags, + uint32_t row_pitch_blocks_or_pixels, + uint32_t rows_in_pixels, + int channel0, int channel1, + uint64_t state_handle) + { + return to_bool(bt_ktx2_transcode_image_level( + ktx2_handle, + level_index, layer_index, face_index, + out_mem_ofs, + out_blocks_or_pixels, + transcoder_texture_format_u32, + decode_flags, + row_pitch_blocks_or_pixels, + rows_in_pixels, + channel0, channel1, + state_handle)); + }, + py::arg("ktx2_handle"), + py::arg("level_index"), + py::arg("layer_index"), + py::arg("face_index"), + py::arg("output_block_mem_ofs"), + py::arg("output_blocks_buf_size_in_blocks_or_pixels"), + py::arg("transcoder_texture_format_u32"), + py::arg("decode_flags"), + py::arg("output_row_pitch_in_blocks_or_pixels") = 0, + py::arg("output_rows_in_pixels") = 0, + py::arg("channel0") = -1, + py::arg("channel1") = -1, + py::arg("state_handle") = 0); + + m.def("read_memory", + [](uint64_t ptr, uint32_t size) { + return py::bytes((const char*)ptr, size); + }, + "Read `size` bytes starting at native memory address `ptr`"); + + m.def("write_memory", + [](uint64_t dest_ptr, py::buffer src) { + py::buffer_info info = src.request(); + memcpy((void*)dest_ptr, info.ptr, info.size * info.itemsize); + }, + "Write bytes/buffer-like object into native memory at address `ptr`"); +} diff --git a/external/basis_universal/python/dds_writer.py b/external/basis_universal/python/dds_writer.py new file mode 100644 index 0000000000..f1c95433c0 --- /dev/null +++ b/external/basis_universal/python/dds_writer.py @@ -0,0 +1,332 @@ +# dds_writer.py +# +# Minimal DDS writer that mirrors the C/C++ save_dds() implementation you provided. +# It writes a DX9-style DDS header, and optionally a DX10 extension header, +# followed by the raw compressed blocks. +# +# No mipmaps, no cubes, no 3D volumes – exactly like the original C code. + +import struct +import sys +from typing import Union + + +# --------------------------------------------------------------------------- +# FourCC helper (same as PIXEL_FMT_FOURCC macro) +# --------------------------------------------------------------------------- +def make_fourcc(a: str, b: str, c: str, d: str) -> int: + return (ord(a) | + (ord(b) << 8) | + (ord(c) << 16) | + (ord(d) << 24)) + + +# --------------------------------------------------------------------------- +# DDS-related constants (only the ones we actually use) +# --------------------------------------------------------------------------- + +# DDSD flags +DDSD_CAPS = 0x00000001 +DDSD_HEIGHT = 0x00000002 +DDSD_WIDTH = 0x00000004 +DDSD_PIXELFORMAT= 0x00001000 +DDSD_LINEARSIZE = 0x00080000 + +# DDPF flags +DDPF_FOURCC = 0x00000004 + +# DDSCAPS flags +DDSCAPS_TEXTURE = 0x00001000 + +# DXGI_FORMAT subset (values must match the C enum) +class DXGI_FORMAT: + UNKNOWN = 0 + BC1_UNORM = 71 + BC3_UNORM = 77 + BC4_UNORM = 80 + BC5_UNORM = 83 + # You can add more as needed; for DX10 header we just write the integer value. + +# DX10 resource dimension +class D3D10_RESOURCE_DIMENSION: + UNKNOWN = 0 + BUFFER = 1 + TEXTURE1D = 2 + TEXTURE2D = 3 + TEXTURE3D = 4 + + +# --------------------------------------------------------------------------- +# DDS writer class +# --------------------------------------------------------------------------- +class DDSWriter: + """ + Python port of the C save_dds() function. + + Usage: + writer = DDSWriter() + ok = writer.save_dds( + filename="out.dds", + width=width, + height=height, + blocks=bc_data, # bytes or bytearray + pixel_format_bpp=4, # e.g. 4 for BC1, 8 for BC3/4/5/etc. + dxgi_format=DXGI_FORMAT.BC1_UNORM, + srgb=False, + force_dx10_header=False, + ) + """ + + DDS_MAGIC = b"DDS " # same as fwrite("DDS ", 4, 1, pFile); + + def save_dds( + self, + filename: str, + width: int, + height: int, + blocks: Union[bytes, bytearray, memoryview], + pixel_format_bpp: int, + dxgi_format: int, + srgb: bool = False, + force_dx10_header: bool = False, + ) -> bool: + """ + Port of: + bool save_dds(const char* pFilename, + uint32_t width, uint32_t height, + const void* pBlocks, + uint32_t pixel_format_bpp, + DXGI_FORMAT dxgi_format, + bool srgb, + bool force_dx10_header); + + The 'blocks' buffer is written as-is (up to computed linear size). + """ + + # srgb is intentionally unused in the original C code (commented logic). + _ = srgb + + # Open file like the C code + try: + f = open(filename, "wb") + except OSError: + print(f"Failed creating file {filename}!", file=sys.stderr) + return False + + try: + # Write the "DDS " magic + f.write(self.DDS_MAGIC) + + # ----------------------------------------------------------------- + # Build DDSURFACEDESC2 equivalent + # ----------------------------------------------------------------- + # We'll pack DDSURFACEDESC2 as 31 uint32's (124 bytes) in little-endian: + # struct DDSURFACEDESC2 { + # uint32 dwSize; + # uint32 dwFlags; + # uint32 dwHeight; + # uint32 dwWidth; + # uint32 lPitch_or_dwLinearSize; + # uint32 dwBackBufferCount; + # uint32 dwMipMapCount; + # uint32 dwAlphaBitDepth; + # uint32 dwUnused0; + # uint32 lpSurface; + # DDCOLORKEY unused0; (2 * uint32) + # DDCOLORKEY unused1; (2 * uint32) + # DDCOLORKEY unused2; (2 * uint32) + # DDCOLORKEY unused3; (2 * uint32) + # DDPIXELFORMAT ddpfPixelFormat; (8 * uint32) + # DDSCAPS2 ddsCaps; (4 * uint32) + # uint32 dwUnused1; + # }; + + dwSize = 124 # sizeof(DDSURFACEDESC2) + + dwFlags = ( + DDSD_WIDTH | + DDSD_HEIGHT | + DDSD_PIXELFORMAT | + DDSD_CAPS + ) + + dwWidth = int(width) + dwHeight = int(height) + + # lPitch (actually LinearSize for compressed formats), same as: + # (((dwWidth + 3) & ~3) * ((dwHeight + 3) & ~3) * pixel_format_bpp) >> 3; + lPitch = ( + ((dwWidth + 3) & ~3) + * ((dwHeight + 3) & ~3) + * int(pixel_format_bpp) + ) >> 3 + + dwFlags |= DDSD_LINEARSIZE + + dwBackBufferCount = 0 + dwMipMapCount = 0 + dwAlphaBitDepth = 0 + dwUnused0 = 0 + lpSurface = 0 + + # DDCOLORKEY unused0..3, all zero + ddcolorkey_zero = [0, 0] * 4 # 4 DDCOLORKEY structs + + # DDPIXELFORMAT + # struct DDPIXELFORMAT { + # uint32 dwSize; + # uint32 dwFlags; + # uint32 dwFourCC; + # uint32 dwRGBBitCount; + # uint32 dwRBitMask; + # uint32 dwGBitMask; + # uint32 dwBBitMask; + # uint32 dwRGBAlphaBitMask; + # }; + ddpf_dwSize = 32 + ddpf_dwFlags = DDPF_FOURCC + ddpf_dwFourCC = 0 + ddpf_dwRGBBitCount = 0 + ddpf_dwRBitMask = 0 + ddpf_dwGBitMask = 0 + ddpf_dwBBitMask = 0 + ddpf_dwRGBAlphaBitMask = 0 + + # DDSCAPS2 + # struct DDSCAPS2 { + # uint32 dwCaps; + # uint32 dwCaps2; + # uint32 dwCaps3; + # uint32 dwCaps4; + # }; + ddsCaps_dwCaps = DDSCAPS_TEXTURE + ddsCaps_dwCaps2 = 0 + ddsCaps_dwCaps3 = 0 + ddsCaps_dwCaps4 = 0 + + dwUnused1 = 0 + + # Decide whether to use legacy FourCC (DXT1/DXT5/ATI1/ATI2) or DX10 header + use_legacy = ( + not force_dx10_header and + dxgi_format in ( + DXGI_FORMAT.BC1_UNORM, + DXGI_FORMAT.BC3_UNORM, + DXGI_FORMAT.BC4_UNORM, + DXGI_FORMAT.BC5_UNORM, + ) + ) + + if use_legacy: + if dxgi_format == DXGI_FORMAT.BC1_UNORM: + ddpf_dwFourCC = make_fourcc('D', 'X', 'T', '1') + elif dxgi_format == DXGI_FORMAT.BC3_UNORM: + ddpf_dwFourCC = make_fourcc('D', 'X', 'T', '5') + elif dxgi_format == DXGI_FORMAT.BC4_UNORM: + ddpf_dwFourCC = make_fourcc('A', 'T', 'I', '1') + elif dxgi_format == DXGI_FORMAT.BC5_UNORM: + ddpf_dwFourCC = make_fourcc('A', 'T', 'I', '2') + else: + # Write DX10 header, FourCC = "DX10" + ddpf_dwFourCC = make_fourcc('D', 'X', '1', '0') + + # Build the 31 uint32's for DDSURFACEDESC2 + header_values = [ + dwSize, + dwFlags, + dwHeight, + dwWidth, + lPitch, + dwBackBufferCount, + dwMipMapCount, + dwAlphaBitDepth, + dwUnused0, + lpSurface, + ] + + header_values.extend(ddcolorkey_zero) # 8 uint32's + + ddpf_values = [ + ddpf_dwSize, + ddpf_dwFlags, + ddpf_dwFourCC, + ddpf_dwRGBBitCount, + ddpf_dwRBitMask, + ddpf_dwGBitMask, + ddpf_dwBBitMask, + ddpf_dwRGBAlphaBitMask, + ] + header_values.extend(ddpf_values) # 8 uint32's + + ddsCaps_values = [ + ddsCaps_dwCaps, + ddsCaps_dwCaps2, + ddsCaps_dwCaps3, + ddsCaps_dwCaps4, + ] + header_values.extend(ddsCaps_values) # 4 uint32's + + header_values.append(dwUnused1) # final uint32 + + if len(header_values) != 31: + raise RuntimeError("Internal error: DDSURFACEDESC2 must contain 31 uint32's") + + # Pack and write DDSURFACEDESC2 + dds_header = struct.pack("<31I", *header_values) + f.write(dds_header) + + # If needed, write the DX10 header (DDS_HEADER_DXT10) + if not use_legacy: + # struct DDS_HEADER_DXT10 { + # DXGI_FORMAT dxgiFormat; + # D3D10_RESOURCE_DIMENSION resourceDimension; + # uint32 miscFlag; + # uint32 arraySize; + # uint32 miscFlags2; + # }; + dxgiFormat = int(dxgi_format) + resourceDimension = D3D10_RESOURCE_DIMENSION.TEXTURE2D + miscFlag = 0 + arraySize = 1 + miscFlags2 = 0 + + dxt10_header = struct.pack( + "<5I", + dxgiFormat, + resourceDimension, + miscFlag, + arraySize, + miscFlags2, + ) + f.write(dxt10_header) + + # ----------------------------------------------------------------- + # Write the actual texture data blocks (pBlocks) + # ----------------------------------------------------------------- + + # C code: fwrite(pBlocks, desc.lPitch, 1, pFile); + # i.e. write exactly lPitch bytes. + data = memoryview(blocks) + if len(data) < lPitch: + raise ValueError( + f"blocks buffer too small: need at least {lPitch} bytes, got {len(data)}" + ) + f.write(data[:lPitch]) + + except Exception as e: + # Mimic the C-style error reporting as much as practical + print(f"Failed writing to DDS file {filename}: {e}", file=sys.stderr) + try: + f.close() + except Exception: + pass + return False + + # Close file + try: + f.close() + except OSError: + print(f"Failed closing DDS file {filename}!", file=sys.stderr) + return False + + return True diff --git a/external/basis_universal/python/explode_ktx2_file.py b/external/basis_universal/python/explode_ktx2_file.py new file mode 100644 index 0000000000..0c414b0602 --- /dev/null +++ b/external/basis_universal/python/explode_ktx2_file.py @@ -0,0 +1,413 @@ +#!/usr/bin/env python3 +""" +explode_ktx2_file.py +FULL LDR/HDR KTX2 EXPLODER + FULL API INTROSPECTION + ASTC + BC7/BC6H OUTPUT + +Usage: + python3 explode_ktx2_file.py input.ktx2 + python3 explode_ktx2_file.py input.ktx2 --info-only +""" + +# Python Dependencies (beyond basisu_py): +# numpy +# pillow +# imageio (v3+) +# wasmtime +# +# System Dependencies: +# OpenImageIO ("oiiotool") -- required for EXR output +# +# Install Python deps: +# pip install numpy pillow imageio wasmtime +# +# On Ubuntu: +# sudo apt install openimageio-tools +# +# On macOS (Homebrew): +# brew install openimageio + +import sys +import os +import numpy as np +import subprocess +import tempfile +import imageio.v3 as iio +from PIL import Image + +from basisu_py import Transcoder +from basisu_py.constants import TranscoderTextureFormat as TF + +# Writers located in same directory as this script +from astc_writer import write_astc_file +from dds_writer import DDSWriter + + +# ============================================================================ +# File-writing helpers +# ============================================================================ +def save_exr(path, rgba32f): + """ + Save float32 RGBA as EXR if possible. + If oiiotool is not available, save TIFF instead (Windows-safe). + """ + import numpy as np + import imageio.v3 as iio + import subprocess, tempfile, os + + # Write temp TIFF + with tempfile.NamedTemporaryFile(suffix=".tiff", delete=False) as tmp: + temp_path = tmp.name + + iio.imwrite(temp_path, rgba32f.astype(np.float32)) + + # Try EXR via oiiotool + try: + subprocess.run(["oiiotool", temp_path, "-o", path], check=True) + os.remove(temp_path) + print(" Wrote EXR:", path) + return + + except Exception: + # --- FALLBACK: save TIFF --- + fallback_path = path + ".tiff" + + # Windows cannot overwrite files via rename(), so remove first + if os.path.exists(fallback_path): + os.remove(fallback_path) + + # os.replace() always overwrites + os.replace(temp_path, fallback_path) + + print(" [Fallback] Wrote TIFF instead:", fallback_path) + + +def save_png(path, rgba8): + img = Image.fromarray(rgba8, mode="RGBA") + img.save(path) + print(f" PNG saved: {path}") + + +# ============================================================================ +# Pretty header +# ============================================================================ +def print_header(title): + print("\n" + "=" * 90) + print(title) + print("=" * 90) + + +# ============================================================================ +# Full top-level metadata dump (ALL API) +# ============================================================================ +def dump_all_top_level(t, h): + print_header("TOP-LEVEL KTX2 METADATA FULL API") + + print("Backend :", t.backend_name) + print("Version :", t.get_version()) + print("Width :", t.get_width(h)) + print("Height :", t.get_height(h)) + print("Levels :", t.get_levels(h)) + print("Faces :", t.get_faces(h)) + + layers = t.get_layers(h) + eff_layers = layers if layers > 0 else 1 + print("Layers (raw) :", layers) + print("Layers (effective) :", eff_layers) + + fmt = t.get_basis_tex_format(h) + print("\nBasisTexFormat :", fmt) + + print("\nKTX2 Format Flags:") + print(" is_etc1s :", t.is_etc1s(h)) + print(" is_uastc_ldr_4x4 :", t.is_uastc_ldr_4x4(h)) + print(" is_xuastc_ldr :", t.is_xuastc_ldr(h)) + print(" is_astc_ldr :", t.is_astc_ldr(h)) + print(" is_hdr :", t.is_hdr(h)) + print(" is_hdr_4x4 :", t.is_hdr_4x4(h)) + print(" is_hdr_6x6 :", t.is_hdr_6x6(h)) + print(" is_ldr :", t.is_ldr(h)) + print(" is_srgb :", t.is_srgb(h)) + print(" is_video :", t.is_video(h)) + print(" has_alpha :", t.has_alpha(h)) + + print("\nBlock Info:") + print(" block_width :", t.get_block_width(h)) + print(" block_height :", t.get_block_height(h)) + + print("\nDFD Info:") + print(" color_model :", t.get_dfd_color_model(h)) + print(" color_primaries :", t.get_dfd_color_primaries(h)) + print(" transfer_func :", t.get_dfd_transfer_func(h)) + print(" flags :", t.get_dfd_flags(h)) + print(" total_samples :", t.get_dfd_total_samples(h)) + print(" channel_id0 :", t.get_dfd_channel_id0(h)) + print(" channel_id1 :", t.get_dfd_channel_id1(h)) + + if t.is_hdr(h): + print(" hdr_nit_multiplier :", t.get_ldr_hdr_upconversion_nit_multiplier(h)) + + +# ============================================================================ +# BasisTexFormat helpers +# ============================================================================ +def dump_basis_tex_format_helpers(t, h): + print_header("BasisTexFormat HELPERS (FULL)") + + fmt = t.get_basis_tex_format(h) + print("basis_tex_format:", fmt) + + print("is_xuastc_ldr :", t.basis_tex_format_is_xuastc_ldr(fmt)) + print("is_astc_ldr :", t.basis_tex_format_is_astc_ldr(fmt)) + print("block width :", t.basis_tex_format_get_block_width(fmt)) + print("block height :", t.basis_tex_format_get_block_height(fmt)) + print("is_hdr :", t.basis_tex_format_is_hdr(fmt)) + print("is_ldr :", t.basis_tex_format_is_ldr(fmt)) + + +# ============================================================================ +# Level / Layer / Face metadata dump +# ============================================================================ +def dump_per_level_info(t, h): + print_header("PER-LEVEL / PER-LAYER / PER-FACE METADATA") + + levels = t.get_levels(h) + faces = t.get_faces(h) + layers = t.get_layers(h) + if layers == 0: + layers = 1 + + for level in range(levels): + for layer in range(layers): + for face in range(faces): + print(f"\nLevel={level}, Layer={layer}, Face={face}") + print(" orig_width :", t.get_level_orig_width(h, level, layer, face)) + print(" orig_height :", t.get_level_orig_height(h, level, layer, face)) + print(" actual_width :", t.get_level_actual_width(h, level, layer, face)) + print(" actual_height:", t.get_level_actual_height(h, level, layer, face)) + print(" blocks_x :", t.get_level_num_blocks_x(h, level, layer, face)) + print(" blocks_y :", t.get_level_num_blocks_y(h, level, layer, face)) + print(" total_blocks :", t.get_level_total_blocks(h, level, layer, face)) + print(" alpha_flag :", t.get_level_alpha_flag(h, level, layer, face)) + print(" iframe_flag :", t.get_level_iframe_flag(h, level, layer, face)) + + +# ============================================================================ +# ASTC Selection +# ============================================================================ +def choose_astc_format(t, h): + fmt = t.get_basis_tex_format(h) + tfmt = t.basis_get_transcoder_texture_format_from_basis_tex_format(fmt) + bw = t.basis_get_block_width(tfmt) + bh = t.basis_get_block_height(tfmt) + + print_header("ASTC SELECTION") + print("ASTC TF:", tfmt) + print(f"Block dims: {bw}x{bh}") + return tfmt, bw, bh + + +# ============================================================================ +# BC Format Selection +# ============================================================================ +def choose_bc_format(t, h): + if t.is_hdr(h): + print_header("HDR -> BC6H") + return TF.TF_BC6H, 8, 95 # DXGI_FORMAT_BC6H_UF16 + else: + print_header("LDR -> BC7") + return TF.TF_BC7_RGBA, 8, 98 # DXGI_FORMAT_BC7_UNORM + + +# ============================================================================ +# Full explode transcoding (using handle API + per-level dims) +# ============================================================================ +def explode_transcode(t, h): + levels = t.get_levels(h) + faces = t.get_faces(h) + layers = t.get_layers(h) + if layers == 0: + layers = 1 + + astc_tfmt, astc_bw, astc_bh = choose_astc_format(t, h) + bc_tfmt, bc_bpp, bc_dxgi = choose_bc_format(t, h) + + ddsw = DDSWriter() + print_header("BEGIN EXPLODE TRANSCODING (handle API)") + + for level in range(levels): + for layer in range(layers): + for face in range(faces): + + print(f"\n- Level={level} Layer={layer} Face={face}") + + ow = t.get_level_orig_width(h, level, layer, face) + oh = t.get_level_orig_height(h, level, layer, face) + print(f" Level orig dims: {ow}x{oh}") + + # ASTC + astc_blocks = t.transcode_tfmt_handle( + h, astc_tfmt, + level=level, layer=layer, face=face, + decode_flags=0, channel0=-1, channel1=-1 + ) + astc_name = f"astc_L{level}_Y{layer}_F{face}.astc" + write_astc_file(astc_name, astc_blocks, astc_bw, astc_bh, ow, oh) + print(" ASTC saved:", astc_name) + + # BC6H / BC7 + bc_blocks = t.transcode_tfmt_handle( + h, bc_tfmt, + level=level, layer=layer, face=face, + decode_flags=0, channel0=-1, channel1=-1 + ) + if t.is_hdr(h): + dds_name = f"bc6h_L{level}_Y{layer}_F{face}.dds" + else: + dds_name = f"bc7_L{level}_Y{layer}_F{face}.dds" + + ddsw.save_dds( + dds_name, + width=ow, height=oh, + blocks=bc_blocks, + pixel_format_bpp=bc_bpp, + dxgi_format=bc_dxgi, + srgb=False, + force_dx10_header=True, + ) + print(" DDS saved :", dds_name) + + print_header("EXPLODE TRANSCODING COMPLETE") + + +# ============================================================================ +# Decode each (Level, Layer, Face) to PNG or EXR +# ============================================================================ +def explode_decode_images(t, h): + print_header("BEGIN EXPLODE IMAGE DECODE (PNG/EXR)") + + levels = t.get_levels(h) + faces = t.get_faces(h) + layers = t.get_layers(h) + if layers == 0: + layers = 1 + + hdr = t.is_hdr(h) + + for level in range(levels): + for layer in range(layers): + for face in range(faces): + + print(f"\n- Decode Level={level} Layer={layer} Face={face}") + + ow = t.get_level_orig_width(h, level, layer, face) + oh = t.get_level_orig_height(h, level, layer, face) + + if hdr: + rgba32f = t.decode_rgba_hdr_handle(h, level, layer, face) + outname = f"exr_L{level}_Y{layer}_F{face}.exr" + save_exr(outname, rgba32f) + else: + rgba8 = t.decode_rgba_handle(h, level, layer, face) + outname = f"png_L{level}_Y{layer}_F{face}.png" + save_png(outname, rgba8) + + print_header("IMAGE DECODE COMPLETE") + +def dump_transcoder_texture_format_helpers(t): + print_header("TranscoderTextureFormat HELPERS (FULL)") + + test_formats = [ + # uncompressed + TF.TF_RGBA32, TF.TF_RGB565, TF.TF_BGR565, + TF.TF_RGBA4444, TF.TF_RGB_HALF, TF.TF_RGBA_HALF, TF.TF_RGB_9E5, + + # basic compressed + TF.TF_ETC1_RGB, TF.TF_ETC2_RGBA, + TF.TF_BC1_RGB, TF.TF_BC3_RGBA, + TF.TF_BC4_R, TF.TF_BC5_RG, + TF.TF_BC7_RGBA, TF.TF_BC6H, + TF.TF_ETC2_EAC_R11, TF.TF_ETC2_EAC_RG11, + TF.TF_FXT1_RGB, + TF.TF_PVRTC1_4_RGB, TF.TF_PVRTC1_4_RGBA, + TF.TF_PVRTC2_4_RGB, TF.TF_PVRTC2_4_RGBA, + TF.TF_ATC_RGB, TF.TF_ATC_RGBA, + + # HDR ASTC + TF.TF_ASTC_HDR_4X4_RGBA, + TF.TF_ASTC_HDR_6X6_RGBA, + + # LDR ASTC + TF.TF_ASTC_LDR_4X4_RGBA, + TF.TF_ASTC_LDR_5X4_RGBA, TF.TF_ASTC_LDR_5X5_RGBA, + TF.TF_ASTC_LDR_6X5_RGBA, TF.TF_ASTC_LDR_6X6_RGBA, + TF.TF_ASTC_LDR_8X5_RGBA, TF.TF_ASTC_LDR_8X6_RGBA, + TF.TF_ASTC_LDR_10X5_RGBA, TF.TF_ASTC_LDR_10X6_RGBA, + TF.TF_ASTC_LDR_8X8_RGBA, TF.TF_ASTC_LDR_10X8_RGBA, + TF.TF_ASTC_LDR_10X10_RGBA, TF.TF_ASTC_LDR_12X10_RGBA, + TF.TF_ASTC_LDR_12X12_RGBA, + ] + + for tfmt in test_formats: + print(f"\nTF={tfmt}") + print(" has_alpha :", t.basis_transcoder_format_has_alpha(tfmt)) + print(" is_hdr :", t.basis_transcoder_format_is_hdr(tfmt)) + print(" is_ldr :", t.basis_transcoder_format_is_ldr(tfmt)) + print(" is_astc :", t.basis_transcoder_texture_format_is_astc(tfmt)) + print(" is_uncompressed :", t.basis_transcoder_format_is_uncompressed(tfmt)) + print(" bytes/block :", t.basis_get_bytes_per_block_or_pixel(tfmt)) + print(" block_width :", t.basis_get_block_width(tfmt)) + print(" block_height :", t.basis_get_block_height(tfmt)) + + +def main(): + if len(sys.argv) < 2: + print("Usage: python explode_ktx2_file.py input.ktx2 [--info-only] [--print-tf]") + return 1 + + args = sys.argv[1:] + info_only = "--info-only" in args + print_tf = "--print-tf" in args or "--transcoder-formats" in args + + # Determine input filename + input_file = None + for a in args: + if not a.startswith("--"): + input_file = a + break + + if input_file is None: + print("Error: No input file provided.") + return 1 + + ktx_bytes = open(input_file, "rb").read() + + t = Transcoder() + h = t.open(ktx_bytes) + t.start_transcoding(h) + + # Full metadata + dump_all_top_level(t, h) + dump_basis_tex_format_helpers(t, h) + dump_per_level_info(t, h) + + # Optional TF helpers + if print_tf: + dump_transcoder_texture_format_helpers(t) + + if info_only: + print_header("INFO-ONLY MODE NO FILES WRITTEN") + t.close(h) + return 0 + + # Full output + explode_transcode(t, h) + explode_decode_images(t, h) + + t.close(h) + print("Success") + return 0 + +if __name__ == "__main__": + sys.exit(main()) + + diff --git a/external/basis_universal/python/lowlevel_test_native/__init__.py b/external/basis_universal/python/lowlevel_test_native/__init__.py new file mode 100644 index 0000000000..143f486c05 --- /dev/null +++ b/external/basis_universal/python/lowlevel_test_native/__init__.py @@ -0,0 +1 @@ +# __init__.py diff --git a/external/basis_universal/python/lowlevel_test_native/basic_test.py b/external/basis_universal/python/lowlevel_test_native/basic_test.py new file mode 100644 index 0000000000..44004e2dd1 --- /dev/null +++ b/external/basis_universal/python/lowlevel_test_native/basic_test.py @@ -0,0 +1,127 @@ +# basic_test.py +import sys +sys.path.append("basisu_py") # make sure Python can load the .so + +import basisu_python as bu +from constants import * + +import ctypes +import math + +def generate_swirl_rgba8(width, height): + """ + Generate a smooth colorful swirl procedural RGBA8 test image. + Returns: a ctypes array of type (c_ubyte * (width * height * 4)) + """ + pixel_count = width * height * 4 + img = (ctypes.c_ubyte * pixel_count)() + + for y in range(height): + for x in range(width): + i = (y * width + x) * 4 + + dx = x - width / 2 + dy = y - height / 2 + + dist = math.hypot(dx, dy) + angle = math.atan2(dy, dx) + + # Color swirl pattern + r = int((math.sin(dist * 0.15) * 0.5 + 0.5) * 255) + g = int((math.sin(angle * 3.0) * 0.5 + 0.5) * 255) + b = int((math.cos(dist * 0.10 + angle * 2.0) * 0.5 + 0.5) * 255) + + img[i + 0] = r & 255 + img[i + 1] = g & 255 + img[i + 2] = b & 255 + img[i + 3] = 255 + + return img + +def generate_test_pattern_rgba8(width, height): + """ + Generate a simple deterministic RGBA8 test pattern: + R = x + G = y + B = x^y + A = 255 + """ + import ctypes + + pixel_count = width * height * 4 + img = (ctypes.c_ubyte * pixel_count)() + + for y in range(height): + for x in range(width): + i = (y * width + x) * 4 + + img[i + 0] = x & 0xFF + img[i + 1] = y & 0xFF + img[i + 2] = (x ^ y) & 0xFF + img[i + 3] = 255 + + return img + +# ------------------------------------------------------------ +# BasisU compression test (NATIVE C++) +# ------------------------------------------------------------ + +print("Native BasisU version:", bu.get_version()) +bu.init() + +# Create comp params +params = bu.new_params() +print("Params handle:", params) + +# Create RGBA8 swirl (64 x 64) +W, H = 512, 512 +pixel_count = W * H * 4 + +# Generate swirl image in PYTHON memory + +img = generate_swirl_rgba8(W, H) +#img = generate_test_pattern_rgba8(W, H) + +# Allocate memory inside NATIVE C++ heap +img_ptr = bu.alloc(pixel_count) + +# Copy Python swirl image ? C++ heap buffer +ctypes.memmove(img_ptr, img, pixel_count) + +# Set into BasisU +pitch = W * 4 +ok = bu.set_image_rgba32(params, 0, img_ptr, W, H, pitch) +print("Set image:", ok) + +# Compress (UASTC LDR 4x4 = 1) +ok = bu.compress( + params, + BasisTexFormat.cASTC_LDR_4x4, # basis_tex_format + BasisQuality.MAX, # quality + BasisEffort.DEFAULT, # effort + BasisFlags.KTX2_OUTPUT | BasisFlags.SRGB | BasisFlags.THREADED | BasisFlags.DEBUG_OUTPUT | BasisFlags.VERBOSE, # flags + 0.0 # rdo +) +print("Compress:", ok) + +# Retrieve compressed data +size = bu.get_comp_data_size(params) +ofs = bu.get_comp_data_ofs(params) + +print("Output size =", size, "ptr =", ofs) + +# Copy bytes out of native memory +byte_ptr = ctypes.cast(ofs, ctypes.POINTER(ctypes.c_ubyte)) +blob = bytes(byte_ptr[i] for i in range(size)) + +print("First 16 bytes:", blob[:16]) + +# Save to KTX2 +with open("out_native.ktx2", "wb") as f: + f.write(blob) + +print("Saved out_native.ktx2") + +# Cleanup +bu.delete_params(params) +bu.free(img_ptr) diff --git a/external/basis_universal/python/lowlevel_test_native/example_capi_python.py b/external/basis_universal/python/lowlevel_test_native/example_capi_python.py new file mode 100644 index 0000000000..d24f48e805 --- /dev/null +++ b/external/basis_universal/python/lowlevel_test_native/example_capi_python.py @@ -0,0 +1,481 @@ +#!/usr/bin/env python3 +# example_capi_python.py +# +# Simple Python port of example_capi.c using native C++ pybind11 bindings: +# - basisu_python (encoder) +# - basisu_transcoder_python (transcoder) +# +# Requires: +# basisu_py/basisu_python*.so +# basisu_py/basisu_transcoder_python*.so +# basisu_py/constants.py + +import sys +import os +import math +import ctypes + +# Make sure Python can see the native .so's and the shared constants +sys.path.append("basisu_py") + +import basisu_python as bu +import basisu_transcoder_python as bt +from constants import BasisTexFormat, BasisFlags +from constants import TranscoderTextureFormat as TF +from constants import TranscodeDecodeFlags as DF + +TRUE = 1 +FALSE = 0 + +# ------------------------------------------------------------ +# Utility: write raw bytes to a file +# ------------------------------------------------------------ + +def write_blob_to_file(filename: str, data: bytes) -> int: + print(f"write_blob_to_file: writing {len(data)} bytes to {filename!r}") + if not filename or data is None: + print(" ERROR: invalid filename or data") + return FALSE + + try: + with open(filename, "wb") as f: + f.write(data) + print(" OK") + return TRUE + except OSError as e: + print(" ERROR:", e) + return FALSE + +# ------------------------------------------------------------ +# TGA writer (24/32bpp) - port of write_tga_image() +# ------------------------------------------------------------ + +def write_tga_image(filename: str, w: int, h: int, has_alpha: bool, pixels_rgba_ptr: int) -> int: + """ + filename: path to TGA file + w, h: image dimensions + has_alpha: True for 32bpp, False for 24bpp + pixels_rgba_ptr: C pointer (uint64) to RGBA or RGB data in native heap + """ + print(f"write_tga_image: {filename!r}, {w}x{h}, has_alpha={has_alpha}, ptr=0x{pixels_rgba_ptr:x}") + if not filename or pixels_rgba_ptr == 0 or w <= 0 or h <= 0: + print(" ERROR: invalid args") + return -1 + + bytes_per_pixel = 4 if has_alpha else 3 + row_bytes = w * bytes_per_pixel + total_bytes = row_bytes * h + + # Create a ctypes buffer that views the native memory + SrcArrayType = ctypes.c_ubyte * total_bytes + src = SrcArrayType.from_address(pixels_rgba_ptr) + + try: + with open(filename, "wb") as f: + header = bytearray(18) + header[2] = 2 # uncompressed true-color + header[12] = w & 0xFF + header[13] = (w >> 8) & 0xFF + header[14] = h & 0xFF + header[15] = (h >> 8) & 0xFF + header[16] = 32 if has_alpha else 24 + header[17] = 8 if has_alpha else 0 # bottom-left origin (with or without alpha) + + f.write(header) + + # temp row buffer for BGRA/BGR + row_buf = bytearray(row_bytes) + + # TGA expects rows bottom-to-top + for y in range(h): + src_y = h - 1 - y + row_start = src_y * row_bytes + src_row = src[row_start:row_start + row_bytes] + + if has_alpha: + # RGBA -> BGRA + for x in range(w): + si = x*4 + di = x*4 + row_buf[di + 0] = src_row[si + 2] # B + row_buf[di + 1] = src_row[si + 1] # G + row_buf[di + 2] = src_row[si + 0] # R + row_buf[di + 3] = src_row[si + 3] # A + else: + # RGB -> BGR + for x in range(w): + si = x*3 + di = x*3 + row_buf[di + 0] = src_row[si + 2] # B + row_buf[di + 1] = src_row[si + 1] # G + row_buf[di + 2] = src_row[si + 0] # R + + f.write(row_buf) + + print(" Wrote TGA:", filename) + return 0 + except OSError as e: + print(" ERROR writing TGA:", e) + return -2 + +# ------------------------------------------------------------ +# ASTC writer - port of write_astc_file() +# ------------------------------------------------------------ + +def write_astc_file(filename: str, + blocks_ptr: int, + block_width: int, + block_height: int, + dim_x: int, + dim_y: int) -> int: + print(f"write_astc_file: {filename!r}, block={block_width}x{block_height}, dim={dim_x}x{dim_y}, ptr=0x{blocks_ptr:x}") + if not filename or blocks_ptr == 0: + print(" ERROR: invalid filename or pointer") + return 0 + + assert dim_x > 0 and dim_y > 0 + assert 4 <= block_width <= 12 + assert 4 <= block_height <= 12 + + num_blocks_x = (dim_x + block_width - 1) // block_width + num_blocks_y = (dim_y + block_height - 1) // block_height + total_blocks = num_blocks_x * num_blocks_y + total_bytes = total_blocks * 16 # 16 bytes per ASTC block + + print(f" num_blocks_x={num_blocks_x}, num_blocks_y={num_blocks_y}, total_blocks={total_blocks}, total_bytes={total_bytes}") + + # View native memory + BlockArray = ctypes.c_ubyte * total_bytes + src = BlockArray.from_address(blocks_ptr) + + try: + with open(filename, "wb") as f: + # Magic + f.write(bytes([0x13, 0xAB, 0xA1, 0x5C])) + + # Block dimensions x,y,z (=1) + f.write(bytes([block_width & 0xFF, block_height & 0xFF, 1])) + + # dim_x (24-bit LE) + f.write(bytes([dim_x & 0xFF, (dim_x >> 8) & 0xFF, (dim_x >> 16) & 0xFF])) + + # dim_y (24-bit LE) + f.write(bytes([dim_y & 0xFF, (dim_y >> 8) & 0xFF, (dim_y >> 16) & 0xFF])) + + # dim_z = 1 (24-bit LE) + f.write(bytes([1, 0, 0])) + + # Block data + f.write(bytes(src)) + + print(" Wrote ASTC:", filename) + return 1 + except OSError as e: + print(" ERROR writing ASTC:", e) + return 0 + +# ------------------------------------------------------------ +# Procedural RGBA pattern (ported & fixed version) +# ------------------------------------------------------------ + +def create_pretty_rgba_pattern(w: int, h: int) -> bytes: + print(f"create_pretty_rgba_pattern: {w}x{h}") + if w <= 0 or h <= 0: + return None + + out = bytearray(w * h * 4) + for y in range(h): + for x in range(w): + fx = x / float(w) + fy = y / float(h) + + # Colorful plasma-type formula + v = math.sin(fx * 12.0 + fy * 4.0) + v += math.sin(fy * 9.0 - fx * 6.0) + v += math.sin((fx + fy) * 7.0) + v = v * 0.25 + 0.5 # scale 0..1 + + L = 1.5 + + r = int(round(255.0 * math.sin(v * 6.28) * L)) + g = int(round(255.0 * (1.0 - v) * L)) + b = int(round(255.0 * v * L)) + + if r < 0: r = 0 + elif r > 255: r = 255 + if g < 0: g = 0 + elif g > 255: g = 255 + if b < 0: b = 0 + elif b > 255: b = 255 + + i = (y * w + x) * 4 + out[i+0] = r + out[i+1] = g + out[i+2] = b + out[i+3] = 255 + + return bytes(out) + +# ------------------------------------------------------------ +# Transcode a KTX2 blob (ported from transcode_ktx2_file) +# ------------------------------------------------------------ + +def transcode_ktx2_file(ktx2_data: bytes) -> int: + if not ktx2_data: + print("transcode_ktx2_file: empty data") + return FALSE + + size = len(ktx2_data) + print(f"transcode_ktx2_file: size={size} bytes") + + if size > 0xFFFFFFFF: + print(" ERROR: size too large for 32-bit length") + return FALSE + + # Allocate memory in transcoder heap and copy KTX2 data + ktx2_data_ofs = bt.alloc(size) + if not ktx2_data_ofs: + print(" ERROR: bt.alloc failed") + return FALSE + + print(f" KTX2 data allocated at 0x{ktx2_data_ofs:x}") + ctypes.memmove(ktx2_data_ofs, ktx2_data, size) + + # Open KTX2 + ktx2_handle = bt.ktx2_open(ktx2_data_ofs, size) + if not ktx2_handle: + print(" ERROR: bt.ktx2_open failed") + bt.free(ktx2_data_ofs) + return FALSE + + print(f" KTX2 handle = 0x{ktx2_handle:x}") + + if not bt.ktx2_is_ldr(ktx2_handle): + print(" ERROR: This sample only handles LDR KTX2 files") + bt.ktx2_close(ktx2_handle) + bt.free(ktx2_data_ofs) + return FALSE + + if not bt.ktx2_start_transcoding(ktx2_handle): + print(" ERROR: bt.ktx2_start_transcoding failed") + bt.ktx2_close(ktx2_handle) + bt.free(ktx2_data_ofs) + return FALSE + + width = bt.ktx2_get_width(ktx2_handle) + height = bt.ktx2_get_height(ktx2_handle) + levels = bt.ktx2_get_levels(ktx2_handle) + faces = bt.ktx2_get_faces(ktx2_handle) + layers = bt.ktx2_get_layers(ktx2_handle) + + basis_tex_format = bt.ktx2_get_basis_tex_format(ktx2_handle) + block_width = bt.ktx2_get_block_width(ktx2_handle) + block_height = bt.ktx2_get_block_height(ktx2_handle) + is_srgb = bt.ktx2_is_srgb(ktx2_handle) + + print(f"KTX2 Dimensions: {width}x{height}, Levels={levels}, Faces={faces}, Layers={layers}") + print(f"basis_tex_format: {basis_tex_format}") + print(f"Block dimensions: {block_width}x{block_height}") + print(f"is sRGB: {is_srgb}") + + if layers < 1: + layers = 1 + + assert width >= 1 and height >= 1 + assert levels >= 1 + assert faces in (1, 6) + + # Optional: separate transcode state (thread-local) + trans_state = bt.ktx2_create_transcode_state() + print(f"trans_state handle = 0x{trans_state:x}") + + for level_index in range(levels): + for layer_index in range(layers): + for face_index in range(faces): + print(f"- Level {level_index}, layer {layer_index}, face {face_index}") + ow = bt.ktx2_get_level_orig_width(ktx2_handle, level_index, layer_index, face_index) + oh = bt.ktx2_get_level_orig_height(ktx2_handle, level_index, layer_index, face_index) + aw = bt.ktx2_get_level_actual_width(ktx2_handle, level_index, layer_index, face_index) + ah = bt.ktx2_get_level_actual_height(ktx2_handle, level_index, layer_index, face_index) + nbx = bt.ktx2_get_level_num_blocks_x(ktx2_handle, level_index, layer_index, face_index) + nby = bt.ktx2_get_level_num_blocks_y(ktx2_handle, level_index, layer_index, face_index) + tblocks = bt.ktx2_get_level_total_blocks(ktx2_handle, level_index, layer_index, face_index) + alpha_flag = bt.ktx2_get_level_alpha_flag(ktx2_handle, level_index, layer_index, face_index) + iframe_flag = bt.ktx2_get_level_iframe_flag(ktx2_handle, level_index, layer_index, face_index) + + print(f" Orig dimensions: {ow}x{oh}, actual: {aw}x{ah}") + print(f" Block dims: {nbx}x{nby}, total blocks: {tblocks}") + print(f" Alpha={alpha_flag}, I-frame={iframe_flag}") + + # 1) Transcode to RGBA32 and write TGA + tga_name = f"transcoded_{level_index}_{layer_index}_{face_index}.tga" + trans_size_rgba = bt.basis_compute_transcoded_image_size_in_bytes(TF.TF_RGBA32, ow, oh) + assert trans_size_rgba > 0 + rgba_ofs = bt.alloc(trans_size_rgba) + print(f" RGBA buf ofs=0x{rgba_ofs:x}, size={trans_size_rgba}") + + decode_flags = 0 + ok = bt.ktx2_transcode_image_level( + ktx2_handle, + level_index, layer_index, face_index, + rgba_ofs, + trans_size_rgba, + TF.TF_RGBA32, + decode_flags, + 0, 0, -1, -1, + trans_state + ) + print(" ktx2_transcode_image_level(RGBA32):", ok) + if not ok: + bt.free(rgba_ofs) + bt.ktx2_destroy_transcode_state(trans_state) + bt.ktx2_close(ktx2_handle) + bt.free(ktx2_data_ofs) + return FALSE + + write_tga_image(tga_name, ow, oh, True, rgba_ofs) + bt.free(rgba_ofs) + + # 2) Transcode to ASTC and write .astc file + astc_name = f"transcoded_{level_index}_{layer_index}_{face_index}.astc" + target_tf = bt.basis_get_transcoder_texture_format_from_basis_tex_format(basis_tex_format) + print(f" Target ASTC TF={target_tf}") + + trans_size_astc = bt.basis_compute_transcoded_image_size_in_bytes(target_tf, ow, oh) + assert trans_size_astc > 0 + astc_ofs = bt.alloc(trans_size_astc) + print(f" ASTC buf ofs=0x{astc_ofs:x}, size={trans_size_astc}") + + ok = bt.ktx2_transcode_image_level( + ktx2_handle, + level_index, layer_index, face_index, + astc_ofs, + trans_size_astc, + target_tf, + 0, 0, 0, -1, -1, + trans_state + ) + print(" ktx2_transcode_image_level(ASTC):", ok) + if not ok: + bt.free(astc_ofs) + bt.ktx2_destroy_transcode_state(trans_state) + bt.ktx2_close(ktx2_handle) + bt.free(ktx2_data_ofs) + return FALSE + + write_astc_file(astc_name, astc_ofs, block_width, block_height, ow, oh) + bt.free(astc_ofs) + + bt.ktx2_destroy_transcode_state(trans_state) + bt.ktx2_close(ktx2_handle) + bt.free(ktx2_data_ofs) + + print("transcode_ktx2_file: success") + return TRUE + +# ------------------------------------------------------------ +# main() equivalent +# ------------------------------------------------------------ + +def main(): + print("example_capi_python:") + + # Init encoder (which initializes transcoder) + print("Calling bu.init() ...") + bu.init() + + print("Calling bt.init() ...") + bt.init() + + # Optional debug control if bound + if hasattr(bu, "enable_debug_printf"): + print("Disabling debug printf from encoder") + bu.enable_debug_printf(False) + + # Generate test image + W, H = 512, 512 + src_image = create_pretty_rgba_pattern(W, H) + if src_image is None: + print("ERROR: create_pretty_rgba_pattern failed") + return 1 + + # Save test image for inspection + print("Writing test_image.tga ...") + # use Python-level TGA writer by allocating a temporary native buffer + tmp_ofs = bt.alloc(len(src_image)) + ctypes.memmove(tmp_ofs, src_image, len(src_image)) + write_tga_image("test_image.tga", W, H, True, tmp_ofs) + bt.free(tmp_ofs) + + # Compress to KTX2 + print("Creating comp_params ...") + comp_params = bu.new_params() + print(" comp_params handle:", comp_params) + + img_ofs = bu.alloc(W * H * 4) + print(f"Allocated encoder image buffer at 0x{img_ofs:x}") + ctypes.memmove(img_ofs, src_image, W * H * 4) + + print("Calling bu.comp_params_set_image_rgba32(...)") + ok = bu.set_image_rgba32(comp_params, 0, img_ofs, W, H, W * 4) + print(" set_image_rgba32:", ok) + if not ok: + print("ERROR: bu_comp_params_set_image_rgba32 failed") + return 1 + + bu.free(img_ofs) + + print("Compressing to XUASTC LDR 8x5 KTX2 ...") + basis_tex_format = BasisTexFormat.cXUASTC_LDR_8x5 + quality_level = 85 + effort_level = 2 + flags = (BasisFlags.KTX2_OUTPUT | + BasisFlags.SRGB | + BasisFlags.THREADED | + BasisFlags.GEN_MIPS_CLAMP | + BasisFlags.PRINT_STATS | + BasisFlags.PRINT_STATUS) + + ok = bu.compress(comp_params, + tex_format=basis_tex_format, + quality=quality_level, + effort=effort_level, + flags=flags, + rdo_quality=0.0) + print(" bu.compress:", ok) + if not ok: + print("ERROR: bu_compress_texture failed") + return 1 + + comp_size = bu.get_comp_data_size(comp_params) + print("Compressed size:", comp_size) + if comp_size == 0: + print("ERROR: bu_comp_params_get_comp_data_size failed") + return 1 + + comp_ofs = bu.get_comp_data_ofs(comp_params) + print(f"Compressed data ptr=0x{comp_ofs:x}") + + # Copy compressed data into Python bytes + CompArray = ctypes.c_ubyte * comp_size + comp_buf = CompArray.from_address(comp_ofs) + comp_bytes = bytes(comp_buf) + + print("Writing test.ktx2 ...") + if not write_blob_to_file("test.ktx2", comp_bytes): + print("ERROR: write_blob_to_file failed") + return 1 + + # Transcode using the native transcoder API + print("Now transcoding test.ktx2 via C API ...") + if not transcode_ktx2_file(comp_bytes): + print("ERROR: transcode_ktx2_file failed") + return 1 + + bu.delete_params(comp_params) + + print("Success") + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/external/basis_universal/python/lowlevel_test_native/test_transcoder_basic.py b/external/basis_universal/python/lowlevel_test_native/test_transcoder_basic.py new file mode 100644 index 0000000000..7aa87720f5 --- /dev/null +++ b/external/basis_universal/python/lowlevel_test_native/test_transcoder_basic.py @@ -0,0 +1,24 @@ +# test_transcoder_basic.py +import sys +import os + +# Make sure Python can find the .so file +sys.path.append("basisu_py") # Adjust if needed + +try: + import basisu_transcoder_python as bt +except ImportError as e: + print("Failed to import basisu_transcoder_python:", e) + raise + +print("Successfully loaded basisu_transcoder_python") + +# Call bt_get_version() via the pybind11 binding +try: + version = bt.get_version() + print("Transcoder version:", version) +except Exception as e: + print("Error calling bt_get_version:", e) + raise + +print("Basic transcoder test complete.") diff --git a/external/basis_universal/python/lowlevel_test_wasm/__init__.py b/external/basis_universal/python/lowlevel_test_wasm/__init__.py new file mode 100644 index 0000000000..143f486c05 --- /dev/null +++ b/external/basis_universal/python/lowlevel_test_wasm/__init__.py @@ -0,0 +1 @@ +# __init__.py diff --git a/external/basis_universal/python/lowlevel_test_wasm/basic_test.py b/external/basis_universal/python/lowlevel_test_wasm/basic_test.py new file mode 100644 index 0000000000..5fbede022c --- /dev/null +++ b/external/basis_universal/python/lowlevel_test_wasm/basic_test.py @@ -0,0 +1,58 @@ +import wasmtime +import ctypes + +# --- Engine --- +engine = wasmtime.Engine() + +# --- Store --- +store = wasmtime.Store(engine) + +# --- WASI config --- +wasi = wasmtime.WasiConfig() +wasi.argv = ["basisu_module_st"] +wasi.inherit_stdout() # <-- tell WASI to use the host stdout +wasi.inherit_stderr() +store.set_wasi(wasi) + +# --- Load module --- +module = wasmtime.Module.from_file(engine, "basisu_py/wasm/basisu_module_st.wasm") + +# --- Linker + WASI --- +linker = wasmtime.Linker(engine) +linker.define_wasi() + +# --- Instantiate --- +instance = linker.instantiate(store, module) +print("Single-threaded WASM instantiated OK") + +# --- Exports --- +exports = instance.exports(store) + +get_version = exports["bu_get_version"] +alloc = exports["bu_alloc"] +free = exports["bu_free"] +memory = exports["memory"] + +# --- Version --- +version = get_version(store) +print("Version =", version) + +# --- Alloc --- +ptr = alloc(store, 64) +print("Allocated ptr =", ptr) + +# --- Access WASM memory properly --- +data_len = memory.data_len(store) +raw_ptr = memory.data_ptr(store) # ctypes pointer +addr = ctypes.addressof(raw_ptr.contents) # convert to integer pointer + +# Create a byte array view into WASM memory +buf = (ctypes.c_ubyte * data_len).from_address(addr) + +# Write TEST at allocated ptr +buf[ptr : ptr + 4] = b"TEST" +print("Wrote TEST into WASM memory.") + +# --- Free --- +free(store, ptr) +print("Memory free OK.") diff --git a/external/basis_universal/python/lowlevel_test_wasm/basisu_wasm.py b/external/basis_universal/python/lowlevel_test_wasm/basisu_wasm.py new file mode 100644 index 0000000000..08b3933555 --- /dev/null +++ b/external/basis_universal/python/lowlevel_test_wasm/basisu_wasm.py @@ -0,0 +1,148 @@ +# basisu_wasm.py +import wasmtime +import ctypes +import sys + +sys.path.append("basisu_py") # our shared .py files + +from constants import * + +class BasisuWasm: + def __init__(self, path): + self.path = path + self.engine = None + self.store = None + self.memory = None + self.exports = None + + # ----------------------------------------------- + # Internal helper: build WASI + Wasmtime engine + # ----------------------------------------------- + def _init_engine(self): + self.engine = wasmtime.Engine() + self.store = wasmtime.Store(self.engine) + + wasi = wasmtime.WasiConfig() + wasi.argv = ["basisu"] + wasi.inherit_stdout() + wasi.inherit_stderr() + self.store.set_wasi(wasi) + + return wasi + + # ----------------------------------------------- + # Create linker and instantiate WASM module + # ----------------------------------------------- + def load(self): + self._init_engine() + + module = wasmtime.Module.from_file(self.engine, self.path) + linker = wasmtime.Linker(self.engine) + linker.define_wasi() + + instance = linker.instantiate(self.store, module) + + self.exports = instance.exports(self.store) + self.memory = self.exports["memory"] + + if "bu_init" in self.exports: + self.exports["bu_init"](self.store) + + print("WASM loaded:", self.path) + + # ----------------------------------------------- + # Read/write WASM linear memory via ctypes + # ----------------------------------------------- + def _wasm_buf(self): + raw_ptr = self.memory.data_ptr(self.store) + length = self.memory.data_len(self.store) + addr = ctypes.addressof(raw_ptr.contents) + return (ctypes.c_ubyte * length).from_address(addr) + + # ----------------------------------------------- + # Exported API accessors + # ----------------------------------------------- + def init(self): + return self.exports["bu_init"](self.store) + + def version(self): + return self.exports["bu_get_version"](self.store) + + def alloc(self, size): + return self.exports["bu_alloc"](self.store, size) + + def free(self, ptr): + return self.exports["bu_free"](self.store, ptr) + + def new_params(self): + return self.exports["bu_new_comp_params"](self.store) + + def delete_params(self, ptr): + return self.exports["bu_delete_comp_params"](self.store, ptr) + + def set_image_rgba32(self, params, image_index, img_ptr, w, h, pitch): + return self.exports["bu_comp_params_set_image_rgba32"]( + self.store, params, image_index, img_ptr, w, h, pitch + ) + + def set_image_float_rgba(self, params, image_index, img_ptr, w, h, pitch): + return self.exports["bu_comp_params_set_image_float_rgba"]( + self.store, params, image_index, img_ptr, w, h, pitch + ) + + # Normally quality_level controls the quality. + # If quality_level==-1, then rdo_quality (a low-level parameter) directly + # controls each codec's quality setting. Normally set to 0. + + def compress_texture_lowlevel(self, params, + tex_format, + quality_level, + effort_level, + flags_and_quality, + rdo_quality): + + return self.exports["bu_compress_texture"]( + self.store, + params, + tex_format, + quality_level, + effort_level, + flags_and_quality, + rdo_quality + ) + + def compress(self, params, + tex_format=BasisTexFormat.cUASTC_LDR_4x4, + quality=BasisQuality.MAX, + effort=BasisEffort.DEFAULT, + flags=BasisFlags.NONE, + rdo_quality=0.0): + + return bool(self.compress_texture_lowlevel( + params, + tex_format, + quality, + effort, + flags, + rdo_quality + )) + + def get_comp_data_ofs(self, params): + return self.exports["bu_comp_params_get_comp_data_ofs"](self.store, params) + + def get_comp_data_size(self, params): + return self.exports["bu_comp_params_get_comp_data_size"](self.store, params) + + # ----------------------------------------------- + # Copy bytes into WASM memory + # ----------------------------------------------- + def write_bytes(self, wasm_ptr, data: bytes): + buf = self._wasm_buf() + buf[wasm_ptr:wasm_ptr+len(data)] = data + + # ----------------------------------------------- + # Read bytes from WASM memory + # ----------------------------------------------- + def read_bytes(self, wasm_ptr, size): + buf = self._wasm_buf() + return bytes(buf[wasm_ptr:wasm_ptr+size]) diff --git a/external/basis_universal/python/lowlevel_test_wasm/compress_test.py b/external/basis_universal/python/lowlevel_test_wasm/compress_test.py new file mode 100644 index 0000000000..285bb105fe --- /dev/null +++ b/external/basis_universal/python/lowlevel_test_wasm/compress_test.py @@ -0,0 +1,63 @@ +# compress_test.py +from .basisu_wasm import * + +# === Load WASM === +codec = BasisuWasm("basisu_py/wasm/basisu_module_st.wasm") +codec.load() + +print("Version =", codec.version()) + +# === Build test image === +W, H = 256, 256 +BYTES_PER_PIXEL = 4 +pitch = W * BYTES_PER_PIXEL + +img = bytearray(W * H * 4) + +for y in range(H): + for x in range(W): + i = (y * W + x) * 4 + img[i + 0] = x & 0xFF # R + img[i + 1] = y & 0xFF # G + img[i + 2] = (x ^ y) & 0xFF # B + img[i + 3] = 255 # A + +# === Upload image to WASM memory === +img_ptr = codec.alloc(len(img)) +codec.write_bytes(img_ptr, img) + +# === Create comp_params === +params = codec.new_params() + +# === Set image into comp_params === +ok = codec.set_image_rgba32(params, 0, img_ptr, W, H, pitch) +print("Set image:", ok) + +# === Compress === +ok = codec.compress( + params, + tex_format=BasisTexFormat.cUASTC_LDR_4x4, + quality=100, + effort=BasisEffort.DEFAULT, + flags=BasisFlags.KTX2_OUTPUT | BasisFlags.SRGB, + rdo_quality=0.0 +) +print("Compress result:", ok) + +# === Retrieve compressed blob === +ofs = codec.get_comp_data_ofs(params) +size = codec.get_comp_data_size(params) +print("Output size =", size) + +comp_data = codec.read_bytes(ofs, size) +print("First 16 bytes:", comp_data[:16]) + +# === Save to KTX2 === +with open("test.ktx2", "wb") as f: + f.write(comp_data) + +print("File written: test.ktx2") + +# === Cleanup === +codec.delete_params(params) +codec.free(img_ptr) diff --git a/external/basis_universal/python/lowlevel_test_wasm/compress_test_float.py b/external/basis_universal/python/lowlevel_test_wasm/compress_test_float.py new file mode 100644 index 0000000000..3f60cb7b4d --- /dev/null +++ b/external/basis_universal/python/lowlevel_test_wasm/compress_test_float.py @@ -0,0 +1,76 @@ +# compress_test_float.py + +from .basisu_wasm import BasisuWasm, BasisTexFormat, BasisEffort, BasisFlags, BasisQuality +import struct # for packing floats + +# === Load WASM === +codec = BasisuWasm("basisu_py/wasm/basisu_module_st.wasm") +codec.load() + +print("Version =", codec.version()) + +# === Build a 256x256 FLOAT RGBA image === +W, H = 256, 256 +BYTES_PER_PIXEL = 16 # float32 * 4 +pitch = W * BYTES_PER_PIXEL + +# Float image stored as bytearray of packed floats +img = bytearray(W * H * BYTES_PER_PIXEL) + +for y in range(H): + for x in range(W): + # Create some float HDR gradient pattern + r = float(x) / W # 0.0 ? 1.0 + g = float(y) / H # 0.0 ? 1.0 + b = float(x ^ y) / 255.0 # quirky pattern + a = 1.0 + + i = (y * W + x) * 4 + + # pack into img bytearray + struct.pack_into("ffff", img, i*4, r, g, b, a) + +print("Created FLOAT RGBA image.") + +# === Upload to WASM memory === +img_ptr = codec.alloc(len(img)) +codec.write_bytes(img_ptr, img) +print("Copied float image into WASM heap at", img_ptr) + +# === Create params === +params = codec.new_params() + +# === Set FLOAT RGBA image === +ok = codec.set_image_float_rgba(params, 0, img_ptr, W, H, pitch) +print("Set float RGBA:", ok) + +# === Compress using HDR UASTC 4x4 === +ok = codec.compress( + params, + tex_format=BasisTexFormat.cUASTC_HDR_4x4, + quality=BasisQuality.MAX, + effort=BasisEffort.DEFAULT, + flags=BasisFlags.KTX2_OUTPUT | BasisFlags.REC2020, # optional: HDR color space + rdo_quality=0.0 +) + +print("Compression result:", ok) + +# === Retrieve compressed HDR KTX2 === +ofs = codec.get_comp_data_ofs(params) +size = codec.get_comp_data_size(params) + +print("Output size =", size) +data = codec.read_bytes(ofs, size) + +print("First 16 bytes:", data[:16]) + +# === Save to test_hdr.ktx2 === +with open("test_hdr.ktx2", "wb") as f: + f.write(data) + +print("Wrote test_hdr.ktx2") + +# === Cleanup === +codec.delete_params(params) +codec.free(img_ptr) diff --git a/external/basis_universal/python/pyproject.toml b/external/basis_universal/python/pyproject.toml new file mode 100644 index 0000000000..e893c577fc --- /dev/null +++ b/external/basis_universal/python/pyproject.toml @@ -0,0 +1,44 @@ +[build-system] +requires = ["setuptools>=65", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "basisu-py" +version = "0.2.0" +description = "Python bindings for Basis Universal encoder/transcoder v2.x with native + WASM backend" +authors = [ + { name = "Binomial LLC", email = "stephanie@binomial.info" } +] +license = { text = "Apache 2.0" } +readme = "README.md" +requires-python = ">=3.8" + +dependencies = [ + "numpy", + "Pillow", + "imageio>=2.22", + "wasmtime", +] + +classifiers = [ + "Programming Language :: Python :: 3", + "Programming Language :: C++", + "Operating System :: OS Independent", + "License :: OSI Approved :: Apache Software License", +] + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.packages.find] +include = ["basisu_py*"] + +[tool.setuptools.package-data] +basisu_py = [ + "*.so", + "*.pyd", + "*.py", + "wasm/*.wasm", + "wasm/*.py", + "README.md", +] diff --git a/external/basis_universal/python/tests/__init__.py b/external/basis_universal/python/tests/__init__.py new file mode 100644 index 0000000000..2badd27e81 --- /dev/null +++ b/external/basis_universal/python/tests/__init__.py @@ -0,0 +1 @@ +# python/tests/__init__.py diff --git a/external/basis_universal/python/tests/test_backend_loading.py b/external/basis_universal/python/tests/test_backend_loading.py new file mode 100644 index 0000000000..598bae84f8 --- /dev/null +++ b/external/basis_universal/python/tests/test_backend_loading.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +import numpy as np +from PIL import Image + +from basisu_py.codec import Encoder, EncoderBackend +from basisu_py.constants import BasisTexFormat + +print("========== BACKEND LOADING TEST ==========\n") + +# -------------------------------------------------------------- +# 1. Test native backend (if available) +# -------------------------------------------------------------- +print("Testing native backend...") + +try: + enc_native = Encoder(backend=EncoderBackend.NATIVE) + print(" [OK] Native backend loaded") +except Exception as e: + print(" [FAIL] Native backend failed to load:", e) + enc_native = None + +# If native loaded, test very basic functionality +if enc_native: + try: + version = enc_native._native.get_version() + print(f" Native get_version() ? {version}") + + ptr = enc_native._native.alloc(16) + print(f" Native alloc() returned ptr = {ptr}") + + enc_native._native.free(ptr) + print(f" Native free() OK") + + print(" [OK] Native basic operations working.\n") + except Exception as e: + print(" [FAIL] Native operations error:", e) +else: + print(" Skipping native basic operations.\n") + +# -------------------------------------------------------------- +# 2. Test WASM backend +# -------------------------------------------------------------- +print("\nTesting WASM backend...") + +try: + enc_wasm = Encoder(backend=EncoderBackend.WASM) + print(" [OK] WASM backend loaded") +except Exception as e: + print(" [FAIL] WASM backend failed to load:", e) + enc_wasm = None + +# If WASM loaded, test basic methods +if enc_wasm and enc_wasm._wasm is not None: + try: + version = enc_wasm._wasm.get_version() + print(f" WASM get_version() ? {version}") + + ptr = enc_wasm._wasm.alloc(16) + print(f" WASM alloc() returned ptr = {ptr}") + + enc_wasm._wasm.free(ptr) + print(f" WASM free() OK") + + print(" [OK] WASM basic operations working.\n") + except Exception as e: + print(" [FAIL] WASM operations error:", e) +else: + print(" Skipping WASM basic operations.\n") + +print("\n========== DONE ==========\n") diff --git a/external/basis_universal/python/tests/test_basic_backend_selection.py b/external/basis_universal/python/tests/test_basic_backend_selection.py new file mode 100644 index 0000000000..6cfb75ced6 --- /dev/null +++ b/external/basis_universal/python/tests/test_basic_backend_selection.py @@ -0,0 +1,7 @@ +from basisu_py import Encoder + +enc = Encoder() # AUTO mode +print("Encoder backend:", enc.backend) +print("Native loaded:", enc._native is not None) +print("WASM loaded:", enc._wasm is not None) +print("Version:", enc._native.get_version() if enc._native else enc._wasm.get_version()) diff --git a/external/basis_universal/python/tests/test_basic_decode.py b/external/basis_universal/python/tests/test_basic_decode.py new file mode 100644 index 0000000000..515c31ac87 --- /dev/null +++ b/external/basis_universal/python/tests/test_basic_decode.py @@ -0,0 +1,19 @@ +from basisu_py import Transcoder +from PIL import Image +import numpy as np + +# Load input file +with open("test.ktx2", "rb") as f: + data = f.read() + +# Decode (AUTO backend) +t = Transcoder() +rgba = t.decode_rgba(data) # returns HxWx4 uint8 NumPy array + +print("Decoded:", rgba.shape, rgba.dtype) + +# Convert to Pillow Image and save +img = Image.fromarray(rgba, mode="RGBA") +img.save("decoded.png") + +print("Wrote decoded.png") \ No newline at end of file diff --git a/external/basis_universal/python/tests/test_basic_transcode.py b/external/basis_universal/python/tests/test_basic_transcode.py new file mode 100644 index 0000000000..fc10a2305d --- /dev/null +++ b/external/basis_universal/python/tests/test_basic_transcode.py @@ -0,0 +1,10 @@ +from basisu_py import Transcoder + +with open("test.ktx2", "rb") as f: + data = f.read() + +t = Transcoder() # AUTO backend +img = t.decode_rgba(data) + +print("Decoded shape:", img.shape) +print("dtype:", img.dtype) diff --git a/external/basis_universal/python/tests/test_basic_wasm_selection.py b/external/basis_universal/python/tests/test_basic_wasm_selection.py new file mode 100644 index 0000000000..bdad65fb11 --- /dev/null +++ b/external/basis_universal/python/tests/test_basic_wasm_selection.py @@ -0,0 +1,6 @@ +from basisu_py import Transcoder +from basisu_py.transcoder import TranscoderBackend + +t = Transcoder(backend=TranscoderBackend.WASM) +print("Backend:", t.backend_name) +t.decode_rgba(open("test.ktx2","rb").read()) diff --git a/external/basis_universal/python/tests/test_compress_swirl.py b/external/basis_universal/python/tests/test_compress_swirl.py new file mode 100644 index 0000000000..a02f7d5754 --- /dev/null +++ b/external/basis_universal/python/tests/test_compress_swirl.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +import numpy as np +from PIL import Image +from math import sin, cos, atan2, hypot + +from basisu_py.codec import Encoder, EncoderBackend +from basisu_py.constants import BasisTexFormat, BasisQuality, BasisEffort, BasisFlags + + +# -------------------------------------------------------------- +# Procedural swirl pattern (RGBA8) +# -------------------------------------------------------------- +def make_swirl_image(w=256, h=256): + arr = np.zeros((h, w, 4), dtype=np.uint8) + + cx = w / 2.0 + cy = h / 2.0 + + for y in range(h): + for x in range(w): + dx = x - cx + dy = y - cy + + dist = hypot(dx, dy) + angle = atan2(dy, dx) + + r = int((sin(dist * 0.15) * 0.5 + 0.5) * 255) + g = int((sin(angle * 3.0) * 0.5 + 0.5) * 255) + b = int((cos(dist * 0.10 + angle * 2.0) * 0.5 + 0.5) * 255) + + arr[y, x] = (r, g, b, 255) + + return arr + + +# -------------------------------------------------------------- +# Test encode using a given backend +# -------------------------------------------------------------- +def compress_swirl(backend, outfile): + print(f"\n========== Testing {backend} backend ==========") + + # Build procedural image + swirl = make_swirl_image(256, 256) + print("Generated swirl image:", swirl.shape) + + # Create encoder + enc = Encoder(backend=backend) + + # Compress + blob = enc.compress( + swirl, + format=BasisTexFormat.cUASTC_LDR_4x4, + quality=BasisQuality.MAX, + effort=BasisEffort.DEFAULT, + flags=BasisFlags.KTX2_OUTPUT | BasisFlags.SRGB + ) + + print(f"Compressed blob size: {len(blob)} bytes") + + # Save output + with open(outfile, "wb") as f: + f.write(blob) + + print(f"Wrote: {outfile}") + print("==============================================") + + +# -------------------------------------------------------------- +# Main +# -------------------------------------------------------------- +if __name__ == "__main__": + # Test native backend + try: + compress_swirl(EncoderBackend.NATIVE, "swirl_native.ktx2") + except Exception as e: + print("Native backend ERROR:", e) + + # Test WASM backend + try: + compress_swirl(EncoderBackend.WASM, "swirl_wasm.ktx2") + except Exception as e: + print("WASM backend ERROR:", e) diff --git a/external/basis_universal/python/tests/test_compress_swirl_hdr.py b/external/basis_universal/python/tests/test_compress_swirl_hdr.py new file mode 100644 index 0000000000..34e0adb9ff --- /dev/null +++ b/external/basis_universal/python/tests/test_compress_swirl_hdr.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +import numpy as np +from math import sin, cos, atan2, hypot +from basisu_py.codec import Encoder, EncoderBackend +from basisu_py.constants import BasisTexFormat, BasisQuality, BasisEffort, BasisFlags + + +# -------------------------------------------------------------- +# Procedural HDR swirl pattern (float32 RGBA) +# -------------------------------------------------------------- +def make_hdr_swirl_image(w=256, h=256): + arr = np.zeros((h, w, 4), dtype=np.float32) + + cx = w / 2.0 + cy = h / 2.0 + + for y in range(h): + for x in range(w): + dx = x - cx + dy = y - cy + dist = hypot(dx, dy) + angle = atan2(dy, dx) + + r = (sin(dist * 0.15) * 0.5 + 0.5) + g = (sin(angle * 3.0) * 0.5 + 0.5) + b = (cos(dist * 0.10 + angle * 2.0) * 0.5 + 0.5) + + arr[y, x] = (r, g, b, 1.0) # full alpha + + return arr + + +# -------------------------------------------------------------- +# Test encode using a given backend +# -------------------------------------------------------------- +def compress_hdr_swirl(backend, outfile): + print(f"\n========== Testing HDR {backend} backend ==========") + + hdr = make_hdr_swirl_image(256, 256) + print("Generated HDR swirl image:", hdr.shape, hdr.dtype) + + enc = Encoder(backend=backend) + + blob = enc.compress( + hdr, + format=-1, # auto-select HDR (UASTC_HDR_4x4) + quality=BasisQuality.MAX, + effort=BasisEffort.DEFAULT, + flags=BasisFlags.KTX2_OUTPUT | BasisFlags.SRGB + ) + + print(f"Compressed blob size: {len(blob)} bytes") + + with open(outfile, "wb") as f: + f.write(blob) + + print(f"Wrote: {outfile}") + print("==============================================") + + +# -------------------------------------------------------------- +# Main +# -------------------------------------------------------------- +if __name__ == "__main__": + # Native backend + try: + compress_hdr_swirl(EncoderBackend.NATIVE, "hdr_swirl_native.ktx2") + except Exception as e: + print("Native HDR backend ERROR:", e) + + # WASM backend + try: + compress_hdr_swirl(EncoderBackend.WASM, "hdr_swirl_wasm.ktx2") + except Exception as e: + print("WASM HDR backend ERROR:", e) diff --git a/external/basis_universal/python/tests/test_transcoder_astc.py b/external/basis_universal/python/tests/test_transcoder_astc.py new file mode 100644 index 0000000000..306834bde0 --- /dev/null +++ b/external/basis_universal/python/tests/test_transcoder_astc.py @@ -0,0 +1,18 @@ +from basisu_py import Transcoder +from astc_writer import write_astc_file + +# Load a .ktx2 +data = open("input.ktx2", "rb").read() +t = Transcoder() + +# Transcode to ASTC +h = t.open(data) +bw = t.get_block_width(h) # or basis_get_block_width(astc_tfmt) +bh = t.get_block_height(h) +tfmt = t.basis_get_transcoder_texture_format_from_basis_tex_format( + t.get_basis_tex_format(h) +) + +blocks = t.transcode_tfmt(data, tfmt) +write_astc_file("output.astc", blocks, bw, bh, t.get_width(h), t.get_height(h)) +t.close(h) diff --git a/external/basis_universal/python/tests/test_transcoder_backend_loading.py b/external/basis_universal/python/tests/test_transcoder_backend_loading.py new file mode 100644 index 0000000000..e0d1ed2f4a --- /dev/null +++ b/external/basis_universal/python/tests/test_transcoder_backend_loading.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +import sys +from basisu_py.transcoder import Transcoder, TranscoderBackend +from basisu_py.constants import BasisTexFormat + +print("========== TESTING TRANSCODER BACKENDS ==========\n") + +# Load some test data (ensure test.ktx2 exists) +try: + test_data = open("test.ktx2", "rb").read() + print("[INFO] Loaded test.ktx2") +except FileNotFoundError: + print("[ERROR] test.ktx2 not found. Create one first via encoder tests.") + sys.exit(1) + + +# ------------------------------------------------------------------- +# 1. Test NATIVE backend +# ------------------------------------------------------------------- +print("\n--- Testing NATIVE transcoder backend ---") + +try: + t_native = Transcoder(TranscoderBackend.NATIVE) + print(" [OK] Native backend loaded") + + version = t_native.get_version() + print(f" Native get_version() = {version}") + + # Open KTX2 + raw = t_native.open(test_data) + print(" [OK] Opened KTX2 (native)") + + # Query some basic properties + print(" Width :", t_native.get_width(raw)) + print(" Height:", t_native.get_height(raw)) + print(" Levels:", t_native.get_levels(raw)) + + # Cleanup + t_native.close(raw) + print(" [OK] Native transcoder basic operations working.") + +except Exception as e: + print(" [FAIL] Native transcoder error:", e) + + +# ------------------------------------------------------------------- +# 2. Test WASM backend +# ------------------------------------------------------------------- +print("\n--- Testing WASM transcoder backend ---") + +try: + t_wasm = Transcoder(TranscoderBackend.WASM) + print(" [OK] WASM backend loaded") + + version = t_wasm.get_version() + print(f" WASM get_version() = {version}") + + raw = t_wasm.open(test_data) + print(" [OK] Opened KTX2 (wasm)") + + print(" Width :", t_wasm.get_width(raw)) + print(" Height:", t_wasm.get_height(raw)) + print(" Levels:", t_wasm.get_levels(raw)) + + t_wasm.close(raw) + print(" [OK] WASM transcoder basic operations working.") + +except Exception as e: + print(" [FAIL] WASM transcoder error:", e) + + +print("\n========== DONE ==========") diff --git a/external/basis_universal/python/tests/test_transcoder_end_to_end.py b/external/basis_universal/python/tests/test_transcoder_end_to_end.py new file mode 100644 index 0000000000..5e4825aa50 --- /dev/null +++ b/external/basis_universal/python/tests/test_transcoder_end_to_end.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +""" +Full end-to-end transcoder test with automatic fallback. + +- Generates a swirl image +- Compresses it using native OR WASM (AUTO mode) +- Writes test.ktx2 +- Decodes it using whichever backends are available: + * AUTO (native if present, otherwise WASM) + * Native (if available) + * WASM (if available) +- Produces PNG outputs for all successful backends +""" + +import numpy as np +from math import sin, cos, atan2, hypot +from PIL import Image +import sys + +from basisu_py.codec import Encoder, EncoderBackend +from basisu_py.transcoder import Transcoder, TranscoderBackend +from basisu_py.constants import ( + BasisTexFormat, + BasisQuality, + BasisEffort, + BasisFlags, +) + + +# ------------------------------------------------------------------- +# Create an RGBA swirl test image +# ------------------------------------------------------------------- +def make_swirl(w=256, h=256): + arr = np.zeros((h, w, 4), dtype=np.uint8) + + cx, cy = w / 2.0, h / 2.0 + + for y in range(h): + for x in range(w): + dx, dy = x - cx, y - cy + dist = hypot(dx, dy) + angle = atan2(dy, dx) + + r = int((sin(dist * 0.15) * 0.5 + 0.5) * 255) + g = int((sin(angle * 3.0) * 0.5 + 0.5) * 255) + b = int((cos(dist * 0.10 + angle * 2.0) * 0.5 + 0.5) * 255) + + arr[y, x] = (r, g, b, 255) + + return arr + + +# ------------------------------------------------------------------- +# Try loading transcoder with a backend, return (success, transcoder) +# ------------------------------------------------------------------- +def try_transcoder(backend): + try: + t = Transcoder(backend) + print(f"[OK] Loaded transcoder backend '{backend}' ({t.backend_name})") + return True, t + except Exception as e: + print(f"[SKIP] Backend '{backend}' unavailable:", e) + return False, None + + +# ------------------------------------------------------------------- +# Try loading encoder with a backend, return blob or None +# ------------------------------------------------------------------- +def try_encoder(backend, img): + try: + enc = Encoder(backend) + print(f"[OK] Loaded encoder backend '{backend}' ({enc.backend_name})") + except Exception as e: + print(f"[SKIP] Encoder backend '{backend}' unavailable:", e) + return None + + try: + print(f"[Test] Compressing swirl -> KTX2 using {enc.backend_name}...") + blob = enc.compress( + img, + format=-1, + quality=BasisQuality.MAX, + effort=BasisEffort.DEFAULT, + flags=BasisFlags.KTX2_OUTPUT | BasisFlags.SRGB + ) + return blob + except Exception as e: + print(f"[FAIL] Compression failed on backend '{backend}':", e) + return None + + +# ------------------------------------------------------------------- +# Decode blob with a given transcoder +# ------------------------------------------------------------------- +def decode_with_backend(name, t, blob): + try: + rgba = t.decode_rgba(blob) + outname = f"decoded_{name}.png" + Image.fromarray(rgba, mode="RGBA").save(outname) + print(f" --> {name}: decoded successfully, wrote {outname}") + except Exception as e: + print(f" [FAIL] decode_rgba on backend '{name}':", e) + + +# ------------------------------------------------------------------- +# Main test +# ------------------------------------------------------------------- +if __name__ == "__main__": + print("========== BasisU End-to-End Compression & Transcoding Test ==========") + + # ------------------------------------------------------- + # Generate swirl test + # ------------------------------------------------------- + img = make_swirl(256, 256) + print("[Test] Generated swirl:", img.shape) + + # ------------------------------------------------------- + # Try AUTO encoder (native if available, else WASM) + # ------------------------------------------------------- + blob = try_encoder(EncoderBackend.AUTO, img) + if blob is None: + print("[FAIL] Could not encode using AUTO backend; aborting.") + sys.exit(1) + + # Save test.ktx2 + with open("test.ktx2", "wb") as f: + f.write(blob) + print("[Test] Wrote: test.ktx2") + + # ------------------------------------------------------- + # Test transcoding using AUTO + # ------------------------------------------------------- + print("\n[Test] Decoding via AUTO backend...") + ok_auto, t_auto = try_transcoder(TranscoderBackend.AUTO) + if ok_auto: + decode_with_backend("auto", t_auto, blob) + + # ------------------------------------------------------- + # Test NATIVE explicitly (if available) + # ------------------------------------------------------- + print("\n[Test] Decoding via NATIVE backend...") + ok_native, t_native = try_transcoder(TranscoderBackend.NATIVE) + if ok_native: + decode_with_backend("native", t_native, blob) + + # ------------------------------------------------------- + # Test WASM explicitly (if available) + # ------------------------------------------------------- + print("\n[Test] Decoding via WASM backend...") + ok_wasm, t_wasm = try_transcoder(TranscoderBackend.WASM) + if ok_wasm: + decode_with_backend("wasm", t_wasm, blob) + + print("\n========== DONE ==========") diff --git a/external/basis_universal/python/tests/test_transcoder_end_to_end_hdr.py b/external/basis_universal/python/tests/test_transcoder_end_to_end_hdr.py new file mode 100644 index 0000000000..d1fceb1011 --- /dev/null +++ b/external/basis_universal/python/tests/test_transcoder_end_to_end_hdr.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +""" +HDR End-to-End Compression & Transcoding Test +Works on all platforms: + - native if available + - WASM fallback otherwise +""" + +import numpy as np +from math import sin, cos, atan2, hypot +from PIL import Image +import subprocess +import tempfile +import os +import imageio.v3 as iio + +from basisu_py.codec import Encoder, EncoderBackend +from basisu_py.transcoder import Transcoder, TranscoderBackend +from basisu_py.constants import ( + BasisTexFormat, + BasisQuality, + BasisEffort, + BasisFlags +) + + +# ------------------------------------------------------------------- +# Save EXR using TIFF temp + oiiotool (as required) +# ------------------------------------------------------------------- +def save_exr(path, rgba32f): + """ + Save float32 RGBA as EXR if possible. + If oiiotool is not available, save TIFF instead (Windows-safe). + """ + import numpy as np + import imageio.v3 as iio + import subprocess, tempfile, os + + # Write temp TIFF + with tempfile.NamedTemporaryFile(suffix=".tiff", delete=False) as tmp: + temp_path = tmp.name + + iio.imwrite(temp_path, rgba32f.astype(np.float32)) + + # Try EXR via oiiotool + try: + subprocess.run(["oiiotool", temp_path, "-o", path], check=True) + os.remove(temp_path) + print(" Wrote EXR:", path) + return + + except Exception: + # --- FALLBACK: save TIFF --- + fallback_path = path + ".tiff" + + # Windows cannot overwrite files via rename(), so remove first + if os.path.exists(fallback_path): + os.remove(fallback_path) + + # os.replace() always overwrites + os.replace(temp_path, fallback_path) + + print(" [Fallback] Wrote TIFF instead:", fallback_path) + +# ------------------------------------------------------------------- +# Generate HDR swirl image (float32) +# ------------------------------------------------------------------- +def make_swirl_hdr(w=256, h=256): + arr = np.zeros((h, w, 4), dtype=np.float32) + cx, cy = w / 2.0, h / 2.0 + + for y in range(h): + for x in range(w): + dx, dy = x - cx, y - cy + dist = hypot(dx, dy) + angle = atan2(dy, dx) + + # HDR values range up to about 4.0 + r = (sin(dist * 0.08) * 0.5 + 0.5) * 4.0 + g = (sin(angle * 2.0) * 0.5 + 0.5) * 4.0 + b = (cos(dist * 0.06 + angle * 1.5) * 0.5 + 0.5) * 4.0 + + arr[y, x] = (r, g, b, 1.0) + + return arr + + +# ------------------------------------------------------------------- +# Try loading a transcoder backend +# ------------------------------------------------------------------- +def try_transcoder(name, backend): + try: + t = Transcoder(backend) + print(f"[OK] Loaded transcoder backend '{name}' ({t.backend_name})") + return t + except Exception as e: + print(f"[SKIP] Backend '{name}' unavailable:", e) + return None + + +# ------------------------------------------------------------------- +# MAIN +# ------------------------------------------------------------------- +if __name__ == "__main__": + print("========== HDR End-to-End Compression & Transcoding Test ==========") + + # ------------------------------------------------------- + # Create HDR test image + # ------------------------------------------------------- + img_hdr = make_swirl_hdr(256, 256) + print("[HDR] swirl:", img_hdr.shape, img_hdr.dtype) + + # ------------------------------------------------------- + # ENCODE using AUTO backend (native ? or WASM) + # ------------------------------------------------------- + try: + enc = Encoder(EncoderBackend.AUTO) + print(f"[HDR] Encoder backend = {enc.backend_name}") + except Exception as e: + print("[FATAL] Could not create encoder:", e) + exit(1) + + try: + print("[HDR] Compressing HDR swirl -> test_hdr.ktx2...") + ktx2_blob = enc.compress( + img_hdr, + format=-1, # auto-select HDR format + quality=BasisQuality.MAX, + effort=BasisEffort.DEFAULT, + flags=BasisFlags.KTX2_OUTPUT + ) + print(" KTX2 size:", len(ktx2_blob)) + open("test_hdr.ktx2", "wb").write(ktx2_blob) + print(" Wrote test_hdr.ktx2") + except Exception as e: + print("[FATAL] Encoding failed:", e) + exit(1) + + # ------------------------------------------------------- + # DECODE using AUTO (native ? or WASM) + # ------------------------------------------------------- + t_auto = try_transcoder("AUTO", TranscoderBackend.AUTO) + if t_auto: + try: + hdr = t_auto.decode_rgba_hdr(ktx2_blob) + print(" AUTO decoded:", hdr.shape, hdr.dtype) + save_exr("decoded_auto_hdr.exr", hdr) + except Exception as e: + print(" [FAIL] AUTO decode failed:", e) + + # ------------------------------------------------------- + # DECODE using NATIVE if available + # ------------------------------------------------------- + t_native = try_transcoder("NATIVE", TranscoderBackend.NATIVE) + if t_native: + try: + hdr_n = t_native.decode_rgba_hdr(ktx2_blob) + print(" Native decoded:", hdr_n.shape, hdr_n.dtype) + save_exr("decoded_native_hdr.exr", hdr_n) + except Exception as e: + print(" [FAIL] Native decode failed:", e) + + # ------------------------------------------------------- + # DECODE using WASM if available + # ------------------------------------------------------- + t_wasm = try_transcoder("WASM", TranscoderBackend.WASM) + if t_wasm: + try: + hdr_w = t_wasm.decode_rgba_hdr(ktx2_blob) + print(" WASM decoded:", hdr_w.shape, hdr_w.dtype) + save_exr("decoded_wasm_hdr.exr", hdr_w) + except Exception as e: + print(" [FAIL] WASM decode failed:", e) + + print("\n========== DONE ==========") diff --git a/external/basis_universal/python/tests/test_transcoder_helpers.py b/external/basis_universal/python/tests/test_transcoder_helpers.py new file mode 100644 index 0000000000..31245c0000 --- /dev/null +++ b/external/basis_universal/python/tests/test_transcoder_helpers.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +import sys +import numpy as np + +from basisu_py.transcoder import Transcoder, TranscoderBackend +from basisu_py.constants import BasisTexFormat, TranscoderTextureFormat + +print("========== TESTING TRANSCODER HELPERS & METADATA ==========\n") + +# ---------------------------------------------------------------------------- +# Load test KTX2 file +# ---------------------------------------------------------------------------- +try: + ktx2_bytes = open("test.ktx2", "rb").read() + print("[INFO] Loaded test.ktx2") +except FileNotFoundError: + print("[ERROR] test.ktx2 not found. Run encoder tests first.") + sys.exit(1) + + +# ---------------------------------------------------------------------------- +# Utility: run helper tests on a given backend +# ---------------------------------------------------------------------------- +def test_backend(name, backend): + print(f"\n=== Testing {name} backend ===") + + try: + t = Transcoder(backend) + except Exception as e: + print(f"[FAIL] Could not initialize {name} backend:", e) + return + + print(f"[OK] {name} backend loaded") + + # Version + try: + ver = t.get_version() + print(f" version = {ver}") + except Exception as e: + print(" [FAIL] get_version() error:", e) + return + + # enable_debug_printf + try: + t.enable_debug_printf(True) + except Exception as e: + print(" [FAIL] enable_debug_printf() failed") + return + + # Open KTX2 + try: + raw = t.open(ktx2_bytes) + print(" [OK] open() success") + except Exception as e: + print(" [FAIL] open() failed:", e) + return + + # ---------------------------------------------------------------------- + # KTX2 top-level metadata + # ---------------------------------------------------------------------- + try: + w = t.get_width(raw) + h = t.get_height(raw) + lv = t.get_levels(raw) + fc = t.get_faces(raw) + la = t.get_layers(raw) + fmt = t.get_basis_tex_format(raw) + + print(f" Width = {w}") + print(f" Height = {h}") + print(f" Levels = {lv}") + print(f" Faces = {fc}") + print(f" Layers = {la}") + print(f" basis_tex_format = {fmt}") + print(f" has_alpha = {t.has_alpha(raw)}") + print(f" is_hdr = {t.is_hdr(raw)}") + print(f" is_ldr = {t.is_ldr(raw)}") + print(f" is_srgb = {t.is_srgb(raw)}") + print(f" is_etc1s = {t.is_etc1s(raw)}") + print(f" is_uastc_ldr_4x4 = {t.is_uastc_ldr_4x4(raw)}") + print(f" is_xuastc_ldr = {t.is_xuastc_ldr(raw)}") + print(f" is_astc_ldr = {t.is_astc_ldr(raw)}") + print(f" block dims = {t.get_block_width(raw)} x {t.get_block_height(raw)}") + + except Exception as e: + print(" [FAIL] get_* metadata error:", e) + t.close(raw) + return + + # ---------------------------------------------------------------------- + # Per-level metadata for each mipmap + # ---------------------------------------------------------------------- + print("\n -- Level Metadata --") + for level in range(lv): + try: + ow = t.get_level_orig_width(raw, level) + oh = t.get_level_orig_height(raw, level) + nbx = t.get_level_num_blocks_x(raw, level) + nby = t.get_level_num_blocks_y(raw, level) + tb = t.get_level_total_blocks(raw, level) + af = t.get_level_alpha_flag(raw, level) + ff = t.get_level_iframe_flag(raw, level) + + print(f" Level {level}: orig={ow}x{oh}, blocks={nbx}x{nby}, total={tb}, alpha={af}, iframe={ff}") + except Exception as e: + print(f" [FAIL] Level {level} metadata error:", e) + + # ---------------------------------------------------------------------- + # Test ALL basis_tex_format helpers on the file's format + # ---------------------------------------------------------------------- + print("\n -- basis_tex_format helpers --") + + try: + print(f" is_xuastc_ldr = {t.basis_tex_format_is_xuastc_ldr(fmt)}") + print(f" is_astc_ldr = {t.basis_tex_format_is_astc_ldr(fmt)}") + print(f" block W/H = {t.basis_tex_format_get_block_width(fmt)} x " + f"{t.basis_tex_format_get_block_height(fmt)}") + print(f" is_hdr = {t.basis_tex_format_is_hdr(fmt)}") + print(f" is_ldr = {t.basis_tex_format_is_ldr(fmt)}") + except Exception as e: + print(" [FAIL] basis_tex_format_* error:", e) + + # ---------------------------------------------------------------------- + # Test transcoder_texture_format helpers using a few common formats + # ---------------------------------------------------------------------- + print("\n -- transcoder_texture_format helpers --") + + test_formats = [ + TranscoderTextureFormat.TF_RGBA32, + TranscoderTextureFormat.TF_RGBA_HALF, + TranscoderTextureFormat.TF_BC7_RGBA, + TranscoderTextureFormat.TF_ETC1_RGB, + ] + + for tfmt in test_formats: + try: + print(f" Format {tfmt}: hdr={t.basis_transcoder_format_is_hdr(tfmt)}, " + f"ldr={t.basis_transcoder_format_is_ldr(tfmt)}, " + f"has_alpha={t.basis_transcoder_format_has_alpha(tfmt)}, " + f"uncompressed={t.basis_transcoder_format_is_uncompressed(tfmt)}, " + f"bytes/pixel or block={t.basis_get_bytes_per_block_or_pixel(tfmt)}") + except Exception as e: + print(" [FAIL] transcoder_texture_format_* error:", e) + + # ---------------------------------------------------------------------- + # Compute transcode buffer sizes + # ---------------------------------------------------------------------- + print("\n -- compute_transcoded_image_size_in_bytes --") + try: + for tfmt in test_formats: + sz = t.basis_compute_transcoded_image_size_in_bytes(tfmt, w, h) + print(f" Format {tfmt}: size = {sz}") + except Exception as e: + print(" [FAIL] size computation error:", e) + + # ---------------------------------------------------------------------- + # Decode RGBA (LDR) + # ---------------------------------------------------------------------- + print("\n -- decode_rgba --") + try: + img_rgba = t.decode_rgba(ktx2_bytes) + print(f" decode_rgba: shape={img_rgba.shape}, dtype={img_rgba.dtype}") + except Exception as e: + print(" [FAIL] decode_rgba error:", e) + + # ---------------------------------------------------------------------- + # Decode HDR if applicable + # ---------------------------------------------------------------------- + if t.is_hdr(raw): + print("\n -- decode_rgba_hdr --") + try: + img_hdr = t.decode_rgba_hdr(ktx2_bytes) + print(f" decode_rgba_hdr: shape={img_hdr.shape}, dtype={img_hdr.dtype}") + except Exception as e: + print(" [FAIL] decode_rgba_hdr error:", e) + else: + print(" Texture is LDR; skipping decode_rgba_hdr().") + + # Cleanup + t.close(raw) + print(f"\n=== {name} backend OK ===\n") + + +# ---------------------------------------------------------------------------- +# Run tests for both backends +# ---------------------------------------------------------------------------- +test_backend("NATIVE", TranscoderBackend.NATIVE) +test_backend("WASM", TranscoderBackend.WASM) + +print("\n========== DONE ==========\n") diff --git a/external/basis_universal/readme_wasi.md b/external/basis_universal/readme_wasi.md new file mode 100644 index 0000000000..270dc81768 --- /dev/null +++ b/external/basis_universal/readme_wasi.md @@ -0,0 +1,167 @@ +# README_WASI.md + +## Building and running Basis Universal under WASI / Wasmtime + +This document describes how to build the `basisu` command-line tool as a WASI (WebAssembly System Interface) executable, and how to run it using Wasmtime. +WASI builds run the encoder inside a secure, portable WebAssembly sandbox with no native dependencies. + +--- + +## 1. Install Wasmtime + +Install Wasmtime using the official installer: + +``` +curl https://wasmtime.dev/install.sh | bash +``` + +Verify: + +``` +wasmtime --version +``` + +--- + +## 2. Install WASI-SDK (WASI toolchain) + +Download the latest WASI SDK from: + +https://github.com/WebAssembly/wasi-sdk/releases/latest +https://github.com/WebAssembly/wasi-sdk/releases + +Example (adjust version if needed): + +``` +wget https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-29/wasi-sdk-29.0-x86_64-linux.tar.gz +tar xf wasi-sdk-29.0-x86_64-linux.tar.gz +``` + +--- + +## 3. Set the WASI_SDK_PATH environment variable + +You must set the path so CMake can find the WASI compiler: + +``` +export WASI_SDK_PATH=/path/to/wasi-sdk-29.0-x86_64-linux +``` + +Example: + +``` +export WASI_SDK_PATH=$HOME/wasi-sdk-29.0-x86_64-linux +``` + +Verify: + +``` +$WASI_SDK_PATH/bin/clang --version +``` + +--- + +## 4. Configure the WASI build using CMake + +WASI builds come in two modes: +- Single-threaded (default) +- Multi-threaded (requires wasi-sdk-pthread.cmake and Wasmtime threading flags) + +Create a fresh build directory and configure using the WASI toolchain file: + +``` +mkdir build +cd build +cmake .. -DCMAKE_TOOLCHAIN_FILE=$WASI_SDK_PATH/share/cmake/wasi-sdk-pthread.cmake -DCMAKE_BUILD_TYPE=Release -DBASISU_WASM_THREADING=ON +``` + +Or for a single threaded build (will run much slower): + +``` +cmake .. -DCMAKE_TOOLCHAIN_FILE=$WASI_SDK_PATH/share/cmake/wasi-sdk.cmake -DCMAKE_BUILD_TYPE=Release -DBASISU_WASM_THREADING=OFF +``` + +--- + +## 5. Build the WASI `.wasm` executable + +Build using: + +``` +make +``` + +This produces: + +``` +bin/basisu.wasm +bin/examples.wasm (if EXAMPLES=ON) +``` + +--- + +## 6. Running `basisu.wasm` with Wasmtime + +WASI programs are sandboxed and cannot access your filesystem unless you explicitly grant permission. + +Use one or more `--dir=` arguments to allow input/output files. + +### Run internal test suite for ETC1S + +``` +bin$ wasmtime run --wasm threads=yes --wasi threads=yes --dir=. --dir=.. --dir=../test_files basisu.wasm -test +``` + +Use backslashes under Windows: "wasmtime run --wasm threads=yes --wasi threads=yes --dir=. --dir=.. --dir=..\test_files basisu.wasm -test" + +For the single threaded wasm executables, "--wasm threads=yes --wasi threads=yes" isn't needed. + +A Windows .cmd batch script example: + +``` +wasmtime --dir=. --dir=.. --dir=..\test_files --dir=d:/dev/test_images::/test_images --dir=d:/dev/test_images/bik::/bik basisu.wasm %* +``` + +A shell script example: + +``` +#!/usr/bin/env bash +wasmtime run --dir=. --dir=../test_files --dir=/mnt/d/dev/test_images::/test_images --dir=/mnt/d/dev/test_images/bik::/test_images/bik --wasm threads=yes --wasi threads=yes ./basisu.wasm "$@" +``` + +### Example: run compression on a PNG to ETC1S + +``` +wasmtime run --wasm threads=yes --wasi threads=yes --dir=. basisu.wasm xmen.png -stats +``` + +### Example: transcode a KTX2 file to .ktx/.png/etc. + +``` +wasmtime run --wasm threads=yes --wasi threads=yes --dir=. basisu.wasm xmen.ktx2 + +``` + +--- + +## Notes + +- WASI builds run inside a secure sandbox with no filesystem access unless explicitly granted via `--dir=`. +- The CMake configuration sets a larger stack size to support ASTC/UASTC compression. +- WASI SDK and Wasmtime can be installed anywhere; just update `WASI_SDK_PATH`. + +--- + +## Summary + +To build and run BasisU under WASI: + +1. Install **Wasmtime** +2. Install **WASI SDK** +3. Set **WASI_SDK_PATH** +4. Run **cmake** using the WASI toolchain in "build" directory +5. Build with **make** +6. Run using **wasmtime** with `--dir=` permissions on .wasm executables in "bin" directory + +This produces a safe, portable, sandboxed version of the Basis Universal encoder that runs anywhere. + diff --git a/external/basis_universal/test_files/base_xuastc_arith.ktx2 b/external/basis_universal/test_files/base_xuastc_arith.ktx2 new file mode 100644 index 0000000000..1b7aaa92ff Binary files /dev/null and b/external/basis_universal/test_files/base_xuastc_arith.ktx2 differ diff --git a/external/basis_universal/test_files/base_xuastc_zstd.ktx2 b/external/basis_universal/test_files/base_xuastc_zstd.ktx2 new file mode 100644 index 0000000000..50cf9584df Binary files /dev/null and b/external/basis_universal/test_files/base_xuastc_zstd.ktx2 differ diff --git a/external/basis_universal/test_files/kodim23.ktx2 b/external/basis_universal/test_files/kodim23.ktx2 new file mode 100644 index 0000000000..0672a0228b Binary files /dev/null and b/external/basis_universal/test_files/kodim23.ktx2 differ diff --git a/external/basis_universal/transcoder/basisu.h b/external/basis_universal/transcoder/basisu.h index 6e2efedd86..d4e339e917 100644 --- a/external/basis_universal/transcoder/basisu.h +++ b/external/basis_universal/transcoder/basisu.h @@ -1,5 +1,5 @@ // basisu.h -// Copyright (C) 2019-2024 Binomial LLC. All Rights Reserved. +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. // Important: If compiling with gcc, be sure strict aliasing is disabled: -fno-strict-aliasing // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,12 +15,16 @@ // limitations under the License. #pragma once +#ifndef BASISD_SUPPORT_XUASTC +#define BASISD_SUPPORT_XUASTC 1 +#endif + #ifdef _MSC_VER #pragma warning (disable : 4201) #pragma warning (disable : 4127) // warning C4127: conditional expression is constant #pragma warning (disable : 4530) // C++ exception handler used, but unwind semantics are not enabled. - + #endif // _MSC_VER #include @@ -40,9 +44,11 @@ #include #include #include +#include #include "basisu_containers.h" +// We never use min/max macros, slam them to off. #ifdef max #undef max #endif @@ -57,6 +63,7 @@ // Set to one to enable debug printf()'s when any errors occur, for development/debugging. Especially useful for WebGL development. #ifndef BASISU_FORCE_DEVEL_MESSAGES +// Do not check in as 1! #define BASISU_FORCE_DEVEL_MESSAGES 0 #endif @@ -93,6 +100,7 @@ namespace basisu typedef basisu::vector int_vec; typedef basisu::vector bool_vec; typedef basisu::vector float_vec; + typedef basisu::vector double_vec; void enable_debug_printf(bool enabled); void debug_printf(const char *pFmt, ...); @@ -109,14 +117,14 @@ namespace basisu #if defined(__GNUC__) && !defined(__clang__) #pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wclass-memaccess" +#pragma GCC diagnostic ignored "-Wclass-memaccess" #endif - + template inline void clear_obj(T& obj) { memset((void *)&obj, 0, sizeof(obj)); } #if defined(__GNUC__) && !defined(__clang__) #pragma GCC diagnostic pop -#endif +#endif constexpr double cPiD = 3.14159265358979323846264338327950288; constexpr float REALLY_SMALL_FLOAT_VAL = .000000125f; @@ -124,7 +132,7 @@ namespace basisu constexpr float BIG_FLOAT_VAL = 1e+30f; template inline T0 lerp(T0 a, T0 b, T1 c) { return a + (b - a) * c; } - + inline float clampf(float value, float low, float high) { if (value < low) value = low; else if (value > high) value = high; return value; } inline float saturate(float value) { return clampf(value, 0, 1.0f); } inline uint8_t minimumub(uint8_t a, uint8_t b) { return (a < b) ? a : b; } @@ -141,6 +149,41 @@ namespace basisu template inline T square(T a) { return a * a; } template inline T sign(T a) { return (a < 0) ? (T)-1 : ((a == 0) ? (T)0 : (T)1); } + inline int imod(int i, int d) + { + assert(i != INT_MIN); + + if (i >= 0) + return i % d; + + int r = (-i) % d; + return (r == 0) ? 0 : d - r; + } + + inline uint8_t safe_cast_uint8(uint32_t x) + { + assert(x <= UINT8_MAX); + return (uint8_t)x; + } + + inline int8_t safe_cast_int8(int32_t x) + { + assert((x >= INT8_MIN) && (x <= INT8_MAX)); + return (int8_t)x; + } + + inline uint16_t safe_cast_uint16(uint32_t x) + { + assert(x <= UINT16_MAX); + return (uint16_t)x; + } + + inline int16_t safe_cast_int16(int32_t x) + { + assert((x >= INT16_MIN) && (x <= INT16_MAX)); + return (int16_t)x; + } + inline bool equal_tol(float a, float b, float t) { return fabsf(a - b) <= ((maximum(fabsf(a), fabsf(b)) + 1.0f) * t); } inline bool equal_tol(double a, double b, double t) { return fabs(a - b) <= ((maximum(fabs(a), fabs(b)) + 1.0f) * t); } @@ -161,27 +204,35 @@ namespace basisu temp = 0; return temp; } - + inline uint32_t iabs(int32_t i) { return (i < 0) ? static_cast(-i) : static_cast(i); } inline uint64_t iabs64(int64_t i) { return (i < 0) ? static_cast(-i) : static_cast(i); } - template inline void clear_vector(T &vec) { vec.erase(vec.begin(), vec.end()); } + template inline void clear_vector(T &vec) { vec.erase(vec.begin(), vec.end()); } template inline typename T::value_type *enlarge_vector(T &vec, size_t n) { size_t cs = vec.size(); vec.resize(cs + n); return &vec[cs]; } inline bool is_pow2(uint32_t x) { return x && ((x & (x - 1U)) == 0U); } inline bool is_pow2(uint64_t x) { return x && ((x & (x - 1U)) == 0U); } + template inline T range_check(T v, T minv, T maxv) { assert(v >= minv && v <= maxv); BASISU_NOTE_UNUSED(minv); BASISU_NOTE_UNUSED(maxv); return v; } + template inline T range_check(T v, T maxv) { assert(v <= maxv); BASISU_NOTE_UNUSED(maxv); return v; } + template inline T open_range_check(T v, T minv, T maxv) { assert(v >= minv && v < maxv); BASISU_NOTE_UNUSED(minv); BASISU_NOTE_UNUSED(maxv); return v; } template inline T open_range_check(T v, T maxv) { assert(v < maxv); BASISU_NOTE_UNUSED(maxv); return v; } // Open interval - inline bool in_bounds(int v, int l, int h) + inline bool is_in_bounds(int v, int l, int h) { return (v >= l) && (v < h); } // Closed interval - inline bool in_range(int v, int l, int h) + inline bool is_in_range(int v, int l, int h) + { + return (v >= l) && (v <= h); + } + + inline bool is_in_range(float v, float l, float h) { return (v >= l) && (v <= h); } @@ -192,7 +243,7 @@ namespace basisu inline uint32_t get_bit(uint32_t src, int ndx) { - assert(in_bounds(ndx, 0, 32)); + assert(is_in_bounds(ndx, 0, 32)); return (src >> ndx) & 1; } @@ -204,7 +255,7 @@ namespace basisu inline uint32_t get_bits(uint32_t val, int low, int high) { const int num_bits = (high - low) + 1; - assert(in_range(num_bits, 1, 32)); + assert(is_in_range(num_bits, 1, 32)); val >>= low; if (num_bits != 32) @@ -213,8 +264,8 @@ namespace basisu return val; } - template inline void append_vector(T &vec, const R *pObjs, size_t n) - { + template inline void append_vector(T &vec, const R *pObjs, size_t n) + { if (n) { if (vec.size()) @@ -265,7 +316,7 @@ namespace basisu for (size_t i = 0; i < vec.size(); i++) vec[i] = obj; } - + inline uint64_t read_be64(const void *p) { uint64_t val = 0; @@ -307,6 +358,14 @@ namespace basisu return (m != 0) ? (y - m) : m; } + inline float posmodf(float x, float y) + { + float m = fmodf(x, y); + if (m < 0.0f) + m += y; + return m; + } + inline bool do_excl_ranges_overlap(int la, int ha, int lb, int hb) { assert(la < ha && lb < hb); @@ -331,7 +390,7 @@ namespace basisu pBytes[2] = (uint8_t)(val >> 16U); pBytes[3] = (uint8_t)(val >> 24U); } - + // Always little endian 1-8 byte unsigned int template struct packed_uint @@ -341,21 +400,21 @@ namespace basisu inline packed_uint() { static_assert(NumBytes <= sizeof(uint64_t), "Invalid NumBytes"); } inline packed_uint(uint64_t v) { *this = v; } inline packed_uint(const packed_uint& other) { *this = other; } - - inline packed_uint& operator= (uint64_t v) - { + + inline packed_uint& operator= (uint64_t v) + { // TODO: Add assert on truncation? - for (uint32_t i = 0; i < NumBytes; i++) - m_bytes[i] = static_cast(v >> (i * 8)); - return *this; + for (uint32_t i = 0; i < NumBytes; i++) + m_bytes[i] = static_cast(v >> (i * 8)); + return *this; } - inline packed_uint& operator= (const packed_uint& rhs) - { - memcpy(m_bytes, rhs.m_bytes, sizeof(m_bytes)); + inline packed_uint& operator= (const packed_uint& rhs) + { + memcpy(m_bytes, rhs.m_bytes, sizeof(m_bytes)); return *this; } - + inline uint64_t get_uint64() const { // Some compilers may warn about this code. It clearly cannot access beyond the end of the m_bytes struct here. @@ -411,7 +470,7 @@ namespace basisu static_assert(NumBytes <= sizeof(uint32_t), "packed_uint too large to use get_uint32"); return static_cast(get_uint64()); } - + inline operator uint32_t() const { static_assert(NumBytes <= sizeof(uint32_t), "packed_uint too large to use operator uint32_t"); @@ -421,14 +480,14 @@ namespace basisu enum eZero { cZero }; enum eNoClamp { cNoClamp }; - + // Rice/Huffman entropy coding - + // This is basically Deflate-style canonical Huffman, except we allow for a lot more symbols. enum { - cHuffmanMaxSupportedCodeSize = 16, cHuffmanMaxSupportedInternalCodeSize = 31, - cHuffmanFastLookupBits = 10, + cHuffmanMaxSupportedCodeSize = 16, cHuffmanMaxSupportedInternalCodeSize = 31, + cHuffmanFastLookupBits = 10, cHuffmanMaxSymsLog2 = 14, cHuffmanMaxSyms = 1 << cHuffmanMaxSymsLog2, // Small zero runs @@ -454,13 +513,13 @@ namespace basisu enum class texture_format { cInvalidTextureFormat = -1, - + // Block-based formats cETC1, // ETC1 cETC1S, // ETC1 (subset: diff colors only, no subblocks) cETC2_RGB, // ETC2 color block (basisu doesn't support ETC2 planar/T/H modes - just basic ETC1) cETC2_RGBA, // ETC2 EAC alpha block followed by ETC2 color block - cETC2_ALPHA, // ETC2 EAC alpha block + cETC2_ALPHA, // ETC2 EAC alpha block cBC1, // DXT1 cBC3, // DXT5 (BC4/DXT5A block followed by a BC1/DXT1 block) cBC4, // DXT5A @@ -479,11 +538,11 @@ namespace basisu cPVRTC2_4_RGBA, cETC2_R11_EAC, cETC2_RG11_EAC, - cUASTC4x4, + cUASTC4x4, cUASTC_HDR_4x4, cBC1_NV, cBC1_AMD, - + // Uncompressed/raw pixels cRGBA32, cRGB565, @@ -492,9 +551,89 @@ namespace basisu cABGR4444, cRGBA_HALF, cRGB_HALF, - cRGB_9E5 + cRGB_9E5, + + // All remaining ASTC LDR block size variants (other than 4x4 which is above). There are 14 total ASTC block sizes, including 4x4. + cASTC_LDR_5x4, + cASTC_LDR_5x5, + cASTC_LDR_6x5, + cASTC_LDR_6x6, + cASTC_LDR_8x5, + cASTC_LDR_8x6, + cASTC_LDR_10x5, + cASTC_LDR_10x6, + cASTC_LDR_8x8, + cASTC_LDR_10x8, + cASTC_LDR_10x10, + cASTC_LDR_12x10, + cASTC_LDR_12x12 }; + inline bool is_astc(texture_format fmt) + { + switch (fmt) + { + case texture_format::cASTC_HDR_4x4: + case texture_format::cASTC_HDR_6x6: + case texture_format::cASTC_LDR_4x4: + case texture_format::cASTC_LDR_5x4: + case texture_format::cASTC_LDR_5x5: + case texture_format::cASTC_LDR_6x5: + case texture_format::cASTC_LDR_6x6: + case texture_format::cASTC_LDR_8x5: + case texture_format::cASTC_LDR_8x6: + case texture_format::cASTC_LDR_10x5: + case texture_format::cASTC_LDR_10x6: + case texture_format::cASTC_LDR_8x8: + case texture_format::cASTC_LDR_10x8: + case texture_format::cASTC_LDR_10x10: + case texture_format::cASTC_LDR_12x10: + case texture_format::cASTC_LDR_12x12: + return true; + default: + break; + } + return false; + } + + inline bool is_hdr_astc(texture_format fmt) + { + switch (fmt) + { + case texture_format::cASTC_HDR_4x4: + case texture_format::cASTC_HDR_6x6: + return true; + default: + break; + } + return false; + } + + inline bool is_ldr_astc(texture_format fmt) + { + switch (fmt) + { + case texture_format::cASTC_LDR_4x4: + case texture_format::cASTC_LDR_5x4: + case texture_format::cASTC_LDR_5x5: + case texture_format::cASTC_LDR_6x5: + case texture_format::cASTC_LDR_6x6: + case texture_format::cASTC_LDR_8x5: + case texture_format::cASTC_LDR_8x6: + case texture_format::cASTC_LDR_10x5: + case texture_format::cASTC_LDR_10x6: + case texture_format::cASTC_LDR_8x8: + case texture_format::cASTC_LDR_10x8: + case texture_format::cASTC_LDR_10x10: + case texture_format::cASTC_LDR_12x10: + case texture_format::cASTC_LDR_12x12: + return true; + default: + break; + } + return false; + } + inline bool is_uncompressed_texture_format(texture_format fmt) { switch (fmt) @@ -555,7 +694,7 @@ namespace basisu default: break; } - + // Everything else is 16 bytes/block. return 16; } @@ -575,10 +714,21 @@ namespace basisu switch (fmt) { - case texture_format::cFXT1_RGB: - return 8; - case texture_format::cASTC_HDR_6x6: - return 6; + case texture_format::cFXT1_RGB: return 8; + case texture_format::cASTC_HDR_6x6: return 6; + case texture_format::cASTC_LDR_5x4: return 5; + case texture_format::cASTC_LDR_5x5: return 5; + case texture_format::cASTC_LDR_6x5: return 6; + case texture_format::cASTC_LDR_6x6: return 6; + case texture_format::cASTC_LDR_8x5: return 8; + case texture_format::cASTC_LDR_8x6: return 8; + case texture_format::cASTC_LDR_10x5: return 10; + case texture_format::cASTC_LDR_10x6: return 10; + case texture_format::cASTC_LDR_8x8: return 8; + case texture_format::cASTC_LDR_10x8: return 10; + case texture_format::cASTC_LDR_10x10: return 10; + case texture_format::cASTC_LDR_12x10: return 12; + case texture_format::cASTC_LDR_12x12: return 12; default: break; } @@ -591,8 +741,19 @@ namespace basisu switch (fmt) { - case texture_format::cASTC_HDR_6x6: - return 6; + case texture_format::cASTC_HDR_6x6: return 6; + case texture_format::cASTC_LDR_5x5: return 5; + case texture_format::cASTC_LDR_6x5: return 5; + case texture_format::cASTC_LDR_6x6: return 6; + case texture_format::cASTC_LDR_8x5: return 5; + case texture_format::cASTC_LDR_8x6: return 6; + case texture_format::cASTC_LDR_10x5: return 5; + case texture_format::cASTC_LDR_10x6: return 6; + case texture_format::cASTC_LDR_8x8: return 8; + case texture_format::cASTC_LDR_10x8: return 8; + case texture_format::cASTC_LDR_10x10: return 10; + case texture_format::cASTC_LDR_12x10: return 10; + case texture_format::cASTC_LDR_12x12: return 12; default: break; } @@ -623,5 +784,38 @@ namespace basisu { return !is_hdr_texture_format(fmt); } + + inline texture_format get_astc_ldr_texture_format(uint32_t width, uint32_t height) + { +#define BU_ASTC_LDR_MATCH_BLOCK_DIM(x, y, f) if ((width == (x)) && (height == (y))) return (f); + BU_ASTC_LDR_MATCH_BLOCK_DIM(4, 4, texture_format::cASTC_LDR_4x4); + BU_ASTC_LDR_MATCH_BLOCK_DIM(5, 4, texture_format::cASTC_LDR_5x4); + + BU_ASTC_LDR_MATCH_BLOCK_DIM(5, 5, texture_format::cASTC_LDR_5x5); + + BU_ASTC_LDR_MATCH_BLOCK_DIM(6, 5, texture_format::cASTC_LDR_6x5); + BU_ASTC_LDR_MATCH_BLOCK_DIM(6, 6, texture_format::cASTC_LDR_6x6); + + BU_ASTC_LDR_MATCH_BLOCK_DIM(8, 5, texture_format::cASTC_LDR_8x5); + BU_ASTC_LDR_MATCH_BLOCK_DIM(8, 6, texture_format::cASTC_LDR_8x6); + BU_ASTC_LDR_MATCH_BLOCK_DIM(10, 5, texture_format::cASTC_LDR_10x5); + BU_ASTC_LDR_MATCH_BLOCK_DIM(10, 6, texture_format::cASTC_LDR_10x6); + + BU_ASTC_LDR_MATCH_BLOCK_DIM(8, 8, texture_format::cASTC_LDR_8x8); + BU_ASTC_LDR_MATCH_BLOCK_DIM(10, 8, texture_format::cASTC_LDR_10x8); + BU_ASTC_LDR_MATCH_BLOCK_DIM(10, 10, texture_format::cASTC_LDR_10x10); + + BU_ASTC_LDR_MATCH_BLOCK_DIM(12, 10, texture_format::cASTC_LDR_12x10); + BU_ASTC_LDR_MATCH_BLOCK_DIM(12, 12, texture_format::cASTC_LDR_12x12); +#undef BU_ASTC_LDR_MATCH_BLOCK_DIM + + return texture_format::cInvalidTextureFormat; + } + inline bool is_valid_astc_block_size(uint32_t width, uint32_t height) + { + return get_astc_ldr_texture_format(width, height) != texture_format::cInvalidTextureFormat; + } + } // namespace basisu + diff --git a/external/basis_universal/transcoder/basisu_astc_cfgs.inl b/external/basis_universal/transcoder/basisu_astc_cfgs.inl new file mode 100644 index 0000000000..b9f138f3f5 --- /dev/null +++ b/external/basis_universal/transcoder/basisu_astc_cfgs.inl @@ -0,0 +1,648 @@ +const uint32_t BU_TOTAL_ASTC_CFGS = 10311; +const uint8_t s_astc_cfg_table[BU_TOTAL_ASTC_CFGS*3] = { +176,72,0,208,72,0,240,72,0,16,73,0,48,73,0,80,73,0,112,73,0,176,130,0,208,130,0,240,130,0,16,131,0,48,131,0,80,131,0,112,131,0,176,132,0,208,132,0, +240,132,0,16,133,0,48,133,0,80,133,0,112,133,0,176,134,0,208,134,0,240,134,0,16,135,0,48,135,0,80,135,0,112,135,0,176,194,0,208,194,0,240,194,0,16,195,0, +48,195,0,80,195,0,112,195,0,176,196,0,208,196,0,240,196,0,16,197,0,48,197,0,80,197,0,112,197,0,176,198,0,208,198,0,240,198,0,16,199,0,48,199,0,80,199,0, +112,199,0,176,2,1,208,2,1,240,2,1,16,3,1,48,3,1,80,3,1,112,3,1,176,4,1,208,4,1,240,4,1,16,5,1,48,5,1,80,5,1,112,5,1,176,6,1, +208,6,1,240,6,1,16,7,1,48,7,1,80,7,1,112,7,1,176,8,1,208,8,1,240,8,1,16,9,1,48,9,1,80,9,1,112,9,1,176,66,1,208,66,1,240,66,1, +16,67,1,48,67,1,80,67,1,112,67,1,176,68,1,208,68,1,240,68,1,16,69,1,48,69,1,80,69,1,112,69,1,176,70,1,208,70,1,240,70,1,16,71,1,48,71,1, +80,71,1,112,71,1,176,72,1,208,72,1,240,72,1,16,73,1,48,73,1,80,73,1,112,73,1,16,1,2,48,1,2,80,1,2,112,1,2,16,17,2,48,17,2,80,17,2, +112,17,2,16,33,2,48,33,2,80,33,2,112,33,2,16,65,2,48,65,2,80,65,2,112,65,2,80,72,2,112,72,2,144,72,2,176,72,2,208,72,2,240,72,2,16,73,2, +48,73,2,80,73,2,112,73,2,16,81,2,48,81,2,80,81,2,112,81,2,10,97,2,42,97,2,73,97,2,105,97,2,16,129,2,48,129,2,80,129,2,112,129,2,80,130,2, +112,130,2,144,130,2,176,130,2,208,130,2,240,130,2,16,131,2,48,131,2,80,131,2,112,131,2,80,132,2,112,132,2,144,132,2,176,132,2,208,132,2,240,132,2,16,133,2, +48,133,2,80,133,2,112,133,2,80,134,2,112,134,2,144,134,2,176,134,2,208,134,2,240,134,2,16,135,2,48,135,2,80,135,2,112,135,2,16,145,2,48,145,2,80,145,2, +112,145,2,10,161,2,42,161,2,73,161,2,105,161,2,16,193,2,48,193,2,80,193,2,112,193,2,80,194,2,112,194,2,144,194,2,176,194,2,208,194,2,240,194,2,16,195,2, +48,195,2,80,195,2,112,195,2,80,196,2,112,196,2,144,196,2,176,196,2,208,196,2,240,196,2,16,197,2,48,197,2,80,197,2,112,197,2,80,198,2,112,198,2,144,198,2, +176,198,2,208,198,2,240,198,2,16,199,2,48,199,2,80,199,2,112,199,2,10,209,2,42,209,2,73,209,2,105,209,2,4,225,2,36,225,2,67,225,2,99,225,2,16,1,3, +48,1,3,80,1,3,112,1,3,80,2,3,112,2,3,144,2,3,176,2,3,208,2,3,240,2,3,16,3,3,48,3,3,80,3,3,112,3,3,80,4,3,112,4,3,144,4,3, +176,4,3,208,4,3,240,4,3,16,5,3,48,5,3,80,5,3,112,5,3,80,6,3,112,6,3,144,6,3,176,6,3,208,6,3,240,6,3,16,7,3,48,7,3,80,7,3, +112,7,3,80,8,3,112,8,3,144,8,3,176,8,3,208,8,3,240,8,3,16,9,3,48,9,3,80,9,3,112,9,3,10,17,3,42,17,3,73,17,3,105,17,3,4,33,3, +36,33,3,67,33,3,99,33,3,16,65,3,48,65,3,80,65,3,112,65,3,80,66,3,112,66,3,144,66,3,176,66,3,208,66,3,240,66,3,15,67,3,45,67,3,76,67,3, +106,67,3,80,68,3,112,68,3,144,68,3,176,68,3,208,68,3,240,68,3,15,69,3,45,69,3,76,69,3,106,69,3,80,70,3,112,70,3,144,70,3,176,70,3,208,70,3, +240,70,3,15,71,3,45,71,3,76,71,3,106,71,3,80,72,3,112,72,3,144,72,3,176,72,3,208,72,3,240,72,3,15,73,3,45,73,3,76,73,3,106,73,3,6,81,3, +37,81,3,69,81,3,100,81,3,176,0,4,208,0,4,240,0,4,16,1,4,48,1,4,80,1,4,112,1,4,176,16,4,208,16,4,240,16,4,16,17,4,48,17,4,80,17,4, +112,17,4,176,32,4,208,32,4,240,32,4,16,33,4,48,33,4,80,33,4,112,33,4,176,64,4,208,64,4,240,64,4,16,65,4,48,65,4,80,65,4,112,65,4,48,72,4, +80,72,4,112,72,4,144,72,4,176,72,4,208,72,4,240,72,4,16,73,4,48,73,4,80,73,4,109,73,4,176,80,4,208,80,4,240,80,4,16,81,4,48,81,4,79,81,4, +110,81,4,170,96,4,202,96,4,233,96,4,8,97,4,40,97,4,71,97,4,102,97,4,176,128,4,208,128,4,240,128,4,16,129,4,48,129,4,80,129,4,112,129,4,48,130,4, +80,130,4,112,130,4,144,130,4,176,130,4,208,130,4,240,130,4,16,131,4,48,131,4,80,131,4,109,131,4,48,132,4,80,132,4,112,132,4,144,132,4,176,132,4,208,132,4, +240,132,4,16,133,4,48,133,4,80,133,4,109,133,4,48,134,4,80,134,4,112,134,4,144,134,4,176,134,4,208,134,4,240,134,4,16,135,4,48,135,4,80,135,4,109,135,4, +176,144,4,208,144,4,240,144,4,16,145,4,48,145,4,79,145,4,110,145,4,170,160,4,202,160,4,233,160,4,8,161,4,40,161,4,71,161,4,102,161,4,176,192,4,208,192,4, +240,192,4,16,193,4,48,193,4,80,193,4,112,193,4,48,194,4,80,194,4,112,194,4,144,194,4,176,194,4,208,194,4,240,194,4,14,195,4,43,195,4,73,195,4,102,195,4, +48,196,4,80,196,4,112,196,4,144,196,4,176,196,4,208,196,4,240,196,4,14,197,4,43,197,4,73,197,4,102,197,4,48,198,4,80,198,4,112,198,4,144,198,4,176,198,4, +208,198,4,240,198,4,14,199,4,43,199,4,73,199,4,102,199,4,170,208,4,202,208,4,233,208,4,8,209,4,40,209,4,71,209,4,102,209,4,164,224,4,196,224,4,227,224,4, +3,225,4,34,225,4,66,225,4,97,225,4,176,0,5,208,0,5,240,0,5,16,1,5,48,1,5,80,1,5,112,1,5,48,2,5,80,2,5,112,2,5,144,2,5,176,2,5, +208,2,5,240,2,5,14,3,5,43,3,5,73,3,5,102,3,5,48,4,5,80,4,5,112,4,5,144,4,5,176,4,5,208,4,5,240,4,5,14,5,5,43,5,5,73,5,5, +102,5,5,48,6,5,80,6,5,112,6,5,144,6,5,176,6,5,208,6,5,240,6,5,14,7,5,43,7,5,73,7,5,102,7,5,48,8,5,80,8,5,112,8,5,144,8,5, +176,8,5,208,8,5,240,8,5,14,9,5,43,9,5,73,9,5,102,9,5,170,16,5,202,16,5,233,16,5,8,17,5,40,17,5,71,17,5,102,17,5,164,32,5,196,32,5, +227,32,5,3,33,5,34,33,5,66,33,5,97,33,5,176,64,5,208,64,5,240,64,5,16,65,5,48,65,5,80,65,5,112,65,5,48,66,5,80,66,5,112,66,5,144,66,5, +175,66,5,204,66,5,235,66,5,9,67,5,38,67,5,69,67,5,99,67,5,48,68,5,80,68,5,112,68,5,144,68,5,175,68,5,204,68,5,235,68,5,9,69,5,38,69,5, +69,69,5,99,69,5,48,70,5,80,70,5,112,70,5,144,70,5,175,70,5,204,70,5,235,70,5,9,71,5,38,71,5,69,71,5,99,71,5,48,72,5,80,72,5,112,72,5, +144,72,5,175,72,5,204,72,5,235,72,5,9,73,5,38,73,5,69,73,5,99,73,5,166,80,5,197,80,5,229,80,5,4,81,5,36,81,5,67,81,5,99,81,5,112,0,6, +144,0,6,176,0,6,208,0,6,240,0,6,16,1,6,48,1,6,80,1,6,112,1,6,112,16,6,144,16,6,176,16,6,208,16,6,240,16,6,16,17,6,48,17,6,80,17,6, +112,17,6,112,32,6,144,32,6,176,32,6,208,32,6,240,32,6,16,33,6,48,33,6,80,33,6,112,33,6,112,64,6,144,64,6,176,64,6,208,64,6,240,64,6,16,65,6, +48,65,6,80,65,6,112,65,6,48,72,6,80,72,6,112,72,6,144,72,6,176,72,6,208,72,6,240,72,6,13,73,6,40,73,6,68,73,6,112,80,6,144,80,6,176,80,6, +208,80,6,239,80,6,14,81,6,44,81,6,76,81,6,106,81,6,106,96,6,138,96,6,169,96,6,200,96,6,231,96,6,6,97,6,37,97,6,69,97,6,100,97,6,112,128,6, +144,128,6,176,128,6,208,128,6,240,128,6,16,129,6,48,129,6,80,129,6,112,129,6,48,130,6,80,130,6,112,130,6,144,130,6,176,130,6,208,130,6,240,130,6,13,131,6, +40,131,6,68,131,6,48,132,6,80,132,6,112,132,6,144,132,6,176,132,6,208,132,6,240,132,6,13,133,6,40,133,6,68,133,6,48,134,6,80,134,6,112,134,6,144,134,6, +176,134,6,208,134,6,240,134,6,13,135,6,40,135,6,68,135,6,112,144,6,144,144,6,176,144,6,208,144,6,239,144,6,14,145,6,44,145,6,76,145,6,106,145,6,106,160,6, +138,160,6,169,160,6,200,160,6,231,160,6,6,161,6,37,161,6,69,161,6,100,161,6,112,192,6,144,192,6,176,192,6,208,192,6,240,192,6,16,193,6,48,193,6,80,193,6, +112,193,6,48,194,6,80,194,6,112,194,6,144,194,6,176,194,6,205,194,6,234,194,6,6,195,6,35,195,6,64,195,6,48,196,6,80,196,6,112,196,6,144,196,6,176,196,6, +205,196,6,234,196,6,6,197,6,35,197,6,64,197,6,48,198,6,80,198,6,112,198,6,144,198,6,176,198,6,205,198,6,234,198,6,6,199,6,35,199,6,64,199,6,106,208,6, +138,208,6,169,208,6,200,208,6,231,208,6,6,209,6,37,209,6,69,209,6,100,209,6,100,224,6,132,224,6,163,224,6,195,224,6,226,224,6,1,225,6,33,225,6,64,225,6, +96,225,6,112,0,7,144,0,7,176,0,7,208,0,7,240,0,7,16,1,7,48,1,7,80,1,7,112,1,7,48,2,7,80,2,7,112,2,7,144,2,7,176,2,7,205,2,7, +234,2,7,6,3,7,35,3,7,64,3,7,48,4,7,80,4,7,112,4,7,144,4,7,176,4,7,205,4,7,234,4,7,6,5,7,35,5,7,64,5,7,48,6,7,80,6,7, +112,6,7,144,6,7,176,6,7,205,6,7,234,6,7,6,7,7,35,7,7,64,7,7,48,8,7,80,8,7,112,8,7,144,8,7,176,8,7,205,8,7,234,8,7,6,9,7, +35,9,7,64,9,7,106,16,7,138,16,7,169,16,7,200,16,7,231,16,7,6,17,7,37,17,7,69,17,7,100,17,7,100,32,7,132,32,7,163,32,7,195,32,7,226,32,7, +1,33,7,33,33,7,64,33,7,96,33,7,112,64,7,144,64,7,176,64,7,208,64,7,240,64,7,16,65,7,48,65,7,80,65,7,111,65,7,48,66,7,80,66,7,111,66,7, +141,66,7,170,66,7,199,66,7,230,66,7,3,67,7,32,67,7,48,68,7,80,68,7,111,68,7,141,68,7,170,68,7,199,68,7,230,68,7,3,69,7,32,69,7,48,70,7, +80,70,7,111,70,7,141,70,7,170,70,7,199,70,7,230,70,7,3,71,7,32,71,7,48,72,7,80,72,7,111,72,7,141,72,7,170,72,7,199,72,7,230,72,7,3,73,7, +32,73,7,102,80,7,133,80,7,164,80,7,196,80,7,227,80,7,3,81,7,34,81,7,65,81,7,97,81,7,80,0,8,112,0,8,144,0,8,176,0,8,208,0,8,240,0,8, +16,1,8,48,1,8,80,1,8,112,1,8,80,16,8,112,16,8,144,16,8,176,16,8,208,16,8,240,16,8,16,17,8,48,17,8,80,17,8,112,17,8,80,32,8,112,32,8, +144,32,8,176,32,8,208,32,8,240,32,8,16,33,8,47,33,8,77,33,8,107,33,8,80,64,8,112,64,8,144,64,8,176,64,8,208,64,8,240,64,8,16,65,8,48,65,8, +80,65,8,112,65,8,16,72,8,48,72,8,80,72,8,112,72,8,144,72,8,176,72,8,205,72,8,232,72,8,1,73,8,80,80,8,112,80,8,144,80,8,175,80,8,206,80,8, +236,80,8,11,81,8,41,81,8,72,81,8,102,81,8,74,96,8,105,96,8,136,96,8,167,96,8,198,96,8,229,96,8,4,97,8,35,97,8,66,97,8,97,97,8,80,128,8, +112,128,8,144,128,8,176,128,8,208,128,8,240,128,8,16,129,8,48,129,8,80,129,8,112,129,8,16,130,8,48,130,8,80,130,8,112,130,8,144,130,8,176,130,8,205,130,8, +232,130,8,1,131,8,16,132,8,48,132,8,80,132,8,112,132,8,144,132,8,176,132,8,205,132,8,232,132,8,1,133,8,16,134,8,48,134,8,80,134,8,112,134,8,144,134,8, +176,134,8,205,134,8,232,134,8,1,135,8,80,144,8,112,144,8,144,144,8,175,144,8,206,144,8,236,144,8,11,145,8,41,145,8,72,145,8,102,145,8,74,160,8,105,160,8, +136,160,8,167,160,8,198,160,8,229,160,8,4,161,8,35,161,8,66,161,8,97,161,8,80,192,8,112,192,8,144,192,8,176,192,8,208,192,8,240,192,8,16,193,8,48,193,8, +80,193,8,112,193,8,16,194,8,48,194,8,80,194,8,112,194,8,143,194,8,170,194,8,198,194,8,227,194,8,16,196,8,48,196,8,80,196,8,112,196,8,143,196,8,170,196,8, +198,196,8,227,196,8,16,198,8,48,198,8,80,198,8,112,198,8,143,198,8,170,198,8,198,198,8,227,198,8,74,208,8,105,208,8,136,208,8,167,208,8,198,208,8,229,208,8, +4,209,8,35,209,8,66,209,8,97,209,8,68,224,8,99,224,8,131,224,8,162,224,8,193,224,8,225,224,8,0,225,8,32,225,8,80,0,9,112,0,9,144,0,9,176,0,9, +208,0,9,240,0,9,16,1,9,48,1,9,80,1,9,112,1,9,16,2,9,48,2,9,80,2,9,112,2,9,143,2,9,170,2,9,198,2,9,227,2,9,16,4,9,48,4,9, +80,4,9,112,4,9,143,4,9,170,4,9,198,4,9,227,4,9,16,6,9,48,6,9,80,6,9,112,6,9,143,6,9,170,6,9,198,6,9,227,6,9,16,8,9,48,8,9, +80,8,9,112,8,9,143,8,9,170,8,9,198,8,9,227,8,9,74,16,9,105,16,9,136,16,9,167,16,9,198,16,9,229,16,9,4,17,9,35,17,9,66,17,9,97,17,9, +68,32,9,99,32,9,131,32,9,162,32,9,193,32,9,225,32,9,0,33,9,32,33,9,80,64,9,112,64,9,144,64,9,176,64,9,208,64,9,240,64,9,15,65,9,46,65,9, +76,65,9,107,65,9,16,66,9,48,66,9,79,66,9,108,66,9,137,66,9,166,66,9,195,66,9,224,66,9,16,68,9,48,68,9,79,68,9,108,68,9,137,68,9,166,68,9, +195,68,9,224,68,9,16,70,9,48,70,9,79,70,9,108,70,9,137,70,9,166,70,9,195,70,9,224,70,9,16,72,9,48,72,9,79,72,9,108,72,9,137,72,9,166,72,9, +195,72,9,224,72,9,70,80,9,101,80,9,132,80,9,163,80,9,195,80,9,226,80,9,1,81,9,32,81,9,64,81,9,80,0,10,112,0,10,144,0,10,176,0,10,208,0,10, +240,0,10,16,1,10,48,1,10,80,1,10,112,1,10,80,16,10,112,16,10,144,16,10,176,16,10,208,16,10,240,16,10,16,17,10,48,17,10,80,17,10,109,17,10,80,32,10, +112,32,10,144,32,10,176,32,10,208,32,10,240,32,10,13,33,10,43,33,10,73,33,10,102,33,10,80,64,10,112,64,10,144,64,10,176,64,10,208,64,10,240,64,10,16,65,10, +48,65,10,80,65,10,112,65,10,16,72,10,48,72,10,80,72,10,112,72,10,144,72,10,170,72,10,195,72,10,80,80,10,112,80,10,143,80,10,173,80,10,203,80,10,234,80,10, +8,81,10,38,81,10,68,81,10,99,81,10,73,96,10,104,96,10,135,96,10,166,96,10,197,96,10,228,96,10,2,97,10,33,97,10,64,97,10,80,128,10,112,128,10,144,128,10, +176,128,10,208,128,10,240,128,10,16,129,10,48,129,10,80,129,10,112,129,10,16,130,10,48,130,10,80,130,10,112,130,10,144,130,10,170,130,10,195,130,10,16,132,10,48,132,10, +80,132,10,112,132,10,144,132,10,170,132,10,195,132,10,16,134,10,48,134,10,80,134,10,112,134,10,144,134,10,170,134,10,195,134,10,80,144,10,112,144,10,143,144,10,173,144,10, +203,144,10,234,144,10,8,145,10,38,145,10,68,145,10,99,145,10,73,160,10,104,160,10,135,160,10,166,160,10,197,160,10,228,160,10,2,161,10,33,161,10,64,161,10,80,192,10, +112,192,10,144,192,10,176,192,10,208,192,10,240,192,10,16,193,10,48,193,10,79,193,10,108,193,10,16,194,10,48,194,10,80,194,10,109,194,10,138,194,10,164,194,10,16,196,10, +48,196,10,80,196,10,109,196,10,138,196,10,164,196,10,16,198,10,48,198,10,80,198,10,109,198,10,138,198,10,164,198,10,73,208,10,104,208,10,135,208,10,166,208,10,197,208,10, +228,208,10,2,209,10,33,209,10,64,209,10,67,224,10,99,224,10,130,224,10,161,224,10,192,224,10,224,224,10,80,0,11,112,0,11,144,0,11,176,0,11,208,0,11,240,0,11, +16,1,11,48,1,11,79,1,11,108,1,11,16,2,11,48,2,11,80,2,11,109,2,11,138,2,11,164,2,11,16,4,11,48,4,11,80,4,11,109,4,11,138,4,11,164,4,11, +16,6,11,48,6,11,80,6,11,109,6,11,138,6,11,164,6,11,16,8,11,48,8,11,80,8,11,109,8,11,138,8,11,164,8,11,73,16,11,104,16,11,135,16,11,166,16,11, +197,16,11,228,16,11,2,17,11,33,17,11,64,17,11,67,32,11,99,32,11,130,32,11,161,32,11,192,32,11,224,32,11,80,64,11,112,64,11,144,64,11,176,64,11,208,64,11, +238,64,11,12,65,11,42,65,11,73,65,11,103,65,11,16,66,11,48,66,11,76,66,11,104,66,11,133,66,11,161,66,11,16,68,11,48,68,11,76,68,11,104,68,11,133,68,11, +161,68,11,16,70,11,48,70,11,76,70,11,104,70,11,133,70,11,161,70,11,16,72,11,48,72,11,76,72,11,104,72,11,133,72,11,161,72,11,69,80,11,100,80,11,131,80,11, +162,80,11,193,80,11,225,80,11,0,81,11,48,0,12,80,0,12,112,0,12,144,0,12,176,0,12,208,0,12,240,0,12,16,1,12,48,1,12,80,1,12,112,1,12,48,16,12, +80,16,12,112,16,12,144,16,12,176,16,12,208,16,12,240,16,12,16,17,12,45,17,12,74,17,12,102,17,12,48,32,12,80,32,12,112,32,12,144,32,12,176,32,12,206,32,12, +236,32,12,9,33,12,38,33,12,68,33,12,97,33,12,48,64,12,80,64,12,112,64,12,144,64,12,176,64,12,208,64,12,240,64,12,16,65,12,48,65,12,80,65,12,111,65,12, +16,72,12,48,72,12,80,72,12,112,72,12,138,72,12,161,72,12,48,80,12,80,80,12,111,80,12,141,80,12,171,80,12,201,80,12,231,80,12,5,81,12,35,81,12,65,81,12, +42,96,12,72,96,12,103,96,12,134,96,12,164,96,12,195,96,12,226,96,12,0,97,12,48,128,12,80,128,12,112,128,12,144,128,12,176,128,12,208,128,12,240,128,12,16,129,12, +48,129,12,80,129,12,111,129,12,16,130,12,48,130,12,80,130,12,112,130,12,138,130,12,161,130,12,16,132,12,48,132,12,80,132,12,112,132,12,138,132,12,161,132,12,16,134,12, +48,134,12,80,134,12,112,134,12,138,134,12,161,134,12,48,144,12,80,144,12,111,144,12,141,144,12,171,144,12,201,144,12,231,144,12,5,145,12,35,145,12,65,145,12,42,160,12, +72,160,12,103,160,12,134,160,12,164,160,12,195,160,12,226,160,12,0,161,12,48,192,12,80,192,12,112,192,12,144,192,12,176,192,12,208,192,12,240,192,12,15,193,12,44,193,12, +74,193,12,103,193,12,16,194,12,48,194,12,78,194,12,105,194,12,132,194,12,16,196,12,48,196,12,78,196,12,105,196,12,132,196,12,16,198,12,48,198,12,78,198,12,105,198,12, +132,198,12,42,208,12,72,208,12,103,208,12,134,208,12,164,208,12,195,208,12,226,208,12,0,209,12,36,224,12,67,224,12,98,224,12,129,224,12,160,224,12,48,0,13,80,0,13, +112,0,13,144,0,13,176,0,13,208,0,13,240,0,13,15,1,13,44,1,13,74,1,13,103,1,13,16,2,13,48,2,13,78,2,13,105,2,13,132,2,13,16,4,13,48,4,13, +78,4,13,105,4,13,132,4,13,16,6,13,48,6,13,78,6,13,105,6,13,132,6,13,16,8,13,48,8,13,78,8,13,105,8,13,132,8,13,42,16,13,72,16,13,103,16,13, +134,16,13,164,16,13,195,16,13,226,16,13,0,17,13,36,32,13,67,32,13,98,32,13,129,32,13,160,32,13,48,64,13,80,64,13,112,64,13,144,64,13,175,64,13,205,64,13, +236,64,13,9,65,13,39,65,13,70,65,13,99,65,13,16,66,13,45,66,13,73,66,13,100,66,13,129,66,13,16,68,13,45,68,13,73,68,13,100,68,13,129,68,13,16,70,13, +45,70,13,73,70,13,100,70,13,129,70,13,16,72,13,45,72,13,73,72,13,100,72,13,129,72,13,37,80,13,68,80,13,99,80,13,130,80,13,161,80,13,192,80,13,48,0,14, +80,0,14,112,0,14,144,0,14,176,0,14,208,0,14,240,0,14,16,1,14,48,1,14,80,1,14,112,1,14,48,16,14,80,16,14,112,16,14,144,16,14,176,16,14,208,16,14, +240,16,14,12,17,14,39,17,14,68,17,14,48,32,14,80,32,14,112,32,14,144,32,14,174,32,14,203,32,14,233,32,14,5,33,14,34,33,14,64,33,14,48,64,14,80,64,14, +112,64,14,144,64,14,176,64,14,208,64,14,240,64,14,16,65,14,48,65,14,77,65,14,103,65,14,16,72,14,48,72,14,80,72,14,106,72,14,131,72,14,48,80,14,79,80,14, +109,80,14,139,80,14,169,80,14,198,80,14,228,80,14,2,81,14,32,81,14,41,96,14,71,96,14,102,96,14,133,96,14,163,96,14,193,96,14,224,96,14,48,128,14,80,128,14, +112,128,14,144,128,14,176,128,14,208,128,14,240,128,14,16,129,14,48,129,14,77,129,14,103,129,14,16,130,14,48,130,14,80,130,14,106,130,14,131,130,14,16,132,14,48,132,14, +80,132,14,106,132,14,131,132,14,16,134,14,48,134,14,80,134,14,106,134,14,131,134,14,48,144,14,79,144,14,109,144,14,139,144,14,169,144,14,198,144,14,228,144,14,2,145,14, +32,145,14,41,160,14,71,160,14,102,160,14,133,160,14,163,160,14,193,160,14,224,160,14,48,192,14,80,192,14,112,192,14,144,192,14,176,192,14,208,192,14,239,192,14,11,193,14, +40,193,14,70,193,14,98,193,14,16,194,14,48,194,14,74,194,14,100,194,14,16,196,14,48,196,14,74,196,14,100,196,14,16,198,14,48,198,14,74,198,14,100,198,14,41,208,14, +71,208,14,102,208,14,133,208,14,163,208,14,193,208,14,224,208,14,35,224,14,66,224,14,97,224,14,128,224,14,48,0,15,80,0,15,112,0,15,144,0,15,176,0,15,208,0,15, +239,0,15,11,1,15,40,1,15,70,1,15,98,1,15,16,2,15,48,2,15,74,2,15,100,2,15,16,4,15,48,4,15,74,4,15,100,4,15,16,6,15,48,6,15,74,6,15, +100,6,15,16,8,15,48,8,15,74,8,15,100,8,15,41,16,15,71,16,15,102,16,15,133,16,15,163,16,15,193,16,15,224,16,15,35,32,15,66,32,15,97,32,15,128,32,15, +48,64,15,80,64,15,112,64,15,144,64,15,173,64,15,203,64,15,233,64,15,6,65,15,36,65,15,66,65,15,96,65,15,16,66,15,43,66,15,70,66,15,97,66,15,16,68,15, +43,68,15,70,68,15,97,68,15,16,70,15,43,70,15,70,70,15,97,70,15,16,72,15,43,72,15,70,72,15,97,72,15,37,80,15,67,80,15,98,80,15,129,80,15,160,80,15, +48,0,16,80,0,16,112,0,16,144,0,16,176,0,16,208,0,16,240,0,16,16,1,16,48,1,16,80,1,16,48,16,16,80,16,16,112,16,16,144,16,16,176,16,16,208,16,16, +236,16,16,6,17,16,33,17,16,48,32,16,80,32,16,112,32,16,143,32,16,171,32,16,200,32,16,229,32,16,1,33,16,48,64,16,80,64,16,112,64,16,144,64,16,176,64,16, +208,64,16,240,64,16,15,65,16,42,65,16,70,65,16,16,72,16,48,72,16,77,72,16,99,72,16,48,80,16,78,80,16,107,80,16,137,80,16,166,80,16,196,80,16,226,80,16, +40,96,16,70,96,16,101,96,16,131,96,16,161,96,16,192,96,16,48,128,16,80,128,16,112,128,16,144,128,16,176,128,16,208,128,16,240,128,16,15,129,16,42,129,16,70,129,16, +16,130,16,48,130,16,77,130,16,99,130,16,16,132,16,48,132,16,77,132,16,99,132,16,16,134,16,48,134,16,77,134,16,99,134,16,48,144,16,78,144,16,107,144,16,137,144,16, +166,144,16,196,144,16,226,144,16,40,160,16,70,160,16,101,160,16,131,160,16,161,160,16,192,160,16,48,192,16,80,192,16,112,192,16,144,192,16,176,192,16,206,192,16,235,192,16, +7,193,16,36,193,16,65,193,16,16,194,16,46,194,16,70,194,16,16,196,16,46,196,16,70,196,16,16,198,16,46,198,16,70,198,16,40,208,16,70,208,16,101,208,16,131,208,16, +161,208,16,192,208,16,35,224,16,65,224,16,96,224,16,128,224,16,48,0,17,80,0,17,112,0,17,144,0,17,176,0,17,206,0,17,235,0,17,7,1,17,36,1,17,65,1,17, +16,2,17,46,2,17,70,2,17,16,4,17,46,4,17,70,4,17,16,6,17,46,6,17,70,6,17,16,8,17,46,8,17,70,8,17,40,16,17,70,16,17,101,16,17,131,16,17, +161,16,17,192,16,17,35,32,17,65,32,17,96,32,17,128,32,17,48,64,17,80,64,17,112,64,17,142,64,17,171,64,17,200,64,17,230,64,17,3,65,17,33,65,17,16,66,17, +41,66,17,67,66,17,16,68,17,41,68,17,67,68,17,16,70,17,41,70,17,67,70,17,16,72,17,41,72,17,67,72,17,36,80,17,67,80,17,97,80,17,128,80,17,48,0,18, +80,0,18,112,0,18,144,0,18,176,0,18,208,0,18,240,0,18,16,1,18,46,1,18,48,16,18,80,16,18,112,16,18,144,16,18,176,16,18,202,16,18,230,16,18,0,17,18, +48,32,18,80,32,18,111,32,18,140,32,18,168,32,18,196,32,18,225,32,18,48,64,18,80,64,18,112,64,18,144,64,18,176,64,18,208,64,18,239,64,18,9,65,18,35,65,18, +16,72,18,48,72,18,71,72,18,47,80,18,76,80,18,105,80,18,135,80,18,164,80,18,193,80,18,39,96,18,69,96,18,99,96,18,130,96,18,160,96,18,48,128,18,80,128,18, +112,128,18,144,128,18,176,128,18,208,128,18,239,128,18,9,129,18,35,129,18,16,130,18,48,130,18,71,130,18,16,132,18,48,132,18,71,132,18,16,134,18,48,134,18,71,134,18, +47,144,18,76,144,18,105,144,18,135,144,18,164,144,18,193,144,18,39,160,18,69,160,18,99,160,18,130,160,18,160,160,18,48,192,18,80,192,18,112,192,18,144,192,18,174,192,18, +202,192,18,231,192,18,3,193,18,16,194,18,43,194,18,66,194,18,16,196,18,43,196,18,66,196,18,16,198,18,43,198,18,66,198,18,39,208,18,69,208,18,99,208,18,130,208,18, +160,208,18,34,224,18,65,224,18,96,224,18,48,0,19,80,0,19,112,0,19,144,0,19,174,0,19,202,0,19,231,0,19,3,1,19,16,2,19,43,2,19,66,2,19,16,4,19, +43,4,19,66,4,19,16,6,19,43,6,19,66,6,19,16,8,19,43,8,19,66,8,19,39,16,19,69,16,19,99,16,19,130,16,19,160,16,19,34,32,19,65,32,19,96,32,19, +48,64,19,80,64,19,110,64,19,140,64,19,169,64,19,198,64,19,227,64,19,0,65,19,16,66,19,38,66,19,64,66,19,16,68,19,38,68,19,64,68,19,16,70,19,38,70,19, +64,70,19,16,72,19,38,72,19,64,72,19,35,80,19,66,80,19,96,80,19,16,0,20,48,0,20,80,0,20,112,0,20,144,0,20,176,0,20,208,0,20,240,0,20,14,1,20, +16,16,20,48,16,20,80,16,20,112,16,20,144,16,20,172,16,20,198,16,20,225,16,20,16,32,20,48,32,20,80,32,20,109,32,20,138,32,20,165,32,20,193,32,20,16,64,20, +48,64,20,80,64,20,112,64,20,144,64,20,176,64,20,207,64,20,234,64,20,3,65,20,16,72,20,48,72,20,65,72,20,16,80,20,46,80,20,75,80,20,104,80,20,133,80,20, +162,80,20,10,96,20,39,96,20,68,96,20,98,96,20,129,96,20,16,128,20,48,128,20,80,128,20,112,128,20,144,128,20,176,128,20,207,128,20,234,128,20,3,129,20,16,130,20, +48,130,20,65,130,20,16,132,20,48,132,20,65,132,20,16,134,20,48,134,20,65,134,20,16,144,20,46,144,20,75,144,20,104,144,20,133,144,20,162,144,20,10,160,20,39,160,20, +68,160,20,98,160,20,129,160,20,16,192,20,48,192,20,80,192,20,112,192,20,144,192,20,171,192,20,199,192,20,228,192,20,16,194,20,40,194,20,16,196,20,40,196,20,16,198,20, +40,198,20,10,208,20,39,208,20,68,208,20,98,208,20,129,208,20,4,224,20,34,224,20,64,224,20,16,0,21,48,0,21,80,0,21,112,0,21,144,0,21,171,0,21,199,0,21, +228,0,21,16,2,21,40,2,21,16,4,21,40,4,21,16,6,21,40,6,21,16,8,21,40,8,21,10,16,21,39,16,21,68,16,21,98,16,21,129,16,21,4,32,21,34,32,21, +64,32,21,16,64,21,48,64,21,79,64,21,108,64,21,138,64,21,166,64,21,195,64,21,225,64,21,15,66,21,36,66,21,15,68,21,36,68,21,15,70,21,36,70,21,15,72,21, +36,72,21,6,80,21,35,80,21,65,80,21,96,80,21,16,1,22,48,1,22,80,1,22,112,1,22,16,17,22,48,17,22,80,17,22,112,17,22,16,33,22,48,33,22,80,33,22, +112,33,22,16,65,22,48,65,22,80,65,22,112,65,22,80,72,22,112,72,22,144,72,22,176,72,22,208,72,22,240,72,22,16,73,22,48,73,22,80,73,22,112,73,22,16,81,22, +48,81,22,80,81,22,112,81,22,10,97,22,42,97,22,73,97,22,105,97,22,16,129,22,48,129,22,80,129,22,112,129,22,80,130,22,112,130,22,144,130,22,176,130,22,208,130,22, +240,130,22,16,131,22,48,131,22,80,131,22,112,131,22,80,132,22,112,132,22,144,132,22,176,132,22,208,132,22,240,132,22,16,133,22,48,133,22,80,133,22,112,133,22,80,134,22, +112,134,22,144,134,22,176,134,22,208,134,22,240,134,22,16,135,22,48,135,22,80,135,22,112,135,22,16,145,22,48,145,22,80,145,22,112,145,22,10,161,22,42,161,22,73,161,22, +105,161,22,16,193,22,48,193,22,80,193,22,112,193,22,80,194,22,112,194,22,144,194,22,176,194,22,208,194,22,240,194,22,16,195,22,48,195,22,80,195,22,112,195,22,80,196,22, +112,196,22,144,196,22,176,196,22,208,196,22,240,196,22,16,197,22,48,197,22,80,197,22,112,197,22,80,198,22,112,198,22,144,198,22,176,198,22,208,198,22,240,198,22,16,199,22, +48,199,22,80,199,22,112,199,22,10,209,22,42,209,22,73,209,22,105,209,22,4,225,22,36,225,22,67,225,22,99,225,22,16,1,23,48,1,23,80,1,23,112,1,23,80,2,23, +112,2,23,144,2,23,176,2,23,208,2,23,240,2,23,16,3,23,48,3,23,80,3,23,112,3,23,80,4,23,112,4,23,144,4,23,176,4,23,208,4,23,240,4,23,16,5,23, +48,5,23,80,5,23,112,5,23,80,6,23,112,6,23,144,6,23,176,6,23,208,6,23,240,6,23,16,7,23,48,7,23,80,7,23,112,7,23,80,8,23,112,8,23,144,8,23, +176,8,23,208,8,23,240,8,23,16,9,23,48,9,23,80,9,23,112,9,23,10,17,23,42,17,23,73,17,23,105,17,23,4,33,23,36,33,23,67,33,23,99,33,23,16,65,23, +48,65,23,80,65,23,112,65,23,80,66,23,112,66,23,144,66,23,176,66,23,208,66,23,240,66,23,15,67,23,45,67,23,76,67,23,106,67,23,80,68,23,112,68,23,144,68,23, +176,68,23,208,68,23,240,68,23,15,69,23,45,69,23,76,69,23,106,69,23,80,70,23,112,70,23,144,70,23,176,70,23,208,70,23,240,70,23,15,71,23,45,71,23,76,71,23, +106,71,23,80,72,23,112,72,23,144,72,23,176,72,23,208,72,23,240,72,23,15,73,23,45,73,23,76,73,23,106,73,23,6,81,23,37,81,23,69,81,23,100,81,23,144,0,24, +176,0,24,208,0,24,240,0,24,16,1,24,48,1,24,80,1,24,112,1,24,144,16,24,176,16,24,208,16,24,240,16,24,16,17,24,48,17,24,80,17,24,112,17,24,144,32,24, +176,32,24,208,32,24,240,32,24,16,33,24,48,33,24,80,33,24,112,33,24,144,64,24,176,64,24,208,64,24,240,64,24,16,65,24,48,65,24,80,65,24,112,65,24,48,72,24, +80,72,24,112,72,24,144,72,24,176,72,24,208,72,24,240,72,24,16,73,24,47,73,24,75,73,24,102,73,24,144,80,24,176,80,24,208,80,24,240,80,24,15,81,24,46,81,24, +77,81,24,108,81,24,138,96,24,170,96,24,201,96,24,232,96,24,7,97,24,39,97,24,70,97,24,101,97,24,144,128,24,176,128,24,208,128,24,240,128,24,16,129,24,48,129,24, +80,129,24,112,129,24,48,130,24,80,130,24,112,130,24,144,130,24,176,130,24,208,130,24,240,130,24,16,131,24,47,131,24,75,131,24,102,131,24,48,132,24,80,132,24,112,132,24, +144,132,24,176,132,24,208,132,24,240,132,24,16,133,24,47,133,24,75,133,24,102,133,24,48,134,24,80,134,24,112,134,24,144,134,24,176,134,24,208,134,24,240,134,24,16,135,24, +47,135,24,75,135,24,102,135,24,144,144,24,176,144,24,208,144,24,240,144,24,15,145,24,46,145,24,77,145,24,108,145,24,138,160,24,170,160,24,201,160,24,232,160,24,7,161,24, +39,161,24,70,161,24,101,161,24,144,192,24,176,192,24,208,192,24,240,192,24,16,193,24,48,193,24,80,193,24,112,193,24,48,194,24,80,194,24,112,194,24,144,194,24,176,194,24, +208,194,24,238,194,24,10,195,24,39,195,24,69,195,24,97,195,24,48,196,24,80,196,24,112,196,24,144,196,24,176,196,24,208,196,24,238,196,24,10,197,24,39,197,24,69,197,24, +97,197,24,48,198,24,80,198,24,112,198,24,144,198,24,176,198,24,208,198,24,238,198,24,10,199,24,39,199,24,69,199,24,97,199,24,138,208,24,170,208,24,201,208,24,232,208,24, +7,209,24,39,209,24,70,209,24,101,209,24,132,224,24,164,224,24,195,224,24,227,224,24,2,225,24,34,225,24,65,225,24,97,225,24,144,0,25,176,0,25,208,0,25,240,0,25, +16,1,25,48,1,25,80,1,25,112,1,25,48,2,25,80,2,25,112,2,25,144,2,25,176,2,25,208,2,25,238,2,25,10,3,25,39,3,25,69,3,25,97,3,25,48,4,25, +80,4,25,112,4,25,144,4,25,176,4,25,208,4,25,238,4,25,10,5,25,39,5,25,69,5,25,97,5,25,48,6,25,80,6,25,112,6,25,144,6,25,176,6,25,208,6,25, +238,6,25,10,7,25,39,7,25,69,7,25,97,7,25,48,8,25,80,8,25,112,8,25,144,8,25,176,8,25,208,8,25,238,8,25,10,9,25,39,9,25,69,9,25,97,9,25, +138,16,25,170,16,25,201,16,25,232,16,25,7,17,25,39,17,25,70,17,25,101,17,25,132,32,25,164,32,25,195,32,25,227,32,25,2,33,25,34,33,25,65,33,25,97,33,25, +144,64,25,176,64,25,208,64,25,240,64,25,16,65,25,48,65,25,80,65,25,112,65,25,48,66,25,80,66,25,112,66,25,143,66,25,172,66,25,202,66,25,232,66,25,6,67,25, +35,67,25,65,67,25,48,68,25,80,68,25,112,68,25,143,68,25,172,68,25,202,68,25,232,68,25,6,69,25,35,69,25,65,69,25,48,70,25,80,70,25,112,70,25,143,70,25, +172,70,25,202,70,25,232,70,25,6,71,25,35,71,25,65,71,25,48,72,25,80,72,25,112,72,25,143,72,25,172,72,25,202,72,25,232,72,25,6,73,25,35,73,25,65,73,25, +134,80,25,165,80,25,196,80,25,228,80,25,3,81,25,35,81,25,66,81,25,98,81,25,80,0,26,112,0,26,144,0,26,176,0,26,208,0,26,240,0,26,16,1,26,48,1,26, +80,1,26,112,1,26,80,16,26,112,16,26,144,16,26,176,16,26,208,16,26,240,16,26,16,17,26,48,17,26,80,17,26,112,17,26,80,32,26,112,32,26,144,32,26,176,32,26, +208,32,26,240,32,26,16,33,26,47,33,26,77,33,26,107,33,26,80,64,26,112,64,26,144,64,26,176,64,26,208,64,26,240,64,26,16,65,26,48,65,26,80,65,26,112,65,26, +16,72,26,48,72,26,80,72,26,112,72,26,144,72,26,176,72,26,205,72,26,232,72,26,1,73,26,80,80,26,112,80,26,144,80,26,175,80,26,206,80,26,236,80,26,11,81,26, +41,81,26,72,81,26,102,81,26,74,96,26,105,96,26,136,96,26,167,96,26,198,96,26,229,96,26,4,97,26,35,97,26,66,97,26,97,97,26,80,128,26,112,128,26,144,128,26, +176,128,26,208,128,26,240,128,26,16,129,26,48,129,26,80,129,26,112,129,26,16,130,26,48,130,26,80,130,26,112,130,26,144,130,26,176,130,26,205,130,26,232,130,26,1,131,26, +16,132,26,48,132,26,80,132,26,112,132,26,144,132,26,176,132,26,205,132,26,232,132,26,1,133,26,16,134,26,48,134,26,80,134,26,112,134,26,144,134,26,176,134,26,205,134,26, +232,134,26,1,135,26,80,144,26,112,144,26,144,144,26,175,144,26,206,144,26,236,144,26,11,145,26,41,145,26,72,145,26,102,145,26,74,160,26,105,160,26,136,160,26,167,160,26, +198,160,26,229,160,26,4,161,26,35,161,26,66,161,26,97,161,26,80,192,26,112,192,26,144,192,26,176,192,26,208,192,26,240,192,26,16,193,26,48,193,26,80,193,26,112,193,26, +16,194,26,48,194,26,80,194,26,112,194,26,143,194,26,170,194,26,198,194,26,227,194,26,16,196,26,48,196,26,80,196,26,112,196,26,143,196,26,170,196,26,198,196,26,227,196,26, +16,198,26,48,198,26,80,198,26,112,198,26,143,198,26,170,198,26,198,198,26,227,198,26,74,208,26,105,208,26,136,208,26,167,208,26,198,208,26,229,208,26,4,209,26,35,209,26, +66,209,26,97,209,26,68,224,26,99,224,26,131,224,26,162,224,26,193,224,26,225,224,26,0,225,26,32,225,26,80,0,27,112,0,27,144,0,27,176,0,27,208,0,27,240,0,27, +16,1,27,48,1,27,80,1,27,112,1,27,16,2,27,48,2,27,80,2,27,112,2,27,143,2,27,170,2,27,198,2,27,227,2,27,16,4,27,48,4,27,80,4,27,112,4,27, +143,4,27,170,4,27,198,4,27,227,4,27,16,6,27,48,6,27,80,6,27,112,6,27,143,6,27,170,6,27,198,6,27,227,6,27,16,8,27,48,8,27,80,8,27,112,8,27, +143,8,27,170,8,27,198,8,27,227,8,27,74,16,27,105,16,27,136,16,27,167,16,27,198,16,27,229,16,27,4,17,27,35,17,27,66,17,27,97,17,27,68,32,27,99,32,27, +131,32,27,162,32,27,193,32,27,225,32,27,0,33,27,32,33,27,80,64,27,112,64,27,144,64,27,176,64,27,208,64,27,240,64,27,15,65,27,46,65,27,76,65,27,107,65,27, +16,66,27,48,66,27,79,66,27,108,66,27,137,66,27,166,66,27,195,66,27,224,66,27,16,68,27,48,68,27,79,68,27,108,68,27,137,68,27,166,68,27,195,68,27,224,68,27, +16,70,27,48,70,27,79,70,27,108,70,27,137,70,27,166,70,27,195,70,27,224,70,27,16,72,27,48,72,27,79,72,27,108,72,27,137,72,27,166,72,27,195,72,27,224,72,27, +70,80,27,101,80,27,132,80,27,163,80,27,195,80,27,226,80,27,1,81,27,32,81,27,64,81,27,48,0,28,80,0,28,112,0,28,144,0,28,176,0,28,208,0,28,240,0,28, +16,1,28,48,1,28,80,1,28,112,1,28,48,16,28,80,16,28,112,16,28,144,16,28,176,16,28,208,16,28,240,16,28,16,17,28,48,17,28,78,17,28,106,17,28,48,32,28, +80,32,28,112,32,28,144,32,28,176,32,28,208,32,28,238,32,28,11,33,28,41,33,28,71,33,28,100,33,28,48,64,28,80,64,28,112,64,28,144,64,28,176,64,28,208,64,28, +240,64,28,16,65,28,48,65,28,80,65,28,112,65,28,16,72,28,48,72,28,80,72,28,112,72,28,143,72,28,166,72,28,48,80,28,80,80,28,112,80,28,142,80,28,172,80,28, +202,80,28,233,80,28,6,81,28,36,81,28,67,81,28,97,81,28,42,96,28,73,96,28,104,96,28,135,96,28,165,96,28,196,96,28,227,96,28,1,97,28,32,97,28,48,128,28, +80,128,28,112,128,28,144,128,28,176,128,28,208,128,28,240,128,28,16,129,28,48,129,28,80,129,28,112,129,28,16,130,28,48,130,28,80,130,28,112,130,28,143,130,28,166,130,28, +16,132,28,48,132,28,80,132,28,112,132,28,143,132,28,166,132,28,16,134,28,48,134,28,80,134,28,112,134,28,143,134,28,166,134,28,48,144,28,80,144,28,112,144,28,142,144,28, +172,144,28,202,144,28,233,144,28,6,145,28,36,145,28,67,145,28,97,145,28,42,160,28,73,160,28,104,160,28,135,160,28,165,160,28,196,160,28,227,160,28,1,161,28,32,161,28, +48,192,28,80,192,28,112,192,28,144,192,28,176,192,28,208,192,28,240,192,28,16,193,28,47,193,28,77,193,28,106,193,28,16,194,28,48,194,28,80,194,28,107,194,28,135,194,28, +161,194,28,16,196,28,48,196,28,80,196,28,107,196,28,135,196,28,161,196,28,16,198,28,48,198,28,80,198,28,107,198,28,135,198,28,161,198,28,42,208,28,73,208,28,104,208,28, +135,208,28,165,208,28,196,208,28,227,208,28,1,209,28,32,209,28,36,224,28,67,224,28,98,224,28,130,224,28,161,224,28,192,224,28,48,0,29,80,0,29,112,0,29,144,0,29, +176,0,29,208,0,29,240,0,29,16,1,29,47,1,29,77,1,29,106,1,29,16,2,29,48,2,29,80,2,29,107,2,29,135,2,29,161,2,29,16,4,29,48,4,29,80,4,29, +107,4,29,135,4,29,161,4,29,16,6,29,48,6,29,80,6,29,107,6,29,135,6,29,161,6,29,16,8,29,48,8,29,80,8,29,107,8,29,135,8,29,161,8,29,42,16,29, +73,16,29,104,16,29,135,16,29,165,16,29,196,16,29,227,16,29,1,17,29,32,17,29,36,32,29,67,32,29,98,32,29,130,32,29,161,32,29,192,32,29,48,64,29,80,64,29, +112,64,29,144,64,29,176,64,29,207,64,29,237,64,29,11,65,29,41,65,29,71,65,29,101,65,29,16,66,29,47,66,29,74,66,29,102,66,29,131,66,29,16,68,29,47,68,29, +74,68,29,102,68,29,131,68,29,16,70,29,47,70,29,74,70,29,102,70,29,131,70,29,16,72,29,47,72,29,74,72,29,102,72,29,131,72,29,38,80,29,68,80,29,100,80,29, +131,80,29,162,80,29,193,80,29,224,80,29,48,0,30,80,0,30,112,0,30,144,0,30,176,0,30,208,0,30,240,0,30,16,1,30,48,1,30,80,1,30,112,1,30,48,16,30, +80,16,30,112,16,30,144,16,30,176,16,30,208,16,30,240,16,30,12,17,30,39,17,30,68,17,30,48,32,30,80,32,30,112,32,30,144,32,30,174,32,30,203,32,30,233,32,30, +5,33,30,34,33,30,64,33,30,48,64,30,80,64,30,112,64,30,144,64,30,176,64,30,208,64,30,240,64,30,16,65,30,48,65,30,77,65,30,103,65,30,16,72,30,48,72,30, +80,72,30,106,72,30,131,72,30,48,80,30,79,80,30,109,80,30,139,80,30,169,80,30,198,80,30,228,80,30,2,81,30,32,81,30,41,96,30,71,96,30,102,96,30,133,96,30, +163,96,30,193,96,30,224,96,30,48,128,30,80,128,30,112,128,30,144,128,30,176,128,30,208,128,30,240,128,30,16,129,30,48,129,30,77,129,30,103,129,30,16,130,30,48,130,30, +80,130,30,106,130,30,131,130,30,16,132,30,48,132,30,80,132,30,106,132,30,131,132,30,16,134,30,48,134,30,80,134,30,106,134,30,131,134,30,48,144,30,79,144,30,109,144,30, +139,144,30,169,144,30,198,144,30,228,144,30,2,145,30,32,145,30,41,160,30,71,160,30,102,160,30,133,160,30,163,160,30,193,160,30,224,160,30,48,192,30,80,192,30,112,192,30, +144,192,30,176,192,30,208,192,30,239,192,30,11,193,30,40,193,30,70,193,30,98,193,30,16,194,30,48,194,30,74,194,30,100,194,30,16,196,30,48,196,30,74,196,30,100,196,30, +16,198,30,48,198,30,74,198,30,100,198,30,41,208,30,71,208,30,102,208,30,133,208,30,163,208,30,193,208,30,224,208,30,35,224,30,66,224,30,97,224,30,128,224,30,48,0,31, +80,0,31,112,0,31,144,0,31,176,0,31,208,0,31,239,0,31,11,1,31,40,1,31,70,1,31,98,1,31,16,2,31,48,2,31,74,2,31,100,2,31,16,4,31,48,4,31, +74,4,31,100,4,31,16,6,31,48,6,31,74,6,31,100,6,31,16,8,31,48,8,31,74,8,31,100,8,31,41,16,31,71,16,31,102,16,31,133,16,31,163,16,31,193,16,31, +224,16,31,35,32,31,66,32,31,97,32,31,128,32,31,48,64,31,80,64,31,112,64,31,144,64,31,173,64,31,203,64,31,233,64,31,6,65,31,36,65,31,66,65,31,96,65,31, +16,66,31,43,66,31,70,66,31,97,66,31,16,68,31,43,68,31,70,68,31,97,68,31,16,70,31,43,70,31,70,70,31,97,70,31,16,72,31,43,72,31,70,72,31,97,72,31, +37,80,31,67,80,31,98,80,31,129,80,31,160,80,31,48,0,32,80,0,32,112,0,32,144,0,32,176,0,32,208,0,32,240,0,32,16,1,32,48,1,32,48,16,32,80,16,32, +112,16,32,144,16,32,176,16,32,205,16,32,233,16,32,3,17,32,48,32,32,80,32,32,112,32,32,142,32,32,170,32,32,198,32,32,227,32,32,48,64,32,80,64,32,112,64,32, +144,64,32,176,64,32,208,64,32,240,64,32,12,65,32,39,65,32,16,72,32,48,72,32,74,72,32,48,80,32,77,80,32,106,80,32,136,80,32,165,80,32,195,80,32,224,80,32, +40,96,32,70,96,32,100,96,32,131,96,32,161,96,32,48,128,32,80,128,32,112,128,32,144,128,32,176,128,32,208,128,32,240,128,32,12,129,32,39,129,32,16,130,32,48,130,32, +74,130,32,16,132,32,48,132,32,74,132,32,16,134,32,48,134,32,74,134,32,48,144,32,77,144,32,106,144,32,136,144,32,165,144,32,195,144,32,224,144,32,40,160,32,70,160,32, +100,160,32,131,160,32,161,160,32,48,192,32,80,192,32,112,192,32,144,192,32,176,192,32,204,192,32,233,192,32,5,193,32,34,193,32,16,194,32,44,194,32,68,194,32,16,196,32, +44,196,32,68,196,32,16,198,32,44,198,32,68,198,32,40,208,32,70,208,32,100,208,32,131,208,32,161,208,32,35,224,32,65,224,32,96,224,32,48,0,33,80,0,33,112,0,33, +144,0,33,176,0,33,204,0,33,233,0,33,5,1,33,34,1,33,16,2,33,44,2,33,68,2,33,16,4,33,44,4,33,68,4,33,16,6,33,44,6,33,68,6,33,16,8,33, +44,8,33,68,8,33,40,16,33,70,16,33,100,16,33,131,16,33,161,16,33,35,32,33,65,32,33,96,32,33,48,64,33,80,64,33,111,64,33,141,64,33,170,64,33,199,64,33, +229,64,33,2,65,33,16,66,33,39,66,33,65,66,33,16,68,33,39,68,33,65,68,33,16,70,33,39,70,33,65,70,33,16,72,33,39,72,33,65,72,33,36,80,33,66,80,33, +97,80,33,128,80,33,16,0,34,48,0,34,80,0,34,112,0,34,144,0,34,176,0,34,208,0,34,240,0,34,14,1,34,16,16,34,48,16,34,80,16,34,112,16,34,144,16,34, +172,16,34,198,16,34,225,16,34,16,32,34,48,32,34,80,32,34,109,32,34,138,32,34,165,32,34,193,32,34,16,64,34,48,64,34,80,64,34,112,64,34,144,64,34,176,64,34, +207,64,34,234,64,34,3,65,34,16,72,34,48,72,34,65,72,34,16,80,34,46,80,34,75,80,34,104,80,34,133,80,34,162,80,34,10,96,34,39,96,34,68,96,34,98,96,34, +129,96,34,16,128,34,48,128,34,80,128,34,112,128,34,144,128,34,176,128,34,207,128,34,234,128,34,3,129,34,16,130,34,48,130,34,65,130,34,16,132,34,48,132,34,65,132,34, +16,134,34,48,134,34,65,134,34,16,144,34,46,144,34,75,144,34,104,144,34,133,144,34,162,144,34,10,160,34,39,160,34,68,160,34,98,160,34,129,160,34,16,192,34,48,192,34, +80,192,34,112,192,34,144,192,34,171,192,34,199,192,34,228,192,34,16,194,34,40,194,34,16,196,34,40,196,34,16,198,34,40,198,34,10,208,34,39,208,34,68,208,34,98,208,34, +129,208,34,4,224,34,34,224,34,64,224,34,16,0,35,48,0,35,80,0,35,112,0,35,144,0,35,171,0,35,199,0,35,228,0,35,16,2,35,40,2,35,16,4,35,40,4,35, +16,6,35,40,6,35,16,8,35,40,8,35,10,16,35,39,16,35,68,16,35,98,16,35,129,16,35,4,32,35,34,32,35,64,32,35,16,64,35,48,64,35,79,64,35,108,64,35, +138,64,35,166,64,35,195,64,35,225,64,35,15,66,35,36,66,35,15,68,35,36,68,35,15,70,35,36,70,35,15,72,35,36,72,35,6,80,35,35,80,35,65,80,35,96,80,35, +16,0,36,48,0,36,80,0,36,112,0,36,144,0,36,176,0,36,208,0,36,16,16,36,48,16,36,80,16,36,112,16,36,141,16,36,165,16,36,16,32,36,48,32,36,78,32,36, +106,32,36,134,32,36,161,32,36,16,64,36,48,64,36,80,64,36,112,64,36,144,64,36,174,64,36,199,64,36,16,72,36,40,72,36,16,80,36,44,80,36,73,80,36,101,80,36, +130,80,36,10,96,36,37,96,36,67,96,36,97,96,36,16,128,36,48,128,36,80,128,36,112,128,36,144,128,36,174,128,36,199,128,36,16,130,36,40,130,36,16,132,36,40,132,36, +16,134,36,40,134,36,16,144,36,44,144,36,73,144,36,101,144,36,130,144,36,10,160,36,37,160,36,67,160,36,97,160,36,16,192,36,48,192,36,80,192,36,112,192,36,140,192,36, +167,192,36,194,192,36,16,194,36,35,194,36,16,196,36,35,196,36,16,198,36,35,198,36,10,208,36,37,208,36,67,208,36,97,208,36,4,224,36,33,224,36,16,0,37,48,0,37, +80,0,37,112,0,37,140,0,37,167,0,37,194,0,37,16,2,37,35,2,37,16,4,37,35,4,37,16,6,37,35,6,37,16,8,37,35,8,37,10,16,37,37,16,37,67,16,37, +97,16,37,4,32,37,33,32,37,16,64,37,48,64,37,77,64,37,106,64,37,135,64,37,163,64,37,192,64,37,12,66,37,32,66,37,12,68,37,32,68,37,12,70,37,32,70,37, +12,72,37,32,72,37,5,80,37,34,80,37,64,80,37,16,0,38,48,0,38,80,0,38,112,0,38,144,0,38,176,0,38,16,16,38,48,16,38,80,16,38,109,16,38,135,16,38, +16,32,38,48,32,38,75,32,38,102,32,38,130,32,38,16,64,38,48,64,38,80,64,38,112,64,38,144,64,38,167,64,38,16,72,38,33,72,38,16,80,38,43,80,38,70,80,38, +99,80,38,128,80,38,9,96,38,36,96,38,65,96,38,16,128,38,48,128,38,80,128,38,112,128,38,144,128,38,167,128,38,16,130,38,33,130,38,16,132,38,33,132,38,16,134,38, +33,134,38,16,144,38,43,144,38,70,144,38,99,144,38,128,144,38,9,160,38,36,160,38,65,160,38,16,192,38,48,192,38,80,192,38,108,192,38,136,192,38,162,192,38,16,194,38, +16,196,38,16,198,38,9,208,38,36,208,38,65,208,38,3,224,38,32,224,38,16,0,39,48,0,39,80,0,39,108,0,39,136,0,39,162,0,39,16,2,39,16,4,39,16,6,39, +16,8,39,9,16,39,36,16,39,65,16,39,3,32,39,32,32,39,16,64,39,47,64,39,75,64,39,103,64,39,132,64,39,160,64,39,10,66,39,10,68,39,10,70,39,10,72,39, +4,80,39,33,80,39,16,0,40,48,0,40,80,0,40,112,0,40,144,0,40,16,16,40,48,16,40,80,16,40,104,16,40,129,16,40,16,32,40,47,32,40,72,32,40,99,32,40, +16,64,40,48,64,40,80,64,40,112,64,40,138,64,40,16,80,40,41,80,40,68,80,40,96,80,40,8,96,40,35,96,40,64,96,40,16,128,40,48,128,40,80,128,40,112,128,40, +138,128,40,16,144,40,41,144,40,68,144,40,96,144,40,8,160,40,35,160,40,64,160,40,16,192,40,48,192,40,78,192,40,105,192,40,132,192,40,8,208,40,35,208,40,64,208,40, +3,224,40,16,0,41,48,0,41,78,0,41,105,0,41,132,0,41,8,16,41,35,16,41,64,16,41,3,32,41,16,64,41,45,64,41,73,64,41,100,64,41,129,64,41,4,80,41, +32,80,41,16,0,42,48,0,42,80,0,42,112,0,42,144,0,42,16,16,42,48,16,42,76,16,42,99,16,42,16,32,42,44,32,42,69,32,42,16,64,42,48,64,42,80,64,42, +108,64,42,132,64,42,15,80,42,39,80,42,66,80,42,7,96,42,34,96,42,16,128,42,48,128,42,80,128,42,108,128,42,132,128,42,15,144,42,39,144,42,66,144,42,7,160,42, +34,160,42,16,192,42,48,192,42,75,192,42,101,192,42,128,192,42,7,208,42,34,208,42,2,224,42,16,0,43,48,0,43,75,0,43,101,0,43,128,0,43,7,16,43,34,16,43, +2,32,43,16,64,43,44,64,43,70,64,43,98,64,43,3,80,43,176,0,44,208,0,44,240,0,44,16,1,44,48,1,44,80,1,44,112,1,44,176,16,44,208,16,44,240,16,44, +16,17,44,48,17,44,80,17,44,112,17,44,176,32,44,208,32,44,240,32,44,16,33,44,48,33,44,80,33,44,112,33,44,176,64,44,208,64,44,240,64,44,16,65,44,48,65,44, +80,65,44,112,65,44,48,72,44,80,72,44,112,72,44,144,72,44,176,72,44,208,72,44,240,72,44,16,73,44,48,73,44,80,73,44,109,73,44,176,80,44,208,80,44,240,80,44, +16,81,44,48,81,44,79,81,44,110,81,44,170,96,44,202,96,44,233,96,44,8,97,44,40,97,44,71,97,44,102,97,44,176,128,44,208,128,44,240,128,44,16,129,44,48,129,44, +80,129,44,112,129,44,48,130,44,80,130,44,112,130,44,144,130,44,176,130,44,208,130,44,240,130,44,16,131,44,48,131,44,80,131,44,109,131,44,48,132,44,80,132,44,112,132,44, +144,132,44,176,132,44,208,132,44,240,132,44,16,133,44,48,133,44,80,133,44,109,133,44,48,134,44,80,134,44,112,134,44,144,134,44,176,134,44,208,134,44,240,134,44,16,135,44, +48,135,44,80,135,44,109,135,44,176,144,44,208,144,44,240,144,44,16,145,44,48,145,44,79,145,44,110,145,44,170,160,44,202,160,44,233,160,44,8,161,44,40,161,44,71,161,44, +102,161,44,176,192,44,208,192,44,240,192,44,16,193,44,48,193,44,80,193,44,112,193,44,48,194,44,80,194,44,112,194,44,144,194,44,176,194,44,208,194,44,240,194,44,14,195,44, +43,195,44,73,195,44,102,195,44,48,196,44,80,196,44,112,196,44,144,196,44,176,196,44,208,196,44,240,196,44,14,197,44,43,197,44,73,197,44,102,197,44,48,198,44,80,198,44, +112,198,44,144,198,44,176,198,44,208,198,44,240,198,44,14,199,44,43,199,44,73,199,44,102,199,44,170,208,44,202,208,44,233,208,44,8,209,44,40,209,44,71,209,44,102,209,44, +164,224,44,196,224,44,227,224,44,3,225,44,34,225,44,66,225,44,97,225,44,176,0,45,208,0,45,240,0,45,16,1,45,48,1,45,80,1,45,112,1,45,48,2,45,80,2,45, +112,2,45,144,2,45,176,2,45,208,2,45,240,2,45,14,3,45,43,3,45,73,3,45,102,3,45,48,4,45,80,4,45,112,4,45,144,4,45,176,4,45,208,4,45,240,4,45, +14,5,45,43,5,45,73,5,45,102,5,45,48,6,45,80,6,45,112,6,45,144,6,45,176,6,45,208,6,45,240,6,45,14,7,45,43,7,45,73,7,45,102,7,45,48,8,45, +80,8,45,112,8,45,144,8,45,176,8,45,208,8,45,240,8,45,14,9,45,43,9,45,73,9,45,102,9,45,170,16,45,202,16,45,233,16,45,8,17,45,40,17,45,71,17,45, +102,17,45,164,32,45,196,32,45,227,32,45,3,33,45,34,33,45,66,33,45,97,33,45,176,64,45,208,64,45,240,64,45,16,65,45,48,65,45,80,65,45,112,65,45,48,66,45, +80,66,45,112,66,45,144,66,45,175,66,45,204,66,45,235,66,45,9,67,45,38,67,45,69,67,45,99,67,45,48,68,45,80,68,45,112,68,45,144,68,45,175,68,45,204,68,45, +235,68,45,9,69,45,38,69,45,69,69,45,99,69,45,48,70,45,80,70,45,112,70,45,144,70,45,175,70,45,204,70,45,235,70,45,9,71,45,38,71,45,69,71,45,99,71,45, +48,72,45,80,72,45,112,72,45,144,72,45,175,72,45,204,72,45,235,72,45,9,73,45,38,73,45,69,73,45,99,73,45,166,80,45,197,80,45,229,80,45,4,81,45,36,81,45, +67,81,45,99,81,45,80,0,46,112,0,46,144,0,46,176,0,46,208,0,46,240,0,46,16,1,46,48,1,46,80,1,46,112,1,46,80,16,46,112,16,46,144,16,46,176,16,46, +208,16,46,240,16,46,16,17,46,48,17,46,80,17,46,112,17,46,80,32,46,112,32,46,144,32,46,176,32,46,208,32,46,240,32,46,16,33,46,47,33,46,77,33,46,107,33,46, +80,64,46,112,64,46,144,64,46,176,64,46,208,64,46,240,64,46,16,65,46,48,65,46,80,65,46,112,65,46,16,72,46,48,72,46,80,72,46,112,72,46,144,72,46,176,72,46, +205,72,46,232,72,46,1,73,46,80,80,46,112,80,46,144,80,46,175,80,46,206,80,46,236,80,46,11,81,46,41,81,46,72,81,46,102,81,46,74,96,46,105,96,46,136,96,46, +167,96,46,198,96,46,229,96,46,4,97,46,35,97,46,66,97,46,97,97,46,80,128,46,112,128,46,144,128,46,176,128,46,208,128,46,240,128,46,16,129,46,48,129,46,80,129,46, +112,129,46,16,130,46,48,130,46,80,130,46,112,130,46,144,130,46,176,130,46,205,130,46,232,130,46,1,131,46,16,132,46,48,132,46,80,132,46,112,132,46,144,132,46,176,132,46, +205,132,46,232,132,46,1,133,46,16,134,46,48,134,46,80,134,46,112,134,46,144,134,46,176,134,46,205,134,46,232,134,46,1,135,46,80,144,46,112,144,46,144,144,46,175,144,46, +206,144,46,236,144,46,11,145,46,41,145,46,72,145,46,102,145,46,74,160,46,105,160,46,136,160,46,167,160,46,198,160,46,229,160,46,4,161,46,35,161,46,66,161,46,97,161,46, +80,192,46,112,192,46,144,192,46,176,192,46,208,192,46,240,192,46,16,193,46,48,193,46,80,193,46,112,193,46,16,194,46,48,194,46,80,194,46,112,194,46,143,194,46,170,194,46, +198,194,46,227,194,46,16,196,46,48,196,46,80,196,46,112,196,46,143,196,46,170,196,46,198,196,46,227,196,46,16,198,46,48,198,46,80,198,46,112,198,46,143,198,46,170,198,46, +198,198,46,227,198,46,74,208,46,105,208,46,136,208,46,167,208,46,198,208,46,229,208,46,4,209,46,35,209,46,66,209,46,97,209,46,68,224,46,99,224,46,131,224,46,162,224,46, +193,224,46,225,224,46,0,225,46,32,225,46,80,0,47,112,0,47,144,0,47,176,0,47,208,0,47,240,0,47,16,1,47,48,1,47,80,1,47,112,1,47,16,2,47,48,2,47, +80,2,47,112,2,47,143,2,47,170,2,47,198,2,47,227,2,47,16,4,47,48,4,47,80,4,47,112,4,47,143,4,47,170,4,47,198,4,47,227,4,47,16,6,47,48,6,47, +80,6,47,112,6,47,143,6,47,170,6,47,198,6,47,227,6,47,16,8,47,48,8,47,80,8,47,112,8,47,143,8,47,170,8,47,198,8,47,227,8,47,74,16,47,105,16,47, +136,16,47,167,16,47,198,16,47,229,16,47,4,17,47,35,17,47,66,17,47,97,17,47,68,32,47,99,32,47,131,32,47,162,32,47,193,32,47,225,32,47,0,33,47,32,33,47, +80,64,47,112,64,47,144,64,47,176,64,47,208,64,47,240,64,47,15,65,47,46,65,47,76,65,47,107,65,47,16,66,47,48,66,47,79,66,47,108,66,47,137,66,47,166,66,47, +195,66,47,224,66,47,16,68,47,48,68,47,79,68,47,108,68,47,137,68,47,166,68,47,195,68,47,224,68,47,16,70,47,48,70,47,79,70,47,108,70,47,137,70,47,166,70,47, +195,70,47,224,70,47,16,72,47,48,72,47,79,72,47,108,72,47,137,72,47,166,72,47,195,72,47,224,72,47,70,80,47,101,80,47,132,80,47,163,80,47,195,80,47,226,80,47, +1,81,47,32,81,47,64,81,47,48,0,48,80,0,48,112,0,48,144,0,48,176,0,48,208,0,48,240,0,48,16,1,48,48,1,48,80,1,48,112,1,48,48,16,48,80,16,48, +112,16,48,144,16,48,176,16,48,208,16,48,240,16,48,16,17,48,45,17,48,74,17,48,102,17,48,48,32,48,80,32,48,112,32,48,144,32,48,176,32,48,206,32,48,236,32,48, +9,33,48,38,33,48,68,33,48,97,33,48,48,64,48,80,64,48,112,64,48,144,64,48,176,64,48,208,64,48,240,64,48,16,65,48,48,65,48,80,65,48,111,65,48,16,72,48, +48,72,48,80,72,48,112,72,48,138,72,48,161,72,48,48,80,48,80,80,48,111,80,48,141,80,48,171,80,48,201,80,48,231,80,48,5,81,48,35,81,48,65,81,48,42,96,48, +72,96,48,103,96,48,134,96,48,164,96,48,195,96,48,226,96,48,0,97,48,48,128,48,80,128,48,112,128,48,144,128,48,176,128,48,208,128,48,240,128,48,16,129,48,48,129,48, +80,129,48,111,129,48,16,130,48,48,130,48,80,130,48,112,130,48,138,130,48,161,130,48,16,132,48,48,132,48,80,132,48,112,132,48,138,132,48,161,132,48,16,134,48,48,134,48, +80,134,48,112,134,48,138,134,48,161,134,48,48,144,48,80,144,48,111,144,48,141,144,48,171,144,48,201,144,48,231,144,48,5,145,48,35,145,48,65,145,48,42,160,48,72,160,48, +103,160,48,134,160,48,164,160,48,195,160,48,226,160,48,0,161,48,48,192,48,80,192,48,112,192,48,144,192,48,176,192,48,208,192,48,240,192,48,15,193,48,44,193,48,74,193,48, +103,193,48,16,194,48,48,194,48,78,194,48,105,194,48,132,194,48,16,196,48,48,196,48,78,196,48,105,196,48,132,196,48,16,198,48,48,198,48,78,198,48,105,198,48,132,198,48, +42,208,48,72,208,48,103,208,48,134,208,48,164,208,48,195,208,48,226,208,48,0,209,48,36,224,48,67,224,48,98,224,48,129,224,48,160,224,48,48,0,49,80,0,49,112,0,49, +144,0,49,176,0,49,208,0,49,240,0,49,15,1,49,44,1,49,74,1,49,103,1,49,16,2,49,48,2,49,78,2,49,105,2,49,132,2,49,16,4,49,48,4,49,78,4,49, +105,4,49,132,4,49,16,6,49,48,6,49,78,6,49,105,6,49,132,6,49,16,8,49,48,8,49,78,8,49,105,8,49,132,8,49,42,16,49,72,16,49,103,16,49,134,16,49, +164,16,49,195,16,49,226,16,49,0,17,49,36,32,49,67,32,49,98,32,49,129,32,49,160,32,49,48,64,49,80,64,49,112,64,49,144,64,49,175,64,49,205,64,49,236,64,49, +9,65,49,39,65,49,70,65,49,99,65,49,16,66,49,45,66,49,73,66,49,100,66,49,129,66,49,16,68,49,45,68,49,73,68,49,100,68,49,129,68,49,16,70,49,45,70,49, +73,70,49,100,70,49,129,70,49,16,72,49,45,72,49,73,72,49,100,72,49,129,72,49,37,80,49,68,80,49,99,80,49,130,80,49,161,80,49,192,80,49,48,0,50,80,0,50, +112,0,50,144,0,50,176,0,50,208,0,50,240,0,50,16,1,50,48,1,50,80,1,50,48,16,50,80,16,50,112,16,50,144,16,50,176,16,50,208,16,50,236,16,50,6,17,50, +33,17,50,48,32,50,80,32,50,112,32,50,143,32,50,171,32,50,200,32,50,229,32,50,1,33,50,48,64,50,80,64,50,112,64,50,144,64,50,176,64,50,208,64,50,240,64,50, +15,65,50,42,65,50,70,65,50,16,72,50,48,72,50,77,72,50,99,72,50,48,80,50,78,80,50,107,80,50,137,80,50,166,80,50,196,80,50,226,80,50,40,96,50,70,96,50, +101,96,50,131,96,50,161,96,50,192,96,50,48,128,50,80,128,50,112,128,50,144,128,50,176,128,50,208,128,50,240,128,50,15,129,50,42,129,50,70,129,50,16,130,50,48,130,50, +77,130,50,99,130,50,16,132,50,48,132,50,77,132,50,99,132,50,16,134,50,48,134,50,77,134,50,99,134,50,48,144,50,78,144,50,107,144,50,137,144,50,166,144,50,196,144,50, +226,144,50,40,160,50,70,160,50,101,160,50,131,160,50,161,160,50,192,160,50,48,192,50,80,192,50,112,192,50,144,192,50,176,192,50,206,192,50,235,192,50,7,193,50,36,193,50, +65,193,50,16,194,50,46,194,50,70,194,50,16,196,50,46,196,50,70,196,50,16,198,50,46,198,50,70,198,50,40,208,50,70,208,50,101,208,50,131,208,50,161,208,50,192,208,50, +35,224,50,65,224,50,96,224,50,128,224,50,48,0,51,80,0,51,112,0,51,144,0,51,176,0,51,206,0,51,235,0,51,7,1,51,36,1,51,65,1,51,16,2,51,46,2,51, +70,2,51,16,4,51,46,4,51,70,4,51,16,6,51,46,6,51,70,6,51,16,8,51,46,8,51,70,8,51,40,16,51,70,16,51,101,16,51,131,16,51,161,16,51,192,16,51, +35,32,51,65,32,51,96,32,51,128,32,51,48,64,51,80,64,51,112,64,51,142,64,51,171,64,51,200,64,51,230,64,51,3,65,51,33,65,51,16,66,51,41,66,51,67,66,51, +16,68,51,41,68,51,67,68,51,16,70,51,41,70,51,67,70,51,16,72,51,41,72,51,67,72,51,36,80,51,67,80,51,97,80,51,128,80,51,16,0,52,48,0,52,80,0,52, +112,0,52,144,0,52,176,0,52,208,0,52,240,0,52,14,1,52,16,16,52,48,16,52,80,16,52,112,16,52,144,16,52,172,16,52,198,16,52,225,16,52,16,32,52,48,32,52, +80,32,52,109,32,52,138,32,52,165,32,52,193,32,52,16,64,52,48,64,52,80,64,52,112,64,52,144,64,52,176,64,52,207,64,52,234,64,52,3,65,52,16,72,52,48,72,52, +65,72,52,16,80,52,46,80,52,75,80,52,104,80,52,133,80,52,162,80,52,10,96,52,39,96,52,68,96,52,98,96,52,129,96,52,16,128,52,48,128,52,80,128,52,112,128,52, +144,128,52,176,128,52,207,128,52,234,128,52,3,129,52,16,130,52,48,130,52,65,130,52,16,132,52,48,132,52,65,132,52,16,134,52,48,134,52,65,134,52,16,144,52,46,144,52, +75,144,52,104,144,52,133,144,52,162,144,52,10,160,52,39,160,52,68,160,52,98,160,52,129,160,52,16,192,52,48,192,52,80,192,52,112,192,52,144,192,52,171,192,52,199,192,52, +228,192,52,16,194,52,40,194,52,16,196,52,40,196,52,16,198,52,40,198,52,10,208,52,39,208,52,68,208,52,98,208,52,129,208,52,4,224,52,34,224,52,64,224,52,16,0,53, +48,0,53,80,0,53,112,0,53,144,0,53,171,0,53,199,0,53,228,0,53,16,2,53,40,2,53,16,4,53,40,4,53,16,6,53,40,6,53,16,8,53,40,8,53,10,16,53, +39,16,53,68,16,53,98,16,53,129,16,53,4,32,53,34,32,53,64,32,53,16,64,53,48,64,53,79,64,53,108,64,53,138,64,53,166,64,53,195,64,53,225,64,53,15,66,53, +36,66,53,15,68,53,36,68,53,15,70,53,36,70,53,15,72,53,36,72,53,6,80,53,35,80,53,65,80,53,96,80,53,16,0,54,48,0,54,80,0,54,112,0,54,144,0,54, +176,0,54,208,0,54,16,16,54,48,16,54,80,16,54,112,16,54,139,16,54,163,16,54,16,32,54,48,32,54,77,32,54,104,32,54,133,32,54,16,64,54,48,64,54,80,64,54, +112,64,54,144,64,54,172,64,54,196,64,54,16,72,54,38,72,54,16,80,54,44,80,54,72,80,54,100,80,54,129,80,54,9,96,54,37,96,54,66,96,54,96,96,54,16,128,54, +48,128,54,80,128,54,112,128,54,144,128,54,172,128,54,196,128,54,16,130,54,38,130,54,16,132,54,38,132,54,16,134,54,38,134,54,16,144,54,44,144,54,72,144,54,100,144,54, +129,144,54,9,160,54,37,160,54,66,160,54,96,160,54,16,192,54,48,192,54,80,192,54,110,192,54,139,192,54,165,192,54,192,192,54,16,194,54,33,194,54,16,196,54,33,196,54, +16,198,54,33,198,54,9,208,54,37,208,54,66,208,54,96,208,54,3,224,54,33,224,54,16,0,55,48,0,55,80,0,55,110,0,55,139,0,55,165,0,55,192,0,55,16,2,55, +33,2,55,16,4,55,33,4,55,16,6,55,33,6,55,16,8,55,33,8,55,9,16,55,37,16,55,66,16,55,96,16,55,3,32,55,33,32,55,16,64,55,48,64,55,76,64,55, +105,64,55,134,64,55,162,64,55,12,66,55,12,68,55,12,70,55,12,72,55,5,80,55,34,80,55,64,80,55,16,0,56,48,0,56,80,0,56,112,0,56,144,0,56,174,0,56, +16,16,56,48,16,56,80,16,56,106,16,56,131,16,56,16,32,56,47,32,56,73,32,56,100,32,56,16,64,56,48,64,56,80,64,56,112,64,56,140,64,56,163,64,56,16,72,56, +16,80,56,41,80,56,69,80,56,97,80,56,8,96,56,35,96,56,64,96,56,16,128,56,48,128,56,80,128,56,112,128,56,140,128,56,163,128,56,16,130,56,16,132,56,16,134,56, +16,144,56,41,144,56,69,144,56,97,144,56,8,160,56,35,160,56,64,160,56,16,192,56,48,192,56,79,192,56,106,192,56,133,192,56,14,194,56,14,196,56,14,198,56,8,208,56, +35,208,56,64,208,56,3,224,56,32,224,56,16,0,57,48,0,57,79,0,57,106,0,57,133,0,57,14,2,57,14,4,57,14,6,57,14,8,57,8,16,57,35,16,57,64,16,57, +3,32,57,32,32,57,16,64,57,46,64,57,73,64,57,101,64,57,130,64,57,9,66,57,9,68,57,9,70,57,9,72,57,4,80,57,32,80,57,16,0,58,48,0,58,80,0,58, +112,0,58,144,0,58,16,16,58,48,16,58,76,16,58,99,16,58,16,32,58,44,32,58,69,32,58,16,64,58,48,64,58,80,64,58,108,64,58,132,64,58,15,80,58,39,80,58, +66,80,58,7,96,58,34,96,58,16,128,58,48,128,58,80,128,58,108,128,58,132,128,58,15,144,58,39,144,58,66,144,58,7,160,58,34,160,58,16,192,58,48,192,58,75,192,58, +101,192,58,128,192,58,7,208,58,34,208,58,2,224,58,16,0,59,48,0,59,75,0,59,101,0,59,128,0,59,7,16,59,34,16,59,2,32,59,16,64,59,44,64,59,70,64,59, +98,64,59,3,80,59,16,0,60,48,0,60,80,0,60,112,0,60,16,16,60,48,16,60,70,16,60,16,32,60,41,32,60,65,32,60,16,64,60,48,64,60,79,64,60,100,64,60, +14,80,60,37,80,60,6,96,60,32,96,60,16,128,60,48,128,60,79,128,60,100,128,60,14,144,60,37,144,60,6,160,60,32,160,60,16,192,60,47,192,60,71,192,60,96,192,60, +6,208,60,32,208,60,1,224,60,16,0,61,47,0,61,71,0,61,96,0,61,6,16,61,32,16,61,1,32,61,16,64,61,41,64,61,67,64,61,3,80,61,16,0,62,48,0,62, +80,0,62,16,16,62,45,16,62,64,16,62,16,32,62,38,32,62,16,64,62,48,64,62,73,64,62,12,80,62,34,80,62,5,96,62,16,128,62,48,128,62,73,128,62,12,144,62, +34,144,62,5,160,62,16,192,62,44,192,62,67,192,62,5,208,62,1,224,62,16,0,63,44,0,63,67,0,63,5,16,63,1,32,63,16,64,63,39,64,63,64,64,63,2,80,63, +16,0,64,48,0,64,78,0,64,16,16,64,40,16,64,16,32,64,35,32,64,16,64,64,48,64,64,67,64,64,11,80,64,32,80,64,4,96,64,16,128,64,48,128,64,67,128,64, +11,144,64,32,144,64,4,160,64,16,192,64,41,192,64,4,208,64,0,224,64,16,0,65,41,0,65,4,16,65,0,32,65,15,64,65,36,64,65,1,80,65,112,0,66,144,0,66, +176,0,66,208,0,66,240,0,66,16,1,66,48,1,66,80,1,66,112,1,66,112,16,66,144,16,66,176,16,66,208,16,66,240,16,66,16,17,66,48,17,66,80,17,66,112,17,66, +112,32,66,144,32,66,176,32,66,208,32,66,240,32,66,16,33,66,48,33,66,80,33,66,112,33,66,112,64,66,144,64,66,176,64,66,208,64,66,240,64,66,16,65,66,48,65,66, +80,65,66,112,65,66,48,72,66,80,72,66,112,72,66,144,72,66,176,72,66,208,72,66,240,72,66,13,73,66,40,73,66,68,73,66,112,80,66,144,80,66,176,80,66,208,80,66, +239,80,66,14,81,66,44,81,66,76,81,66,106,81,66,106,96,66,138,96,66,169,96,66,200,96,66,231,96,66,6,97,66,37,97,66,69,97,66,100,97,66,112,128,66,144,128,66, +176,128,66,208,128,66,240,128,66,16,129,66,48,129,66,80,129,66,112,129,66,48,130,66,80,130,66,112,130,66,144,130,66,176,130,66,208,130,66,240,130,66,13,131,66,40,131,66, +68,131,66,48,132,66,80,132,66,112,132,66,144,132,66,176,132,66,208,132,66,240,132,66,13,133,66,40,133,66,68,133,66,48,134,66,80,134,66,112,134,66,144,134,66,176,134,66, +208,134,66,240,134,66,13,135,66,40,135,66,68,135,66,112,144,66,144,144,66,176,144,66,208,144,66,239,144,66,14,145,66,44,145,66,76,145,66,106,145,66,106,160,66,138,160,66, +169,160,66,200,160,66,231,160,66,6,161,66,37,161,66,69,161,66,100,161,66,112,192,66,144,192,66,176,192,66,208,192,66,240,192,66,16,193,66,48,193,66,80,193,66,112,193,66, +48,194,66,80,194,66,112,194,66,144,194,66,176,194,66,205,194,66,234,194,66,6,195,66,35,195,66,64,195,66,48,196,66,80,196,66,112,196,66,144,196,66,176,196,66,205,196,66, +234,196,66,6,197,66,35,197,66,64,197,66,48,198,66,80,198,66,112,198,66,144,198,66,176,198,66,205,198,66,234,198,66,6,199,66,35,199,66,64,199,66,106,208,66,138,208,66, +169,208,66,200,208,66,231,208,66,6,209,66,37,209,66,69,209,66,100,209,66,100,224,66,132,224,66,163,224,66,195,224,66,226,224,66,1,225,66,33,225,66,64,225,66,96,225,66, +112,0,67,144,0,67,176,0,67,208,0,67,240,0,67,16,1,67,48,1,67,80,1,67,112,1,67,48,2,67,80,2,67,112,2,67,144,2,67,176,2,67,205,2,67,234,2,67, +6,3,67,35,3,67,64,3,67,48,4,67,80,4,67,112,4,67,144,4,67,176,4,67,205,4,67,234,4,67,6,5,67,35,5,67,64,5,67,48,6,67,80,6,67,112,6,67, +144,6,67,176,6,67,205,6,67,234,6,67,6,7,67,35,7,67,64,7,67,48,8,67,80,8,67,112,8,67,144,8,67,176,8,67,205,8,67,234,8,67,6,9,67,35,9,67, +64,9,67,106,16,67,138,16,67,169,16,67,200,16,67,231,16,67,6,17,67,37,17,67,69,17,67,100,17,67,100,32,67,132,32,67,163,32,67,195,32,67,226,32,67,1,33,67, +33,33,67,64,33,67,96,33,67,112,64,67,144,64,67,176,64,67,208,64,67,240,64,67,16,65,67,48,65,67,80,65,67,111,65,67,48,66,67,80,66,67,111,66,67,141,66,67, +170,66,67,199,66,67,230,66,67,3,67,67,32,67,67,48,68,67,80,68,67,111,68,67,141,68,67,170,68,67,199,68,67,230,68,67,3,69,67,32,69,67,48,70,67,80,70,67, +111,70,67,141,70,67,170,70,67,199,70,67,230,70,67,3,71,67,32,71,67,48,72,67,80,72,67,111,72,67,141,72,67,170,72,67,199,72,67,230,72,67,3,73,67,32,73,67, +102,80,67,133,80,67,164,80,67,196,80,67,227,80,67,3,81,67,34,81,67,65,81,67,97,81,67,48,0,68,80,0,68,112,0,68,144,0,68,176,0,68,208,0,68,240,0,68, +16,1,68,48,1,68,80,1,68,112,1,68,48,16,68,80,16,68,112,16,68,144,16,68,176,16,68,208,16,68,240,16,68,16,17,68,48,17,68,78,17,68,106,17,68,48,32,68, +80,32,68,112,32,68,144,32,68,176,32,68,208,32,68,238,32,68,11,33,68,41,33,68,71,33,68,100,33,68,48,64,68,80,64,68,112,64,68,144,64,68,176,64,68,208,64,68, +240,64,68,16,65,68,48,65,68,80,65,68,112,65,68,16,72,68,48,72,68,80,72,68,112,72,68,143,72,68,166,72,68,48,80,68,80,80,68,112,80,68,142,80,68,172,80,68, +202,80,68,233,80,68,6,81,68,36,81,68,67,81,68,97,81,68,42,96,68,73,96,68,104,96,68,135,96,68,165,96,68,196,96,68,227,96,68,1,97,68,32,97,68,48,128,68, +80,128,68,112,128,68,144,128,68,176,128,68,208,128,68,240,128,68,16,129,68,48,129,68,80,129,68,112,129,68,16,130,68,48,130,68,80,130,68,112,130,68,143,130,68,166,130,68, +16,132,68,48,132,68,80,132,68,112,132,68,143,132,68,166,132,68,16,134,68,48,134,68,80,134,68,112,134,68,143,134,68,166,134,68,48,144,68,80,144,68,112,144,68,142,144,68, +172,144,68,202,144,68,233,144,68,6,145,68,36,145,68,67,145,68,97,145,68,42,160,68,73,160,68,104,160,68,135,160,68,165,160,68,196,160,68,227,160,68,1,161,68,32,161,68, +48,192,68,80,192,68,112,192,68,144,192,68,176,192,68,208,192,68,240,192,68,16,193,68,47,193,68,77,193,68,106,193,68,16,194,68,48,194,68,80,194,68,107,194,68,135,194,68, +161,194,68,16,196,68,48,196,68,80,196,68,107,196,68,135,196,68,161,196,68,16,198,68,48,198,68,80,198,68,107,198,68,135,198,68,161,198,68,42,208,68,73,208,68,104,208,68, +135,208,68,165,208,68,196,208,68,227,208,68,1,209,68,32,209,68,36,224,68,67,224,68,98,224,68,130,224,68,161,224,68,192,224,68,48,0,69,80,0,69,112,0,69,144,0,69, +176,0,69,208,0,69,240,0,69,16,1,69,47,1,69,77,1,69,106,1,69,16,2,69,48,2,69,80,2,69,107,2,69,135,2,69,161,2,69,16,4,69,48,4,69,80,4,69, +107,4,69,135,4,69,161,4,69,16,6,69,48,6,69,80,6,69,107,6,69,135,6,69,161,6,69,16,8,69,48,8,69,80,8,69,107,8,69,135,8,69,161,8,69,42,16,69, +73,16,69,104,16,69,135,16,69,165,16,69,196,16,69,227,16,69,1,17,69,32,17,69,36,32,69,67,32,69,98,32,69,130,32,69,161,32,69,192,32,69,48,64,69,80,64,69, +112,64,69,144,64,69,176,64,69,207,64,69,237,64,69,11,65,69,41,65,69,71,65,69,101,65,69,16,66,69,47,66,69,74,66,69,102,66,69,131,66,69,16,68,69,47,68,69, +74,68,69,102,68,69,131,68,69,16,70,69,47,70,69,74,70,69,102,70,69,131,70,69,16,72,69,47,72,69,74,72,69,102,72,69,131,72,69,38,80,69,68,80,69,100,80,69, +131,80,69,162,80,69,193,80,69,224,80,69,48,0,70,80,0,70,112,0,70,144,0,70,176,0,70,208,0,70,240,0,70,16,1,70,48,1,70,80,1,70,48,16,70,80,16,70, +112,16,70,144,16,70,176,16,70,208,16,70,236,16,70,6,17,70,33,17,70,48,32,70,80,32,70,112,32,70,143,32,70,171,32,70,200,32,70,229,32,70,1,33,70,48,64,70, +80,64,70,112,64,70,144,64,70,176,64,70,208,64,70,240,64,70,15,65,70,42,65,70,70,65,70,16,72,70,48,72,70,77,72,70,99,72,70,48,80,70,78,80,70,107,80,70, +137,80,70,166,80,70,196,80,70,226,80,70,40,96,70,70,96,70,101,96,70,131,96,70,161,96,70,192,96,70,48,128,70,80,128,70,112,128,70,144,128,70,176,128,70,208,128,70, +240,128,70,15,129,70,42,129,70,70,129,70,16,130,70,48,130,70,77,130,70,99,130,70,16,132,70,48,132,70,77,132,70,99,132,70,16,134,70,48,134,70,77,134,70,99,134,70, +48,144,70,78,144,70,107,144,70,137,144,70,166,144,70,196,144,70,226,144,70,40,160,70,70,160,70,101,160,70,131,160,70,161,160,70,192,160,70,48,192,70,80,192,70,112,192,70, +144,192,70,176,192,70,206,192,70,235,192,70,7,193,70,36,193,70,65,193,70,16,194,70,46,194,70,70,194,70,16,196,70,46,196,70,70,196,70,16,198,70,46,198,70,70,198,70, +40,208,70,70,208,70,101,208,70,131,208,70,161,208,70,192,208,70,35,224,70,65,224,70,96,224,70,128,224,70,48,0,71,80,0,71,112,0,71,144,0,71,176,0,71,206,0,71, +235,0,71,7,1,71,36,1,71,65,1,71,16,2,71,46,2,71,70,2,71,16,4,71,46,4,71,70,4,71,16,6,71,46,6,71,70,6,71,16,8,71,46,8,71,70,8,71, +40,16,71,70,16,71,101,16,71,131,16,71,161,16,71,192,16,71,35,32,71,65,32,71,96,32,71,128,32,71,48,64,71,80,64,71,112,64,71,142,64,71,171,64,71,200,64,71, +230,64,71,3,65,71,33,65,71,16,66,71,41,66,71,67,66,71,16,68,71,41,68,71,67,68,71,16,70,71,41,70,71,67,70,71,16,72,71,41,72,71,67,72,71,36,80,71, +67,80,71,97,80,71,128,80,71,16,0,72,48,0,72,80,0,72,112,0,72,144,0,72,176,0,72,208,0,72,240,0,72,16,16,72,48,16,72,80,16,72,112,16,72,144,16,72, +170,16,72,195,16,72,16,32,72,48,32,72,80,32,72,108,32,72,137,32,72,164,32,72,16,64,72,48,64,72,80,64,72,112,64,72,144,64,72,176,64,72,204,64,72,231,64,72, +16,72,72,45,72,72,16,80,72,46,80,72,74,80,72,103,80,72,132,80,72,161,80,72,10,96,72,38,96,72,68,96,72,98,96,72,128,96,72,16,128,72,48,128,72,80,128,72, +112,128,72,144,128,72,176,128,72,204,128,72,231,128,72,16,130,72,45,130,72,16,132,72,45,132,72,16,134,72,45,134,72,16,144,72,46,144,72,74,144,72,103,144,72,132,144,72, +161,144,72,10,160,72,38,160,72,68,160,72,98,160,72,128,160,72,16,192,72,48,192,72,80,192,72,112,192,72,143,192,72,170,192,72,197,192,72,226,192,72,16,194,72,38,194,72, +16,196,72,38,196,72,16,198,72,38,198,72,10,208,72,38,208,72,68,208,72,98,208,72,128,208,72,4,224,72,33,224,72,64,224,72,16,0,73,48,0,73,80,0,73,112,0,73, +143,0,73,170,0,73,197,0,73,226,0,73,16,2,73,38,2,73,16,4,73,38,4,73,16,6,73,38,6,73,16,8,73,38,8,73,10,16,73,38,16,73,68,16,73,98,16,73, +128,16,73,4,32,73,33,32,73,64,32,73,16,64,73,48,64,73,79,64,73,107,64,73,137,64,73,165,64,73,194,64,73,224,64,73,14,66,73,35,66,73,14,68,73,35,68,73, +14,70,73,35,70,73,14,72,73,35,72,73,6,80,73,35,80,73,65,80,73,16,0,74,48,0,74,80,0,74,112,0,74,144,0,74,176,0,74,16,16,74,48,16,74,80,16,74, +109,16,74,135,16,74,16,32,74,48,32,74,75,32,74,102,32,74,130,32,74,16,64,74,48,64,74,80,64,74,112,64,74,144,64,74,167,64,74,16,72,74,33,72,74,16,80,74, +43,80,74,70,80,74,99,80,74,128,80,74,9,96,74,36,96,74,65,96,74,16,128,74,48,128,74,80,128,74,112,128,74,144,128,74,167,128,74,16,130,74,33,130,74,16,132,74, +33,132,74,16,134,74,33,134,74,16,144,74,43,144,74,70,144,74,99,144,74,128,144,74,9,160,74,36,160,74,65,160,74,16,192,74,48,192,74,80,192,74,108,192,74,136,192,74, +162,192,74,16,194,74,16,196,74,16,198,74,9,208,74,36,208,74,65,208,74,3,224,74,32,224,74,16,0,75,48,0,75,80,0,75,108,0,75,136,0,75,162,0,75,16,2,75, +16,4,75,16,6,75,16,8,75,9,16,75,36,16,75,65,16,75,3,32,75,32,32,75,16,64,75,47,64,75,75,64,75,103,64,75,132,64,75,160,64,75,10,66,75,10,68,75, +10,70,75,10,72,75,4,80,75,33,80,75,16,0,76,48,0,76,80,0,76,112,0,76,144,0,76,16,16,76,48,16,76,77,16,76,100,16,76,16,32,76,45,32,76,70,32,76, +96,32,76,16,64,76,48,64,76,80,64,76,109,64,76,135,64,76,16,80,76,40,80,76,67,80,76,8,96,76,34,96,76,16,128,76,48,128,76,80,128,76,109,128,76,135,128,76, +16,144,76,40,144,76,67,144,76,8,160,76,34,160,76,16,192,76,48,192,76,76,192,76,102,192,76,130,192,76,8,208,76,34,208,76,2,224,76,16,0,77,48,0,77,76,0,77, +102,0,77,130,0,77,8,16,77,34,16,77,2,32,77,16,64,77,44,64,77,71,64,77,99,64,77,4,80,77,32,80,77,16,0,78,48,0,78,80,0,78,112,0,78,16,16,78, +48,16,78,70,16,78,16,32,78,41,32,78,65,32,78,16,64,78,48,64,78,79,64,78,100,64,78,14,80,78,37,80,78,6,96,78,32,96,78,16,128,78,48,128,78,79,128,78, +100,128,78,14,144,78,37,144,78,6,160,78,32,160,78,16,192,78,47,192,78,71,192,78,96,192,78,6,208,78,32,208,78,1,224,78,16,0,79,47,0,79,71,0,79,96,0,79, +6,16,79,32,16,79,1,32,79,16,64,79,41,64,79,67,64,79,3,80,79,16,0,80,48,0,80,80,0,80,16,16,80,44,16,80,16,32,80,37,32,80,16,64,80,48,64,80, +71,64,80,12,80,80,34,80,80,5,96,80,16,128,80,48,128,80,71,128,80,12,144,80,34,144,80,5,160,80,16,192,80,43,192,80,66,192,80,5,208,80,1,224,80,16,0,81, +43,0,81,66,0,81,5,16,81,1,32,81,16,64,81,38,64,81,64,64,81,2,80,81,16,0,82,48,0,82,16,16,82,38,16,82,16,32,82,33,32,82,16,64,82,47,64,82, +10,80,82,4,96,82,16,128,82,47,128,82,10,144,82,4,160,82,16,192,82,39,192,82,4,208,82,0,224,82,16,0,83,39,0,83,4,16,83,0,32,83,15,64,83,35,64,83, +1,80,83,16,0,84,48,0,84,16,16,84,32,16,84,14,32,84,16,64,84,41,64,84,8,80,84,3,96,84,16,128,84,41,128,84,8,144,84,3,160,84,16,192,84,35,192,84, +3,208,84,16,0,85,35,0,85,3,16,85,13,64,85,32,64,85,0,80,85,16,0,86,46,0,86,16,16,86,11,32,86,16,64,86,35,64,86,6,80,86,1,96,86,16,128,86, +35,128,86,6,144,86,1,160,86,16,192,86,1,208,86,16,0,87,1,16,87,11,64,87,80,0,88,112,0,88,144,0,88,176,0,88,208,0,88,240,0,88,16,1,88,48,1,88, +80,1,88,112,1,88,80,16,88,112,16,88,144,16,88,176,16,88,208,16,88,240,16,88,16,17,88,48,17,88,80,17,88,112,17,88,80,32,88,112,32,88,144,32,88,176,32,88, +208,32,88,240,32,88,16,33,88,47,33,88,77,33,88,107,33,88,80,64,88,112,64,88,144,64,88,176,64,88,208,64,88,240,64,88,16,65,88,48,65,88,80,65,88,112,65,88, +16,72,88,48,72,88,80,72,88,112,72,88,144,72,88,176,72,88,205,72,88,232,72,88,1,73,88,80,80,88,112,80,88,144,80,88,175,80,88,206,80,88,236,80,88,11,81,88, +41,81,88,72,81,88,102,81,88,74,96,88,105,96,88,136,96,88,167,96,88,198,96,88,229,96,88,4,97,88,35,97,88,66,97,88,97,97,88,80,128,88,112,128,88,144,128,88, +176,128,88,208,128,88,240,128,88,16,129,88,48,129,88,80,129,88,112,129,88,16,130,88,48,130,88,80,130,88,112,130,88,144,130,88,176,130,88,205,130,88,232,130,88,1,131,88, +16,132,88,48,132,88,80,132,88,112,132,88,144,132,88,176,132,88,205,132,88,232,132,88,1,133,88,16,134,88,48,134,88,80,134,88,112,134,88,144,134,88,176,134,88,205,134,88, +232,134,88,1,135,88,80,144,88,112,144,88,144,144,88,175,144,88,206,144,88,236,144,88,11,145,88,41,145,88,72,145,88,102,145,88,74,160,88,105,160,88,136,160,88,167,160,88, +198,160,88,229,160,88,4,161,88,35,161,88,66,161,88,97,161,88,80,192,88,112,192,88,144,192,88,176,192,88,208,192,88,240,192,88,16,193,88,48,193,88,80,193,88,112,193,88, +16,194,88,48,194,88,80,194,88,112,194,88,143,194,88,170,194,88,198,194,88,227,194,88,16,196,88,48,196,88,80,196,88,112,196,88,143,196,88,170,196,88,198,196,88,227,196,88, +16,198,88,48,198,88,80,198,88,112,198,88,143,198,88,170,198,88,198,198,88,227,198,88,74,208,88,105,208,88,136,208,88,167,208,88,198,208,88,229,208,88,4,209,88,35,209,88, +66,209,88,97,209,88,68,224,88,99,224,88,131,224,88,162,224,88,193,224,88,225,224,88,0,225,88,32,225,88,80,0,89,112,0,89,144,0,89,176,0,89,208,0,89,240,0,89, +16,1,89,48,1,89,80,1,89,112,1,89,16,2,89,48,2,89,80,2,89,112,2,89,143,2,89,170,2,89,198,2,89,227,2,89,16,4,89,48,4,89,80,4,89,112,4,89, +143,4,89,170,4,89,198,4,89,227,4,89,16,6,89,48,6,89,80,6,89,112,6,89,143,6,89,170,6,89,198,6,89,227,6,89,16,8,89,48,8,89,80,8,89,112,8,89, +143,8,89,170,8,89,198,8,89,227,8,89,74,16,89,105,16,89,136,16,89,167,16,89,198,16,89,229,16,89,4,17,89,35,17,89,66,17,89,97,17,89,68,32,89,99,32,89, +131,32,89,162,32,89,193,32,89,225,32,89,0,33,89,32,33,89,80,64,89,112,64,89,144,64,89,176,64,89,208,64,89,240,64,89,15,65,89,46,65,89,76,65,89,107,65,89, +16,66,89,48,66,89,79,66,89,108,66,89,137,66,89,166,66,89,195,66,89,224,66,89,16,68,89,48,68,89,79,68,89,108,68,89,137,68,89,166,68,89,195,68,89,224,68,89, +16,70,89,48,70,89,79,70,89,108,70,89,137,70,89,166,70,89,195,70,89,224,70,89,16,72,89,48,72,89,79,72,89,108,72,89,137,72,89,166,72,89,195,72,89,224,72,89, +70,80,89,101,80,89,132,80,89,163,80,89,195,80,89,226,80,89,1,81,89,32,81,89,64,81,89,48,0,90,80,0,90,112,0,90,144,0,90,176,0,90,208,0,90,240,0,90, +16,1,90,48,1,90,80,1,90,112,1,90,48,16,90,80,16,90,112,16,90,144,16,90,176,16,90,208,16,90,240,16,90,12,17,90,39,17,90,68,17,90,48,32,90,80,32,90, +112,32,90,144,32,90,174,32,90,203,32,90,233,32,90,5,33,90,34,33,90,64,33,90,48,64,90,80,64,90,112,64,90,144,64,90,176,64,90,208,64,90,240,64,90,16,65,90, +48,65,90,77,65,90,103,65,90,16,72,90,48,72,90,80,72,90,106,72,90,131,72,90,48,80,90,79,80,90,109,80,90,139,80,90,169,80,90,198,80,90,228,80,90,2,81,90, +32,81,90,41,96,90,71,96,90,102,96,90,133,96,90,163,96,90,193,96,90,224,96,90,48,128,90,80,128,90,112,128,90,144,128,90,176,128,90,208,128,90,240,128,90,16,129,90, +48,129,90,77,129,90,103,129,90,16,130,90,48,130,90,80,130,90,106,130,90,131,130,90,16,132,90,48,132,90,80,132,90,106,132,90,131,132,90,16,134,90,48,134,90,80,134,90, +106,134,90,131,134,90,48,144,90,79,144,90,109,144,90,139,144,90,169,144,90,198,144,90,228,144,90,2,145,90,32,145,90,41,160,90,71,160,90,102,160,90,133,160,90,163,160,90, +193,160,90,224,160,90,48,192,90,80,192,90,112,192,90,144,192,90,176,192,90,208,192,90,239,192,90,11,193,90,40,193,90,70,193,90,98,193,90,16,194,90,48,194,90,74,194,90, +100,194,90,16,196,90,48,196,90,74,196,90,100,196,90,16,198,90,48,198,90,74,198,90,100,198,90,41,208,90,71,208,90,102,208,90,133,208,90,163,208,90,193,208,90,224,208,90, +35,224,90,66,224,90,97,224,90,128,224,90,48,0,91,80,0,91,112,0,91,144,0,91,176,0,91,208,0,91,239,0,91,11,1,91,40,1,91,70,1,91,98,1,91,16,2,91, +48,2,91,74,2,91,100,2,91,16,4,91,48,4,91,74,4,91,100,4,91,16,6,91,48,6,91,74,6,91,100,6,91,16,8,91,48,8,91,74,8,91,100,8,91,41,16,91, +71,16,91,102,16,91,133,16,91,163,16,91,193,16,91,224,16,91,35,32,91,66,32,91,97,32,91,128,32,91,48,64,91,80,64,91,112,64,91,144,64,91,173,64,91,203,64,91, +233,64,91,6,65,91,36,65,91,66,65,91,96,65,91,16,66,91,43,66,91,70,66,91,97,66,91,16,68,91,43,68,91,70,68,91,97,68,91,16,70,91,43,70,91,70,70,91, +97,70,91,16,72,91,43,72,91,70,72,91,97,72,91,37,80,91,67,80,91,98,80,91,129,80,91,160,80,91,16,0,92,48,0,92,80,0,92,112,0,92,144,0,92,176,0,92, +208,0,92,240,0,92,14,1,92,16,16,92,48,16,92,80,16,92,112,16,92,144,16,92,172,16,92,198,16,92,225,16,92,16,32,92,48,32,92,80,32,92,109,32,92,138,32,92, +165,32,92,193,32,92,16,64,92,48,64,92,80,64,92,112,64,92,144,64,92,176,64,92,207,64,92,234,64,92,3,65,92,16,72,92,48,72,92,65,72,92,16,80,92,46,80,92, +75,80,92,104,80,92,133,80,92,162,80,92,10,96,92,39,96,92,68,96,92,98,96,92,129,96,92,16,128,92,48,128,92,80,128,92,112,128,92,144,128,92,176,128,92,207,128,92, +234,128,92,3,129,92,16,130,92,48,130,92,65,130,92,16,132,92,48,132,92,65,132,92,16,134,92,48,134,92,65,134,92,16,144,92,46,144,92,75,144,92,104,144,92,133,144,92, +162,144,92,10,160,92,39,160,92,68,160,92,98,160,92,129,160,92,16,192,92,48,192,92,80,192,92,112,192,92,144,192,92,171,192,92,199,192,92,228,192,92,16,194,92,40,194,92, +16,196,92,40,196,92,16,198,92,40,198,92,10,208,92,39,208,92,68,208,92,98,208,92,129,208,92,4,224,92,34,224,92,64,224,92,16,0,93,48,0,93,80,0,93,112,0,93, +144,0,93,171,0,93,199,0,93,228,0,93,16,2,93,40,2,93,16,4,93,40,4,93,16,6,93,40,6,93,16,8,93,40,8,93,10,16,93,39,16,93,68,16,93,98,16,93, +129,16,93,4,32,93,34,32,93,64,32,93,16,64,93,48,64,93,79,64,93,108,64,93,138,64,93,166,64,93,195,64,93,225,64,93,15,66,93,36,66,93,15,68,93,36,68,93, +15,70,93,36,70,93,15,72,93,36,72,93,6,80,93,35,80,93,65,80,93,96,80,93,16,0,94,48,0,94,80,0,94,112,0,94,144,0,94,176,0,94,16,16,94,48,16,94, +80,16,94,109,16,94,135,16,94,16,32,94,48,32,94,75,32,94,102,32,94,130,32,94,16,64,94,48,64,94,80,64,94,112,64,94,144,64,94,167,64,94,16,72,94,33,72,94, +16,80,94,43,80,94,70,80,94,99,80,94,128,80,94,9,96,94,36,96,94,65,96,94,16,128,94,48,128,94,80,128,94,112,128,94,144,128,94,167,128,94,16,130,94,33,130,94, +16,132,94,33,132,94,16,134,94,33,134,94,16,144,94,43,144,94,70,144,94,99,144,94,128,144,94,9,160,94,36,160,94,65,160,94,16,192,94,48,192,94,80,192,94,108,192,94, +136,192,94,162,192,94,16,194,94,16,196,94,16,198,94,9,208,94,36,208,94,65,208,94,3,224,94,32,224,94,16,0,95,48,0,95,80,0,95,108,0,95,136,0,95,162,0,95, +16,2,95,16,4,95,16,6,95,16,8,95,9,16,95,36,16,95,65,16,95,3,32,95,32,32,95,16,64,95,47,64,95,75,64,95,103,64,95,132,64,95,160,64,95,10,66,95, +10,68,95,10,70,95,10,72,95,4,80,95,33,80,95,16,0,96,48,0,96,80,0,96,112,0,96,144,0,96,16,16,96,48,16,96,76,16,96,99,16,96,16,32,96,44,32,96, +69,32,96,16,64,96,48,64,96,80,64,96,108,64,96,132,64,96,15,80,96,39,80,96,66,80,96,7,96,96,34,96,96,16,128,96,48,128,96,80,128,96,108,128,96,132,128,96, +15,144,96,39,144,96,66,144,96,7,160,96,34,160,96,16,192,96,48,192,96,75,192,96,101,192,96,128,192,96,7,208,96,34,208,96,2,224,96,16,0,97,48,0,97,75,0,97, +101,0,97,128,0,97,7,16,97,34,16,97,2,32,97,16,64,97,44,64,97,70,64,97,98,64,97,3,80,97,16,0,98,48,0,98,80,0,98,16,16,98,47,16,98,67,16,98, +16,32,98,39,32,98,16,64,98,48,64,98,76,64,98,13,80,98,35,80,98,6,96,98,16,128,98,48,128,98,76,128,98,13,144,98,35,144,98,6,160,98,16,192,98,45,192,98, +69,192,98,6,208,98,1,224,98,16,0,99,45,0,99,69,0,99,6,16,99,1,32,99,16,64,99,40,64,99,66,64,99,2,80,99,16,0,100,48,0,100,78,0,100,16,16,100, +40,16,100,16,32,100,35,32,100,16,64,100,48,64,100,67,64,100,11,80,100,32,80,100,4,96,100,16,128,100,48,128,100,67,128,100,11,144,100,32,144,100,4,160,100,16,192,100, +41,192,100,4,208,100,0,224,100,16,0,101,41,0,101,4,16,101,0,32,101,15,64,101,36,64,101,1,80,101,16,0,102,48,0,102,16,16,102,33,16,102,14,32,102,16,64,102, +42,64,102,9,80,102,3,96,102,16,128,102,42,128,102,9,144,102,3,160,102,16,192,102,36,192,102,3,208,102,16,0,103,36,0,103,3,16,103,13,64,103,33,64,103,0,80,103, +16,0,104,46,0,104,16,16,104,11,32,104,16,64,104,35,64,104,6,80,104,1,96,104,16,128,104,35,128,104,6,144,104,1,160,104,16,192,104,1,208,104,16,0,105,1,16,105, +11,64,105,80,0,110,112,0,110,144,0,110,176,0,110,208,0,110,240,0,110,16,1,110,48,1,110,80,1,110,112,1,110,80,16,110,112,16,110,144,16,110,176,16,110,208,16,110, +240,16,110,16,17,110,48,17,110,80,17,110,109,17,110,80,32,110,112,32,110,144,32,110,176,32,110,208,32,110,240,32,110,13,33,110,43,33,110,73,33,110,102,33,110,80,64,110, +112,64,110,144,64,110,176,64,110,208,64,110,240,64,110,16,65,110,48,65,110,80,65,110,112,65,110,16,72,110,48,72,110,80,72,110,112,72,110,144,72,110,170,72,110,195,72,110, +80,80,110,112,80,110,143,80,110,173,80,110,203,80,110,234,80,110,8,81,110,38,81,110,68,81,110,99,81,110,73,96,110,104,96,110,135,96,110,166,96,110,197,96,110,228,96,110, +2,97,110,33,97,110,64,97,110,80,128,110,112,128,110,144,128,110,176,128,110,208,128,110,240,128,110,16,129,110,48,129,110,80,129,110,112,129,110,16,130,110,48,130,110,80,130,110, +112,130,110,144,130,110,170,130,110,195,130,110,16,132,110,48,132,110,80,132,110,112,132,110,144,132,110,170,132,110,195,132,110,16,134,110,48,134,110,80,134,110,112,134,110,144,134,110, +170,134,110,195,134,110,80,144,110,112,144,110,143,144,110,173,144,110,203,144,110,234,144,110,8,145,110,38,145,110,68,145,110,99,145,110,73,160,110,104,160,110,135,160,110,166,160,110, +197,160,110,228,160,110,2,161,110,33,161,110,64,161,110,80,192,110,112,192,110,144,192,110,176,192,110,208,192,110,240,192,110,16,193,110,48,193,110,79,193,110,108,193,110,16,194,110, +48,194,110,80,194,110,109,194,110,138,194,110,164,194,110,16,196,110,48,196,110,80,196,110,109,196,110,138,196,110,164,196,110,16,198,110,48,198,110,80,198,110,109,198,110,138,198,110, +164,198,110,73,208,110,104,208,110,135,208,110,166,208,110,197,208,110,228,208,110,2,209,110,33,209,110,64,209,110,67,224,110,99,224,110,130,224,110,161,224,110,192,224,110,224,224,110, +80,0,111,112,0,111,144,0,111,176,0,111,208,0,111,240,0,111,16,1,111,48,1,111,79,1,111,108,1,111,16,2,111,48,2,111,80,2,111,109,2,111,138,2,111,164,2,111, +16,4,111,48,4,111,80,4,111,109,4,111,138,4,111,164,4,111,16,6,111,48,6,111,80,6,111,109,6,111,138,6,111,164,6,111,16,8,111,48,8,111,80,8,111,109,8,111, +138,8,111,164,8,111,73,16,111,104,16,111,135,16,111,166,16,111,197,16,111,228,16,111,2,17,111,33,17,111,64,17,111,67,32,111,99,32,111,130,32,111,161,32,111,192,32,111, +224,32,111,80,64,111,112,64,111,144,64,111,176,64,111,208,64,111,238,64,111,12,65,111,42,65,111,73,65,111,103,65,111,16,66,111,48,66,111,76,66,111,104,66,111,133,66,111, +161,66,111,16,68,111,48,68,111,76,68,111,104,68,111,133,68,111,161,68,111,16,70,111,48,70,111,76,70,111,104,70,111,133,70,111,161,70,111,16,72,111,48,72,111,76,72,111, +104,72,111,133,72,111,161,72,111,69,80,111,100,80,111,131,80,111,162,80,111,193,80,111,225,80,111,0,81,111,48,0,112,80,0,112,112,0,112,144,0,112,176,0,112,208,0,112, +240,0,112,16,1,112,48,1,112,48,16,112,80,16,112,112,16,112,144,16,112,176,16,112,205,16,112,233,16,112,3,17,112,48,32,112,80,32,112,112,32,112,142,32,112,170,32,112, +198,32,112,227,32,112,48,64,112,80,64,112,112,64,112,144,64,112,176,64,112,208,64,112,240,64,112,12,65,112,39,65,112,16,72,112,48,72,112,74,72,112,48,80,112,77,80,112, +106,80,112,136,80,112,165,80,112,195,80,112,224,80,112,40,96,112,70,96,112,100,96,112,131,96,112,161,96,112,48,128,112,80,128,112,112,128,112,144,128,112,176,128,112,208,128,112, +240,128,112,12,129,112,39,129,112,16,130,112,48,130,112,74,130,112,16,132,112,48,132,112,74,132,112,16,134,112,48,134,112,74,134,112,48,144,112,77,144,112,106,144,112,136,144,112, +165,144,112,195,144,112,224,144,112,40,160,112,70,160,112,100,160,112,131,160,112,161,160,112,48,192,112,80,192,112,112,192,112,144,192,112,176,192,112,204,192,112,233,192,112,5,193,112, +34,193,112,16,194,112,44,194,112,68,194,112,16,196,112,44,196,112,68,196,112,16,198,112,44,198,112,68,198,112,40,208,112,70,208,112,100,208,112,131,208,112,161,208,112,35,224,112, +65,224,112,96,224,112,48,0,113,80,0,113,112,0,113,144,0,113,176,0,113,204,0,113,233,0,113,5,1,113,34,1,113,16,2,113,44,2,113,68,2,113,16,4,113,44,4,113, +68,4,113,16,6,113,44,6,113,68,6,113,16,8,113,44,8,113,68,8,113,40,16,113,70,16,113,100,16,113,131,16,113,161,16,113,35,32,113,65,32,113,96,32,113,48,64,113, +80,64,113,111,64,113,141,64,113,170,64,113,199,64,113,229,64,113,2,65,113,16,66,113,39,66,113,65,66,113,16,68,113,39,68,113,65,68,113,16,70,113,39,70,113,65,70,113, +16,72,113,39,72,113,65,72,113,36,80,113,66,80,113,97,80,113,128,80,113,16,0,114,48,0,114,80,0,114,112,0,114,144,0,114,176,0,114,208,0,114,16,16,114,48,16,114, +80,16,114,112,16,114,139,16,114,163,16,114,16,32,114,48,32,114,77,32,114,104,32,114,133,32,114,16,64,114,48,64,114,80,64,114,112,64,114,144,64,114,172,64,114,196,64,114, +16,72,114,38,72,114,16,80,114,44,80,114,72,80,114,100,80,114,129,80,114,9,96,114,37,96,114,66,96,114,96,96,114,16,128,114,48,128,114,80,128,114,112,128,114,144,128,114, +172,128,114,196,128,114,16,130,114,38,130,114,16,132,114,38,132,114,16,134,114,38,134,114,16,144,114,44,144,114,72,144,114,100,144,114,129,144,114,9,160,114,37,160,114,66,160,114, +96,160,114,16,192,114,48,192,114,80,192,114,110,192,114,139,192,114,165,192,114,192,192,114,16,194,114,33,194,114,16,196,114,33,196,114,16,198,114,33,198,114,9,208,114,37,208,114, +66,208,114,96,208,114,3,224,114,33,224,114,16,0,115,48,0,115,80,0,115,110,0,115,139,0,115,165,0,115,192,0,115,16,2,115,33,2,115,16,4,115,33,4,115,16,6,115, +33,6,115,16,8,115,33,8,115,9,16,115,37,16,115,66,16,115,96,16,115,3,32,115,33,32,115,16,64,115,48,64,115,76,64,115,105,64,115,134,64,115,162,64,115,12,66,115, +12,68,115,12,70,115,12,72,115,5,80,115,34,80,115,64,80,115,16,0,116,48,0,116,80,0,116,112,0,116,144,0,116,16,16,116,48,16,116,77,16,116,100,16,116,16,32,116, +45,32,116,70,32,116,96,32,116,16,64,116,48,64,116,80,64,116,109,64,116,135,64,116,16,80,116,40,80,116,67,80,116,8,96,116,34,96,116,16,128,116,48,128,116,80,128,116, +109,128,116,135,128,116,16,144,116,40,144,116,67,144,116,8,160,116,34,160,116,16,192,116,48,192,116,76,192,116,102,192,116,130,192,116,8,208,116,34,208,116,2,224,116,16,0,117, +48,0,117,76,0,117,102,0,117,130,0,117,8,16,117,34,16,117,2,32,117,16,64,117,44,64,117,71,64,117,99,64,117,4,80,117,32,80,117,16,0,118,48,0,118,80,0,118, +16,16,118,47,16,118,67,16,118,16,32,118,39,32,118,16,64,118,48,64,118,76,64,118,13,80,118,35,80,118,6,96,118,16,128,118,48,128,118,76,128,118,13,144,118,35,144,118, +6,160,118,16,192,118,45,192,118,69,192,118,6,208,118,1,224,118,16,0,119,45,0,119,69,0,119,6,16,119,1,32,119,16,64,119,40,64,119,66,64,119,2,80,119,16,0,120, +48,0,120,16,16,120,39,16,120,16,32,120,34,32,120,16,64,120,48,64,120,10,80,120,4,96,120,16,128,120,48,128,120,10,144,120,4,160,120,16,192,120,40,192,120,4,208,120, +0,224,120,16,0,121,40,0,121,4,16,121,0,32,121,15,64,121,36,64,121,1,80,121,16,0,122,48,0,122,16,16,122,13,32,122,16,64,122,39,64,122,8,80,122,2,96,122, +16,128,122,39,128,122,8,144,122,2,160,122,16,192,122,34,192,122,2,208,122,16,0,123,34,0,123,2,16,123,12,64,123,32,64,123,0,80,123,16,0,124,16,16,124,10,32,124, +16,64,124,5,80,124,1,96,124,16,128,124,5,144,124,1,160,124,16,192,124,1,208,124,16,0,125,1,16,125,10,64,125,48,0,132,80,0,132,112,0,132,144,0,132,176,0,132, +208,0,132,240,0,132,16,1,132,48,1,132,80,1,132,112,1,132,48,16,132,80,16,132,112,16,132,144,16,132,176,16,132,208,16,132,240,16,132,16,17,132,45,17,132,74,17,132, +102,17,132,48,32,132,80,32,132,112,32,132,144,32,132,176,32,132,206,32,132,236,32,132,9,33,132,38,33,132,68,33,132,97,33,132,48,64,132,80,64,132,112,64,132,144,64,132, +176,64,132,208,64,132,240,64,132,16,65,132,48,65,132,80,65,132,111,65,132,16,72,132,48,72,132,80,72,132,112,72,132,138,72,132,161,72,132,48,80,132,80,80,132,111,80,132, +141,80,132,171,80,132,201,80,132,231,80,132,5,81,132,35,81,132,65,81,132,42,96,132,72,96,132,103,96,132,134,96,132,164,96,132,195,96,132,226,96,132,0,97,132,48,128,132, +80,128,132,112,128,132,144,128,132,176,128,132,208,128,132,240,128,132,16,129,132,48,129,132,80,129,132,111,129,132,16,130,132,48,130,132,80,130,132,112,130,132,138,130,132,161,130,132, +16,132,132,48,132,132,80,132,132,112,132,132,138,132,132,161,132,132,16,134,132,48,134,132,80,134,132,112,134,132,138,134,132,161,134,132,48,144,132,80,144,132,111,144,132,141,144,132, +171,144,132,201,144,132,231,144,132,5,145,132,35,145,132,65,145,132,42,160,132,72,160,132,103,160,132,134,160,132,164,160,132,195,160,132,226,160,132,0,161,132,48,192,132,80,192,132, +112,192,132,144,192,132,176,192,132,208,192,132,240,192,132,15,193,132,44,193,132,74,193,132,103,193,132,16,194,132,48,194,132,78,194,132,105,194,132,132,194,132,16,196,132,48,196,132, +78,196,132,105,196,132,132,196,132,16,198,132,48,198,132,78,198,132,105,198,132,132,198,132,42,208,132,72,208,132,103,208,132,134,208,132,164,208,132,195,208,132,226,208,132,0,209,132, +36,224,132,67,224,132,98,224,132,129,224,132,160,224,132,48,0,133,80,0,133,112,0,133,144,0,133,176,0,133,208,0,133,240,0,133,15,1,133,44,1,133,74,1,133,103,1,133, +16,2,133,48,2,133,78,2,133,105,2,133,132,2,133,16,4,133,48,4,133,78,4,133,105,4,133,132,4,133,16,6,133,48,6,133,78,6,133,105,6,133,132,6,133,16,8,133, +48,8,133,78,8,133,105,8,133,132,8,133,42,16,133,72,16,133,103,16,133,134,16,133,164,16,133,195,16,133,226,16,133,0,17,133,36,32,133,67,32,133,98,32,133,129,32,133, +160,32,133,48,64,133,80,64,133,112,64,133,144,64,133,175,64,133,205,64,133,236,64,133,9,65,133,39,65,133,70,65,133,99,65,133,16,66,133,45,66,133,73,66,133,100,66,133, +129,66,133,16,68,133,45,68,133,73,68,133,100,68,133,129,68,133,16,70,133,45,70,133,73,70,133,100,70,133,129,70,133,16,72,133,45,72,133,73,72,133,100,72,133,129,72,133, +37,80,133,68,80,133,99,80,133,130,80,133,161,80,133,192,80,133,16,0,134,48,0,134,80,0,134,112,0,134,144,0,134,176,0,134,208,0,134,240,0,134,14,1,134,16,16,134, +48,16,134,80,16,134,112,16,134,144,16,134,172,16,134,198,16,134,225,16,134,16,32,134,48,32,134,80,32,134,109,32,134,138,32,134,165,32,134,193,32,134,16,64,134,48,64,134, +80,64,134,112,64,134,144,64,134,176,64,134,207,64,134,234,64,134,3,65,134,16,72,134,48,72,134,65,72,134,16,80,134,46,80,134,75,80,134,104,80,134,133,80,134,162,80,134, +10,96,134,39,96,134,68,96,134,98,96,134,129,96,134,16,128,134,48,128,134,80,128,134,112,128,134,144,128,134,176,128,134,207,128,134,234,128,134,3,129,134,16,130,134,48,130,134, +65,130,134,16,132,134,48,132,134,65,132,134,16,134,134,48,134,134,65,134,134,16,144,134,46,144,134,75,144,134,104,144,134,133,144,134,162,144,134,10,160,134,39,160,134,68,160,134, +98,160,134,129,160,134,16,192,134,48,192,134,80,192,134,112,192,134,144,192,134,171,192,134,199,192,134,228,192,134,16,194,134,40,194,134,16,196,134,40,196,134,16,198,134,40,198,134, +10,208,134,39,208,134,68,208,134,98,208,134,129,208,134,4,224,134,34,224,134,64,224,134,16,0,135,48,0,135,80,0,135,112,0,135,144,0,135,171,0,135,199,0,135,228,0,135, +16,2,135,40,2,135,16,4,135,40,4,135,16,6,135,40,6,135,16,8,135,40,8,135,10,16,135,39,16,135,68,16,135,98,16,135,129,16,135,4,32,135,34,32,135,64,32,135, +16,64,135,48,64,135,79,64,135,108,64,135,138,64,135,166,64,135,195,64,135,225,64,135,15,66,135,36,66,135,15,68,135,36,68,135,15,70,135,36,70,135,15,72,135,36,72,135, +6,80,135,35,80,135,65,80,135,96,80,135,16,0,136,48,0,136,80,0,136,112,0,136,144,0,136,174,0,136,16,16,136,48,16,136,80,16,136,106,16,136,131,16,136,16,32,136, +47,32,136,73,32,136,100,32,136,16,64,136,48,64,136,80,64,136,112,64,136,140,64,136,163,64,136,16,72,136,16,80,136,41,80,136,69,80,136,97,80,136,8,96,136,35,96,136, +64,96,136,16,128,136,48,128,136,80,128,136,112,128,136,140,128,136,163,128,136,16,130,136,16,132,136,16,134,136,16,144,136,41,144,136,69,144,136,97,144,136,8,160,136,35,160,136, +64,160,136,16,192,136,48,192,136,79,192,136,106,192,136,133,192,136,14,194,136,14,196,136,14,198,136,8,208,136,35,208,136,64,208,136,3,224,136,32,224,136,16,0,137,48,0,137, +79,0,137,106,0,137,133,0,137,14,2,137,14,4,137,14,6,137,14,8,137,8,16,137,35,16,137,64,16,137,3,32,137,32,32,137,16,64,137,46,64,137,73,64,137,101,64,137, +130,64,137,9,66,137,9,68,137,9,70,137,9,72,137,4,80,137,32,80,137,16,0,138,48,0,138,80,0,138,112,0,138,16,16,138,48,16,138,70,16,138,16,32,138,41,32,138, +65,32,138,16,64,138,48,64,138,79,64,138,100,64,138,14,80,138,37,80,138,6,96,138,32,96,138,16,128,138,48,128,138,79,128,138,100,128,138,14,144,138,37,144,138,6,160,138, +32,160,138,16,192,138,47,192,138,71,192,138,96,192,138,6,208,138,32,208,138,1,224,138,16,0,139,47,0,139,71,0,139,96,0,139,6,16,139,32,16,139,1,32,139,16,64,139, +41,64,139,67,64,139,3,80,139,16,0,140,48,0,140,78,0,140,16,16,140,40,16,140,16,32,140,35,32,140,16,64,140,48,64,140,67,64,140,11,80,140,32,80,140,4,96,140, +16,128,140,48,128,140,67,128,140,11,144,140,32,144,140,4,160,140,16,192,140,41,192,140,4,208,140,0,224,140,16,0,141,41,0,141,4,16,141,0,32,141,15,64,141,36,64,141, +1,80,141,16,0,142,48,0,142,16,16,142,13,32,142,16,64,142,39,64,142,8,80,142,2,96,142,16,128,142,39,128,142,8,144,142,2,160,142,16,192,142,34,192,142,2,208,142, +16,0,143,34,0,143,2,16,143,12,64,143,32,64,143,0,80,143,16,0,144,16,16,144,9,32,144,16,64,144,5,80,144,0,96,144,16,128,144,5,144,144,0,160,144,15,192,144, +0,208,144,15,0,145,0,16,145,9,64,145,48,0,154,80,0,154,112,0,154,144,0,154,176,0,154,208,0,154,240,0,154,16,1,154,48,1,154,80,1,154,112,1,154,48,16,154, +80,16,154,112,16,154,144,16,154,176,16,154,208,16,154,240,16,154,12,17,154,39,17,154,68,17,154,48,32,154,80,32,154,112,32,154,144,32,154,174,32,154,203,32,154,233,32,154, +5,33,154,34,33,154,64,33,154,48,64,154,80,64,154,112,64,154,144,64,154,176,64,154,208,64,154,240,64,154,16,65,154,48,65,154,77,65,154,103,65,154,16,72,154,48,72,154, +80,72,154,106,72,154,131,72,154,48,80,154,79,80,154,109,80,154,139,80,154,169,80,154,198,80,154,228,80,154,2,81,154,32,81,154,41,96,154,71,96,154,102,96,154,133,96,154, +163,96,154,193,96,154,224,96,154,48,128,154,80,128,154,112,128,154,144,128,154,176,128,154,208,128,154,240,128,154,16,129,154,48,129,154,77,129,154,103,129,154,16,130,154,48,130,154, +80,130,154,106,130,154,131,130,154,16,132,154,48,132,154,80,132,154,106,132,154,131,132,154,16,134,154,48,134,154,80,134,154,106,134,154,131,134,154,48,144,154,79,144,154,109,144,154, +139,144,154,169,144,154,198,144,154,228,144,154,2,145,154,32,145,154,41,160,154,71,160,154,102,160,154,133,160,154,163,160,154,193,160,154,224,160,154,48,192,154,80,192,154,112,192,154, +144,192,154,176,192,154,208,192,154,239,192,154,11,193,154,40,193,154,70,193,154,98,193,154,16,194,154,48,194,154,74,194,154,100,194,154,16,196,154,48,196,154,74,196,154,100,196,154, +16,198,154,48,198,154,74,198,154,100,198,154,41,208,154,71,208,154,102,208,154,133,208,154,163,208,154,193,208,154,224,208,154,35,224,154,66,224,154,97,224,154,128,224,154,48,0,155, +80,0,155,112,0,155,144,0,155,176,0,155,208,0,155,239,0,155,11,1,155,40,1,155,70,1,155,98,1,155,16,2,155,48,2,155,74,2,155,100,2,155,16,4,155,48,4,155, +74,4,155,100,4,155,16,6,155,48,6,155,74,6,155,100,6,155,16,8,155,48,8,155,74,8,155,100,8,155,41,16,155,71,16,155,102,16,155,133,16,155,163,16,155,193,16,155, +224,16,155,35,32,155,66,32,155,97,32,155,128,32,155,48,64,155,80,64,155,112,64,155,144,64,155,173,64,155,203,64,155,233,64,155,6,65,155,36,65,155,66,65,155,96,65,155, +16,66,155,43,66,155,70,66,155,97,66,155,16,68,155,43,68,155,70,68,155,97,68,155,16,70,155,43,70,155,70,70,155,97,70,155,16,72,155,43,72,155,70,72,155,97,72,155, +37,80,155,67,80,155,98,80,155,129,80,155,160,80,155,16,0,156,48,0,156,80,0,156,112,0,156,144,0,156,176,0,156,208,0,156,16,16,156,48,16,156,80,16,156,112,16,156, +141,16,156,165,16,156,16,32,156,48,32,156,78,32,156,106,32,156,134,32,156,161,32,156,16,64,156,48,64,156,80,64,156,112,64,156,144,64,156,174,64,156,199,64,156,16,72,156, +40,72,156,16,80,156,44,80,156,73,80,156,101,80,156,130,80,156,10,96,156,37,96,156,67,96,156,97,96,156,16,128,156,48,128,156,80,128,156,112,128,156,144,128,156,174,128,156, +199,128,156,16,130,156,40,130,156,16,132,156,40,132,156,16,134,156,40,134,156,16,144,156,44,144,156,73,144,156,101,144,156,130,144,156,10,160,156,37,160,156,67,160,156,97,160,156, +16,192,156,48,192,156,80,192,156,112,192,156,140,192,156,167,192,156,194,192,156,16,194,156,35,194,156,16,196,156,35,196,156,16,198,156,35,198,156,10,208,156,37,208,156,67,208,156, +97,208,156,4,224,156,33,224,156,16,0,157,48,0,157,80,0,157,112,0,157,140,0,157,167,0,157,194,0,157,16,2,157,35,2,157,16,4,157,35,4,157,16,6,157,35,6,157, +16,8,157,35,8,157,10,16,157,37,16,157,67,16,157,97,16,157,4,32,157,33,32,157,16,64,157,48,64,157,77,64,157,106,64,157,135,64,157,163,64,157,192,64,157,12,66,157, +32,66,157,12,68,157,32,68,157,12,70,157,32,70,157,12,72,157,32,72,157,5,80,157,34,80,157,64,80,157,16,0,158,48,0,158,80,0,158,112,0,158,144,0,158,16,16,158, +48,16,158,76,16,158,99,16,158,16,32,158,44,32,158,69,32,158,16,64,158,48,64,158,80,64,158,108,64,158,132,64,158,15,80,158,39,80,158,66,80,158,7,96,158,34,96,158, +16,128,158,48,128,158,80,128,158,108,128,158,132,128,158,15,144,158,39,144,158,66,144,158,7,160,158,34,160,158,16,192,158,48,192,158,75,192,158,101,192,158,128,192,158,7,208,158, +34,208,158,2,224,158,16,0,159,48,0,159,75,0,159,101,0,159,128,0,159,7,16,159,34,16,159,2,32,159,16,64,159,44,64,159,70,64,159,98,64,159,3,80,159,16,0,160, +48,0,160,80,0,160,16,16,160,44,16,160,16,32,160,37,32,160,16,64,160,48,64,160,71,64,160,12,80,160,34,80,160,5,96,160,16,128,160,48,128,160,71,128,160,12,144,160, +34,144,160,5,160,160,16,192,160,43,192,160,66,192,160,5,208,160,1,224,160,16,0,161,43,0,161,66,0,161,5,16,161,1,32,161,16,64,161,38,64,161,64,64,161,2,80,161, +16,0,162,48,0,162,16,16,162,33,16,162,14,32,162,16,64,162,42,64,162,9,80,162,3,96,162,16,128,162,42,128,162,9,144,162,3,160,162,16,192,162,36,192,162,3,208,162, +16,0,163,36,0,163,3,16,163,13,64,163,33,64,163,0,80,163,16,0,164,16,16,164,10,32,164,16,64,164,5,80,164,1,96,164,16,128,164,5,144,164,1,160,164,16,192,164, +1,208,164,16,0,165,1,16,165,10,64,165,48,0,176,80,0,176,112,0,176,144,0,176,176,0,176,208,0,176,240,0,176,16,1,176,48,1,176,80,1,176,48,16,176,80,16,176, +112,16,176,144,16,176,176,16,176,208,16,176,236,16,176,6,17,176,33,17,176,48,32,176,80,32,176,112,32,176,143,32,176,171,32,176,200,32,176,229,32,176,1,33,176,48,64,176, +80,64,176,112,64,176,144,64,176,176,64,176,208,64,176,240,64,176,15,65,176,42,65,176,70,65,176,16,72,176,48,72,176,77,72,176,99,72,176,48,80,176,78,80,176,107,80,176, +137,80,176,166,80,176,196,80,176,226,80,176,40,96,176,70,96,176,101,96,176,131,96,176,161,96,176,192,96,176,48,128,176,80,128,176,112,128,176,144,128,176,176,128,176,208,128,176, +240,128,176,15,129,176,42,129,176,70,129,176,16,130,176,48,130,176,77,130,176,99,130,176,16,132,176,48,132,176,77,132,176,99,132,176,16,134,176,48,134,176,77,134,176,99,134,176, +48,144,176,78,144,176,107,144,176,137,144,176,166,144,176,196,144,176,226,144,176,40,160,176,70,160,176,101,160,176,131,160,176,161,160,176,192,160,176,48,192,176,80,192,176,112,192,176, +144,192,176,176,192,176,206,192,176,235,192,176,7,193,176,36,193,176,65,193,176,16,194,176,46,194,176,70,194,176,16,196,176,46,196,176,70,196,176,16,198,176,46,198,176,70,198,176, +40,208,176,70,208,176,101,208,176,131,208,176,161,208,176,192,208,176,35,224,176,65,224,176,96,224,176,128,224,176,48,0,177,80,0,177,112,0,177,144,0,177,176,0,177,206,0,177, +235,0,177,7,1,177,36,1,177,65,1,177,16,2,177,46,2,177,70,2,177,16,4,177,46,4,177,70,4,177,16,6,177,46,6,177,70,6,177,16,8,177,46,8,177,70,8,177, +40,16,177,70,16,177,101,16,177,131,16,177,161,16,177,192,16,177,35,32,177,65,32,177,96,32,177,128,32,177,48,64,177,80,64,177,112,64,177,142,64,177,171,64,177,200,64,177, +230,64,177,3,65,177,33,65,177,16,66,177,41,66,177,67,66,177,16,68,177,41,68,177,67,68,177,16,70,177,41,70,177,67,70,177,16,72,177,41,72,177,67,72,177,36,80,177, +67,80,177,97,80,177,128,80,177,16,0,178,48,0,178,80,0,178,112,0,178,144,0,178,176,0,178,16,16,178,48,16,178,80,16,178,109,16,178,135,16,178,16,32,178,48,32,178, +75,32,178,102,32,178,130,32,178,16,64,178,48,64,178,80,64,178,112,64,178,144,64,178,167,64,178,16,72,178,33,72,178,16,80,178,43,80,178,70,80,178,99,80,178,128,80,178, +9,96,178,36,96,178,65,96,178,16,128,178,48,128,178,80,128,178,112,128,178,144,128,178,167,128,178,16,130,178,33,130,178,16,132,178,33,132,178,16,134,178,33,134,178,16,144,178, +43,144,178,70,144,178,99,144,178,128,144,178,9,160,178,36,160,178,65,160,178,16,192,178,48,192,178,80,192,178,108,192,178,136,192,178,162,192,178,16,194,178,16,196,178,16,198,178, +9,208,178,36,208,178,65,208,178,3,224,178,32,224,178,16,0,179,48,0,179,80,0,179,108,0,179,136,0,179,162,0,179,16,2,179,16,4,179,16,6,179,16,8,179,9,16,179, +36,16,179,65,16,179,3,32,179,32,32,179,16,64,179,47,64,179,75,64,179,103,64,179,132,64,179,160,64,179,10,66,179,10,68,179,10,70,179,10,72,179,4,80,179,33,80,179, +16,0,180,48,0,180,80,0,180,112,0,180,16,16,180,48,16,180,70,16,180,16,32,180,41,32,180,65,32,180,16,64,180,48,64,180,79,64,180,100,64,180,14,80,180,37,80,180, +6,96,180,32,96,180,16,128,180,48,128,180,79,128,180,100,128,180,14,144,180,37,144,180,6,160,180,32,160,180,16,192,180,47,192,180,71,192,180,96,192,180,6,208,180,32,208,180, +1,224,180,16,0,181,47,0,181,71,0,181,96,0,181,6,16,181,32,16,181,1,32,181,16,64,181,41,64,181,67,64,181,3,80,181,16,0,182,48,0,182,16,16,182,38,16,182, +16,32,182,33,32,182,16,64,182,47,64,182,10,80,182,4,96,182,16,128,182,47,128,182,10,144,182,4,160,182,16,192,182,39,192,182,4,208,182,0,224,182,16,0,183,39,0,183, +4,16,183,0,32,183,15,64,183,35,64,183,1,80,183,16,0,184,46,0,184,16,16,184,11,32,184,16,64,184,35,64,184,6,80,184,1,96,184,16,128,184,35,128,184,6,144,184, +1,160,184,16,192,184,1,208,184,16,0,185,1,16,185,11,64,185,48,0,198,80,0,198,112,0,198,144,0,198,176,0,198,208,0,198,240,0,198,16,1,198,46,1,198,48,16,198, +80,16,198,112,16,198,144,16,198,176,16,198,202,16,198,230,16,198,0,17,198,48,32,198,80,32,198,111,32,198,140,32,198,168,32,198,196,32,198,225,32,198,48,64,198,80,64,198, +112,64,198,144,64,198,176,64,198,208,64,198,239,64,198,9,65,198,35,65,198,16,72,198,48,72,198,71,72,198,47,80,198,76,80,198,105,80,198,135,80,198,164,80,198,193,80,198, +39,96,198,69,96,198,99,96,198,130,96,198,160,96,198,48,128,198,80,128,198,112,128,198,144,128,198,176,128,198,208,128,198,239,128,198,9,129,198,35,129,198,16,130,198,48,130,198, +71,130,198,16,132,198,48,132,198,71,132,198,16,134,198,48,134,198,71,134,198,47,144,198,76,144,198,105,144,198,135,144,198,164,144,198,193,144,198,39,160,198,69,160,198,99,160,198, +130,160,198,160,160,198,48,192,198,80,192,198,112,192,198,144,192,198,174,192,198,202,192,198,231,192,198,3,193,198,16,194,198,43,194,198,66,194,198,16,196,198,43,196,198,66,196,198, +16,198,198,43,198,198,66,198,198,39,208,198,69,208,198,99,208,198,130,208,198,160,208,198,34,224,198,65,224,198,96,224,198,48,0,199,80,0,199,112,0,199,144,0,199,174,0,199, +202,0,199,231,0,199,3,1,199,16,2,199,43,2,199,66,2,199,16,4,199,43,4,199,66,4,199,16,6,199,43,6,199,66,6,199,16,8,199,43,8,199,66,8,199,39,16,199, +69,16,199,99,16,199,130,16,199,160,16,199,34,32,199,65,32,199,96,32,199,48,64,199,80,64,199,110,64,199,140,64,199,169,64,199,198,64,199,227,64,199,0,65,199,16,66,199, +38,66,199,64,66,199,16,68,199,38,68,199,64,68,199,16,70,199,38,70,199,64,70,199,16,72,199,38,72,199,64,72,199,35,80,199,66,80,199,96,80,199,16,0,200,48,0,200, +80,0,200,112,0,200,144,0,200,16,16,200,48,16,200,80,16,200,104,16,200,129,16,200,16,32,200,47,32,200,72,32,200,99,32,200,16,64,200,48,64,200,80,64,200,112,64,200, +138,64,200,16,80,200,41,80,200,68,80,200,96,80,200,8,96,200,35,96,200,64,96,200,16,128,200,48,128,200,80,128,200,112,128,200,138,128,200,16,144,200,41,144,200,68,144,200, +96,144,200,8,160,200,35,160,200,64,160,200,16,192,200,48,192,200,78,192,200,105,192,200,132,192,200,8,208,200,35,208,200,64,208,200,3,224,200,16,0,201,48,0,201,78,0,201, +105,0,201,132,0,201,8,16,201,35,16,201,64,16,201,3,32,201,16,64,201,45,64,201,73,64,201,100,64,201,129,64,201,4,80,201,32,80,201,16,0,202,48,0,202,80,0,202, +16,16,202,45,16,202,64,16,202,16,32,202,38,32,202,16,64,202,48,64,202,73,64,202,12,80,202,34,80,202,5,96,202,16,128,202,48,128,202,73,128,202,12,144,202,34,144,202, +5,160,202,16,192,202,44,192,202,67,192,202,5,208,202,1,224,202,16,0,203,44,0,203,67,0,203,5,16,203,1,32,203,16,64,203,39,64,203,64,64,203,2,80,203,16,0,204, +48,0,204,16,16,204,32,16,204,14,32,204,16,64,204,41,64,204,8,80,204,3,96,204,16,128,204,41,128,204,8,144,204,3,160,204,16,192,204,35,192,204,3,208,204,16,0,205, +35,0,205,3,16,205,13,64,205,32,64,205,0,80,205,16,0,220,48,0,220,80,0,220,112,0,220,144,0,220,176,0,220,208,0,220,240,0,220,14,1,220,16,16,220,48,16,220, +80,16,220,112,16,220,144,16,220,172,16,220,198,16,220,225,16,220,16,32,220,48,32,220,80,32,220,109,32,220,138,32,220,165,32,220,193,32,220,16,64,220,48,64,220,80,64,220, +112,64,220,144,64,220,176,64,220,207,64,220,234,64,220,3,65,220,16,72,220,48,72,220,65,72,220,16,80,220,46,80,220,75,80,220,104,80,220,133,80,220,162,80,220,10,96,220, +39,96,220,68,96,220,98,96,220,129,96,220,16,128,220,48,128,220,80,128,220,112,128,220,144,128,220,176,128,220,207,128,220,234,128,220,3,129,220,16,130,220,48,130,220,65,130,220, +16,132,220,48,132,220,65,132,220,16,134,220,48,134,220,65,134,220,16,144,220,46,144,220,75,144,220,104,144,220,133,144,220,162,144,220,10,160,220,39,160,220,68,160,220,98,160,220, +129,160,220,16,192,220,48,192,220,80,192,220,112,192,220,144,192,220,171,192,220,199,192,220,228,192,220,16,194,220,40,194,220,16,196,220,40,196,220,16,198,220,40,198,220,10,208,220, +39,208,220,68,208,220,98,208,220,129,208,220,4,224,220,34,224,220,64,224,220,16,0,221,48,0,221,80,0,221,112,0,221,144,0,221,171,0,221,199,0,221,228,0,221,16,2,221, +40,2,221,16,4,221,40,4,221,16,6,221,40,6,221,16,8,221,40,8,221,10,16,221,39,16,221,68,16,221,98,16,221,129,16,221,4,32,221,34,32,221,64,32,221,16,64,221, +48,64,221,79,64,221,108,64,221,138,64,221,166,64,221,195,64,221,225,64,221,15,66,221,36,66,221,15,68,221,36,68,221,15,70,221,36,70,221,15,72,221,36,72,221,6,80,221, +35,80,221,65,80,221,96,80,221,16,0,222,48,0,222,80,0,222,112,0,222,144,0,222,16,16,222,48,16,222,76,16,222,99,16,222,16,32,222,44,32,222,69,32,222,16,64,222, +48,64,222,80,64,222,108,64,222,132,64,222,15,80,222,39,80,222,66,80,222,7,96,222,34,96,222,16,128,222,48,128,222,80,128,222,108,128,222,132,128,222,15,144,222,39,144,222, +66,144,222,7,160,222,34,160,222,16,192,222,48,192,222,75,192,222,101,192,222,128,192,222,7,208,222,34,208,222,2,224,222,16,0,223,48,0,223,75,0,223,101,0,223,128,0,223, +7,16,223,34,16,223,2,32,223,16,64,223,44,64,223,70,64,223,98,64,223,3,80,223,16,0,224,48,0,224,78,0,224,16,16,224,40,16,224,16,32,224,35,32,224,16,64,224, +48,64,224,67,64,224,11,80,224,32,80,224,4,96,224,16,128,224,48,128,224,67,128,224,11,144,224,32,144,224,4,160,224,16,192,224,41,192,224,4,208,224,0,224,224,16,0,225, +41,0,225,4,16,225,0,32,225,15,64,225,36,64,225,1,80,225,16,0,226,46,0,226,16,16,226,11,32,226,16,64,226,35,64,226,6,80,226,1,96,226,16,128,226,35,128,226, +6,144,226,1,160,226,16,192,226,1,208,226,16,0,227,1,16,227,11,64,227 +}; diff --git a/external/basis_universal/transcoder/basisu_astc_hdr_core.h b/external/basis_universal/transcoder/basisu_astc_hdr_core.h index 8e63f72718..143e29e956 100644 --- a/external/basis_universal/transcoder/basisu_astc_hdr_core.h +++ b/external/basis_universal/transcoder/basisu_astc_hdr_core.h @@ -1,6 +1,5 @@ // File: basisu_astc_hdr_core.h #pragma once -#include "basisu_astc_helpers.h" namespace basist { @@ -109,7 +108,10 @@ namespace basist const uint32_t TOTAL_BLOCK_MODE_DECS = 75; extern const block_mode_desc g_block_mode_descs[TOTAL_BLOCK_MODE_DECS]; - void copy_weight_grid(bool dual_plane, uint32_t grid_x, uint32_t grid_y, const uint8_t* transcode_weights, astc_helpers::log_astc_block& decomp_blk); + const uint32_t UASTC_6x6_HDR_SIG0 = 0xABCD; // original release + const uint32_t UASTC_6x6_HDR_SIG1 = 0xABCE; // 2x2->4x4 weight grid upsampling change, not backwards compatible with older decoders + + void copy_weight_grid(bool dual_plane, uint32_t grid_x, uint32_t grid_y, const uint8_t* transcode_weights, astc_helpers::log_astc_block& decomp_blk, bool orig_behavior); enum class encoding_type { @@ -181,7 +183,7 @@ namespace basist bool m_hq_ls; bool m_brute_force_weight4_assignment; - + fast_bc6h_params() { init(); @@ -203,3 +205,4 @@ namespace basist } // namespace astc_6x6_hdr } // namespace basist + diff --git a/external/basis_universal/transcoder/basisu_astc_helpers.h b/external/basis_universal/transcoder/basisu_astc_helpers.h index 37918054d3..32b16087a2 100644 --- a/external/basis_universal/transcoder/basisu_astc_helpers.h +++ b/external/basis_universal/transcoder/basisu_astc_helpers.h @@ -10,12 +10,22 @@ namespace astc_helpers { - const uint32_t MAX_WEIGHT_VALUE = 64; // grid texel weights must range from [0,64] const uint32_t MIN_GRID_DIM = 2; // the minimum dimension of a block's weight grid const uint32_t MIN_BLOCK_DIM = 4, MAX_BLOCK_DIM = 12; // the valid block dimensions in texels + const uint32_t MAX_BLOCK_PIXELS = MAX_BLOCK_DIM * MAX_BLOCK_DIM; const uint32_t MAX_GRID_WEIGHTS = 64; // a block may have a maximum of 64 weight grid values - const uint32_t NUM_MODE11_ENDPOINTS = 6, NUM_MODE7_ENDPOINTS = 4; - + const uint32_t MAX_CEM_ENDPOINT_VALS = 8; // see Table 94. ASTC LDR/HDR color endpoint modes (max 8 values to encode any CEM, minimum 2) + + // The number of BISE values needed to encode endpoints for each CEM. + const uint32_t NUM_MODE0_ENDPOINTS = 2, NUM_MODE4_ENDPOINTS = 4; + const uint32_t NUM_MODE6_ENDPOINTS = 4, NUM_MODE8_ENDPOINTS = 6, NUM_MODE9_ENDPOINTS = 6; // LDR RGB + const uint32_t NUM_MODE10_ENDPOINTS = 6, NUM_MODE12_ENDPOINTS = 8, NUM_MODE13_ENDPOINTS = 8; // LDR RGBA + const uint32_t NUM_MODE11_ENDPOINTS = 6, NUM_MODE7_ENDPOINTS = 4; // hdr + + const uint32_t MAX_WEIGHTS = 32; // max supported # of weights (or "selectors") in any mode, i.e. the max # of colors per endpoint pair + const uint32_t MAX_WEIGHT_INTERPOLANT_VALUE = 64; // grid texel weights must range from [0,64], i.e. the weight interpolant range is [0,64] + + // 14 unique block dimensions supported by ASTC static const uint32_t NUM_ASTC_BLOCK_SIZES = 14; extern const uint8_t g_astc_block_sizes[NUM_ASTC_BLOCK_SIZES][2]; @@ -70,6 +80,29 @@ namespace astc_helpers const uint32_t TOTAL_ISE_RANGES = 21; + enum + { + cBLOCK_SIZE_4x4 = 0, // 16 samples + cBLOCK_SIZE_5x4 = 1, // 20 samples + cBLOCK_SIZE_5x5 = 2, // 25 samples + cBLOCK_SIZE_6x5 = 3, // 30 samples + + cBLOCK_SIZE_6x6 = 4, // 36 samples + cBLOCK_SIZE_8x5 = 5, // 40 samples + cBLOCK_SIZE_8x6 = 6, // 48 samples + cBLOCK_SIZE_10x5 = 7, // 50 samples + + cBLOCK_SIZE_10x6 = 8, // 60 samples + cBLOCK_SIZE_8x8 = 9, // 64 samples + cBLOCK_SIZE_10x8 = 10, // 80 samples + cBLOCK_SIZE_10x10 = 11, // 100 samples + + cBLOCK_SIZE_12x10 = 12, // 120 samples + cBLOCK_SIZE_12x12 = 13, // 144 samples + + cTOTAL_BLOCK_SIZES = 14 + }; + // Valid endpoint ISE ranges const uint32_t FIRST_VALID_ENDPOINT_ISE_RANGE = BISE_6_LEVELS; // 4 const uint32_t LAST_VALID_ENDPOINT_ISE_RANGE = BISE_256_LEVELS; // 20 @@ -79,11 +112,11 @@ namespace astc_helpers const uint32_t FIRST_VALID_WEIGHT_ISE_RANGE = BISE_2_LEVELS; // 0 const uint32_t LAST_VALID_WEIGHT_ISE_RANGE = BISE_32_LEVELS; // 11 const uint32_t TOTAL_WEIGHT_ISE_RANGES = LAST_VALID_WEIGHT_ISE_RANGE - FIRST_VALID_WEIGHT_ISE_RANGE + 1; - + // The ISE range table. extern const int8_t g_ise_range_table[TOTAL_ISE_RANGES][3]; // 0=bits (0 to 8), 1=trits (0 or 1), 2=quints (0 or 1) - // Possible Color Component Select values, used in dual plane mode. + // Possible Color Component Select values, used in dual plane mode. // The CCS component will be interpolated using the 2nd weight plane. enum ccs { @@ -92,7 +125,7 @@ namespace astc_helpers CCS_RGA_B = 2, CCS_RGB_A = 3 }; - + struct astc_block { uint32_t m_vals[4]; @@ -106,14 +139,14 @@ namespace astc_helpers struct log_astc_block { bool m_error_flag; - + bool m_solid_color_flag_ldr, m_solid_color_flag_hdr; uint8_t m_user_mode; // user defined value, not used in this module - + // Rest is only valid if !m_solid_color_flag_ldr && !m_solid_color_flag_hdr uint8_t m_grid_width, m_grid_height; // weight grid dimensions, not the dimension of the block - + bool m_dual_plane; uint8_t m_weight_ise_range; // 0-11 @@ -123,16 +156,16 @@ namespace astc_helpers uint8_t m_num_partitions; // or the # of subsets, 1-4 (1-3 if dual plane mode) uint16_t m_partition_id; // 10-bits, must be 0 if m_num_partitions==1 - + uint8_t m_color_endpoint_modes[MAX_PARTITIONS]; // each subset's CEM's - + union { // ISE weight grid values. In dual plane mode, the order is p0,p1, p0,p1, etc. uint8_t m_weights[MAX_GRID_WEIGHTS]; uint16_t m_solid_color[4]; }; - + // ISE endpoint values // Endpoint order examples: // 1 subset LA : LL0 LH0 AL0 AH0 @@ -142,13 +175,13 @@ namespace astc_helpers // 2 subset RGB : RL0 RH0 GL0 GH0 BL0 BH0 RL1 RH1 GL1 GH1 BL1 BH1 // 2 subset RGBA : RL0 RH0 GL0 GH0 BL0 BH0 AL0 AH0 RL1 RH1 GL1 GH1 BL1 BH1 AL1 AH1 uint8_t m_endpoints[MAX_ENDPOINTS]; - + void clear() { memset(this, 0, sizeof(*this)); } }; - + // Open interval inline int bounds_check(int v, int l, int h) { (void)v; (void)l; (void)h; assert(v >= l && v < h); return v; } inline uint32_t bounds_check(uint32_t v, uint32_t l, uint32_t h) { (void)v; (void)l; (void)h; assert(v >= l && v < h); return v; } @@ -166,8 +199,8 @@ namespace astc_helpers } // Returns the number of levels in the given ISE range. - inline uint32_t get_ise_levels(uint32_t ise_range) - { + inline uint32_t get_ise_levels(uint32_t ise_range) + { assert(ise_range < TOTAL_ISE_RANGES); return (1 + 2 * g_ise_range_table[ise_range][1] + 4 * g_ise_range_table[ise_range][2]) << g_ise_range_table[ise_range][0]; } @@ -180,10 +213,10 @@ namespace astc_helpers total_bits += (g_ise_range_table[range][2] * 7 * count + 2) / 3; return total_bits; } - + inline uint32_t weight_interpolate(uint32_t l, uint32_t h, uint32_t w) { - assert(w <= MAX_WEIGHT_VALUE); + assert(w <= MAX_WEIGHT_INTERPOLANT_VALUE); return (l * (64 - w) + h * w + 32) >> 6; } @@ -198,9 +231,15 @@ namespace astc_helpers inline pack_stats() { clear(); } inline void clear() { memset(this, 0, sizeof(*this)); } }; - + + enum + { + cValidateEarlyOutAtEndpointISEChecks = 1, + cValidateSkipFinalEndpointWeightPacking = 2, + }; + // Packs a logical to physical ASTC block. Note this does not validate the block's dimensions (use is_valid_block_size()), just the grid dimensions. - bool pack_astc_block(astc_block &phys_block, const log_astc_block& log_block, int* pExpected_endpoint_range = nullptr, pack_stats *pStats = nullptr); + bool pack_astc_block(astc_block &phys_block, const log_astc_block& log_block, int* pExpected_endpoint_range = nullptr, pack_stats *pStats = nullptr, uint32_t validate_flags = 0); // Pack LDR void extent (really solid color) blocks. For LDR, pass in (val | (val << 8)) for each component. void pack_void_extent_ldr(astc_block& blk, uint16_t r, uint16_t g, uint16_t b, uint16_t a, pack_stats *pStats = nullptr); @@ -209,10 +248,10 @@ namespace astc_helpers void pack_void_extent_hdr(astc_block& blk, uint16_t rh, uint16_t gh, uint16_t bh, uint16_t ah, pack_stats* pStats = nullptr); // These helpers are all quite slow, but are useful for table preparation. - + // Dequantizes ISE encoded endpoint val to [0,255] uint32_t dequant_bise_endpoint(uint32_t val, uint32_t ise_range); // ISE ranges 4-11 - + // Dequantizes ISE encoded weight val to [0,64] uint32_t dequant_bise_weight(uint32_t val, uint32_t ise_range); // ISE ranges 0-10 @@ -231,12 +270,21 @@ namespace astc_helpers bool is_cem_ldr(uint32_t mode); inline bool is_cem_hdr(uint32_t mode) { return !is_cem_ldr(mode); } + bool does_cem_have_alpha(uint32_t mode); + // True if the passed in dimensions are a valid ASTC block size. There are 14 supported configs, from 4x4 (8bpp) to 12x12 (.89bpp). bool is_valid_block_size(uint32_t w, uint32_t h); + + // w/h must be a valid ASTC block size, or it returns cBLOCK_SIZE_4x4 + uint32_t get_block_size_index(uint32_t w, uint32_t h); + + float get_bitrate_from_block_size(uint32_t w, uint32_t h); + + uint32_t get_texel_partition_from_table(uint32_t block_width, uint32_t block_height, uint32_t seed, uint32_t subsets, uint32_t x, uint32_t y); bool block_has_any_hdr_cems(const log_astc_block& log_blk); bool block_has_any_ldr_cems(const log_astc_block& log_blk); - + // Returns the # of endpoint values for the given CEM. inline uint32_t get_num_cem_values(uint32_t cem) { assert(cem <= 15); return 2 + 2 * (cem >> 2); } @@ -245,17 +293,28 @@ namespace astc_helpers basisu::vector m_val_to_ise; // [0-255] or [0-64] value to nearest ISE symbol, array size is [256] or [65] basisu::vector m_ISE_to_val; // ASTC encoded ISE symbol to [0,255] or [0,64] value, [levels] basisu::vector m_ISE_to_rank; // returns the level rank index given an ISE symbol, [levels] - basisu::vector m_rank_to_ISE; // returns the ISE symbol given a level rank, inverse of pISE_to_rank, [levels] + basisu::vector m_rank_to_ISE; // returns the ISE symbol given a level rank, inverse of pISE_to_rank, [levels] - void init(bool weight_flag, uint32_t num_levels, bool init_rank_tabs) + void init(bool weight_flag, uint32_t num_levels) { - m_val_to_ise.resize(weight_flag ? (MAX_WEIGHT_VALUE + 1) : 256); + m_val_to_ise.resize(weight_flag ? (MAX_WEIGHT_INTERPOLANT_VALUE + 1) : 256); m_ISE_to_val.resize(num_levels); - if (init_rank_tabs) - { - m_ISE_to_rank.resize(num_levels); - m_rank_to_ISE.resize(num_levels); - } + m_ISE_to_rank.resize(num_levels); + m_rank_to_ISE.resize(num_levels); + } + + uint32_t get_rank_to_val(uint32_t rank) const + { + const uint32_t ise = m_rank_to_ISE[rank]; + const uint32_t val = m_ISE_to_val[ise]; + return val; + } + + uint32_t get_val_to_rank(uint32_t val) + { + const uint32_t ise = m_val_to_ise[val]; + const uint32_t rank = m_ISE_to_rank[ise]; + return rank; } }; @@ -263,6 +322,7 @@ namespace astc_helpers { dequant_table m_weights[TOTAL_WEIGHT_ISE_RANGES]; dequant_table m_endpoints[TOTAL_ENDPOINT_ISE_RANGES]; + bool m_initialized_flag = false; const dequant_table& get_weight_tab(uint32_t range) const { @@ -288,16 +348,19 @@ namespace astc_helpers return m_endpoints[range - FIRST_VALID_ENDPOINT_ISE_RANGE]; } - void init(bool init_rank_tabs) + void init() { + if (m_initialized_flag) + return; + for (uint32_t range = FIRST_VALID_WEIGHT_ISE_RANGE; range <= LAST_VALID_WEIGHT_ISE_RANGE; range++) { const uint32_t num_levels = get_ise_levels(range); dequant_table& tab = get_weight_tab(range); - tab.init(true, num_levels, init_rank_tabs); + tab.init(true, num_levels); - create_quant_tables(tab.m_val_to_ise.data(), tab.m_ISE_to_val.data(), init_rank_tabs ? tab.m_ISE_to_rank.data() : nullptr, init_rank_tabs ? tab.m_rank_to_ISE.data() : nullptr, range, true); + create_quant_tables(tab.m_val_to_ise.data(), tab.m_ISE_to_val.data(), tab.m_ISE_to_rank.data(), tab.m_rank_to_ISE.data(), range, true); } for (uint32_t range = FIRST_VALID_ENDPOINT_ISE_RANGE; range <= LAST_VALID_ENDPOINT_ISE_RANGE; range++) @@ -305,15 +368,17 @@ namespace astc_helpers const uint32_t num_levels = get_ise_levels(range); dequant_table& tab = get_endpoint_tab(range); - tab.init(false, num_levels, init_rank_tabs); + tab.init(false, num_levels); - create_quant_tables(tab.m_val_to_ise.data(), tab.m_ISE_to_val.data(), init_rank_tabs ? tab.m_ISE_to_rank.data() : nullptr, init_rank_tabs ? tab.m_rank_to_ISE.data() : nullptr, range, false); + create_quant_tables(tab.m_val_to_ise.data(), tab.m_ISE_to_val.data(), tab.m_ISE_to_rank.data(), tab.m_rank_to_ISE.data(), range, false); } + + m_initialized_flag = true; } }; extern dequant_tables g_dequant_tables; - void init_tables(bool init_rank_tabs); + void init_tables(); struct weighted_sample { @@ -333,9 +398,22 @@ namespace astc_helpers const uint8_t* pSrc_weights, // these are dequantized [0,64] weights, NOT ISE symbols, [wy][wx] uint8_t* pDst_weights); // [by][bx] - // Procedurally returns the texel partition/subset index given the block coordinate and config. + void upsample_weight_grid_xuastc_ldr( + uint32_t bx, uint32_t by, // destination/to dimension + uint32_t wx, uint32_t wy, // source/from dimension + const uint8_t* pSrc_weights0, // these are dequantized [0,64] weights, NOT ISE symbols, [wy][wx] + uint8_t* pDst_weights0, // [by][bx] + const uint8_t* pSrc_weights1, // these are dequantized [0,64] weights, NOT ISE symbols, [wy][wx] + uint8_t* pDst_weights1); // [by][bx] + + bool is_small_block(uint32_t block_width, uint32_t block_height); + + // Procedurally returns the texel partition/subset index given the block coordinate and config (very slow). int compute_texel_partition(uint32_t seedIn, uint32_t xIn, uint32_t yIn, uint32_t zIn, int num_partitions, bool small_block); + // Returns the texel partition/subset index given the block coordinate and config - table lookup, but currently ONLY 2-3 SUBSETS to save RAM. + int get_precomputed_texel_partition(uint32_t block_width, uint32_t block_height, uint32_t seed, uint32_t x, uint32_t y, uint32_t num_partitions); + void blue_contract( int r, int g, int b, int a, int& dr, int& dg, int& db, int& da); @@ -372,7 +450,7 @@ namespace astc_helpers const int MAX_RGB9E5 = 0xff80; void unpack_rgb9e5(uint32_t packed, float& r, float& g, float& b); uint32_t pack_rgb9e5(float r, float g, float b); - + enum decode_mode { cDecodeModeSRGB8 = 0, // returns uint8_t's, not valid on HDR blocks @@ -385,11 +463,71 @@ namespace astc_helpers // pPixels must point to either 32-bit pixel values (SRGB8/LDR8/9E5) or 64-bit pixel values (HDR16) bool decode_block(const log_astc_block& log_blk, void* pPixels, uint32_t blk_width, uint32_t blk_height, decode_mode dec_mode); + // Assuming the ASTC logical block is valid, this checks for the extra XUASTC LDR constraints. + bool is_block_xuastc_ldr(const log_astc_block& log_blk); + + // XUASTC LDR only - primary assumption is the logical block comes directly from our supercompressor. DO NOT call on general ASTC blocks. + bool decode_block_xuastc_ldr(const log_astc_block& log_blk, void* pPixels, uint32_t blk_width, uint32_t blk_height, decode_mode dec_mode, const uint8_t* pUpsampled_weights_to_use = nullptr, uint32_t start_x = 0, uint32_t start_y = 0, uint32_t end_x = 0, uint32_t end_y = 0); + void decode_bise(uint32_t ise_range, uint8_t* pVals, uint32_t num_vals, const uint8_t *pBits128, uint32_t bit_ofs); // Unpack a physical ASTC encoded GPU texture block to a logical block description. bool unpack_block(const void* pASTC_block, log_astc_block& log_blk, uint32_t blk_width, uint32_t blk_height); + uint8_t& get_weight(log_astc_block& log_block, uint32_t plane_index, uint32_t idx); + uint8_t get_weight(const log_astc_block& log_block, uint32_t plane_index, uint32_t idx); + void extract_weights(const log_astc_block& log_block, uint8_t* pWeights, uint32_t plane_index); + void set_weights(log_astc_block& log_block, const uint8_t* pWeights, uint32_t plane_index); + uint32_t get_total_weights(const log_astc_block& log_block); + + uint8_t* get_endpoints(log_astc_block& log_block, uint32_t partition_index); + const uint8_t* get_endpoints(const log_astc_block& log_block, uint32_t partition_index); + + const char* get_cem_name(uint32_t cem_index); + bool cem_is_ldr_direct(uint32_t cem_index); + bool cem_is_ldr_base_scale(uint32_t cem_index); + bool cem_is_ldr_base_plus_ofs(uint32_t cem_index); + + bool cem_supports_bc(uint32_t cem); + + void bit_transfer_signed_dec(int& a, int& b); + void bit_transfer_signed_enc(int& a, int& b); + + bool cem8_or_12_used_blue_contraction(uint32_t cem_index, const uint8_t* pEndpoint_vals, uint32_t endpoint_ise_index); + bool cem9_or_13_used_blue_contraction(uint32_t cem_index, const uint8_t* pEndpoint_vals, uint32_t endpoint_ise_index); + bool used_blue_contraction(uint32_t cem_index, const uint8_t* pEndpoint_vals, uint32_t endpoint_ise_index); + + uint32_t get_base_cem_without_alpha(uint32_t cem); + + int apply_delta_to_bise_endpoint_val(uint32_t endpoint_ise_range, int ise_val, int delta); + + // index range: [0,NUM_ASTC_BLOCK_SIZES-1] + void get_astc_block_size_by_index(uint32_t index, uint32_t& width, uint32_t& height); + + // -1 if invalid + int find_astc_block_size_index(uint32_t width, uint32_t height); + + // 8-bit linear8 or sRGB8, le/he are [0,255], w is [0,64] + inline int channel_interpolate(int le, int he, int w, bool astc_srgb_decode) + { + assert((w >= 0) && (w <= 64)); + assert((le >= 0) && (le <= 255)); + assert((he >= 0) && (he <= 255)); + + if (astc_srgb_decode) + { + le = (le << 8) | 0x80; + he = (he << 8) | 0x80; + } + else + { + le = (le << 8) | le; + he = (he << 8) | he; + } + + return astc_helpers::weight_interpolate(le, he, w) >> 8; + } + } // namespace astc_helpers #endif // BASISU_ASTC_HELPERS_HEADER @@ -403,11 +541,11 @@ namespace astc_helpers template inline T my_min(T a, T b) { return (a < b) ? a : b; } template inline T my_max(T a, T b) { return (a > b) ? a : b; } - const uint8_t g_astc_block_sizes[NUM_ASTC_BLOCK_SIZES][2] = { - { 4, 4 }, { 5, 4 }, { 5, 5 }, { 6, 5 }, - { 6, 6 }, { 8, 5 }, { 8, 6 }, { 10, 5 }, - { 10, 6 }, { 8, 8 }, { 10, 8 }, { 10, 10 }, - { 12, 10 }, { 12, 12 } + const uint8_t g_astc_block_sizes[NUM_ASTC_BLOCK_SIZES][2] = { + { 4, 4 }, { 5, 4 }, { 5, 5 }, { 6, 5 }, + { 6, 6 }, { 8, 5 }, { 8, 6 }, { 10, 5 }, + { 10, 6 }, { 8, 8 }, { 10, 8 }, { 10, 10 }, + { 12, 10 }, { 12, 12 } }; const int8_t g_ise_range_table[TOTAL_ISE_RANGES][3] = @@ -436,7 +574,7 @@ namespace astc_helpers { 6, 1, 0 }, // 0..191 19 { 8, 0, 0 }, // 0..255 20 }; - + static inline void astc_set_bits_1_to_9(uint32_t* pDst, uint32_t& bit_offset, uint32_t code, uint32_t codesize) { uint8_t* pBuf = reinterpret_cast(pDst); @@ -551,10 +689,10 @@ namespace astc_helpers // Now interleave the 8 encoded trit bits with the bits to form the encoded output. See table 94. astc_set_bits(pOutput, bit_pos, bits[0] | (astc_extract_bits(T, 0, 1) << n) | (bits[1] << (2 + n)), n * 2 + 2); - + astc_set_bits(pOutput, bit_pos, astc_extract_bits(T, 2, 3) | (bits[2] << 2) | (astc_extract_bits(T, 4, 4) << (2 + n)) | (bits[3] << (3 + n)) | (astc_extract_bits(T, 5, 6) << (3 + n * 2)) | (bits[4] << (5 + n * 2)) | (astc_extract_bits(T, 7, 7) << (5 + n * 3)), n * 3 + 6); - + if (pStats) *pStats += n * 5 + 8; } @@ -582,7 +720,7 @@ namespace astc_helpers if (group_size) { - // Range has trits or quints - pack each group of 5 or 3 values + // Range has trits or quints - pack each group of 5 or 3 values const int total_groups = (group_size == 5) ? ((num_vals + 4) / 5) : ((num_vals + 2) / 3); for (int group_index = 0; group_index < total_groups; group_index++) @@ -593,7 +731,7 @@ namespace astc_helpers for (int i = 0; i < limit; i++) vals[i] = pSrc_vals[group_index * group_size + i]; - // Note this always writes a group of 3 or 5 bits values, even for incomplete groups. So it can write more than needed. + // Note this always writes a group of 3 or 5 bits values, even for incomplete groups. So it can write more than needed. // get_ise_sequence_bits() returns the # of bits that must be written for proper decoding. if (group_size == 5) astc_encode_trits(temp, vals, bit_pos, num_bits, pStats); @@ -632,14 +770,14 @@ namespace astc_helpers const uint32_t P = log_block.m_weight_ise_range >= 6; // high precision const uint32_t Dp_P = (log_block.m_dual_plane << 1) | P; // pack dual plane+high precision bits - + // See Tables 81-82 // Compute p from weight range uint32_t p = 2 + log_block.m_weight_ise_range - (P ? 6 : 0); - + // Rearrange p's bits to p0 p2 p1 p = (p >> 1) + ((p & 1) << 2); - + // Try encoding each row of table 82. // W+4 H+2 @@ -676,7 +814,7 @@ namespace astc_helpers config_bits = (Dp_P << 9) | ((W) << 7) | ((H - 2) << 5) | ((p & 4) << 2) | 12 | (p & 3); return true; } - + // 12 H+2 if ((W == 12) && is_packable(H - 2, 2)) { @@ -704,7 +842,7 @@ namespace astc_helpers config_bits = (Dp_P << 9) | (0b1101 << 5) | (p << 2); return true; } - + // W+6 H+6 (no dual plane or high prec) if ((!Dp_P) && is_packable(W - 6, 2) && is_packable(H - 6, 2)) { @@ -715,9 +853,19 @@ namespace astc_helpers // Failed: unsupported weight grid dimensions or config. return false; } - - bool pack_astc_block(astc_block& phys_block, const log_astc_block& log_block, int* pExpected_endpoint_range, pack_stats *pStats) + + bool pack_astc_block(astc_block& phys_block, const log_astc_block& log_block, int* pExpected_endpoint_range, pack_stats *pStats, uint32_t validate_flags) { + // Basic sanity checking + if (!log_block.m_dual_plane) + { + assert(log_block.m_color_component_selector == 0); + } + else + { + assert(log_block.m_color_component_selector <= 3); + } + memset(&phys_block, 0, sizeof(phys_block)); if (pExpected_endpoint_range) @@ -726,7 +874,7 @@ namespace astc_helpers assert(!log_block.m_error_flag); if (log_block.m_error_flag) return false; - + if (log_block.m_solid_color_flag_ldr) { pack_void_extent_ldr(phys_block, log_block.m_solid_color[0], log_block.m_solid_color[1], log_block.m_solid_color[2], log_block.m_solid_color[3], pStats); @@ -737,7 +885,7 @@ namespace astc_helpers pack_void_extent_hdr(phys_block, log_block.m_solid_color[0], log_block.m_solid_color[1], log_block.m_solid_color[2], log_block.m_solid_color[3], pStats); return true; } - + if ((log_block.m_num_partitions < 1) || (log_block.m_num_partitions > MAX_PARTITIONS)) return false; @@ -753,7 +901,7 @@ namespace astc_helpers return false; // TODO: sanity check grid width/height vs. block's physical width/height - + uint32_t config_bits = 0; if (!get_config_bits(log_block, config_bits)) return false; @@ -794,7 +942,7 @@ namespace astc_helpers if (highest_cem > 15) return false; - + // Ensure CEM range is contiguous if (((highest_cem >> 2) > (1 + (lowest_cem >> 2)))) return false; @@ -809,7 +957,7 @@ namespace astc_helpers for (uint32_t j = 0; j < log_block.m_num_partitions; j++) { const int M = log_block.m_color_endpoint_modes[j] & 3; - + const int C = (log_block.m_color_endpoint_modes[j] >> 2) - ((encoded_cem & 3) - 1); if ((C & 1) != C) return false; @@ -850,7 +998,7 @@ namespace astc_helpers return false; total_extra_bits += 2; - + uint32_t ccs_bit_pos = 128 - (int)total_weight_bits - (int)total_extra_bits; astc_set_bits(&phys_block.m_vals[0], ccs_bit_pos, log_block.m_color_component_selector, 2); if (pStats) @@ -868,6 +1016,9 @@ namespace astc_helpers if (total_cem_vals > MAX_ENDPOINTS) return false; + + if (validate_flags & cValidateEarlyOutAtEndpointISEChecks) + return true; int endpoint_ise_range = -1; for (int k = 20; k > 0; k--) @@ -898,6 +1049,9 @@ namespace astc_helpers pStats->m_weight_bits += get_ise_sequence_bits(total_grid_weights, log_block.m_weight_ise_range); } + if (validate_flags & cValidateSkipFinalEndpointWeightPacking) + return true; + // Pack endpoints forwards encode_bise(&phys_block.m_vals[0], log_block.m_endpoints, bit_pos, total_cem_vals, endpoint_ise_range); @@ -1094,12 +1248,12 @@ namespace astc_helpers uint32_t u = 0; switch (ise_range) { - case 0: + case 0: { u = val ? 63 : 0; break; } - case 1: // 0-2 + case 1: // 0-2 { const uint8_t s_tab_0_2[3] = { 0, 32, 63 }; u = s_tab_0_2[val]; @@ -1140,7 +1294,7 @@ namespace astc_helpers const uint32_t num_bits = g_ise_range_table[ise_range][0]; const uint32_t num_trits = g_ise_range_table[ise_range][1]; BASISU_NOTE_UNUSED(num_trits); const uint32_t num_quints = g_ise_range_table[ise_range][2]; BASISU_NOTE_UNUSED(num_quints); - + // compute Table 103 row index const int range_index = num_bits * 2 + (num_quints ? 1 : 0); @@ -1153,11 +1307,11 @@ namespace astc_helpers // Now dequantize // See Table 103. ASTC weight unquantization parameters static const uint32_t C_table[5] = { 50, 28, 23, 13, 11 }; - + const uint32_t a = bits & 1, b = (bits >> 1) & 1, c = (bits >> 2) & 1; const uint32_t A = (a == 0) ? 0 : 0x7F; - + uint32_t B = 0; if (range_index == 4) B = ((b << 6) | (b << 2) | (b << 0)); @@ -1210,7 +1364,7 @@ namespace astc_helpers uint32_t find_nearest_bise_weight(int v, uint32_t ise_range) { assert(ise_range >= FIRST_VALID_WEIGHT_ISE_RANGE && ise_range <= LAST_VALID_WEIGHT_ISE_RANGE); - assert(v <= (int)MAX_WEIGHT_VALUE); + assert(v <= (int)MAX_WEIGHT_INTERPOLANT_VALUE); const uint32_t total_levels = get_ise_levels(ise_range); int best_e = INT_MAX, best_index = 0; @@ -1237,7 +1391,7 @@ namespace astc_helpers uint32_t ise_range, // ise range, [4,20] for endpoints, [0,11] for weights bool weight_flag) // false if block endpoints, true if weights { - const uint32_t num_dequant_vals = weight_flag ? (MAX_WEIGHT_VALUE + 1) : 256; + const uint32_t num_dequant_vals = weight_flag ? (MAX_WEIGHT_INTERPOLANT_VALUE + 1) : 256; for (uint32_t i = 0; i < num_dequant_vals; i++) { @@ -1273,22 +1427,22 @@ namespace astc_helpers for (uint32_t i = 0; i < num_levels; i++) { uint32_t v = weight_flag ? astc_helpers::dequant_bise_weight(i, ise_range) : astc_helpers::dequant_bise_endpoint(i, ise_range); - + // Low=ISE value // High=dequantized value vals[i] = (v << 16) | i; } - + // Sorts by dequantized value std::sort(vals, vals + num_levels); - + for (uint32_t rank = 0; rank < num_levels; rank++) { uint32_t ise_val = (uint8_t)vals[rank]; if (pISE_to_rank) pISE_to_rank[ise_val] = (uint8_t)rank; - + if (pRank_to_ISE) pRank_to_ISE[rank] = (uint8_t)ise_val; } @@ -1314,17 +1468,17 @@ namespace astc_helpers pDst[15] = (uint8_t)(ah >> 8); if (pStats) - pStats->m_header_bits += 128; + pStats->m_header_bits += 16 + 64; } // rh-ah are half-floats - void pack_void_extent_hdr(astc_block& blk, uint16_t rh, uint16_t gh, uint16_t bh, uint16_t ah, pack_stats *pStats) + void pack_void_extent_hdr(astc_block& blk, uint16_t rh, uint16_t gh, uint16_t bh, uint16_t ah, pack_stats *pStats) { uint8_t* pDst = (uint8_t*)&blk.m_vals[0]; memset(pDst, 0xFF, 16); pDst[0] = 0b11111100; - + pDst[8] = (uint8_t)rh; pDst[9] = (uint8_t)(rh >> 8); pDst[10] = (uint8_t)gh; @@ -1335,9 +1489,9 @@ namespace astc_helpers pDst[15] = (uint8_t)(ah >> 8); if (pStats) - pStats->m_header_bits += 128; + pStats->m_header_bits += 8 + 64; } - + bool is_cem_ldr(uint32_t mode) { switch (mode) @@ -1356,39 +1510,132 @@ namespace astc_helpers default: break; } + + return false; + } + + bool does_cem_have_alpha(uint32_t mode) + { + switch (mode) + { + case CEM_LDR_LUM_ALPHA_DIRECT: + case CEM_LDR_LUM_ALPHA_BASE_PLUS_OFS: + case CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A: + case CEM_LDR_RGBA_DIRECT: + case CEM_LDR_RGBA_BASE_PLUS_OFFSET: + case CEM_HDR_RGB_LDR_ALPHA: + case CEM_HDR_RGB_HDR_ALPHA: + return true; + default: + break; + } return false; } bool is_valid_block_size(uint32_t w, uint32_t h) { - assert((w >= MIN_BLOCK_DIM) && (w <= MAX_BLOCK_DIM)); - assert((h >= MIN_BLOCK_DIM) && (h <= MAX_BLOCK_DIM)); - -#define SIZECHK(x, y) if ((w == (x)) && (h == (y))) return true; - SIZECHK(4, 4); - SIZECHK(5, 4); +#define BU_ASTC_HELPERS_SIZECHK(x, y) if ((w == (x)) && (h == (y))) return true; + BU_ASTC_HELPERS_SIZECHK(4, 4); // 0 + BU_ASTC_HELPERS_SIZECHK(5, 4); // 1 - SIZECHK(5, 5); + BU_ASTC_HELPERS_SIZECHK(5, 5); // 2 - SIZECHK(6, 5); - SIZECHK(6, 6); + BU_ASTC_HELPERS_SIZECHK(6, 5); // 3 + BU_ASTC_HELPERS_SIZECHK(6, 6); // 4 - SIZECHK(8, 5); - SIZECHK(8, 6); - SIZECHK(10, 5); - SIZECHK(10, 6); + BU_ASTC_HELPERS_SIZECHK(8, 5); // 5 + BU_ASTC_HELPERS_SIZECHK(8, 6); // 6 + BU_ASTC_HELPERS_SIZECHK(10, 5); // 7 + BU_ASTC_HELPERS_SIZECHK(10, 6); // 8 - SIZECHK(8, 8); - SIZECHK(10, 8); - SIZECHK(10, 10); + BU_ASTC_HELPERS_SIZECHK(8, 8); // 9 + BU_ASTC_HELPERS_SIZECHK(10, 8); // 10 + BU_ASTC_HELPERS_SIZECHK(10, 10); // 11 - SIZECHK(12, 10); - SIZECHK(12, 12); -#undef SIZECHK + BU_ASTC_HELPERS_SIZECHK(12, 10); // 12 + BU_ASTC_HELPERS_SIZECHK(12, 12); // 13 +#undef BU_ASTC_HELPERS_SIZECHK return false; } + + uint32_t get_block_size_index(uint32_t w, uint32_t h) + { + assert(is_valid_block_size(w, h)); + + const uint32_t t = w * h; + + if (t <= 36) + { + if (t == 36) + return cBLOCK_SIZE_6x6; + else if (t == 16) + return cBLOCK_SIZE_4x4; + else if (t == 25) + return cBLOCK_SIZE_5x5; + else if (t == 20) + return cBLOCK_SIZE_5x4; + else if (t == 30) + return cBLOCK_SIZE_6x5; + } + else if (t <= 64) + { + if (t == 64) + return cBLOCK_SIZE_8x8; + else if (t == 60) + return cBLOCK_SIZE_10x6; + else if (t == 50) + return cBLOCK_SIZE_10x5; + else if (t == 48) + return cBLOCK_SIZE_8x6; + else if (t == 40) + return cBLOCK_SIZE_8x5; + } + else + { + if (t == 80) + return cBLOCK_SIZE_10x8; + else if (t == 100) + return cBLOCK_SIZE_10x10; + else if (t == 120) + return cBLOCK_SIZE_12x10; + else if (t == 144) + return cBLOCK_SIZE_12x12; + } + + assert(0); + return cBLOCK_SIZE_4x4; + } + + // returns the standard ASTC bitrates given a valid block size from the ASTC spec. + // 0=invalid block size + float get_bitrate_from_block_size(uint32_t w, uint32_t h) + { +#define BU_ASTC_HELPERS_BLOCK_BITRATE(x, y, b) if ((w == (x)) && (h == (y))) return (b); + BU_ASTC_HELPERS_BLOCK_BITRATE(4, 4, 8.0f); + BU_ASTC_HELPERS_BLOCK_BITRATE(5, 4, 6.4f); + + BU_ASTC_HELPERS_BLOCK_BITRATE(5, 5, 5.12f); + + BU_ASTC_HELPERS_BLOCK_BITRATE(6, 5, 4.27f); + BU_ASTC_HELPERS_BLOCK_BITRATE(6, 6, 3.56f); + + BU_ASTC_HELPERS_BLOCK_BITRATE(8, 5, 3.20f); + BU_ASTC_HELPERS_BLOCK_BITRATE(8, 6, 2.67f); + BU_ASTC_HELPERS_BLOCK_BITRATE(10, 5, 2.56f); + BU_ASTC_HELPERS_BLOCK_BITRATE(10, 6, 2.13f); + + BU_ASTC_HELPERS_BLOCK_BITRATE(8, 8, 2.00f); + BU_ASTC_HELPERS_BLOCK_BITRATE(10, 8, 1.60f); + BU_ASTC_HELPERS_BLOCK_BITRATE(10, 10, 1.28f); + + BU_ASTC_HELPERS_BLOCK_BITRATE(12, 10, 1.07f); + BU_ASTC_HELPERS_BLOCK_BITRATE(12, 12, .89f); +#undef BU_ASTC_HELPERS_BLOCK_BITRATE + + return 0.0f; + } bool block_has_any_hdr_cems(const log_astc_block& log_blk) { @@ -1411,20 +1658,19 @@ namespace astc_helpers return false; } - + dequant_tables g_dequant_tables; - - void precompute_texel_partitions_4x4(); - void precompute_texel_partitions_6x6(); - - void init_tables(bool init_rank_tabs) + + void precompute_texel_partitions(); + + // TODO: this is called twice when using the encoder, first init_rank_tabs=false then init_rank_tabs=true. + void init_tables() { - g_dequant_tables.init(init_rank_tabs); - - precompute_texel_partitions_4x4(); - precompute_texel_partitions_6x6(); + g_dequant_tables.init(); + + precompute_texel_partitions(); } - + void compute_upsample_weights( int block_width, int block_height, int weight_grid_width, int weight_grid_height, @@ -1474,6 +1720,8 @@ namespace astc_helpers if (total_src_weights == total_dst_weights) { + assert((bx == wx) && (by == wy)); + memcpy(pDst_weights, pSrc_weights, total_src_weights); return; } @@ -1507,6 +1755,78 @@ namespace astc_helpers } } + void upsample_weight_grid_xuastc_ldr( + uint32_t bx, uint32_t by, // destination/to dimension + uint32_t wx, uint32_t wy, // source/from dimension + const uint8_t* pSrc_weights0, // these are dequantized [0,64] weights, NOT ISE symbols, [wy][wx] + uint8_t* pDst_weights0, // [by][bx] + const uint8_t* pSrc_weights1, // these are dequantized [0,64] weights, NOT ISE symbols, [wy][wx] + uint8_t* pDst_weights1) // [by][bx] + { + assert((bx >= 2) && (by >= 2) && (bx <= 12) && (by <= 12)); + assert((wx >= 2) && (wy >= 2) && (wx <= bx) && (wy <= by)); + + assert((bx != wx) || (by != wy)); + + const uint32_t scaleX = (1024 + bx / 2) / (bx - 1); + const uint32_t scaleY = (1024 + by / 2) / (by - 1); + + const uint32_t gYUInc = scaleY * (wy - 1); + const uint32_t gXUInc = scaleX * (wx - 1); + + uint32_t gYU = 32; + for (uint32_t texel_y = 0; texel_y < by; texel_y++) + { + const uint32_t gY = gYU >> 6; + gYU += gYUInc; + + const uint32_t jY = gY >> 4; + const uint32_t fY = gY & 0xf; + + uint32_t gXU = 32; + for (uint32_t texel_x = 0; texel_x < bx; texel_x++) + { + const uint32_t gX = gXU >> 6; + gXU += gXUInc; + + const uint32_t jX = gX >> 4; + const uint32_t fX = gX & 0xf; + + const uint32_t w11 = (fX * fY + 8) >> 4; + const uint32_t w10 = fY - w11; + const uint32_t w01 = fX - w11; + const uint32_t w00 = 16 - fX - fY + w11; + + assert(w00 || w01 || w10 || w11); + + const uint32_t sx = jX, sy = jY; + + { + uint32_t total0 = 8; + + if (w00) total0 += pSrc_weights0[sx + sy * wx] * w00; + if (w01) total0 += pSrc_weights0[sx + 1 + sy * wx] * w01; + if (w10) total0 += pSrc_weights0[sx + (sy + 1) * wx] * w10; + if (w11) total0 += pSrc_weights0[sx + 1 + (sy + 1) * wx] * w11; + + pDst_weights0[texel_x + texel_y * bx] = (uint8_t)(total0 >> 4); + } + + if (pDst_weights1) + { + uint32_t total1 = 8; + + if (w00) total1 += pSrc_weights1[sx + sy * wx] * w00; + if (w01) total1 += pSrc_weights1[sx + 1 + sy * wx] * w01; + if (w10) total1 += pSrc_weights1[sx + (sy + 1) * wx] * w10; + if (w11) total1 += pSrc_weights1[sx + 1 + (sy + 1) * wx] * w11; + + pDst_weights1[texel_x + texel_y * bx] = (uint8_t)(total1 >> 4); + } + } // texel_x + } // texel_y + } + inline uint32_t hash52(uint32_t v) { uint32_t p = v; @@ -1516,6 +1836,16 @@ namespace astc_helpers return p; } + bool is_small_block(uint32_t block_width, uint32_t block_height) + { + assert((block_width >= MIN_BLOCK_DIM) && (block_width <= MAX_BLOCK_DIM)); + assert((block_height >= MIN_BLOCK_DIM) && (block_height <= MAX_BLOCK_DIM)); + + const uint32_t num_blk_pixels = block_width * block_height; + + return num_blk_pixels < 31; + } + // small_block = num_blk_pixels < 31 int compute_texel_partition(uint32_t seedIn, uint32_t xIn, uint32_t yIn, uint32_t zIn, int num_partitions, bool small_block) { @@ -1582,76 +1912,108 @@ namespace astc_helpers : (c >= d) ? 2 : 3; } + + // Precomputed partition patterns for each 10-bit seed and small/large block sizes for 2-3 subsets. + // This costs 144KB of RAM and some init, but considering the sheer complexity of compute_texel_partition() and how hotly it's called in the compressors and transcoders that's worth it. + // Byte packing: + // low 4 bits=small blocks (on valid up to 6x5) + // high 4 bits=large blocks (6x6 or larger) - // 4x4, 2 and 3 subsets - static uint32_t g_texel_partitions_4x4[1024][2]; - - // 6x6, 2 and 3 subsets (2 subsets low 4 bits, 3 subsets high 4 bits) - static uint8_t g_texel_partitions_6x6[1024][6 * 6]; + static uint8_t g_texel_partitions[NUM_PARTITION_PATTERNS][12][12]; // [seed][y][x] - void precompute_texel_partitions_4x4() + void sanity_check_texel_partition_tables() { - for (uint32_t p = 0; p < 1024; p++) +#if 0 +#if defined(_DEBUG) || defined(DEBUG) + // sanity checking + for (uint32_t i = 0; i < cTOTAL_BLOCK_SIZES; i++) { - uint32_t v2 = 0, v3 = 0; + const uint32_t bw = g_astc_block_sizes[i][0], bh = g_astc_block_sizes[i][1]; + const bool is_small_block_flag = is_small_block(bw, bh); - for (uint32_t y = 0; y < 4; y++) + assert(get_block_size_index(bw, bh) == i); + + for (uint32_t s = 0; s < NUM_PARTITION_PATTERNS; s++) { - for (uint32_t x = 0; x < 4; x++) + for (uint32_t y = 0; y < bh; y++) { - const uint32_t shift = x * 2 + y * 8; - v2 |= (compute_texel_partition(p, x, y, 0, 2, true) << shift); - v3 |= (compute_texel_partition(p, x, y, 0, 3, true) << shift); - } - } + for (uint32_t x = 0; x < bw; x++) + { + const uint32_t k2 = compute_texel_partition(s, x, y, 0, 2, is_small_block_flag); + const uint32_t k3 = compute_texel_partition(s, x, y, 0, 3, is_small_block_flag); - g_texel_partitions_4x4[p][0] = v2; - g_texel_partitions_4x4[p][1] = v3; + assert(get_precomputed_texel_partition(bw, bh, s, x, y, 2) == (int)k2); + assert(get_precomputed_texel_partition(bw, bh, s, x, y, 3) == (int)k3); + } // x + } // y + } // s } + printf("precompute_texel_partitions: Sanity check OK\n"); +#endif +#endif } - - void precompute_texel_partitions_6x6() + + void precompute_texel_partition() { - for (uint32_t p = 0; p < 1024; p++) + for (uint32_t seed = 0; seed < NUM_PARTITION_PATTERNS; seed++) { - for (uint32_t y = 0; y < 6; y++) + for (uint32_t y = 0; y < MAX_BLOCK_DIM; y++) { - for (uint32_t x = 0; x < 6; x++) + for (uint32_t x = 0; x < MAX_BLOCK_DIM; x++) { - const uint32_t p2 = compute_texel_partition(p, x, y, 0, 2, false); - const uint32_t p3 = compute_texel_partition(p, x, y, 0, 3, false); + uint32_t k = 0; - assert((p2 <= 1) && (p3 <= 2)); - g_texel_partitions_6x6[p][x + y * 6] = (uint8_t)((p3 << 4) | p2); - } - } - } - } + // small block (width*height<31) + if ((x <= 6) && (y <= 5)) + { + uint32_t v2 = compute_texel_partition(seed, x, y, 0, 2, true); assert(v2 <= 1); + uint32_t v3 = compute_texel_partition(seed, x, y, 0, 3, true); assert(v3 <= 2); + k |= v2 | (v3 << 2); + } + + // not small block + { + uint32_t v2 = compute_texel_partition(seed, x, y, 0, 2, false); assert(v2 <= 1); + uint32_t v3 = compute_texel_partition(seed, x, y, 0, 3, false); assert(v3 <= 2); + k |= ((v2 | (v3 << 2)) << 4); + } - static inline int get_precompute_texel_partitions_4x4(uint32_t seed, uint32_t x, uint32_t y, uint32_t num_partitions) - { - assert(g_texel_partitions_4x4[1][0]); - assert(seed < 1024); - assert((x <= 3) && (y <= 3)); - assert((num_partitions >= 2) && (num_partitions <= 3)); + assert(k <= 255); - const uint32_t shift = x * 2 + y * 8; - return (g_texel_partitions_4x4[seed][num_partitions - 2] >> shift) & 3; + g_texel_partitions[seed][y][x] = (uint8_t)k; + } // x + } // y + } // seed } + + int get_precomputed_texel_partition(uint32_t block_width, uint32_t block_height, uint32_t seed, uint32_t x, uint32_t y, uint32_t subsets) + { + assert(seed < NUM_PARTITION_PATTERNS); + assert((subsets >= 2) && (subsets <= 3)); + assert((x < block_width) && (y < block_height)); + + const uint32_t v = g_texel_partitions[seed][y][x]; + + uint32_t shift = (subsets == 3) ? 2 : 0; + shift += ((block_width * block_height) >= 31) * 4; + uint32_t res = (v >> shift) & 3; - static inline int get_precompute_texel_partitions_6x6(uint32_t seed, uint32_t x, uint32_t y, uint32_t num_partitions) + // sanity checking + assert(res == (uint32_t)compute_texel_partition(seed, x, y, 0, subsets, is_small_block(block_width, block_height))); + + return res; + } + + void precompute_texel_partitions() { - assert(g_texel_partitions_6x6[0][0]); - assert(seed < 1024); - assert((x <= 5) && (y <= 5)); - assert((num_partitions >= 2) && (num_partitions <= 3)); + if (!g_texel_partitions[0][0][0]) + precompute_texel_partition(); - const uint32_t shift = (num_partitions == 3) ? 4 : 0; - return (g_texel_partitions_6x6[seed][x + y * 6] >> shift) & 3; + sanity_check_texel_partition_tables(); } void blue_contract( - int r, int g, int b, int a, + int r, int g, int b, int a, int &dr, int &dg, int &db, int &da) { dr = (r + b) >> 1; @@ -1666,7 +2028,7 @@ namespace astc_helpers b |= (a & 0x80); a >>= 1; a &= 0x3F; - if ((a & 0x20) != 0) + if ((a & 0x20) != 0) a -= 0x40; } @@ -1900,7 +2262,7 @@ namespace astc_helpers e0_g = y0; e1_g = y1; e0_b = y0; e1_b = y1; e0_a = 0x780; e1_a = 0x780; - + break; } case CEM_HDR_LUM_SMALL_RANGE: @@ -1917,11 +2279,11 @@ namespace astc_helpers y0 = ((v1 & 0xF0) << 4) | ((v0 & 0x7F) << 1); d = (v1 & 0x0F) << 1; } - + y1 = y0 + d; - if (y1 > 0xFFF) + if (y1 > 0xFFF) y1 = 0xFFF; - + e0_r = y0; e1_r = y1; e0_g = y0; e1_g = y1; e0_b = y0; e1_b = y1; @@ -1932,36 +2294,36 @@ namespace astc_helpers case CEM_HDR_RGB_BASE_SCALE: { int v2 = pE[2], v3 = pE[3]; - + int modeval = ((v0 & 0xC0) >> 6) | ((v1 & 0x80) >> 5) | ((v2 & 0x80) >> 4); - + int majcomp, mode; - if ((modeval & 0xC) != 0xC) + if ((modeval & 0xC) != 0xC) { - majcomp = modeval >> 2; + majcomp = modeval >> 2; mode = modeval & 3; } - else if (modeval != 0xF) + else if (modeval != 0xF) { - majcomp = modeval & 3; + majcomp = modeval & 3; mode = 4; } - else + else { - majcomp = 0; + majcomp = 0; mode = 5; } - int red = v0 & 0x3f; + int red = v0 & 0x3f; int green = v1 & 0x1f; - int blue = v2 & 0x1f; + int blue = v2 & 0x1f; int scale = v3 & 0x1f; - int x0 = (v1 >> 6) & 1; - int x1 = (v1 >> 5) & 1; + int x0 = (v1 >> 6) & 1; + int x1 = (v1 >> 5) & 1; int x2 = (v2 >> 6) & 1; - int x3 = (v2 >> 5) & 1; - int x4 = (v3 >> 7) & 1; + int x3 = (v2 >> 5) & 1; + int x4 = (v3 >> 7) & 1; int x5 = (v3 >> 6) & 1; int x6 = (v3 >> 5) & 1; @@ -1985,25 +2347,25 @@ namespace astc_helpers if (ohm & 0x02) red |= x5 << 10; static const int s_shamts[6] = { 1,1,2,3,4,5 }; - + const int shamt = s_shamts[mode]; - red <<= shamt; - green <<= shamt; - blue <<= shamt; + red <<= shamt; + green <<= shamt; + blue <<= shamt; scale <<= shamt; - if (mode != 5) - { - green = red - green; - blue = red - blue; + if (mode != 5) + { + green = red - green; + blue = red - blue; } - if (majcomp == 1) + if (majcomp == 1) std::swap(red, green); - if (majcomp == 2) + if (majcomp == 2) std::swap(red, blue); - + e1_r = clamp(red, 0, 0xFFF); e1_g = clamp(green, 0, 0xFFF); e1_b = clamp(blue, 0, 0xFFF); @@ -2027,7 +2389,7 @@ namespace astc_helpers e0_a = 0x780; e1_a = 0x780; - if (majcomp == 3) + if (majcomp == 3) { e0_r = v0 << 4; e0_g = v2 << 4; @@ -2128,12 +2490,16 @@ namespace astc_helpers v7 &= (0x3F >> mode); v7 ^= (0x20 >> mode); v7 -= (0x20 >> mode); - v6 <<= (4 - mode); - v7 <<= (4 - mode); + + //v6 <<= (4 - mode); // undefined behavior if neg + v6 = ((uint32_t)v6) << (4 - mode); + + //v7 <<= (4 - mode); // undefined behavior if neg + v7 = ((uint32_t)v7) << (4 - mode); v7 += v6; v7 = clamp(v7, 0, 0xFFF); - e0_a = v6; + e0_a = v6; e1_a = v7; } } @@ -2152,7 +2518,7 @@ namespace astc_helpers } } } - + static inline bool is_half_inf_or_nan(half_float v) { return get_bits(v, 10, 14) == 31; @@ -2265,7 +2631,7 @@ namespace astc_helpers x.u = m | (e << 23) | (s << 31); return x.f; } - + // See https://registry.khronos.org/OpenGL/extensions/EXT/EXT_texture_shared_exponent.txt const int RGB9E5_EXPONENT_BITS = 5, RGB9E5_MANTISSA_BITS = 9, RGB9E5_EXP_BIAS = 15, RGB9E5_MAX_VALID_BIASED_EXP = 31; const int MAX_RGB9E5_EXP = (RGB9E5_MAX_VALID_BIASED_EXP - RGB9E5_EXP_BIAS); @@ -2273,7 +2639,7 @@ namespace astc_helpers const int MAX_RGB9E5_MANTISSA = (RGB9E5_MANTISSA_VALUES - 1); //const int MAX_RGB9E5 = (int)(((float)MAX_RGB9E5_MANTISSA) / RGB9E5_MANTISSA_VALUES * (1 << MAX_RGB9E5_EXP)); const int EPSILON_RGB9E5 = (int)((1.0f / (float)RGB9E5_MANTISSA_VALUES) / (float)(1 << RGB9E5_EXP_BIAS)); - + void unpack_rgb9e5(uint32_t packed, float& r, float& g, float& b) { int x = packed & 511; @@ -2287,9 +2653,9 @@ namespace astc_helpers g = y * scale; b = z * scale; } - + // floor_log2 is not correct for the denorm and zero values, but we are going to do a max of this value with the minimum rgb9e5 exponent that will hide these problem cases. - static inline int floor_log2(float x) + static inline int floor_log2(float x) { union float754 { @@ -2325,7 +2691,7 @@ namespace astc_helpers exp_shared += 1; assert(exp_shared <= RGB9E5_MAX_VALID_BIASED_EXP); } - else + else { assert(maxm <= MAX_RGB9E5_MANTISSA); } @@ -2337,7 +2703,7 @@ namespace astc_helpers assert((rm >= 0) && (rm <= MAX_RGB9E5_MANTISSA)); assert((gm >= 0) && (gm <= MAX_RGB9E5_MANTISSA)); assert((bm >= 0) && (bm <= MAX_RGB9E5_MANTISSA)); - + return rm | (gm << 9) | (bm << 18) | (exp_shared << 27); } @@ -2348,7 +2714,7 @@ namespace astc_helpers if (!x) return 17; - + uint32_t n = 0; while ((x & 0x10000) == 0) { @@ -2426,17 +2792,8 @@ namespace astc_helpers return texel; } - // Important: pPixels is either 32-bit/texel or 64-bit/texel. - bool decode_block(const log_astc_block& log_blk, void* pPixels, uint32_t blk_width, uint32_t blk_height, decode_mode dec_mode) + static void write_error_block(void* pPixels, uint32_t num_blk_pixels, decode_mode dec_mode) { - assert(is_valid_block_size(blk_width, blk_height)); - - assert(g_dequant_tables.m_endpoints[0].m_ISE_to_val.size()); - if (!g_dequant_tables.m_endpoints[0].m_ISE_to_val.size()) - return false; - - const uint32_t num_blk_pixels = blk_width * blk_height; - // Write block error color if (dec_mode == cDecodeModeHDR16) { @@ -2455,9 +2812,32 @@ namespace astc_helpers for (uint32_t i = 0; i < num_blk_pixels; i++) ((uint32_t*)pPixels)[i] = 0xFFFF00FF; } + } + + // Important: pPixels is either 32-bit/texel or 64-bit/texel. + bool decode_block(const log_astc_block& log_blk, void* pPixels, uint32_t blk_width, uint32_t blk_height, decode_mode dec_mode) + { + assert(is_valid_block_size(blk_width, blk_height)); + + // Basic sanity checking + if (!log_blk.m_dual_plane) + { + assert(log_blk.m_color_component_selector == 0); + } + else + { + assert(log_blk.m_color_component_selector <= 3); + } + + assert(g_dequant_tables.m_endpoints[0].m_ISE_to_val.size()); + if (!g_dequant_tables.m_endpoints[0].m_ISE_to_val.size()) + return false; + const uint32_t num_blk_pixels = blk_width * blk_height; + if (log_blk.m_error_flag) { + write_error_block(pPixels, num_blk_pixels, dec_mode); // Should this return false? It's not an invalid logical block config, though. return false; } @@ -2510,7 +2890,7 @@ namespace astc_helpers float r = half_to_float(log_blk.m_solid_color[0]); float g = half_to_float(log_blk.m_solid_color[1]); float b = half_to_float(log_blk.m_solid_color[2]); - + const uint32_t packed = pack_rgb9e5(r, g, b); for (uint32_t i = 0; i < num_blk_pixels; i++) @@ -2518,36 +2898,71 @@ namespace astc_helpers } else { + write_error_block(pPixels, num_blk_pixels, dec_mode); return false; } return true; } - + // Sanity check block's config if ((log_blk.m_grid_width < 2) || (log_blk.m_grid_height < 2)) + { + write_error_block(pPixels, num_blk_pixels, dec_mode); return false; + } + if ((log_blk.m_grid_width > blk_width) || (log_blk.m_grid_height > blk_height)) + { + write_error_block(pPixels, num_blk_pixels, dec_mode); return false; + } if ((log_blk.m_endpoint_ise_range < FIRST_VALID_ENDPOINT_ISE_RANGE) || (log_blk.m_endpoint_ise_range > LAST_VALID_ENDPOINT_ISE_RANGE)) + { + write_error_block(pPixels, num_blk_pixels, dec_mode); return false; + } + if ((log_blk.m_weight_ise_range < FIRST_VALID_WEIGHT_ISE_RANGE) || (log_blk.m_weight_ise_range > LAST_VALID_WEIGHT_ISE_RANGE)) + { + write_error_block(pPixels, num_blk_pixels, dec_mode); return false; + } + if ((log_blk.m_num_partitions < 1) || (log_blk.m_num_partitions > MAX_PARTITIONS)) + { + write_error_block(pPixels, num_blk_pixels, dec_mode); return false; + } + if ((log_blk.m_dual_plane) && (log_blk.m_num_partitions > MAX_DUAL_PLANE_PARTITIONS)) + { + write_error_block(pPixels, num_blk_pixels, dec_mode); return false; + } + if (log_blk.m_partition_id >= NUM_PARTITION_PATTERNS) + { + write_error_block(pPixels, num_blk_pixels, dec_mode); return false; + } + if ((log_blk.m_num_partitions == 1) && (log_blk.m_partition_id > 0)) + { + write_error_block(pPixels, num_blk_pixels, dec_mode); return false; + } + if (log_blk.m_color_component_selector > 3) + { + write_error_block(pPixels, num_blk_pixels, dec_mode); return false; + } const uint32_t total_endpoint_levels = get_ise_levels(log_blk.m_endpoint_ise_range); const uint32_t total_weight_levels = get_ise_levels(log_blk.m_weight_ise_range); - + bool is_ldr_endpoints[MAX_PARTITIONS]; // Check CEM's @@ -2555,15 +2970,21 @@ namespace astc_helpers for (uint32_t i = 0; i < log_blk.m_num_partitions; i++) { if (log_blk.m_color_endpoint_modes[i] > 15) + { + write_error_block(pPixels, num_blk_pixels, dec_mode); return false; + } total_cem_vals += get_num_cem_values(log_blk.m_color_endpoint_modes[i]); - + is_ldr_endpoints[i] = is_cem_ldr(log_blk.m_color_endpoint_modes[i]); } if (total_cem_vals > MAX_ENDPOINTS) + { + write_error_block(pPixels, num_blk_pixels, dec_mode); return false; + } const dequant_table& endpoint_dequant_tab = g_dequant_tables.get_endpoint_tab(log_blk.m_endpoint_ise_range); const uint8_t* pEndpoint_dequant = endpoint_dequant_tab.m_ISE_to_val.data(); @@ -2573,21 +2994,28 @@ namespace astc_helpers for (uint32_t i = 0; i < total_cem_vals; i++) { if (log_blk.m_endpoints[i] >= total_endpoint_levels) + { + write_error_block(pPixels, num_blk_pixels, dec_mode); return false; + } + dequantized_endpoints[i] = pEndpoint_dequant[log_blk.m_endpoints[i]]; } - + // Dequantize weights to [0,64] uint8_t dequantized_weights[2][12 * 12]; - + const dequant_table& weight_dequant_tab = g_dequant_tables.get_weight_tab(log_blk.m_weight_ise_range); const uint8_t* pWeight_dequant = weight_dequant_tab.m_ISE_to_val.data(); - + const uint32_t total_weight_vals = (log_blk.m_dual_plane ? 2 : 1) * log_blk.m_grid_width * log_blk.m_grid_height; for (uint32_t i = 0; i < total_weight_vals; i++) { if (log_blk.m_weights[i] >= total_weight_levels) + { + write_error_block(pPixels, num_blk_pixels, dec_mode); return false; + } const uint32_t plane_index = log_blk.m_dual_plane ? (i & 1) : 0; const uint32_t grid_index = log_blk.m_dual_plane ? (i >> 1) : i; @@ -2617,10 +3045,9 @@ namespace astc_helpers // Decode texels const bool small_block = num_blk_pixels < 31; - const bool use_precomputed_texel_partitions_4x4 = (blk_width == 4) && (blk_height == 4) && (log_blk.m_num_partitions >= 2) && (log_blk.m_num_partitions <= 3); - const bool use_precomputed_texel_partitions_6x6 = (blk_width == 6) && (blk_height == 6) && (log_blk.m_num_partitions >= 2) && (log_blk.m_num_partitions <= 3); + const bool use_precomputed_texel_partitions = (log_blk.m_num_partitions >= 2) && (log_blk.m_num_partitions <= 3); const uint32_t ccs = log_blk.m_dual_plane ? log_blk.m_color_component_selector : UINT32_MAX; - + bool success = true; if (dec_mode == cDecodeModeRGB9E5) @@ -2631,14 +3058,15 @@ namespace astc_helpers for (uint32_t x = 0; x < blk_width; x++) { const uint32_t pixel_index = x + y * blk_width; - + uint32_t subset = 0; if (log_blk.m_num_partitions > 1) { - if (use_precomputed_texel_partitions_4x4) - subset = get_precompute_texel_partitions_4x4(log_blk.m_partition_id, x, y, log_blk.m_num_partitions); - else if (use_precomputed_texel_partitions_6x6) - subset = get_precompute_texel_partitions_6x6(log_blk.m_partition_id, x, y, log_blk.m_num_partitions); + if (use_precomputed_texel_partitions) + { + subset = get_precomputed_texel_partition(blk_width, blk_height, log_blk.m_partition_id, x, y, log_blk.m_num_partitions); + //assert((int)subset == compute_texel_partition(log_blk.m_partition_id, x, y, 0, log_blk.m_num_partitions, small_block)); // extra paranoia + } else subset = compute_texel_partition(log_blk.m_partition_id, x, y, 0, log_blk.m_num_partitions, small_block); } @@ -2680,7 +3108,7 @@ namespace astc_helpers if (is_half_inf_or_nan((half_float)comp[c])) comp[c] = 0x7BFF; } - + } // c uint32_t packed; @@ -2697,21 +3125,22 @@ namespace astc_helpers else if (dec_mode == cDecodeModeHDR16) { // Note: must round towards zero when converting float to half for ASTC (18.19 Weight Application) - + // returns half floats for (uint32_t y = 0; y < blk_height; y++) { for (uint32_t x = 0; x < blk_width; x++) { const uint32_t pixel_index = x + y * blk_width; - + uint32_t subset = 0; if (log_blk.m_num_partitions > 1) { - if (use_precomputed_texel_partitions_4x4) - subset = get_precompute_texel_partitions_4x4(log_blk.m_partition_id, x, y, log_blk.m_num_partitions); - else if (use_precomputed_texel_partitions_6x6) - subset = get_precompute_texel_partitions_6x6(log_blk.m_partition_id, x, y, log_blk.m_num_partitions); + if (use_precomputed_texel_partitions) + { + subset = get_precomputed_texel_partition(blk_width, blk_height, log_blk.m_partition_id, x, y, log_blk.m_num_partitions); + //assert((int)subset == compute_texel_partition(log_blk.m_partition_id, x, y, 0, log_blk.m_num_partitions, small_block)); // extra paranoia + } else subset = compute_texel_partition(log_blk.m_partition_id, x, y, 0, log_blk.m_num_partitions, small_block); } @@ -2751,13 +3180,13 @@ namespace astc_helpers int he = endpoints[subset][c][1] << 4; int qlog16 = weight_interpolate(le, he, w); - + o = qlog16_to_half(qlog16); if (is_half_inf_or_nan(o)) o = 0x7BFF; } - + ((half_float*)pPixels)[pixel_index * 4 + c] = o; } @@ -2776,17 +3205,18 @@ namespace astc_helpers uint32_t subset = 0; if (log_blk.m_num_partitions > 1) { - if (use_precomputed_texel_partitions_4x4) - subset = get_precompute_texel_partitions_4x4(log_blk.m_partition_id, x, y, log_blk.m_num_partitions); - else if (use_precomputed_texel_partitions_6x6) - subset = get_precompute_texel_partitions_6x6(log_blk.m_partition_id, x, y, log_blk.m_num_partitions); + if (use_precomputed_texel_partitions) + { + subset = get_precomputed_texel_partition(blk_width, blk_height, log_blk.m_partition_id, x, y, log_blk.m_num_partitions); + //assert((int)subset == compute_texel_partition(log_blk.m_partition_id, x, y, 0, log_blk.m_num_partitions, small_block)); // extra paranoia + } else subset = compute_texel_partition(log_blk.m_partition_id, x, y, 0, log_blk.m_num_partitions, small_block); } if (!is_ldr_endpoints[subset]) { - ((uint32_t*)pPixels)[pixel_index * 4] = 0xFFFF00FF; + ((uint32_t*)pPixels)[pixel_index] = 0xFFFF00FF; success = false; } else @@ -2801,6 +3231,9 @@ namespace astc_helpers // FIXME: the spec is apparently wrong? this matches ARM's and Google's decoder //if ((dec_mode == cDecodeModeSRGB8) && (c <= 2)) // See https://github.com/ARM-software/astc-encoder/issues/447 + // See latest spec with recent (2023-2024) fixes: + // https://raw.githubusercontent.com/KhronosGroup/DataFormat/refs/heads/main/astc.txt + // "For _LDR endpoint modes_, each color component C is calculated from the corresponding 8 - bit endpoint components C~0~and C~1~as follows" - does this mean alpha too? I guess so. (8/15/2025.) if (dec_mode == cDecodeModeSRGB8) { le = (le << 8) | 0x80; @@ -2814,8 +3247,8 @@ namespace astc_helpers uint32_t k = weight_interpolate(le, he, w); - // FIXME: This is what the spec says to do in LDR mode, but this is not what ARM's decoder does - // See decompress_symbolic_block(), decode_texel() and unorm16_to_sf16. + // FIXME (old comment - before 2023/2024 ARM etc. spec fixes): This is what the spec says to do in LDR mode, but this is not what ARM's decoder does + // See decompress_symbolic_block(), decode_texel() and unorm16_to_sf16. // It seems to effectively divide by 65535.0 and convert to FP16, then back to float, mul by 255.0, add .5 and then convert to 8-bit. ((uint8_t*)pPixels)[pixel_index * 4 + c] = (uint8_t)(k >> 8); } @@ -2824,10 +3257,428 @@ namespace astc_helpers } // x } // y } - + return success; } + bool is_block_xuastc_ldr(const log_astc_block& log_blk) + { + if (log_blk.m_error_flag) + return false; + + if (log_blk.m_solid_color_flag_ldr) + return true; + + if (log_blk.m_solid_color_flag_hdr) + return false; + + if (log_blk.m_num_partitions > 3) + return false; + + if ((log_blk.m_dual_plane) && (log_blk.m_num_partitions > 1)) + return false; + + // TODO: Check partition pattern ID against unique set. + + for (uint32_t i = 1; i < log_blk.m_num_partitions; i++) + if (log_blk.m_color_endpoint_modes[0] != log_blk.m_color_endpoint_modes[i]) + return false; + + switch (log_blk.m_color_endpoint_modes[0]) + { + case CEM_LDR_LUM_DIRECT: + case CEM_LDR_LUM_ALPHA_DIRECT: + case CEM_LDR_RGB_BASE_SCALE: + case CEM_LDR_RGB_DIRECT: + case CEM_LDR_RGB_BASE_PLUS_OFFSET: + case CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A: + case CEM_LDR_RGBA_DIRECT: + case CEM_LDR_RGBA_BASE_PLUS_OFFSET: + { + break; + } + default: + { + return false; + } + } + + return true; + } + + // ~2x faster than decode_block(), but XUASTC LDR only. + // pUpsampled_weights_to_use must be at block res, [0,64], single plane blocks ONLY + bool decode_block_xuastc_ldr(const log_astc_block& log_blk, void* pPixels, uint32_t blk_width, uint32_t blk_height, decode_mode dec_mode, + const uint8_t* pUpsampled_weights_to_use, uint32_t start_x, uint32_t start_y, uint32_t end_x, uint32_t end_y) + { + if (!end_x) + end_x = blk_width; + + if (!end_y) + end_y = blk_height; + + assert(start_x < end_x); + assert(start_y < end_y); + assert(end_x <= blk_width); + assert(end_y <= blk_height); + + assert(g_dequant_tables.m_endpoints[0].m_ISE_to_val.size()); + assert((dec_mode == cDecodeModeSRGB8) || (dec_mode == cDecodeModeLDR8)); + assert(is_valid_block_size(blk_width, blk_height)); + assert(!log_blk.m_error_flag && !log_blk.m_solid_color_flag_hdr); + + if (!log_blk.m_solid_color_flag_ldr) + { + assert(((log_blk.m_num_partitions >= 1) && (log_blk.m_num_partitions <= 3))); + assert((log_blk.m_grid_width >= 2) & (log_blk.m_grid_height >= 2)); + assert((log_blk.m_grid_width <= blk_width) && (log_blk.m_grid_height <= blk_height)); + assert((log_blk.m_grid_width * log_blk.m_grid_height) <= MAX_GRID_WEIGHTS); + assert((log_blk.m_num_partitions > 1) || (log_blk.m_partition_id == 0)); + } + + assert(is_block_xuastc_ldr(log_blk)); + + const uint32_t num_blk_pixels = blk_width * blk_height; + + // Handle solid color blocks + if (log_blk.m_solid_color_flag_ldr) + { + // Convert LDR pixels to 8-bits + uint32_t x; + + ((uint8_t*)&x)[0] = (uint8_t)(log_blk.m_solid_color[0] >> 8); + ((uint8_t*)&x)[1] = (uint8_t)(log_blk.m_solid_color[1] >> 8); + ((uint8_t*)&x)[2] = (uint8_t)(log_blk.m_solid_color[2] >> 8); + ((uint8_t*)&x)[3] = (uint8_t)(log_blk.m_solid_color[3] >> 8); + + uint32_t* pDst = (uint32_t*)pPixels; + + uint32_t i = 0; + while ((i + 3) < num_blk_pixels) + { + pDst[i] = x; + pDst[i + 1] = x; + pDst[i + 2] = x; + pDst[i + 3] = x; + + i += 4; + } + + while (i < num_blk_pixels) + pDst[i++] = x; + + return true; + } + + const dequant_table& endpoint_dequant_tab = g_dequant_tables.get_endpoint_tab(log_blk.m_endpoint_ise_range); + const uint8_t* pEndpoint_dequant = endpoint_dequant_tab.m_ISE_to_val.data(); + + const dequant_table& weight_dequant_tab = g_dequant_tables.get_weight_tab(log_blk.m_weight_ise_range); + const uint8_t* pWeight_dequant = weight_dequant_tab.m_ISE_to_val.data(); + + // Check CEM's + const uint32_t num_cem_vals = get_num_cem_values(log_blk.m_color_endpoint_modes[0]); + const uint32_t total_cem_vals = num_cem_vals * log_blk.m_num_partitions; + + assert(total_cem_vals <= MAX_ENDPOINTS); + + // Dequantized endpoints to [0,255] + uint8_t dequantized_endpoints[MAX_ENDPOINTS]; + + for (uint32_t i = 0; i < total_cem_vals; i++) + { + assert(log_blk.m_endpoints[i] < endpoint_dequant_tab.m_ISE_to_val.size_u32()); + dequantized_endpoints[i] = pEndpoint_dequant[log_blk.m_endpoints[i]]; + } + + // Decode CEM's + int endpoints[4][4][2]; // [subset][comp][l/h] + + uint32_t endpoint_val_index = 0; + const uint32_t cem_index = log_blk.m_color_endpoint_modes[0]; + + uint32_t alpha_mask = 0xFF; + + for (uint32_t subset = 0; subset < log_blk.m_num_partitions; subset++) + { + assert(log_blk.m_color_endpoint_modes[subset] == cem_index); + + decode_endpoint(cem_index, &endpoints[subset][0], &dequantized_endpoints[endpoint_val_index]); + + alpha_mask &= endpoints[subset][3][0]; + alpha_mask &= endpoints[subset][3][1]; + + endpoint_val_index += num_cem_vals; + } + + const bool any_alpha = alpha_mask != 255; + + // Dequantize weights to [0,64] + uint8_t upsampled_weights[2][12 * 12]; + + const uint32_t total_weight_vals = (log_blk.m_dual_plane ? 2 : 1) * log_blk.m_grid_width * log_blk.m_grid_height; + + // Upsample weight grid. [0,64] weights + const uint8_t(*pUpsampled_weights)[12 * 12]; + + uint8_t dequantized_weights[2][12 * 12]; + + // For simplicity, ignore any passed in weights if dual plane + if ((pUpsampled_weights_to_use) && (!log_blk.m_dual_plane)) + { + // Caller is jamming in already unpacked weights for the first plane to save time + pUpsampled_weights = reinterpret_cast(pUpsampled_weights_to_use); + } + else + { + if (log_blk.m_dual_plane) + { + for (uint32_t i = 0; i < total_weight_vals; i++) + { + const uint32_t plane_index = i & 1; + const uint32_t grid_index = i >> 1; + + assert(log_blk.m_weights[i] < weight_dequant_tab.m_ISE_to_val.size_u32()); + dequantized_weights[plane_index][grid_index] = pWeight_dequant[log_blk.m_weights[i]]; + } + } + else + { + for (uint32_t i = 0; i < total_weight_vals; i++) + { + assert(log_blk.m_weights[i] < weight_dequant_tab.m_ISE_to_val.size_u32()); + dequantized_weights[0][i] = pWeight_dequant[log_blk.m_weights[i]]; + } + } + + pUpsampled_weights = &dequantized_weights[0]; + + if ((log_blk.m_grid_width < blk_width) || (log_blk.m_grid_height < blk_height)) + { + upsample_weight_grid_xuastc_ldr(blk_width, blk_height, + log_blk.m_grid_width, log_blk.m_grid_height, + &dequantized_weights[0][0], &upsampled_weights[0][0], + log_blk.m_dual_plane ? &dequantized_weights[1][0] : nullptr, log_blk.m_dual_plane ? &upsampled_weights[1][0] : nullptr); + + pUpsampled_weights = &upsampled_weights[0]; + } + } + + // Decode texels + const uint32_t ccs = log_blk.m_dual_plane ? log_blk.m_color_component_selector : UINT32_MAX; + + const uint8_t *pPart = &g_texel_partitions[log_blk.m_partition_id][0][0]; // [seed][y][x] + + const bool large_block = (num_blk_pixels >= 31); + uint32_t part_shift = (log_blk.m_num_partitions == 3) ? 2 : 0; + part_shift += large_block * 4; + + //uint32_t pixel_index = 0; + + if (log_blk.m_num_partitions == 1) + { + // alpha, 1 subset + int le0 = endpoints[0][0][0], he0 = endpoints[0][0][1]; + int le1 = endpoints[0][1][0], he1 = endpoints[0][1][1]; + int le2 = endpoints[0][2][0], he2 = endpoints[0][2][1]; + int le3 = endpoints[0][3][0], he3 = endpoints[0][3][1]; + + if (dec_mode == cDecodeModeSRGB8) + { + le0 = (le0 << 8) | 0x80; he0 = (he0 << 8) | 0x80; + le1 = (le1 << 8) | 0x80; he1 = (he1 << 8) | 0x80; + le2 = (le2 << 8) | 0x80; he2 = (he2 << 8) | 0x80; + le3 = (le3 << 8) | 0x80; he3 = (he3 << 8) | 0x80; + } + else + { + le0 = (le0 << 8) | le0; he0 = (he0 << 8) | he0; + le1 = (le1 << 8) | le1; he1 = (he1 << 8) | he1; + le2 = (le2 << 8) | le2; he2 = (he2 << 8) | he2; + le3 = (le3 << 8) | le3; he3 = (he3 << 8) | he3; + } + + // no subsets + if (!any_alpha) + { + if (!log_blk.m_dual_plane) + { + for (uint32_t y = start_y; y < end_y; y++) + { + for (uint32_t x = start_x; x < end_x; x++) + { + const uint32_t pixel_index = x + y * blk_width; + + const uint32_t w0 = pUpsampled_weights[0][pixel_index]; + const uint32_t w1 = pUpsampled_weights[0][pixel_index]; + const uint32_t w2 = pUpsampled_weights[0][pixel_index]; + + const uint32_t k0 = weight_interpolate(le0, he0, w0); + const uint32_t k1 = weight_interpolate(le1, he1, w1); + const uint32_t k2 = weight_interpolate(le2, he2, w2); + + ((uint8_t*)pPixels)[pixel_index * 4 + 0] = (uint8_t)(k0 >> 8); + ((uint8_t*)pPixels)[pixel_index * 4 + 1] = (uint8_t)(k1 >> 8); + ((uint8_t*)pPixels)[pixel_index * 4 + 2] = (uint8_t)(k2 >> 8); + ((uint8_t*)pPixels)[pixel_index * 4 + 3] = 255; + } // x + } // y + } + else + { + for (uint32_t y = start_y; y < end_y; y++) + { + for (uint32_t x = start_x; x < end_x; x++) + { + const uint32_t pixel_index = x + y * blk_width; + + const uint32_t w0 = pUpsampled_weights[(0 == ccs) ? 1 : 0][pixel_index]; + const uint32_t w1 = pUpsampled_weights[(1 == ccs) ? 1 : 0][pixel_index]; + const uint32_t w2 = pUpsampled_weights[(2 == ccs) ? 1 : 0][pixel_index]; + + const uint32_t k0 = weight_interpolate(le0, he0, w0); + const uint32_t k1 = weight_interpolate(le1, he1, w1); + const uint32_t k2 = weight_interpolate(le2, he2, w2); + + ((uint8_t*)pPixels)[pixel_index * 4 + 0] = (uint8_t)(k0 >> 8); + ((uint8_t*)pPixels)[pixel_index * 4 + 1] = (uint8_t)(k1 >> 8); + ((uint8_t*)pPixels)[pixel_index * 4 + 2] = (uint8_t)(k2 >> 8); + ((uint8_t*)pPixels)[pixel_index * 4 + 3] = 255; + } // x + } // y + } + } + else // (!any_alpha) + { + for (uint32_t y = start_y; y < end_y; y++) + { + for (uint32_t x = start_x; x < end_x; x++) + { + const uint32_t pixel_index = x + y * blk_width; + + const uint32_t w0 = pUpsampled_weights[(0 == ccs) ? 1 : 0][pixel_index]; + const uint32_t w1 = pUpsampled_weights[(1 == ccs) ? 1 : 0][pixel_index]; + const uint32_t w2 = pUpsampled_weights[(2 == ccs) ? 1 : 0][pixel_index]; + const uint32_t w3 = pUpsampled_weights[(3 == ccs) ? 1 : 0][pixel_index]; + + const uint32_t k0 = weight_interpolate(le0, he0, w0); + const uint32_t k1 = weight_interpolate(le1, he1, w1); + const uint32_t k2 = weight_interpolate(le2, he2, w2); + const uint32_t k3 = weight_interpolate(le3, he3, w3); + + ((uint8_t*)pPixels)[pixel_index * 4 + 0] = (uint8_t)(k0 >> 8); + ((uint8_t*)pPixels)[pixel_index * 4 + 1] = (uint8_t)(k1 >> 8); + ((uint8_t*)pPixels)[pixel_index * 4 + 2] = (uint8_t)(k2 >> 8); + ((uint8_t*)pPixels)[pixel_index * 4 + 3] = (uint8_t)(k3 >> 8); + + } // x + } // y + } + } + else + { + for (uint32_t subset = 0; subset < log_blk.m_num_partitions; subset++) + { + int le0 = endpoints[subset][0][0], he0 = endpoints[subset][0][1]; + int le1 = endpoints[subset][1][0], he1 = endpoints[subset][1][1]; + int le2 = endpoints[subset][2][0], he2 = endpoints[subset][2][1]; + int le3 = endpoints[subset][3][0], he3 = endpoints[subset][3][1]; + + if (dec_mode == cDecodeModeSRGB8) + { + le0 = (le0 << 8) | 0x80; he0 = (he0 << 8) | 0x80; + le1 = (le1 << 8) | 0x80; he1 = (he1 << 8) | 0x80; + le2 = (le2 << 8) | 0x80; he2 = (he2 << 8) | 0x80; + le3 = (le3 << 8) | 0x80; he3 = (he3 << 8) | 0x80; + } + else + { + le0 = (le0 << 8) | le0; he0 = (he0 << 8) | he0; + le1 = (le1 << 8) | le1; he1 = (he1 << 8) | he1; + le2 = (le2 << 8) | le2; he2 = (he2 << 8) | he2; + le3 = (le3 << 8) | le3; he3 = (he3 << 8) | he3; + } + + endpoints[subset][0][0] = le0, endpoints[subset][0][1] = he0; + endpoints[subset][1][0] = le1, endpoints[subset][1][1] = he1; + endpoints[subset][2][0] = le2, endpoints[subset][2][1] = he2; + endpoints[subset][3][0] = le3, endpoints[subset][3][1] = he3; + } + + // subsets + if (!any_alpha) + { + // no alpha, sRGB + for (uint32_t y = start_y; y < end_y; y++) + { + for (uint32_t x = start_x; x < end_x; x++) + { + const uint32_t pixel_index = x + y * blk_width; + + const uint32_t v = pPart[y * 12 + x]; + const uint32_t subset = (v >> part_shift) & 3; + + const uint32_t w0 = pUpsampled_weights[(0 == ccs) ? 1 : 0][pixel_index]; + const uint32_t w1 = pUpsampled_weights[(1 == ccs) ? 1 : 0][pixel_index]; + const uint32_t w2 = pUpsampled_weights[(2 == ccs) ? 1 : 0][pixel_index]; + + int le0 = endpoints[subset][0][0], he0 = endpoints[subset][0][1]; + int le1 = endpoints[subset][1][0], he1 = endpoints[subset][1][1]; + int le2 = endpoints[subset][2][0], he2 = endpoints[subset][2][1]; + + const uint32_t k0 = weight_interpolate(le0, he0, w0); + const uint32_t k1 = weight_interpolate(le1, he1, w1); + const uint32_t k2 = weight_interpolate(le2, he2, w2); + + ((uint8_t*)pPixels)[pixel_index * 4 + 0] = (uint8_t)(k0 >> 8); + ((uint8_t*)pPixels)[pixel_index * 4 + 1] = (uint8_t)(k1 >> 8); + ((uint8_t*)pPixels)[pixel_index * 4 + 2] = (uint8_t)(k2 >> 8); + ((uint8_t*)pPixels)[pixel_index * 4 + 3] = 255; + } // x + } // y + } + else + { + // alpha + for (uint32_t y = start_y; y < end_y; y++) + { + for (uint32_t x = start_x; x < end_x; x++) + { + const uint32_t pixel_index = x + y * blk_width; + + const uint32_t v = pPart[y * 12 + x]; + const uint32_t subset = (v >> part_shift) & 3; + + const uint32_t w0 = pUpsampled_weights[(0 == ccs) ? 1 : 0][pixel_index]; + const uint32_t w1 = pUpsampled_weights[(1 == ccs) ? 1 : 0][pixel_index]; + const uint32_t w2 = pUpsampled_weights[(2 == ccs) ? 1 : 0][pixel_index]; + const uint32_t w3 = pUpsampled_weights[(3 == ccs) ? 1 : 0][pixel_index]; + + int le0 = endpoints[subset][0][0], he0 = endpoints[subset][0][1]; + int le1 = endpoints[subset][1][0], he1 = endpoints[subset][1][1]; + int le2 = endpoints[subset][2][0], he2 = endpoints[subset][2][1]; + int le3 = endpoints[subset][3][0], he3 = endpoints[subset][3][1]; + + const uint32_t k0 = weight_interpolate(le0, he0, w0); + const uint32_t k1 = weight_interpolate(le1, he1, w1); + const uint32_t k2 = weight_interpolate(le2, he2, w2); + const uint32_t k3 = weight_interpolate(le3, he3, w3); + + ((uint8_t*)pPixels)[pixel_index * 4 + 0] = (uint8_t)(k0 >> 8); + ((uint8_t*)pPixels)[pixel_index * 4 + 1] = (uint8_t)(k1 >> 8); + ((uint8_t*)pPixels)[pixel_index * 4 + 2] = (uint8_t)(k2 >> 8); + ((uint8_t*)pPixels)[pixel_index * 4 + 3] = (uint8_t)(k3 >> 8); + + } // x + } // y + + } + + } // if (log_blk.m_num_partitions == 1) + + return true; + } + //------------------------------------------------ // Physical to logical block decoding @@ -3220,7 +4071,7 @@ namespace astc_helpers return *this; } }; - + static bool decode_void_extent(const uint128& bits, log_astc_block& log_blk) { if (bits.get_bits(10, 2) != 0b11) @@ -3232,9 +4083,9 @@ namespace astc_helpers const uint32_t min_t = bits.next_bits(bit_ofs, 13); const uint32_t max_t = bits.next_bits(bit_ofs, 13); assert(bit_ofs == 64); - + const bool all_extents_all_ones = (min_s == 0x1FFF) && (max_s == 0x1FFF) && (min_t == 0x1FFF) && (max_t == 0x1FFF); - + if (!all_extents_all_ones && ((min_s >= max_s) || (min_t >= max_t))) return false; @@ -3256,7 +4107,7 @@ namespace astc_helpers if (is_half_inf_or_nan(log_blk.m_solid_color[c])) return false; } - + return true; } @@ -3269,7 +4120,7 @@ namespace astc_helpers { // Dp_ofs, P_ofs, W_ofs, W_size, H_ofs, H_size, W_bias, H_bias, p0_ofs, p1_ofs, p2_ofs; { 10, 9, 7, 2, 5, 2, 4, 2, 4, 0, 1 }, // 4 2 - { 10, 9, 7, 2, 5, 2, 8, 2, 4, 0, 1 }, // 8 2 + { 10, 9, 7, 2, 5, 2, 8, 2, 4, 0, 1 }, // 8 2 { 10, 9, 5, 2, 7, 2, 2, 8, 4, 0, 1 }, // 2 8 { 10, 9, 5, 2, 7, 1, 2, 6, 4, 0, 1 }, // 2 6 @@ -3291,14 +4142,14 @@ namespace astc_helpers // Reserved if ((bits.get_bits(0, 2) == 0) && (bits.get_bits(6, 3) == 0b111)) { - if (bits.get_bits(2, 4) != 0b1111) + if (bits.get_bits(2, 4) != 0b1111) return false; } // Void extent if (bits.get_bits(0, 9) == 0b111111100) return decode_void_extent(bits, log_blk); - + // Check rows const uint32_t x0_2 = bits.get_bits(0, 2), x2_2 = bits.get_bits(2, 2); const uint32_t x5_4 = bits.get_bits(5, 4), x8_1 = bits.get_bits(8, 1); @@ -3344,7 +4195,7 @@ namespace astc_helpers if (r.Dp_ofs >= 0) Dp = bits.get_bits(r.Dp_ofs, 1) != 0; - + if (r.W_size) W += bits.get_bits(r.W_ofs, r.W_size); @@ -3353,7 +4204,7 @@ namespace astc_helpers assert((W >= MIN_GRID_DIM) && (W <= MAX_BLOCK_DIM)); assert((H >= MIN_GRID_DIM) && (H <= MAX_BLOCK_DIM)); - + int p0 = bits.get_bits(r.p0_ofs, 1); int p1 = bits.get_bits(r.p1_ofs, 1); int p2 = bits.get_bits(r.p2_ofs, 1); @@ -3361,10 +4212,10 @@ namespace astc_helpers uint32_t p = p0 | (p1 << 1) | (p2 << 2); if (p < 2) return false; - + log_blk.m_grid_width = (uint8_t)W; log_blk.m_grid_height = (uint8_t)H; - + log_blk.m_weight_ise_range = (uint8_t)((p - 2) + (P * BISE_10_LEVELS)); assert(log_blk.m_weight_ise_range <= LAST_VALID_WEIGHT_ISE_RANGE); @@ -3480,7 +4331,7 @@ namespace astc_helpers static void decode_bise(uint32_t ise_range, uint8_t* pVals, uint32_t num_vals, const uint128& bits, uint32_t bit_ofs) { assert(num_vals && (ise_range < TOTAL_ISE_RANGES)); - + const uint32_t bits_per_val = g_ise_range_table[ise_range][0]; if (g_ise_range_table[ise_range][1]) @@ -3521,24 +4372,24 @@ namespace astc_helpers return decode_bise(ise_range, pVals, num_vals, bits, bit_ofs); } - + // Decodes a physical ASTC block to a logical ASTC block. // blk_width/blk_height are only used to validate the weight grid's dimensions. bool unpack_block(const void* pASTC_block, log_astc_block& log_blk, uint32_t blk_width, uint32_t blk_height) { assert(is_valid_block_size(blk_width, blk_height)); - + const uint8_t* pS = (uint8_t*)pASTC_block; log_blk.clear(); log_blk.m_error_flag = true; - + const uint128 bits( (uint64_t)read_le_dword(pS) | (((uint64_t)read_le_dword(pS + sizeof(uint32_t))) << 32), (uint64_t)read_le_dword(pS + sizeof(uint32_t) * 2) | (((uint64_t)read_le_dword(pS + sizeof(uint32_t) * 3)) << 32)); - + const uint128 rev_bits(bits.get_reversed_bits()); - + if (!decode_config(bits, log_blk)) return false; @@ -3552,16 +4403,16 @@ namespace astc_helpers // Check grid dimensions if ((log_blk.m_grid_width > blk_width) || (log_blk.m_grid_height > blk_height)) return false; - + // Now we have the grid width/height, dual plane, weight ISE range - + const uint32_t total_grid_weights = (log_blk.m_dual_plane ? 2 : 1) * (log_blk.m_grid_width * log_blk.m_grid_height); const uint32_t total_weight_bits = get_ise_sequence_bits(total_grid_weights, log_blk.m_weight_ise_range); - + // 18.24 Illegal Encodings if ((!total_grid_weights) || (total_grid_weights > MAX_GRID_WEIGHTS) || (total_weight_bits < 24) || (total_weight_bits > 96)) return false; - + const uint32_t end_of_weight_bit_ofs = 128 - total_weight_bits; uint32_t total_extra_bits = 0; @@ -3598,9 +4449,9 @@ namespace astc_helpers return false; uint32_t cem_bit_pos = end_of_weight_bit_ofs - total_extra_bits; - + uint32_t c[4] = { 0 }, m[4] = { 0 }; - + cem_bits >>= 2; for (uint32_t i = 0; i < log_blk.m_num_partitions; i++, cem_bits >>= 1) c[i] = cem_bits & 1; @@ -3666,7 +4517,7 @@ namespace astc_helpers // config+num_parts+total_extra_bits (CEM extra+CCS) uint32_t total_config_bits = config_bit_pos + total_extra_bits; - + // Compute number of remaining bits in block const int num_remaining_bits = 128 - (int)total_config_bits - (int)total_weight_bits; if (num_remaining_bits < 0) @@ -3681,6 +4532,7 @@ namespace astc_helpers return false; // Infer endpoint ISE range based off the # of values we need to encode, and the # of remaining bits in the block + // TODO: Optimize int endpoint_ise_range = -1; for (int k = 20; k > 0; k--) { @@ -3709,6 +4561,289 @@ namespace astc_helpers return true; } + // Misc. helpers + + uint8_t get_weight(const log_astc_block& log_block, uint32_t plane_index, uint32_t i) + { + const uint32_t num_planes = log_block.m_dual_plane ? 2 : 1; + assert(plane_index < num_planes); + assert(i < (uint32_t)(log_block.m_grid_width * log_block.m_grid_height)); + + const uint32_t idx = i * num_planes + plane_index; + assert(idx < MAX_GRID_WEIGHTS); + + return log_block.m_weights[idx]; + } + + uint8_t &get_weight(log_astc_block& log_block, uint32_t plane_index, uint32_t i) + { + const uint32_t num_planes = log_block.m_dual_plane ? 2 : 1; + assert(plane_index < num_planes); + assert(i < (uint32_t)(log_block.m_grid_width * log_block.m_grid_height)); + + const uint32_t idx = i * num_planes + plane_index; + assert(idx < MAX_GRID_WEIGHTS); + + return log_block.m_weights[idx]; + } + + void extract_weights(const log_astc_block& log_block, uint8_t* pWeights, uint32_t plane_index) + { + const uint32_t num_planes = log_block.m_dual_plane ? 2 : 1; + assert(plane_index < num_planes); + + const uint32_t num_weights = log_block.m_grid_width * log_block.m_grid_height; + for (uint32_t i = 0; i < num_weights; i++) + pWeights[i] = log_block.m_weights[i * num_planes + plane_index]; + } + + void set_weights(log_astc_block& log_block, const uint8_t* pWeights, uint32_t plane_index) + { + const uint32_t num_planes = log_block.m_dual_plane ? 2 : 1; + assert(plane_index < num_planes); + + const uint32_t num_weights = log_block.m_grid_width * log_block.m_grid_height; + for (uint32_t i = 0; i < num_weights; i++) + log_block.m_weights[i * num_planes + plane_index] = pWeights[i]; + } + + uint32_t get_total_weights(const log_astc_block& log_block) + { + return (log_block.m_dual_plane ? 2 : 1) * (log_block.m_grid_width * log_block.m_grid_height); + } + + // Returns a pointer to the beginning of a partition's/subset's endpoint values. + uint8_t *get_endpoints(log_astc_block& log_block, uint32_t partition_index) + { + assert(partition_index < log_block.m_num_partitions); + + uint32_t ofs = 0; + + for (uint32_t i = 0; i != partition_index; ++i) + ofs += get_num_cem_values(log_block.m_color_endpoint_modes[i]); + + assert(ofs < MAX_ENDPOINTS); + + return log_block.m_endpoints + ofs; + } + + const uint8_t* get_endpoints(const log_astc_block& log_block, uint32_t partition_index) + { + assert(partition_index < log_block.m_num_partitions); + + uint32_t ofs = 0; + + for (uint32_t i = 0; i != partition_index; ++i) + ofs += get_num_cem_values(log_block.m_color_endpoint_modes[i]); + + assert(ofs < MAX_ENDPOINTS); + + return log_block.m_endpoints + ofs; + } + + const char* get_cem_name(uint32_t cem_index) + { + static const char *s_cem_names[16] = + { + "CEM_LDR_LUM_DIRECT (0)", + "CEM_LDR_LUM_BASE_PLUS_OFS (1)", + "CEM_HDR_LUM_LARGE_RANGE (2)", + "CEM_HDR_LUM_SMALL_RANGE (3)", + "CEM_LDR_LUM_ALPHA_DIRECT (4)", + "CEM_LDR_LUM_ALPHA_BASE_PLUS_OFS (5)", + "CEM_LDR_RGB_BASE_SCALE (6)", + "CEM_HDR_RGB_BASE_SCALE (7)", + "CEM_LDR_RGB_DIRECT (8)", + "CEM_LDR_RGB_BASE_PLUS_OFFSET (9)", + "CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A (10)", + "CEM_HDR_RGB (11)", + "CEM_LDR_RGBA_DIRECT (12)", + "CEM_LDR_RGBA_BASE_PLUS_OFFSET (13)", + "CEM_HDR_RGB_LDR_ALPHA (14)", + "CEM_HDR_RGB_HDR_ALPHA (15)" + }; + + assert(cem_index < std::size(s_cem_names)); + const char *p = s_cem_names[cem_index]; + assert(p); + return p; + } + + bool cem_is_ldr_direct(uint32_t cem_index) + { + return (cem_index == CEM_LDR_RGB_DIRECT) || (cem_index == CEM_LDR_RGBA_DIRECT); + } + + bool cem_is_ldr_base_scale(uint32_t cem_index) + { + return (cem_index == CEM_LDR_RGB_BASE_SCALE) || (cem_index == CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A); + } + + bool cem_is_ldr_base_plus_ofs(uint32_t cem_index) + { + return (cem_index == CEM_LDR_RGB_BASE_PLUS_OFFSET) || (cem_index == CEM_LDR_RGBA_BASE_PLUS_OFFSET); + } + + bool cem_supports_bc(uint32_t cem) + { + switch (cem) + { + case CEM_LDR_RGB_DIRECT: + case CEM_LDR_RGBA_DIRECT: + case CEM_LDR_RGB_BASE_PLUS_OFFSET: + case CEM_LDR_RGBA_BASE_PLUS_OFFSET: + return true; + default: + break; + } + return false; + } + + // input: + // a=[0,255] + // b=[0,255] + // output: + // a=from, converted to -32 to 31 + // b=to, shifted right by 1 and 1 bit added to MSB, so [0,255] + void bit_transfer_signed_dec(int& a, int& b) + { + assert((a >= 0) && (a <= 255)); + assert((b >= 0) && (b <= 255)); + + b >>= 1; + b |= (a & 0x80); + + a >>= 1; + a &= 0x3F; + if ((a & 0x20) != 0) + a -= 0x40; + } + + // transfers a bit from b to a, prepares a for encoding + // input: + // a=[-32,31] (6-bits, 2's complement) + // b=[0,255] (8-bits) + // output: + // a=[0,255] (preserve top 2 bits) + // b=[0,255] + void bit_transfer_signed_enc(int& a, int& b) + { + assert((a >= -32) && (a <= 31)); + assert((b >= 0) && (b <= 255)); + + // extract MSB of b + bool bit_to_transfer = (b & 0x80) != 0; + b = (b << 1) & 0xFF; // 7 bits to 8 + + a &= 0x3F; // 6 bits + a <<= 1; // 6 to 7 bits + if (bit_to_transfer) + a |= 0x80; // set MSB + } + + // RGB or RGBA direct + bool cem8_or_12_used_blue_contraction(uint32_t cem_index, const uint8_t* pEndpoint_vals, uint32_t endpoint_ise_index) + { + assert((cem_index == CEM_LDR_RGB_DIRECT) || (cem_index == CEM_LDR_RGBA_DIRECT)); + (void)(cem_index); + + const auto& endpoint_dequant_tab = g_dequant_tables.get_endpoint_tab(endpoint_ise_index).m_ISE_to_val; + + uint8_t dequantized_endpoints[6]; + for (uint32_t i = 0; i < 6; i++) + dequantized_endpoints[i] = endpoint_dequant_tab[pEndpoint_vals[i]]; + + uint32_t s0 = dequantized_endpoints[0] + dequantized_endpoints[2] + dequantized_endpoints[4]; + uint32_t s1 = dequantized_endpoints[1] + dequantized_endpoints[3] + dequantized_endpoints[5]; + + return s1 < s0; + } + + // RGB or RGBA base plus offset + bool cem9_or_13_used_blue_contraction(uint32_t cem_index, const uint8_t* pEndpoint_vals, uint32_t endpoint_ise_index) + { + assert((cem_index == CEM_LDR_RGB_BASE_PLUS_OFFSET) || (cem_index == CEM_LDR_RGBA_BASE_PLUS_OFFSET)); + (void)(cem_index); + + const auto& endpoint_dequant_tab = g_dequant_tables.get_endpoint_tab(endpoint_ise_index).m_ISE_to_val; + + int dequantized_endpoints[6]; + for (uint32_t i = 0; i < 6; i++) + dequantized_endpoints[i] = endpoint_dequant_tab[pEndpoint_vals[i]]; + + bit_transfer_signed_dec(dequantized_endpoints[1], dequantized_endpoints[0]); + bit_transfer_signed_dec(dequantized_endpoints[3], dequantized_endpoints[2]); + bit_transfer_signed_dec(dequantized_endpoints[5], dequantized_endpoints[4]); + + int s = dequantized_endpoints[1] + dequantized_endpoints[3] + dequantized_endpoints[5]; + + return s < 0; + } + + bool used_blue_contraction(uint32_t cem_index, const uint8_t* pEndpoint_vals, uint32_t endpoint_ise_index) + { + assert(is_cem_ldr(cem_index)); + + bool used_blue_contraction_flag = false; + + if ((cem_index == 8) || (cem_index == 12)) + used_blue_contraction_flag = cem8_or_12_used_blue_contraction(cem_index, pEndpoint_vals, endpoint_ise_index); + else if ((cem_index == 9) || (cem_index == 13)) + used_blue_contraction_flag = cem9_or_13_used_blue_contraction(cem_index, pEndpoint_vals, endpoint_ise_index); + + return used_blue_contraction_flag; + } + + uint32_t get_base_cem_without_alpha(uint32_t cem) + { + assert(is_cem_ldr(cem)); + + switch (cem) + { + case CEM_LDR_LUM_ALPHA_DIRECT: return CEM_LDR_LUM_DIRECT; + case CEM_LDR_RGBA_DIRECT: return CEM_LDR_RGB_DIRECT; + case CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A: return CEM_LDR_RGB_BASE_SCALE; + case CEM_LDR_RGBA_BASE_PLUS_OFFSET: return CEM_LDR_RGB_BASE_PLUS_OFFSET; + default: + break; + } + + return cem; + } + + int apply_delta_to_bise_endpoint_val(uint32_t endpoint_ise_range, int ise_val, int delta) + { + if (delta == 0) + return ise_val; + + uint32_t num_ise_levels = astc_helpers::get_ise_levels(endpoint_ise_range); + + const auto& ISE_to_rank = astc_helpers::g_dequant_tables.get_endpoint_tab(endpoint_ise_range).m_ISE_to_rank; + const auto& rank_to_ISE = astc_helpers::g_dequant_tables.get_endpoint_tab(endpoint_ise_range).m_rank_to_ISE; + + int cur_rank = ISE_to_rank[ise_val]; + int new_rank = basisu::clamp(cur_rank + delta, 0, (int)num_ise_levels - 1); + + return rank_to_ISE[new_rank]; + } + + void get_astc_block_size_by_index(uint32_t index, uint32_t& width, uint32_t& height) + { + assert(index < NUM_ASTC_BLOCK_SIZES); + + width = g_astc_block_sizes[index][0]; + height = g_astc_block_sizes[index][1]; + } + + int find_astc_block_size_index(uint32_t width, uint32_t height) + { + for (uint32_t i = 0; i < NUM_ASTC_BLOCK_SIZES; i++) + if ((width == g_astc_block_sizes[i][0]) && (height == g_astc_block_sizes[i][1])) + return i; + + return -1; + } + } // namespace astc_helpers #endif //BASISU_ASTC_HELPERS_IMPLEMENTATION diff --git a/external/basis_universal/transcoder/basisu_containers.h b/external/basis_universal/transcoder/basisu_containers.h index 88026c7198..dc816f3d10 100644 --- a/external/basis_universal/transcoder/basisu_containers.h +++ b/external/basis_universal/transcoder/basisu_containers.h @@ -159,8 +159,8 @@ namespace basisu static inline void construct(T** p) { memset(p, 0, sizeof(T*)); } static inline void construct(T** p, T* init) { *p = init; } static inline void construct_array(T** p, size_t n) { memset(p, 0, sizeof(T*) * n); } - static inline void destruct(T** p) { p; } - static inline void destruct_array(T** p, size_t n) { p, n; } + static inline void destruct(T** p) { (void)p; } + static inline void destruct_array(T** p, size_t n) { (void)p, (void)n; } }; #define BASISU_DEFINE_BUILT_IN_TYPE(X) \ @@ -169,8 +169,8 @@ namespace basisu static inline void construct(X* p) { memset(p, 0, sizeof(X)); } \ static inline void construct(X* p, const X& init) { memcpy(p, &init, sizeof(X)); } \ static inline void construct_array(X* p, size_t n) { memset(p, 0, sizeof(X) * n); } \ - static inline void destruct(X* p) { p; } \ - static inline void destruct_array(X* p, size_t n) { p, n; } }; + static inline void destruct(X* p) { (void)p; } \ + static inline void destruct_array(X* p, size_t n) { (void)p, (void)n; } }; BASISU_DEFINE_BUILT_IN_TYPE(bool) BASISU_DEFINE_BUILT_IN_TYPE(char) @@ -272,7 +272,7 @@ namespace basisu size_t c = a + b; return c < a; } - + // Returns false on overflow, true if OK. template inline bool can_fit_into_size_t(T val) @@ -294,7 +294,7 @@ namespace basisu template class writable_span; - + template class readable_span { @@ -304,7 +304,7 @@ namespace basisu using const_pointer = const T*; using const_reference = const T&; using const_iterator = const T*; - + inline readable_span() : m_p(nullptr), m_size(0) @@ -941,7 +941,7 @@ namespace basisu inline iterator begin() const { return m_p; } inline iterator end() const { assert(m_p || !m_size); return m_p + m_size; } - + inline const_iterator cbegin() const { return m_p; } inline const_iterator cend() const { assert(m_p || !m_size); return m_p + m_size; } @@ -1506,7 +1506,7 @@ namespace basisu #if defined(__GNUC__) && !defined(__clang__) #pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wclass-memaccess" +#pragma GCC diagnostic ignored "-Wclass-memaccess" #endif if ((m_p) && (other.m_p)) { @@ -1563,6 +1563,52 @@ namespace basisu set(ws); } + // mostly to ease porting from std::vector, not particularly optimized + inline void assign(size_t new_size, const T& init) + { + assert(!m_p || (&init < m_p) || (&init >= (m_p + m_size))); + + // Blow away existing contents + resize(0); + + if (new_size) + { + resize(new_size); + + for (size_t i = 0; i < new_size; ++i) + m_p[i] = init; + } + } + + // mostly to ease porting from std::vector, not particularly optimized + template + inline void assign(const R* pBegin, const R* pEnd) + { + assert(!m_p || + (reinterpret_cast(pEnd) <= reinterpret_cast(m_p)) || + (reinterpret_cast(pBegin) >= reinterpret_cast(m_p + m_size)) + ); + + // Blow away existing contents + resize(0); + + if ((!pBegin) || (!pEnd) || (pEnd <= pBegin)) + { + assert(0); + return; + } + + const size_t new_size = static_cast(static_cast(pEnd - pBegin)); + + if (new_size) + { + resize(new_size); + + for (size_t i = 0; i < new_size; ++i) + m_p[i] = static_cast(*pBegin++); + } + } + // Set contents of vector to contents of the readable span bool set(const readable_span& rs) { @@ -1647,7 +1693,7 @@ namespace basisu { #if defined(__GNUC__) && !defined(__clang__) #pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wclass-memaccess" +#pragma GCC diagnostic ignored "-Wclass-memaccess" #endif if ((m_p) && (other.m_p)) memcpy((void *)m_p, other.m_p, other.m_size * sizeof(T)); @@ -2147,7 +2193,7 @@ namespace basisu if (!try_insert(p, obj)) container_abort("vector::insert() failed!\n"); } - + // push_front() isn't going to be very fast - it's only here for usability. inline void push_front(const T& obj) { @@ -2228,7 +2274,7 @@ namespace basisu #if defined(__GNUC__) && !defined(__clang__) #pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wclass-memaccess" +#pragma GCC diagnostic ignored "-Wclass-memaccess" #endif memmove((void *)pDst, pSrc, num_to_move * sizeof(T)); @@ -2239,7 +2285,7 @@ namespace basisu } else { - // Type is not bitwise copyable or movable. + // Type is not bitwise copyable or movable. // Move them down one at a time by using the equals operator, and destroying anything that's left over at the end. T* pDst_end = pDst + num_to_move; @@ -2482,7 +2528,7 @@ namespace basisu { #if defined(__GNUC__) && !defined(__clang__) #pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wclass-memaccess" +#pragma GCC diagnostic ignored "-Wclass-memaccess" #endif memset(m_p, *reinterpret_cast(&o), m_size); @@ -2770,6 +2816,7 @@ namespace basisu m_grow_threshold = 0; } + // Destroys elements/empties container but doesn't free memory. inline void reset() { if (!m_num_valid) @@ -2798,7 +2845,7 @@ namespace basisu } else if (sizeof(node) <= 16) { - memset(&m_values[0], 0, m_values.size_in_bytes()); + memset((void *)&m_values[0], 0, m_values.size_in_bytes()); } else { @@ -2829,6 +2876,11 @@ namespace basisu return m_num_valid; } + inline uint32_t size_u32() + { + return static_cast(m_num_valid); + } + inline size_t get_table_size() { return m_values.size(); @@ -3102,7 +3154,7 @@ namespace basisu { return try_insert(result, std::move(v.first), std::move(v.second)); } - + inline const_iterator find(const Key& k) const { return const_iterator(*this, find_index(k)); @@ -3183,12 +3235,12 @@ namespace basisu static inline void construct_value_type(value_type* pDst, const Key& k, const Value& v) { if (BASISU_IS_BITWISE_COPYABLE(Key)) - memcpy(&pDst->first, &k, sizeof(Key)); + memcpy((void *)&pDst->first, &k, sizeof(Key)); else scalar_type::construct(&pDst->first, k); if (BASISU_IS_BITWISE_COPYABLE(Value)) - memcpy(&pDst->second, &v, sizeof(Value)); + memcpy((void *)&pDst->second, &v, sizeof(Value)); else scalar_type::construct(&pDst->second, v); } @@ -3197,17 +3249,17 @@ namespace basisu { if ((BASISU_IS_BITWISE_COPYABLE(Key)) && (BASISU_IS_BITWISE_COPYABLE(Value))) { - memcpy(pDst, pSrc, sizeof(value_type)); + memcpy((void *)pDst, pSrc, sizeof(value_type)); } else { if (BASISU_IS_BITWISE_COPYABLE(Key)) - memcpy(&pDst->first, &pSrc->first, sizeof(Key)); + memcpy((void *)&pDst->first, &pSrc->first, sizeof(Key)); else scalar_type::construct(&pDst->first, pSrc->first); if (BASISU_IS_BITWISE_COPYABLE(Value)) - memcpy(&pDst->second, &pSrc->second, sizeof(Value)); + memcpy((void *)&pDst->second, &pSrc->second, sizeof(Value)); else scalar_type::construct(&pDst->second, pSrc->second); } @@ -3227,14 +3279,14 @@ namespace basisu if (BASISU_IS_BITWISE_COPYABLE_OR_MOVABLE(Key) && BASISU_IS_BITWISE_COPYABLE_OR_MOVABLE(Value)) { - memcpy(pDst, pSrc, sizeof(node)); + memcpy((void *)pDst, pSrc, sizeof(node)); assert(pDst->state == cStateValid); } else { if (BASISU_IS_BITWISE_COPYABLE_OR_MOVABLE(Key)) - memcpy(&pDst->first, &pSrc->first, sizeof(Key)); + memcpy((void*)&pDst->first, &pSrc->first, sizeof(Key)); else { new ((void*)&pDst->first) Key(std::move(pSrc->first)); @@ -3242,7 +3294,7 @@ namespace basisu } if (BASISU_IS_BITWISE_COPYABLE_OR_MOVABLE(Value)) - memcpy(&pDst->second, &pSrc->second, sizeof(Value)); + memcpy((void*)&pDst->second, &pSrc->second, sizeof(Value)); else { new ((void*)&pDst->second) Value(std::move(pSrc->second)); @@ -3583,7 +3635,7 @@ namespace basisu // Not checking for is MOVABLE because the caller could later destruct k and/or v (what state do we set them to?) if (BASISU_IS_BITWISE_COPYABLE(Key)) { - memcpy(&pDst->first, &k, sizeof(Key)); + memcpy((void *)&pDst->first, (const void *)&k, sizeof(Key)); } else { @@ -3593,7 +3645,7 @@ namespace basisu if (BASISU_IS_BITWISE_COPYABLE(Value)) { - memcpy(&pDst->second, &v, sizeof(Value)); + memcpy((void *)&pDst->second, (const void*)&v, sizeof(Value)); } else { @@ -3721,11 +3773,11 @@ namespace basisu va_list args; va_start(args, pFmt); -#ifdef _WIN32 +#ifdef _WIN32 vsprintf_s(buf, sizeof(buf), pFmt, args); #else vsnprintf(buf, sizeof(buf), pFmt, args); -#endif +#endif va_end(args); return std::string(buf); @@ -3893,7 +3945,7 @@ namespace basisu std::size_t copy_size = std::min(list.size(), N); std::copy_n(list.begin(), copy_size, m_data); // Copy up to min(list.size(), N) - if (list.size() < N) + if (list.size() < N) { // Initialize the rest of the array std::fill(m_data + copy_size, m_data + N, T{}); @@ -3907,7 +3959,7 @@ namespace basisu return m_data[index]; } - BASISU_FORCE_INLINE const T& operator[](std::size_t index) const + BASISU_FORCE_INLINE const T& operator[](std::size_t index) const { if (index >= N) container_abort("fixed_array: Index out of bounds."); @@ -3950,26 +4002,26 @@ namespace basisu { return writable_span(m_data, N); } - + private: BASISU_FORCE_INLINE void initialize_array() { - if constexpr (std::is_integral::value || std::is_floating_point::value) + if constexpr (std::is_integral::value || std::is_floating_point::value) memset(m_data, 0, sizeof(m_data)); - else + else std::fill(m_data, m_data + N, T{}); } BASISU_FORCE_INLINE T& access_element(std::size_t index) { - if (index >= N) + if (index >= N) container_abort("fixed_array: Index out of bounds."); return m_data[index]; } BASISU_FORCE_INLINE const T& access_element(std::size_t index) const { - if (index >= N) + if (index >= N) container_abort("fixed_array: Index out of bounds."); return m_data[index]; } @@ -4046,6 +4098,9 @@ namespace basisu inline uint32_t get_width() const { return m_width; } inline uint32_t get_height() const { return m_height; } + inline uint32_t get_cols() const { return m_width; } + inline uint32_t get_rows() const { return m_height; } + inline const T& operator() (uint32_t x, uint32_t y) const { assert(x < m_width && y < m_height); return m_values[x + y * m_width]; } inline T& operator() (uint32_t x, uint32_t y) { assert(x < m_width && y < m_height); return m_values[x + y * m_width]; } @@ -4054,9 +4109,23 @@ namespace basisu inline const T& operator[] (uint32_t i) const { return m_values[i]; } inline T& operator[] (uint32_t i) { return m_values[i]; } + inline const T& at(int x, int y) const { return (*this)((uint32_t)x, (uint32_t)y); } + inline T& at(int x, int y) { return (*this)((uint32_t)x, (uint32_t)y); } + inline const T& at_clamped(int x, int y) const { return (*this)(clamp(x, 0, m_width - 1), clamp(y, 0, m_height - 1)); } inline T& at_clamped(int x, int y) { return (*this)(clamp(x, 0, m_width - 1), clamp(y, 0, m_height - 1)); } + inline const T& at_row_col(int y, int x) const { return (*this)(clamp(x, 0, m_width - 1), clamp(y, 0, m_height - 1)); } + inline T& at_row_col(int y, int x) { return (*this)(clamp(x, 0, m_width - 1), clamp(y, 0, m_height - 1)); } + + inline void set_clipped(int x, int y, const T& val) + { + if ( ((uint32_t)x >= m_width) || ((uint32_t)y >= m_height) ) + return; + + m_values[x + y * m_width] = val; + } + void clear() { m_width = 0; @@ -4141,9 +4210,18 @@ namespace basisu return true; } + vector2D& resize_rows_cols(uint32_t rows, uint32_t cols) + { + return resize(cols, rows); + } + + bool try_resize_rows_cols(uint32_t rows, uint32_t cols) + { + return try_resize(cols, rows); + } + const vector2D& extract_block_clamped(T* pDst, uint32_t src_x, uint32_t src_y, uint32_t w, uint32_t h) const { - // HACK HACK if (((src_x + w) > m_width) || ((src_y + h) > m_height)) { // Slower clamping case @@ -4165,8 +4243,87 @@ namespace basisu return *this; } + + const vector2D& extract_block_clamped(T* pDst, uint32_t src_x, uint32_t src_y, uint32_t w, uint32_t h, uint32_t override_height) const + { + assert(override_height && (override_height <= m_height)); + + if (((src_x + w) > m_width) || ((src_y + h) > minimum(m_height, override_height))) + { + // Slower clamping case + for (uint32_t y = 0; y < h; y++) + for (uint32_t x = 0; x < w; x++) + *pDst++ = at_clamped(src_x + x, minimum(src_y + y, override_height - 1)); + } + else + { + const T* pSrc = &m_values[src_x + src_y * m_width]; + + for (uint32_t y = 0; y < h; y++) + { + memcpy(pDst, pSrc, w * sizeof(T)); + pSrc += m_width; + pDst += w; + } + } + + return *this; + } }; + // Explictly primitive container intended for POD's, simple usage. + // push_back() and resize() will refuse to push anymore and just return when full. + template + class static_vector + { + T m_data[N]; + uint32_t m_size; + + public: + static_vector() : m_size(0) { } + + inline void reserve(size_t reserve_size) + { + (void)(reserve_size); + + assert(reserve_size <= N); + } + + inline void push_back(const T& value) + { + // Should never happen. + if (m_size >= N) + { + assert(0); + fprintf(stderr, "basisu::static_vector overflow!\n"); + return; + } + + m_data[m_size++] = value; + } + + inline std::size_t size() const { return m_size; } + inline uint32_t size_u32() const { return m_size; } + inline constexpr std::size_t capacity() const { return N; } + + inline bool empty() const { return !m_size; } + + inline T& operator[](std::size_t i) { return m_data[i]; } + inline const T& operator[](std::size_t i) const { return m_data[i]; } + + inline void resize(size_t new_size) + { + if (new_size > N) + { + assert(0); + fprintf(stderr, "basisu::static_vector overflow!\n"); + return; + } + + m_size = (uint32_t)new_size; + } + }; + } // namespace basisu namespace std diff --git a/external/basis_universal/transcoder/basisu_containers_impl.h b/external/basis_universal/transcoder/basisu_containers_impl.h index 4c85ed3dfa..2ac13029dc 100644 --- a/external/basis_universal/transcoder/basisu_containers_impl.h +++ b/external/basis_universal/transcoder/basisu_containers_impl.h @@ -14,7 +14,7 @@ namespace basisu #ifdef _MSC_VER __declspec(noreturn) #else - [[noreturn]] + [[noreturn]] #endif void container_abort(const char* pMsg, ...) { @@ -42,12 +42,12 @@ namespace basisu assert(m_size <= m_capacity); assert(min_new_capacity >= m_size); assert(element_size); - + // Basic sanity check min_new_capacity if (!can_fit_into_size_t((uint64_t)min_new_capacity * element_size)) { assert(0); - + if (nofail_flag) return false; @@ -100,7 +100,7 @@ namespace basisu } const size_t desired_size = static_cast(desired_size_u64); - + size_t actual_size = 0; BASISU_NOTE_UNUSED(actual_size); @@ -109,6 +109,7 @@ namespace basisu void* new_p = realloc(m_p, desired_size); if (!new_p) { + fprintf(stderr, "elemental_vector::increase_capacity: Allocation failed!\n"); assert(0); if (nofail_flag) @@ -133,7 +134,9 @@ namespace basisu void* new_p = malloc(desired_size); if (!new_p) { + fprintf(stderr, "elemental_vector::increase_capacity: Allocation failed!\n"); assert(0); + if (nofail_flag) return false; @@ -269,7 +272,7 @@ namespace basisu s.insert(i); k.push_back(i); } - + for (uint32_t i = 0; i < k.size(); i++) { uint32_t r = rand() ^ (rand() << 15); @@ -315,7 +318,7 @@ namespace basisu { typedef basisu::hash_map< uint32_t, basisu::vector > hm; hm q; - + basisu::vector a, b; a.push_back(1); b.push_back(2); diff --git a/external/basis_universal/transcoder/basisu_etc1_mods.inl b/external/basis_universal/transcoder/basisu_etc1_mods.inl new file mode 100644 index 0000000000..572a816d00 --- /dev/null +++ b/external/basis_universal/transcoder/basisu_etc1_mods.inl @@ -0,0 +1,257 @@ +static const uint8_t g_etc1_mod_tabs[255][8] = { +{0,0,0,0,0,0,0,0,}, +{0,0,0,0,0,0,0,0,}, +{0,0,0,0,0,0,0,0,}, +{0,0,0,0,0,0,0,0,}, +{0,0,0,0,0,0,1,1,}, +{0,0,0,0,0,0,0,1,}, +{0,0,0,0,0,0,0,0,}, +{0,0,0,0,0,0,0,0,}, +{0,0,0,0,0,0,0,0,}, +{0,0,0,0,0,0,0,0,}, +{0,0,0,0,0,0,0,0,}, +{0,0,0,0,0,0,0,0,}, +{0,0,0,0,0,0,0,1,}, +{0,0,0,0,0,0,1,1,}, +{0,0,0,0,0,1,1,1,}, +{0,0,0,0,0,1,1,1,}, +{0,0,0,0,1,1,1,1,}, +{0,0,0,0,1,1,1,1,}, +{0,0,0,0,1,1,1,1,}, +{0,0,0,0,1,1,1,1,}, +{0,0,0,1,1,1,1,1,}, +{0,0,0,1,1,1,1,1,}, +{0,0,0,1,1,1,1,1,}, +{0,0,0,1,1,1,1,2,}, +{0,0,0,1,1,1,2,2,}, +{0,0,0,1,1,1,2,2,}, +{0,0,0,1,1,2,2,2,}, +{0,0,1,1,1,2,2,2,}, +{0,0,1,1,1,2,2,2,}, +{0,0,1,1,1,2,2,2,}, +{0,0,1,1,1,2,2,2,}, +{0,0,1,1,2,2,2,2,}, +{0,0,1,1,2,2,2,2,}, +{0,0,1,1,2,2,2,2,}, +{0,0,1,1,2,2,2,2,}, +{0,0,1,1,2,2,2,2,}, +{0,0,1,1,2,2,2,3,}, +{0,0,1,1,2,2,3,3,}, +{0,0,1,2,2,2,3,3,}, +{0,0,1,2,2,2,3,3,}, +{0,0,1,2,2,2,3,3,}, +{0,1,1,2,2,3,3,3,}, +{0,1,1,2,2,3,3,3,}, +{0,1,1,2,2,3,3,3,}, +{0,1,1,2,2,3,3,3,}, +{0,1,1,2,2,3,3,3,}, +{0,1,1,2,2,3,3,3,}, +{0,1,1,2,2,3,3,3,}, +{0,1,1,2,3,3,3,3,}, +{0,1,1,2,3,3,3,3,}, +{0,1,1,2,3,3,3,3,}, +{0,1,2,2,3,3,3,3,}, +{0,1,2,2,3,3,3,4,}, +{0,1,2,2,3,3,3,4,}, +{0,1,2,2,3,3,4,4,}, +{0,1,2,2,3,3,4,4,}, +{0,1,2,2,3,3,4,4,}, +{0,1,2,2,3,3,4,4,}, +{0,1,2,2,3,3,4,4,}, +{0,1,2,2,3,4,4,4,}, +{0,1,2,3,3,4,4,4,}, +{0,1,2,3,3,4,4,4,}, +{0,1,2,3,3,4,4,4,}, +{0,1,2,3,3,4,4,4,}, +{0,1,2,3,3,4,4,4,}, +{0,1,2,3,3,4,4,4,}, +{0,1,2,3,3,4,4,4,}, +{0,1,2,3,3,4,4,4,}, +{0,1,2,3,3,4,4,4,}, +{0,1,2,3,3,4,4,4,}, +{0,1,2,3,4,4,4,4,}, +{0,1,2,3,4,4,4,5,}, +{0,1,2,3,4,4,4,5,}, +{0,1,2,3,4,4,4,5,}, +{0,1,2,3,4,4,5,5,}, +{0,1,2,3,4,4,5,5,}, +{0,1,2,3,4,4,5,5,}, +{0,1,2,3,4,4,5,5,}, +{0,2,2,3,4,4,5,5,}, +{0,2,2,3,4,4,5,5,}, +{0,2,2,3,4,4,5,5,}, +{0,2,3,3,4,5,5,5,}, +{0,2,3,3,4,5,5,5,}, +{0,2,3,3,4,5,5,5,}, +{0,2,3,3,4,5,5,5,}, +{1,2,3,3,4,5,5,5,}, +{1,2,3,3,4,5,5,5,}, +{1,2,3,4,4,5,5,5,}, +{1,2,3,4,4,5,5,5,}, +{1,2,3,4,4,5,5,5,}, +{1,2,3,4,4,5,5,5,}, +{1,2,3,4,4,5,5,5,}, +{1,2,3,4,4,5,5,5,}, +{1,2,3,4,4,5,5,5,}, +{1,2,3,4,4,5,5,5,}, +{1,2,3,4,4,5,5,6,}, +{1,2,3,4,5,5,5,6,}, +{1,2,3,4,5,5,5,6,}, +{1,2,3,4,5,5,5,6,}, +{1,2,3,4,5,5,6,6,}, +{1,2,3,4,5,5,6,6,}, +{1,2,3,4,5,5,6,6,}, +{1,2,3,4,5,5,6,6,}, +{1,2,3,4,5,5,6,6,}, +{1,2,3,4,5,5,6,6,}, +{1,2,3,4,5,5,6,6,}, +{1,2,3,4,5,5,6,6,}, +{1,2,3,4,5,5,6,6,}, +{1,2,3,4,5,6,6,6,}, +{1,2,3,4,5,6,6,6,}, +{1,2,3,4,5,6,6,6,}, +{1,2,3,4,5,6,6,6,}, +{1,2,3,4,5,6,6,6,}, +{1,2,3,4,5,6,6,6,}, +{1,2,3,4,5,6,6,6,}, +{1,2,3,4,5,6,6,6,}, +{1,2,4,4,5,6,6,6,}, +{1,2,4,4,5,6,6,6,}, +{1,2,4,4,5,6,6,6,}, +{1,2,4,4,5,6,6,6,}, +{1,2,4,5,5,6,6,6,}, +{1,2,4,5,5,6,6,6,}, +{1,3,4,5,5,6,6,6,}, +{1,3,4,5,5,6,6,6,}, +{1,3,4,5,5,6,6,6,}, +{1,3,4,5,5,6,6,6,}, +{1,3,4,5,5,6,6,6,}, +{1,3,4,5,5,6,6,6,}, +{1,3,4,5,6,6,6,6,}, +{1,3,4,5,6,6,6,6,}, +{1,3,4,5,6,6,6,6,}, +{1,3,4,5,6,6,6,6,}, +{1,3,4,5,6,6,6,6,}, +{1,3,4,5,6,6,6,6,}, +{1,3,4,5,6,6,6,6,}, +{1,3,4,5,6,6,6,6,}, +{1,3,4,5,6,6,6,6,}, +{1,3,4,5,6,6,6,6,}, +{1,3,4,5,6,6,6,6,}, +{1,3,4,5,6,6,6,6,}, +{1,3,4,5,6,6,6,6,}, +{1,3,4,5,6,6,6,6,}, +{1,3,4,5,6,6,6,6,}, +{1,3,4,5,6,6,6,6,}, +{1,3,4,5,6,6,6,6,}, +{1,3,4,5,6,6,6,6,}, +{1,3,4,5,6,6,6,6,}, +{1,3,4,5,6,6,6,7,}, +{1,3,4,5,6,6,6,7,}, +{1,3,4,5,6,6,6,7,}, +{1,3,4,5,6,6,6,7,}, +{1,3,4,5,6,6,6,7,}, +{1,3,4,5,6,6,6,7,}, +{1,3,4,5,6,6,6,7,}, +{1,3,4,5,6,6,7,7,}, +{1,3,4,5,6,6,7,7,}, +{1,3,4,5,6,6,7,7,}, +{1,3,4,5,6,6,7,7,}, +{1,3,4,5,6,6,7,7,}, +{1,3,4,6,6,6,7,7,}, +{1,3,5,6,6,6,7,7,}, +{1,3,5,6,6,6,7,7,}, +{1,3,5,6,6,6,7,7,}, +{2,3,5,6,6,6,7,7,}, +{2,3,5,6,6,6,7,7,}, +{2,3,5,6,6,6,7,7,}, +{2,3,5,6,6,6,7,7,}, +{2,3,5,6,6,7,7,7,}, +{2,3,5,6,6,7,7,7,}, +{2,3,5,6,6,7,7,7,}, +{2,3,5,6,6,7,7,7,}, +{2,3,5,6,6,7,7,7,}, +{2,3,5,6,6,7,7,7,}, +{2,3,5,6,6,7,7,7,}, +{2,3,5,6,6,7,7,7,}, +{2,4,5,6,6,7,7,7,}, +{2,4,5,6,6,7,7,7,}, +{2,4,5,6,6,7,7,7,}, +{2,4,5,6,6,7,7,7,}, +{2,4,5,6,6,7,7,7,}, +{2,4,5,6,6,7,7,7,}, +{2,4,5,6,6,7,7,7,}, +{2,4,5,6,6,7,7,7,}, +{2,4,5,6,6,7,7,7,}, +{2,4,5,6,6,7,7,7,}, +{2,4,5,6,6,7,7,7,}, +{2,4,5,6,6,7,7,7,}, +{2,4,5,6,6,7,7,7,}, +{2,4,5,6,6,7,7,7,}, +{2,4,5,6,6,7,7,7,}, +{2,4,5,6,6,7,7,7,}, +{2,4,5,6,6,7,7,7,}, +{2,4,5,6,6,7,7,7,}, +{2,4,5,6,6,7,7,7,}, +{2,4,5,6,6,7,7,7,}, +{2,4,5,6,6,7,7,7,}, +{2,4,5,6,6,7,7,7,}, +{2,4,5,6,7,7,7,7,}, +{2,4,5,6,7,7,7,7,}, +{2,4,5,6,7,7,7,7,}, +{2,4,5,6,7,7,7,7,}, +{2,4,5,6,7,7,7,7,}, +{2,4,5,6,7,7,7,7,}, +{2,4,5,6,7,7,7,7,}, +{2,4,5,6,7,7,7,7,}, +{2,4,5,6,7,7,7,7,}, +{2,4,5,6,7,7,7,7,}, +{2,4,5,6,7,7,7,7,}, +{2,4,5,6,7,7,7,7,}, +{2,4,5,6,7,7,7,7,}, +{2,4,5,6,7,7,7,7,}, +{2,4,5,6,7,7,7,7,}, +{2,4,6,6,7,7,7,7,}, +{2,4,6,6,7,7,7,7,}, +{2,4,6,6,7,7,7,7,}, +{2,4,6,6,7,7,7,7,}, +{2,4,6,6,7,7,7,7,}, +{2,4,6,6,7,7,7,7,}, +{2,4,6,6,7,7,7,7,}, +{2,4,6,6,7,7,7,7,}, +{2,4,6,6,7,7,7,7,}, +{2,4,6,6,7,7,7,7,}, +{2,4,6,6,7,7,7,7,}, +{2,4,6,6,7,7,7,7,}, +{2,4,6,6,7,7,7,7,}, +{2,4,6,6,7,7,7,7,}, +{2,4,6,6,7,7,7,7,}, +{2,4,6,6,7,7,7,7,}, +{2,4,6,6,7,7,7,7,}, +{2,4,6,6,7,7,7,7,}, +{2,4,6,6,7,7,7,7,}, +{2,4,6,6,7,7,7,7,}, +{2,4,6,6,7,7,7,7,}, +{2,4,6,6,7,7,7,7,}, +{2,4,6,6,7,7,7,7,}, +{2,4,6,6,7,7,7,7,}, +{2,4,6,6,7,7,7,7,}, +{2,4,6,6,7,7,7,7,}, +{2,4,6,6,7,7,7,7,}, +{2,4,6,6,7,7,7,7,}, +{2,4,6,6,7,7,7,7,}, +{2,4,6,6,7,7,7,7,}, +{2,5,6,6,7,7,7,7,}, +{2,5,6,6,7,7,7,7,}, +{2,5,6,6,7,7,7,7,}, +{2,5,6,6,7,7,7,7,}, +{2,5,6,6,7,7,7,7,}, +{2,5,6,7,7,7,7,7,}, +{2,5,6,7,7,7,7,7,}, +{2,5,6,7,7,7,7,7,}, +{2,5,6,7,7,7,7,7,}, +{2,5,6,7,7,7,7,7,}, +{2,5,6,7,7,7,7,7,}, +{2,5,6,7,7,7,7,7,}, +{2,5,6,7,7,7,7,7,} +}; diff --git a/external/basis_universal/transcoder/basisu_file_headers.h b/external/basis_universal/transcoder/basisu_file_headers.h index ddd117a0c9..10462774f4 100644 --- a/external/basis_universal/transcoder/basisu_file_headers.h +++ b/external/basis_universal/transcoder/basisu_file_headers.h @@ -1,5 +1,5 @@ // basis_file_headers.h -// Copyright (C) 2019-2024 Binomial LLC. All Rights Reserved. +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -39,7 +39,7 @@ namespace basist basisu::packed_uint<2> m_orig_height; // The original image height (may not be a multiple of 4 pixels) basisu::packed_uint<2> m_num_blocks_x; // The slice's block X dimensions. Each block is 4x4 or 6x6 pixels. The slice's pixel resolution may or may not be a power of 2. - basisu::packed_uint<2> m_num_blocks_y; // The slice's block Y dimensions. + basisu::packed_uint<2> m_num_blocks_y; // The slice's block Y dimensions. basisu::packed_uint<4> m_file_ofs; // Offset from the start of the file to the start of the slice's data basisu::packed_uint<4> m_file_size; // The size of the compressed slice data in bytes @@ -59,16 +59,16 @@ namespace basist // Set if any slices contain alpha (for ETC1S, if the odd slices contain alpha data) cBASISHeaderFlagHasAlphaSlices = 4, - // For ETC1S files, this will be true if the file utilizes a codebook from another .basis file. + // For ETC1S files, this will be true if the file utilizes a codebook from another .basis file. cBASISHeaderFlagUsesGlobalCodebook = 8, - // Set if the texture data is sRGB, otherwise it's linear. + // Set if the texture data is sRGB, otherwise it's linear. // In reality, we have no idea if the texture data is actually linear or sRGB. This is the m_perceptual parameter passed to the compressor. cBASISHeaderFlagSRGB = 16, }; // The image type field attempts to describe how to interpret the image data in a Basis file. - // The encoder library doesn't really do anything special or different with these texture types, this is mostly here for the benefit of the user. + // The encoder library doesn't really do anything special or different with these texture types, this is mostly here for the benefit of the user. // We do make sure the various constraints are followed (2DArray/cubemap/videoframes/volume implies that each image has the same resolution and # of mipmap levels, etc., cubemap implies that the # of image slices is a multiple of 6) enum basis_texture_type { @@ -88,14 +88,113 @@ namespace basist enum class basis_tex_format { + // Original LDR formats cETC1S = 0, - cUASTC4x4 = 1, + cUASTC_LDR_4x4 = 1, + + // HDR formats cUASTC_HDR_4x4 = 2, cASTC_HDR_6x6 = 3, - cASTC_HDR_6x6_INTERMEDIATE = 4, + cUASTC_HDR_6x6_INTERMEDIATE = 4, // TODO: rename to UASTC_HDR_6x6 + + // XUASTC (supercompressed) LDR variants (the standard ASTC block sizes) + cXUASTC_LDR_4x4 = 5, + cXUASTC_LDR_5x4 = 6, + cXUASTC_LDR_5x5 = 7, + cXUASTC_LDR_6x5 = 8, + + cXUASTC_LDR_6x6 = 9, + cXUASTC_LDR_8x5 = 10, + cXUASTC_LDR_8x6 = 11, + cXUASTC_LDR_10x5 = 12, + + cXUASTC_LDR_10x6 = 13, + cXUASTC_LDR_8x8 = 14, + cXUASTC_LDR_10x8 = 15, + cXUASTC_LDR_10x10 = 16, + + cXUASTC_LDR_12x10 = 17, + cXUASTC_LDR_12x12 = 18, + + // Standard (non-supercompressed) ASTC LDR variants (the standard ASTC block sizes) + cASTC_LDR_4x4 = 19, + cASTC_LDR_5x4 = 20, + cASTC_LDR_5x5 = 21, + cASTC_LDR_6x5 = 22, + + cASTC_LDR_6x6 = 23, + cASTC_LDR_8x5 = 24, + cASTC_LDR_8x6 = 25, + cASTC_LDR_10x5 = 26, + + cASTC_LDR_10x6 = 27, + cASTC_LDR_8x8 = 28, + cASTC_LDR_10x8 = 29, + cASTC_LDR_10x10 = 30, + + cASTC_LDR_12x10 = 31, + cASTC_LDR_12x12 = 32, + cTotalFormats }; + // True if the basis_tex_format is XUASTC LDR 4x4-12x12. + inline bool basis_tex_format_is_xuastc_ldr(basis_tex_format tex_fmt) + { + return ((uint32_t)tex_fmt >= (uint32_t)basis_tex_format::cXUASTC_LDR_4x4) && ((uint32_t)tex_fmt <= (uint32_t)basis_tex_format::cXUASTC_LDR_12x12); + } + + // True if the basis_tex_format is ASTC LDR 4x4-12x12. + inline bool basis_tex_format_is_astc_ldr(basis_tex_format tex_fmt) + { + return ((uint32_t)tex_fmt >= (uint32_t)basis_tex_format::cASTC_LDR_4x4) && ((uint32_t)tex_fmt <= (uint32_t)basis_tex_format::cASTC_LDR_12x12); + } + + inline void get_basis_tex_format_block_size(basis_tex_format tex_fmt, uint32_t &width, uint32_t &height) + { + switch (tex_fmt) + { + case basis_tex_format::cETC1S: width = 4; height = 4; break; + case basis_tex_format::cUASTC_LDR_4x4: width = 4; height = 4; break; + case basis_tex_format::cUASTC_HDR_4x4: width = 4; height = 4; break; + case basis_tex_format::cASTC_HDR_6x6: width = 6; height = 6; break; + case basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE: width = 6; height = 6; break; + case basis_tex_format::cXUASTC_LDR_4x4: width = 4; height = 4; break; + case basis_tex_format::cXUASTC_LDR_5x4: width = 5; height = 4; break; + case basis_tex_format::cXUASTC_LDR_5x5: width = 5; height = 5; break; + case basis_tex_format::cXUASTC_LDR_6x5: width = 6; height = 5; break; + case basis_tex_format::cXUASTC_LDR_6x6: width = 6; height = 6; break; + case basis_tex_format::cXUASTC_LDR_8x5: width = 8; height = 5; break; + case basis_tex_format::cXUASTC_LDR_8x6: width = 8; height = 6; break; + case basis_tex_format::cXUASTC_LDR_10x5: width = 10; height = 5; break; + case basis_tex_format::cXUASTC_LDR_10x6: width = 10; height = 6; break; + case basis_tex_format::cXUASTC_LDR_8x8: width = 8; height = 8; break; + case basis_tex_format::cXUASTC_LDR_10x8: width = 10; height = 8; break; + case basis_tex_format::cXUASTC_LDR_10x10: width = 10; height = 10; break; + case basis_tex_format::cXUASTC_LDR_12x10: width = 12; height = 10; break; + case basis_tex_format::cXUASTC_LDR_12x12: width = 12; height = 12; break; + case basis_tex_format::cASTC_LDR_4x4: width = 4; height = 4; break; + case basis_tex_format::cASTC_LDR_5x4: width = 5; height = 4; break; + case basis_tex_format::cASTC_LDR_5x5: width = 5; height = 5; break; + case basis_tex_format::cASTC_LDR_6x5: width = 6; height = 5; break; + case basis_tex_format::cASTC_LDR_6x6: width = 6; height = 6; break; + case basis_tex_format::cASTC_LDR_8x5: width = 8; height = 5; break; + case basis_tex_format::cASTC_LDR_8x6: width = 8; height = 6; break; + case basis_tex_format::cASTC_LDR_10x5: width = 10; height = 5; break; + case basis_tex_format::cASTC_LDR_10x6: width = 10; height = 6; break; + case basis_tex_format::cASTC_LDR_8x8: width = 8; height = 8; break; + case basis_tex_format::cASTC_LDR_10x8: width = 10; height = 8; break; + case basis_tex_format::cASTC_LDR_10x10: width = 10; height = 10; break; + case basis_tex_format::cASTC_LDR_12x10: width = 12; height = 10; break; + case basis_tex_format::cASTC_LDR_12x12: width = 12; height = 12; break; + default: + assert(0); + width = 0; + height = 0; + break; + } + } + struct basis_file_header { enum @@ -115,7 +214,7 @@ namespace basist basisu::packed_uint<3> m_total_slices; // The total # of compressed slices (1 slice per image, or 2 for alpha .basis files) basisu::packed_uint<3> m_total_images; // The total # of images - + basisu::packed_uint<1> m_tex_format; // enum basis_tex_format basisu::packed_uint<2> m_flags; // enum basist::header_flags basisu::packed_uint<1> m_tex_type; // enum basist::basis_texture_type @@ -125,11 +224,11 @@ namespace basist basisu::packed_uint<4> m_userdata0; // For client use basisu::packed_uint<4> m_userdata1; // For client use - basisu::packed_uint<2> m_total_endpoints; // The number of endpoints in the endpoint codebook + basisu::packed_uint<2> m_total_endpoints; // The number of endpoints in the endpoint codebook basisu::packed_uint<4> m_endpoint_cb_file_ofs; // The compressed endpoint codebook's file offset relative to the start of the file basisu::packed_uint<3> m_endpoint_cb_file_size; // The compressed endpoint codebook's size in bytes - basisu::packed_uint<2> m_total_selectors; // The number of selectors in the endpoint codebook + basisu::packed_uint<2> m_total_selectors; // The number of selectors in the endpoint codebook basisu::packed_uint<4> m_selector_cb_file_ofs; // The compressed selectors codebook's file offset relative to the start of the file basisu::packed_uint<3> m_selector_cb_file_size; // The compressed selector codebook's size in bytes @@ -137,7 +236,7 @@ namespace basist basisu::packed_uint<4> m_tables_file_size; // The file size in bytes of the compressed huffman codelength tables basisu::packed_uint<4> m_slice_desc_file_ofs; // The file offset to the slice description array, usually follows the header - + basisu::packed_uint<4> m_extended_file_ofs; // The file offset of the "extended" header and compressed data, for future use basisu::packed_uint<4> m_extended_file_size; // The file size in bytes of the "extended" header and compressed data, for future use }; diff --git a/external/basis_universal/transcoder/basisu_idct.h b/external/basis_universal/transcoder/basisu_idct.h new file mode 100644 index 0000000000..33b77d0013 --- /dev/null +++ b/external/basis_universal/transcoder/basisu_idct.h @@ -0,0 +1,1446 @@ +// ------------------------------------------------------------ +// 1D ORTHONORMAL IDCT (DCT-III), SIZE 2, FLOAT +// out[x*dst_stride] = sum_k C[k][x] * src[k*src_stride] +// C[k][x] = alpha(k) * cos(pi * (2*x+1) * k / (2*N)), +// alpha(0) = sqrt(1/N), alpha(k>0) = sqrt(2/N) +static inline void idct_1d_2( + const float* src, int src_stride, + float* dst, int dst_stride) +{ + float s0 = 0.0f; + float s1 = 0.0f; + + { + float v = src[0 * src_stride]; + if (v != 0.0f) + { + s0 += 7.071067691e-01f * v; + s1 += 7.071067691e-01f * v; + } + } + + { + float v = src[1 * src_stride]; + if (v != 0.0f) + { + s0 += 7.071067691e-01f * v; + s1 += -7.071067691e-01f * v; + } + } + + dst[0 * dst_stride] = s0; + dst[1 * dst_stride] = s1; +} + +// ------------------------------------------------------------ +// 1D ORTHONORMAL IDCT (DCT-III), SIZE 3, FLOAT +// out[x*dst_stride] = sum_k C[k][x] * src[k*src_stride] +// C[k][x] = alpha(k) * cos(pi * (2*x+1) * k / (2*N)), +// alpha(0) = sqrt(1/N), alpha(k>0) = sqrt(2/N) +static inline void idct_1d_3( + const float* src, int src_stride, + float* dst, int dst_stride) +{ + float s0 = 0.0f; + float s1 = 0.0f; + float s2 = 0.0f; + + { + float v = src[0 * src_stride]; + if (v != 0.0f) + { + s0 += 5.773502588e-01f * v; + s1 += 5.773502588e-01f * v; + s2 += 5.773502588e-01f * v; + } + } + + { + float v = src[1 * src_stride]; + if (v != 0.0f) + { + s0 += 7.071067691e-01f * v; + s2 += -7.071068883e-01f * v; + } + } + + { + float v = src[2 * src_stride]; + if (v != 0.0f) + { + s0 += 4.082482755e-01f * v; + s1 += -8.164966106e-01f * v; + s2 += 4.082486033e-01f * v; + } + } + + dst[0 * dst_stride] = s0; + dst[1 * dst_stride] = s1; + dst[2 * dst_stride] = s2; +} + +// ------------------------------------------------------------ +// 1D ORTHONORMAL IDCT (DCT-III), SIZE 4, FLOAT +// out[x*dst_stride] = sum_k C[k][x] * src[k*src_stride] +// C[k][x] = alpha(k) * cos(pi * (2*x+1) * k / (2*N)), +// alpha(0) = sqrt(1/N), alpha(k>0) = sqrt(2/N) +static inline void idct_1d_4( + const float* src, int src_stride, + float* dst, int dst_stride) +{ + float s0 = 0.0f; + float s1 = 0.0f; + float s2 = 0.0f; + float s3 = 0.0f; + + { + float v = src[0 * src_stride]; + if (v != 0.0f) + { + s0 += 5.000000000e-01f * v; + s1 += 5.000000000e-01f * v; + s2 += 5.000000000e-01f * v; + s3 += 5.000000000e-01f * v; + } + } + + { + float v = src[1 * src_stride]; + if (v != 0.0f) + { + s0 += 6.532814503e-01f * v; + s1 += 2.705980539e-01f * v; + s2 += -2.705981135e-01f * v; + s3 += -6.532815099e-01f * v; + } + } + + { + float v = src[2 * src_stride]; + if (v != 0.0f) + { + s0 += 4.999999702e-01f * v; + s1 += -4.999999702e-01f * v; + s2 += -4.999999106e-01f * v; + s3 += 5.000001788e-01f * v; + } + } + + { + float v = src[3 * src_stride]; + if (v != 0.0f) + { + s0 += 2.705980539e-01f * v; + s1 += -6.532814503e-01f * v; + s2 += 6.532815099e-01f * v; + s3 += -2.705983818e-01f * v; + } + } + + dst[0 * dst_stride] = s0; + dst[1 * dst_stride] = s1; + dst[2 * dst_stride] = s2; + dst[3 * dst_stride] = s3; +} + +// ------------------------------------------------------------ +// 1D ORTHONORMAL IDCT (DCT-III), SIZE 5, FLOAT +// out[x*dst_stride] = sum_k C[k][x] * src[k*src_stride] +// C[k][x] = alpha(k) * cos(pi * (2*x+1) * k / (2*N)), +// alpha(0) = sqrt(1/N), alpha(k>0) = sqrt(2/N) +static inline void idct_1d_5( + const float* src, int src_stride, + float* dst, int dst_stride) +{ + float s0 = 0.0f; + float s1 = 0.0f; + float s2 = 0.0f; + float s3 = 0.0f; + float s4 = 0.0f; + + { + float v = src[0 * src_stride]; + if (v != 0.0f) + { + s0 += 4.472135901e-01f * v; + s1 += 4.472135901e-01f * v; + s2 += 4.472135901e-01f * v; + s3 += 4.472135901e-01f * v; + s4 += 4.472135901e-01f * v; + } + } + + { + float v = src[1 * src_stride]; + if (v != 0.0f) + { + s0 += 6.015009880e-01f * v; + s1 += 3.717480302e-01f * v; + s3 += -3.717481494e-01f * v; + s4 += -6.015009284e-01f * v; + } + } + + { + float v = src[2 * src_stride]; + if (v != 0.0f) + { + s0 += 5.116672516e-01f * v; + s1 += -1.954395324e-01f * v; + s2 += -6.324555278e-01f * v; + s3 += -1.954392791e-01f * v; + s4 += 5.116672516e-01f * v; + } + } + + { + float v = src[3 * src_stride]; + if (v != 0.0f) + { + s0 += 3.717480302e-01f * v; + s1 += -6.015009284e-01f * v; + s3 += 6.015008688e-01f * v; + s4 += -3.717483282e-01f * v; + } + } + + { + float v = src[4 * src_stride]; + if (v != 0.0f) + { + s0 += 1.954394877e-01f * v; + s1 += -5.116672516e-01f * v; + s2 += 6.324555278e-01f * v; + s3 += -5.116675496e-01f * v; + s4 += 1.954394132e-01f * v; + } + } + + dst[0 * dst_stride] = s0; + dst[1 * dst_stride] = s1; + dst[2 * dst_stride] = s2; + dst[3 * dst_stride] = s3; + dst[4 * dst_stride] = s4; +} + +// ------------------------------------------------------------ +// 1D ORTHONORMAL IDCT (DCT-III), SIZE 6, FLOAT +// out[x*dst_stride] = sum_k C[k][x] * src[k*src_stride] +// C[k][x] = alpha(k) * cos(pi * (2*x+1) * k / (2*N)), +// alpha(0) = sqrt(1/N), alpha(k>0) = sqrt(2/N) +static inline void idct_1d_6( + const float* src, int src_stride, + float* dst, int dst_stride) +{ + float s0 = 0.0f; + float s1 = 0.0f; + float s2 = 0.0f; + float s3 = 0.0f; + float s4 = 0.0f; + float s5 = 0.0f; + + { + float v = src[0 * src_stride]; + if (v != 0.0f) + { + s0 += 4.082483053e-01f * v; + s1 += 4.082483053e-01f * v; + s2 += 4.082483053e-01f * v; + s3 += 4.082483053e-01f * v; + s4 += 4.082483053e-01f * v; + s5 += 4.082483053e-01f * v; + } + } + + { + float v = src[1 * src_stride]; + if (v != 0.0f) + { + s0 += 5.576775074e-01f * v; + s1 += 4.082482755e-01f * v; + s2 += 1.494291872e-01f * v; + s3 += -1.494293064e-01f * v; + s4 += -4.082482755e-01f * v; + s5 += -5.576775670e-01f * v; + } + } + + { + float v = src[2 * src_stride]; + if (v != 0.0f) + { + s0 += 4.999999702e-01f * v; + s2 += -5.000000596e-01f * v; + s3 += -4.999999106e-01f * v; + s5 += 5.000000596e-01f * v; + } + } + + { + float v = src[3 * src_stride]; + if (v != 0.0f) + { + s0 += 4.082482755e-01f * v; + s1 += -4.082482755e-01f * v; + s2 += -4.082483053e-01f * v; + s3 += 4.082484245e-01f * v; + s4 += 4.082480669e-01f * v; + s5 += -4.082485437e-01f * v; + } + } + + { + float v = src[4 * src_stride]; + if (v != 0.0f) + { + s0 += 2.886750996e-01f * v; + s1 += -5.773502588e-01f * v; + s2 += 2.886753380e-01f * v; + s3 += 2.886748910e-01f * v; + s4 += -5.773502588e-01f * v; + s5 += 2.886753976e-01f * v; + } + } + + { + float v = src[5 * src_stride]; + if (v != 0.0f) + { + s0 += 1.494291872e-01f * v; + s1 += -4.082483053e-01f * v; + s2 += 5.576775074e-01f * v; + s3 += -5.576776266e-01f * v; + s4 += 4.082483053e-01f * v; + s5 += -1.494295001e-01f * v; + } + } + + dst[0 * dst_stride] = s0; + dst[1 * dst_stride] = s1; + dst[2 * dst_stride] = s2; + dst[3 * dst_stride] = s3; + dst[4 * dst_stride] = s4; + dst[5 * dst_stride] = s5; +} + +// ------------------------------------------------------------ +// 1D ORTHONORMAL IDCT (DCT-III), SIZE 7, FLOAT +// out[x*dst_stride] = sum_k C[k][x] * src[k*src_stride] +// C[k][x] = alpha(k) * cos(pi * (2*x+1) * k / (2*N)), +// alpha(0) = sqrt(1/N), alpha(k>0) = sqrt(2/N) +static inline void idct_1d_7( + const float* src, int src_stride, + float* dst, int dst_stride) +{ + float s0 = 0.0f; + float s1 = 0.0f; + float s2 = 0.0f; + float s3 = 0.0f; + float s4 = 0.0f; + float s5 = 0.0f; + float s6 = 0.0f; + + { + float v = src[0 * src_stride]; + if (v != 0.0f) + { + s0 += 3.779644668e-01f * v; + s1 += 3.779644668e-01f * v; + s2 += 3.779644668e-01f * v; + s3 += 3.779644668e-01f * v; + s4 += 3.779644668e-01f * v; + s5 += 3.779644668e-01f * v; + s6 += 3.779644668e-01f * v; + } + } + + { + float v = src[1 * src_stride]; + if (v != 0.0f) + { + s0 += 5.211208463e-01f * v; + s1 += 4.179065228e-01f * v; + s2 += 2.319205552e-01f * v; + s4 += -2.319206595e-01f * v; + s5 += -4.179066122e-01f * v; + s6 += -5.211208463e-01f * v; + } + } + + { + float v = src[2 * src_stride]; + if (v != 0.0f) + { + s0 += 4.815880954e-01f * v; + s1 += 1.189424619e-01f * v; + s2 += -3.332694173e-01f * v; + s3 += -5.345224738e-01f * v; + s4 += -3.332692087e-01f * v; + s5 += 1.189427450e-01f * v; + s6 += 4.815880954e-01f * v; + } + } + + { + float v = src[3 * src_stride]; + if (v != 0.0f) + { + s0 += 4.179065228e-01f * v; + s1 += -2.319206595e-01f * v; + s2 += -5.211208463e-01f * v; + s4 += 5.211208463e-01f * v; + s5 += 2.319205403e-01f * v; + s6 += -4.179067314e-01f * v; + } + } + + { + float v = src[4 * src_stride]; + if (v != 0.0f) + { + s0 += 3.332692981e-01f * v; + s1 += -4.815880954e-01f * v; + s2 += -1.189422309e-01f * v; + s3 += 5.345224738e-01f * v; + s4 += -1.189426631e-01f * v; + s5 += -4.815878570e-01f * v; + s6 += 3.332692981e-01f * v; + } + } + + { + float v = src[5 * src_stride]; + if (v != 0.0f) + { + s0 += 2.319205552e-01f * v; + s1 += -5.211208463e-01f * v; + s2 += 4.179064631e-01f * v; + s4 += -4.179064035e-01f * v; + s5 += 5.211209059e-01f * v; + s6 += -2.319207191e-01f * v; + } + } + + { + float v = src[6 * src_stride]; + if (v != 0.0f) + { + s0 += 1.189424619e-01f * v; + s1 += -3.332692087e-01f * v; + s2 += 4.815881252e-01f * v; + s3 += -5.345224738e-01f * v; + s4 += 4.815881550e-01f * v; + s5 += -3.332694471e-01f * v; + s6 += 1.189431697e-01f * v; + } + } + + dst[0 * dst_stride] = s0; + dst[1 * dst_stride] = s1; + dst[2 * dst_stride] = s2; + dst[3 * dst_stride] = s3; + dst[4 * dst_stride] = s4; + dst[5 * dst_stride] = s5; + dst[6 * dst_stride] = s6; +} + +// ------------------------------------------------------------ +// 1D ORTHONORMAL IDCT (DCT-III), SIZE 8, FLOAT +// out[x*dst_stride] = sum_k C[k][x] * src[k*src_stride] +// C[k][x] = alpha(k) * cos(pi * (2*x+1) * k / (2*N)), +// alpha(0) = sqrt(1/N), alpha(k>0) = sqrt(2/N) +static inline void idct_1d_8( + const float* src, int src_stride, + float* dst, int dst_stride) +{ + float s0 = 0.0f; + float s1 = 0.0f; + float s2 = 0.0f; + float s3 = 0.0f; + float s4 = 0.0f; + float s5 = 0.0f; + float s6 = 0.0f; + float s7 = 0.0f; + + { + float v = src[0 * src_stride]; + if (v != 0.0f) + { + s0 += 3.535533845e-01f * v; + s1 += 3.535533845e-01f * v; + s2 += 3.535533845e-01f * v; + s3 += 3.535533845e-01f * v; + s4 += 3.535533845e-01f * v; + s5 += 3.535533845e-01f * v; + s6 += 3.535533845e-01f * v; + s7 += 3.535533845e-01f * v; + } + } + + { + float v = src[1 * src_stride]; + if (v != 0.0f) + { + s0 += 4.903926253e-01f * v; + s1 += 4.157347977e-01f * v; + s2 += 2.777850926e-01f * v; + s3 += 9.754511714e-02f * v; + s4 += -9.754516184e-02f * v; + s5 += -2.777851820e-01f * v; + s6 += -4.157348275e-01f * v; + s7 += -4.903926551e-01f * v; + } + } + + { + float v = src[2 * src_stride]; + if (v != 0.0f) + { + s0 += 4.619397521e-01f * v; + s1 += 1.913417131e-01f * v; + s2 += -1.913417578e-01f * v; + s3 += -4.619398117e-01f * v; + s4 += -4.619397521e-01f * v; + s5 += -1.913415641e-01f * v; + s6 += 1.913418025e-01f * v; + s7 += 4.619397819e-01f * v; + } + } + + { + float v = src[3 * src_stride]; + if (v != 0.0f) + { + s0 += 4.157347977e-01f * v; + s1 += -9.754516184e-02f * v; + s2 += -4.903926551e-01f * v; + s3 += -2.777850032e-01f * v; + s4 += 2.777852118e-01f * v; + s5 += 4.903926253e-01f * v; + s6 += 9.754503518e-02f * v; + s7 += -4.157348871e-01f * v; + } + } + + { + float v = src[4 * src_stride]; + if (v != 0.0f) + { + s0 += 3.535533845e-01f * v; + s1 += -3.535533845e-01f * v; + s2 += -3.535533249e-01f * v; + s3 += 3.535535038e-01f * v; + s4 += 3.535533845e-01f * v; + s5 += -3.535536230e-01f * v; + s6 += -3.535532653e-01f * v; + s7 += 3.535534143e-01f * v; + } + } + + { + float v = src[5 * src_stride]; + if (v != 0.0f) + { + s0 += 2.777850926e-01f * v; + s1 += -4.903926551e-01f * v; + s2 += 9.754520655e-02f * v; + s3 += 4.157346785e-01f * v; + s4 += -4.157348871e-01f * v; + s5 += -9.754510969e-02f * v; + s6 += 4.903926551e-01f * v; + s7 += -2.777854204e-01f * v; + } + } + + { + float v = src[6 * src_stride]; + if (v != 0.0f) + { + s0 += 1.913417131e-01f * v; + s1 += -4.619397521e-01f * v; + s2 += 4.619397819e-01f * v; + s3 += -1.913419515e-01f * v; + s4 += -1.913414896e-01f * v; + s5 += 4.619396627e-01f * v; + s6 += -4.619398713e-01f * v; + s7 += 1.913419515e-01f * v; + } + } + + { + float v = src[7 * src_stride]; + if (v != 0.0f) + { + s0 += 9.754511714e-02f * v; + s1 += -2.777850032e-01f * v; + s2 += 4.157346785e-01f * v; + s3 += -4.903925955e-01f * v; + s4 += 4.903927147e-01f * v; + s5 += -4.157347977e-01f * v; + s6 += 2.777855694e-01f * v; + s7 += -9.754577279e-02f * v; + } + } + + dst[0 * dst_stride] = s0; + dst[1 * dst_stride] = s1; + dst[2 * dst_stride] = s2; + dst[3 * dst_stride] = s3; + dst[4 * dst_stride] = s4; + dst[5 * dst_stride] = s5; + dst[6 * dst_stride] = s6; + dst[7 * dst_stride] = s7; +} + +// ------------------------------------------------------------ +// 1D ORTHONORMAL IDCT (DCT-III), SIZE 9, FLOAT +// out[x*dst_stride] = sum_k C[k][x] * src[k*src_stride] +// C[k][x] = alpha(k) * cos(pi * (2*x+1) * k / (2*N)), +// alpha(0) = sqrt(1/N), alpha(k>0) = sqrt(2/N) +static inline void idct_1d_9( + const float* src, int src_stride, + float* dst, int dst_stride) +{ + float s0 = 0.0f; + float s1 = 0.0f; + float s2 = 0.0f; + float s3 = 0.0f; + float s4 = 0.0f; + float s5 = 0.0f; + float s6 = 0.0f; + float s7 = 0.0f; + float s8 = 0.0f; + + { + float v = src[0 * src_stride]; + if (v != 0.0f) + { + s0 += 3.333333433e-01f * v; + s1 += 3.333333433e-01f * v; + s2 += 3.333333433e-01f * v; + s3 += 3.333333433e-01f * v; + s4 += 3.333333433e-01f * v; + s5 += 3.333333433e-01f * v; + s6 += 3.333333433e-01f * v; + s7 += 3.333333433e-01f * v; + s8 += 3.333333433e-01f * v; + } + } + + { + float v = src[1 * src_stride]; + if (v != 0.0f) + { + s0 += 4.642428160e-01f * v; + s1 += 4.082482755e-01f * v; + s2 += 3.030129671e-01f * v; + s3 += 1.612297893e-01f * v; + s5 += -1.612298936e-01f * v; + s6 += -3.030129969e-01f * v; + s7 += -4.082482755e-01f * v; + s8 += -4.642428458e-01f * v; + } + } + + { + float v = src[2 * src_stride]; + if (v != 0.0f) + { + s0 += 4.429753423e-01f * v; + s1 += 2.357022464e-01f * v; + s2 += -8.185859025e-02f * v; + s3 += -3.611168861e-01f * v; + s4 += -4.714045227e-01f * v; + s5 += -3.611167669e-01f * v; + s6 += -8.185851574e-02f * v; + s7 += 2.357022166e-01f * v; + s8 += 4.429753721e-01f * v; + } + } + + { + float v = src[3 * src_stride]; + if (v != 0.0f) + { + s0 += 4.082482755e-01f * v; + s2 += -4.082482755e-01f * v; + s3 += -4.082482159e-01f * v; + s5 += 4.082483649e-01f * v; + s6 += 4.082482755e-01f * v; + s8 += -4.082485437e-01f * v; + } + } + + { + float v = src[4 * src_stride]; + if (v != 0.0f) + { + s0 += 3.611168265e-01f * v; + s1 += -2.357022911e-01f * v; + s2 += -4.429753125e-01f * v; + s3 += 8.185874671e-02f * v; + s4 += 4.714045227e-01f * v; + s5 += 8.185835928e-02f * v; + s6 += -4.429753721e-01f * v; + s7 += -2.357023507e-01f * v; + s8 += 3.611169457e-01f * v; + } + } + + { + float v = src[5 * src_stride]; + if (v != 0.0f) + { + s0 += 3.030129671e-01f * v; + s1 += -4.082482755e-01f * v; + s2 += -1.612298042e-01f * v; + s3 += 4.642428458e-01f * v; + s5 += -4.642428160e-01f * v; + s6 += 1.612296849e-01f * v; + s7 += 4.082482159e-01f * v; + s8 += -3.030129373e-01f * v; + } + } + + { + float v = src[6 * src_stride]; + if (v != 0.0f) + { + s0 += 2.357022464e-01f * v; + s1 += -4.714045227e-01f * v; + s2 += 2.357022166e-01f * v; + s3 += 2.357020825e-01f * v; + s4 += -4.714045227e-01f * v; + s5 += 2.357024848e-01f * v; + s6 += 2.357022017e-01f * v; + s7 += -4.714045227e-01f * v; + s8 += 2.357031256e-01f * v; + } + } + + { + float v = src[7 * src_stride]; + if (v != 0.0f) + { + s0 += 1.612297893e-01f * v; + s1 += -4.082482159e-01f * v; + s2 += 4.642428458e-01f * v; + s3 += -3.030130565e-01f * v; + s5 += 3.030129075e-01f * v; + s6 += -4.642427862e-01f * v; + s7 += 4.082485735e-01f * v; + s8 += -1.612301171e-01f * v; + } + } + + { + float v = src[8 * src_stride]; + if (v != 0.0f) + { + s0 += 8.185850084e-02f * v; + s1 += -2.357022166e-01f * v; + s2 += 3.611166775e-01f * v; + s3 += -4.429752231e-01f * v; + s4 += 4.714045227e-01f * v; + s5 += -4.429754615e-01f * v; + s6 += 3.611168563e-01f * v; + s7 += -2.357021123e-01f * v; + s8 += 8.185899258e-02f * v; + } + } + + dst[0 * dst_stride] = s0; + dst[1 * dst_stride] = s1; + dst[2 * dst_stride] = s2; + dst[3 * dst_stride] = s3; + dst[4 * dst_stride] = s4; + dst[5 * dst_stride] = s5; + dst[6 * dst_stride] = s6; + dst[7 * dst_stride] = s7; + dst[8 * dst_stride] = s8; +} + +// ------------------------------------------------------------ +// 1D ORTHONORMAL IDCT (DCT-III), SIZE 10, FLOAT +// out[x*dst_stride] = sum_k C[k][x] * src[k*src_stride] +// C[k][x] = alpha(k) * cos(pi * (2*x+1) * k / (2*N)), +// alpha(0) = sqrt(1/N), alpha(k>0) = sqrt(2/N) +static inline void idct_1d_10( + const float* src, int src_stride, + float* dst, int dst_stride) +{ + float s0 = 0.0f; + float s1 = 0.0f; + float s2 = 0.0f; + float s3 = 0.0f; + float s4 = 0.0f; + float s5 = 0.0f; + float s6 = 0.0f; + float s7 = 0.0f; + float s8 = 0.0f; + float s9 = 0.0f; + + { + float v = src[0 * src_stride]; + if (v != 0.0f) + { + s0 += 3.162277639e-01f * v; + s1 += 3.162277639e-01f * v; + s2 += 3.162277639e-01f * v; + s3 += 3.162277639e-01f * v; + s4 += 3.162277639e-01f * v; + s5 += 3.162277639e-01f * v; + s6 += 3.162277639e-01f * v; + s7 += 3.162277639e-01f * v; + s8 += 3.162277639e-01f * v; + s9 += 3.162277639e-01f * v; + } + } + + { + float v = src[1 * src_stride]; + if (v != 0.0f) + { + s0 += 4.417076707e-01f * v; + s1 += 3.984702229e-01f * v; + s2 += 3.162277639e-01f * v; + s3 += 2.030306906e-01f * v; + s4 += 6.995963305e-02f * v; + s5 += -6.995966285e-02f * v; + s6 += -2.030307651e-01f * v; + s7 += -3.162277639e-01f * v; + s8 += -3.984702528e-01f * v; + s9 += -4.417076707e-01f * v; + } + } + + { + float v = src[2 * src_stride]; + if (v != 0.0f) + { + s0 += 4.253254235e-01f * v; + s1 += 2.628655434e-01f * v; + s3 += -2.628656328e-01f * v; + s4 += -4.253253937e-01f * v; + s5 += -4.253253639e-01f * v; + s6 += -2.628654838e-01f * v; + s8 += 2.628656626e-01f * v; + s9 += 4.253254235e-01f * v; + } + } + + { + float v = src[3 * src_stride]; + if (v != 0.0f) + { + s0 += 3.984702229e-01f * v; + s1 += 6.995963305e-02f * v; + s2 += -3.162277639e-01f * v; + s3 += -4.417076409e-01f * v; + s4 += -2.030306011e-01f * v; + s5 += 2.030307949e-01f * v; + s6 += 4.417076707e-01f * v; + s7 += 3.162277639e-01f * v; + s8 += -6.995979697e-02f * v; + s9 += -3.984701931e-01f * v; + } + } + + { + float v = src[4 * src_stride]; + if (v != 0.0f) + { + s0 += 3.618034124e-01f * v; + s1 += -1.381966174e-01f * v; + s2 += -4.472135901e-01f * v; + s3 += -1.381964386e-01f * v; + s4 += 3.618033826e-01f * v; + s5 += 3.618032932e-01f * v; + s6 += -1.381967962e-01f * v; + s7 += -4.472135901e-01f * v; + s8 += -1.381963789e-01f * v; + s9 += 3.618034124e-01f * v; + } + } + + { + float v = src[5 * src_stride]; + if (v != 0.0f) + { + s0 += 3.162277639e-01f * v; + s1 += -3.162277639e-01f * v; + s2 += -3.162277043e-01f * v; + s3 += 3.162278533e-01f * v; + s4 += 3.162277639e-01f * v; + s5 += -3.162276745e-01f * v; + s6 += -3.162276447e-01f * v; + s7 += 3.162280619e-01f * v; + s8 += 3.162278533e-01f * v; + s9 += -3.162281811e-01f * v; + } + } + + { + float v = src[6 * src_stride]; + if (v != 0.0f) + { + s0 += 2.628655434e-01f * v; + s1 += -4.253253937e-01f * v; + s3 += 4.253253639e-01f * v; + s4 += -2.628657520e-01f * v; + s5 += -2.628654242e-01f * v; + s6 += 4.253254235e-01f * v; + s8 += -4.253252745e-01f * v; + s9 += 2.628654540e-01f * v; + } + } + + { + float v = src[7 * src_stride]; + if (v != 0.0f) + { + s0 += 2.030306906e-01f * v; + s1 += -4.417076409e-01f * v; + s2 += 3.162278533e-01f * v; + s3 += 6.995949894e-02f * v; + s4 += -3.984701633e-01f * v; + s5 += 3.984702528e-01f * v; + s6 += -6.996008009e-02f * v; + s7 += -3.162274361e-01f * v; + s8 += 4.417077899e-01f * v; + s9 += -2.030310780e-01f * v; + } + } + + { + float v = src[8 * src_stride]; + if (v != 0.0f) + { + s0 += 1.381965876e-01f * v; + s1 += -3.618033826e-01f * v; + s2 += 4.472135901e-01f * v; + s3 += -3.618035913e-01f * v; + s4 += 1.381965429e-01f * v; + s5 += 1.381962299e-01f * v; + s6 += -3.618031442e-01f * v; + s7 += 4.472135901e-01f * v; + s8 += -3.618036509e-01f * v; + s9 += 1.381966770e-01f * v; + } + } + + { + float v = src[9 * src_stride]; + if (v != 0.0f) + { + s0 += 6.995963305e-02f * v; + s1 += -2.030306011e-01f * v; + s2 += 3.162277639e-01f * v; + s3 += -3.984701633e-01f * v; + s4 += 4.417076409e-01f * v; + s5 += -4.417076409e-01f * v; + s6 += 3.984701931e-01f * v; + s7 += -3.162280619e-01f * v; + s8 += 2.030308247e-01f * v; + s9 += -6.995939463e-02f * v; + } + } + + dst[0 * dst_stride] = s0; + dst[1 * dst_stride] = s1; + dst[2 * dst_stride] = s2; + dst[3 * dst_stride] = s3; + dst[4 * dst_stride] = s4; + dst[5 * dst_stride] = s5; + dst[6 * dst_stride] = s6; + dst[7 * dst_stride] = s7; + dst[8 * dst_stride] = s8; + dst[9 * dst_stride] = s9; +} + +// ------------------------------------------------------------ +// 1D ORTHONORMAL IDCT (DCT-III), SIZE 11, FLOAT +// out[x*dst_stride] = sum_k C[k][x] * src[k*src_stride] +// C[k][x] = alpha(k) * cos(pi * (2*x+1) * k / (2*N)), +// alpha(0) = sqrt(1/N), alpha(k>0) = sqrt(2/N) +static inline void idct_1d_11( + const float* src, int src_stride, + float* dst, int dst_stride) +{ + float s0 = 0.0f; + float s1 = 0.0f; + float s2 = 0.0f; + float s3 = 0.0f; + float s4 = 0.0f; + float s5 = 0.0f; + float s6 = 0.0f; + float s7 = 0.0f; + float s8 = 0.0f; + float s9 = 0.0f; + float s10 = 0.0f; + + { + float v = src[0 * src_stride]; + if (v != 0.0f) + { + s0 += 3.015113473e-01f * v; + s1 += 3.015113473e-01f * v; + s2 += 3.015113473e-01f * v; + s3 += 3.015113473e-01f * v; + s4 += 3.015113473e-01f * v; + s5 += 3.015113473e-01f * v; + s6 += 3.015113473e-01f * v; + s7 += 3.015113473e-01f * v; + s8 += 3.015113473e-01f * v; + s9 += 3.015113473e-01f * v; + s10 += 3.015113473e-01f * v; + } + } + + { + float v = src[1 * src_stride]; + if (v != 0.0f) + { + s0 += 4.220612943e-01f * v; + s1 += 3.878683746e-01f * v; + s2 += 3.222526908e-01f * v; + s3 += 2.305300087e-01f * v; + s4 += 1.201311573e-01f * v; + s6 += -1.201311946e-01f * v; + s7 += -2.305300087e-01f * v; + s8 += -3.222527206e-01f * v; + s9 += -3.878683746e-01f * v; + s10 += -4.220612943e-01f * v; + } + } + + { + float v = src[2 * src_stride]; + if (v != 0.0f) + { + s0 += 4.091291726e-01f * v; + s1 += 2.792335451e-01f * v; + s2 += 6.068321317e-02f * v; + s3 += -1.771336049e-01f * v; + s4 += -3.587117195e-01f * v; + s5 += -4.264014363e-01f * v; + s6 += -3.587116897e-01f * v; + s7 += -1.771335900e-01f * v; + s8 += 6.068333238e-02f * v; + s9 += 2.792335451e-01f * v; + s10 += 4.091292024e-01f * v; + } + } + + { + float v = src[3 * src_stride]; + if (v != 0.0f) + { + s0 += 3.878683746e-01f * v; + s1 += 1.201311573e-01f * v; + s2 += -2.305300087e-01f * v; + s3 += -4.220612943e-01f * v; + s4 += -3.222526908e-01f * v; + s6 += 3.222527504e-01f * v; + s7 += 4.220612645e-01f * v; + s8 += 2.305298299e-01f * v; + s9 += -1.201310679e-01f * v; + s10 += -3.878685534e-01f * v; + } + } + + { + float v = src[4 * src_stride]; + if (v != 0.0f) + { + s0 += 3.587117195e-01f * v; + s1 += -6.068325043e-02f * v; + s2 += -4.091292024e-01f * v; + s3 += -2.792334855e-01f * v; + s4 += 1.771336049e-01f * v; + s5 += 4.264014363e-01f * v; + s6 += 1.771334559e-01f * v; + s7 += -2.792335153e-01f * v; + s8 += -4.091291428e-01f * v; + s9 += -6.068325043e-02f * v; + s10 += 3.587118387e-01f * v; + } + } + + { + float v = src[5 * src_stride]; + if (v != 0.0f) + { + s0 += 3.222526908e-01f * v; + s1 += -2.305300087e-01f * v; + s2 += -3.878683448e-01f * v; + s3 += 1.201313213e-01f * v; + s4 += 4.220612645e-01f * v; + s6 += -4.220612943e-01f * v; + s7 += -1.201310530e-01f * v; + s8 += 3.878682852e-01f * v; + s9 += 2.305295914e-01f * v; + s10 += -3.222530484e-01f * v; + } + } + + { + float v = src[6 * src_stride]; + if (v != 0.0f) + { + s0 += 2.792335451e-01f * v; + s1 += -3.587117195e-01f * v; + s2 += -1.771335900e-01f * v; + s3 += 4.091292024e-01f * v; + s4 += 6.068318710e-02f * v; + s5 += -4.264014363e-01f * v; + s6 += 6.068341061e-02f * v; + s7 += 4.091290832e-01f * v; + s8 += -1.771339774e-01f * v; + s9 += -3.587118387e-01f * v; + s10 += 2.792341411e-01f * v; + } + } + + { + float v = src[7 * src_stride]; + if (v != 0.0f) + { + s0 += 2.305300087e-01f * v; + s1 += -4.220612943e-01f * v; + s2 += 1.201313213e-01f * v; + s3 += 3.222525418e-01f * v; + s4 += -3.878685534e-01f * v; + s6 += 3.878683150e-01f * v; + s7 += -3.222530484e-01f * v; + s8 += -1.201303899e-01f * v; + s9 += 4.220611751e-01f * v; + s10 += -2.305305302e-01f * v; + } + } + + { + float v = src[8 * src_stride]; + if (v != 0.0f) + { + s0 += 1.771335304e-01f * v; + s1 += -4.091291726e-01f * v; + s2 += 3.587118089e-01f * v; + s3 += -6.068347394e-02f * v; + s4 += -2.792334855e-01f * v; + s5 += 4.264014363e-01f * v; + s6 += -2.792337239e-01f * v; + s7 += -6.068337709e-02f * v; + s8 += 3.587115407e-01f * v; + s9 += -4.091291726e-01f * v; + s10 += 1.771339774e-01f * v; + } + } + + { + float v = src[9 * src_stride]; + if (v != 0.0f) + { + s0 += 1.201311573e-01f * v; + s1 += -3.222526908e-01f * v; + s2 += 4.220612645e-01f * v; + s3 += -3.878685534e-01f * v; + s4 += 2.305301726e-01f * v; + s6 += -2.305298299e-01f * v; + s7 += 3.878681958e-01f * v; + s8 += -4.220613837e-01f * v; + s9 += 3.222527504e-01f * v; + s10 += -1.201314703e-01f * v; + } + } + + { + float v = src[10 * src_stride]; + if (v != 0.0f) + { + s0 += 6.068321317e-02f * v; + s1 += -1.771335900e-01f * v; + s2 += 2.792334557e-01f * v; + s3 += -3.587115407e-01f * v; + s4 += 4.091290832e-01f * v; + s5 += -4.264014363e-01f * v; + s6 += 4.091292620e-01f * v; + s7 += -3.587118387e-01f * v; + s8 += 2.792330980e-01f * v; + s9 += -1.771344692e-01f * v; + s10 += 6.068423390e-02f * v; + } + } + + dst[0 * dst_stride] = s0; + dst[1 * dst_stride] = s1; + dst[2 * dst_stride] = s2; + dst[3 * dst_stride] = s3; + dst[4 * dst_stride] = s4; + dst[5 * dst_stride] = s5; + dst[6 * dst_stride] = s6; + dst[7 * dst_stride] = s7; + dst[8 * dst_stride] = s8; + dst[9 * dst_stride] = s9; + dst[10 * dst_stride] = s10; +} + +// ------------------------------------------------------------ +// 1D ORTHONORMAL IDCT (DCT-III), SIZE 12, FLOAT +// out[x*dst_stride] = sum_k C[k][x] * src[k*src_stride] +// C[k][x] = alpha(k) * cos(pi * (2*x+1) * k / (2*N)), +// alpha(0) = sqrt(1/N), alpha(k>0) = sqrt(2/N) +static inline void idct_1d_12( + const float* src, int src_stride, + float* dst, int dst_stride) +{ + float s0 = 0.0f; + float s1 = 0.0f; + float s2 = 0.0f; + float s3 = 0.0f; + float s4 = 0.0f; + float s5 = 0.0f; + float s6 = 0.0f; + float s7 = 0.0f; + float s8 = 0.0f; + float s9 = 0.0f; + float s10 = 0.0f; + float s11 = 0.0f; + + { + float v = src[0 * src_stride]; + if (v != 0.0f) + { + s0 += 2.886751294e-01f * v; + s1 += 2.886751294e-01f * v; + s2 += 2.886751294e-01f * v; + s3 += 2.886751294e-01f * v; + s4 += 2.886751294e-01f * v; + s5 += 2.886751294e-01f * v; + s6 += 2.886751294e-01f * v; + s7 += 2.886751294e-01f * v; + s8 += 2.886751294e-01f * v; + s9 += 2.886751294e-01f * v; + s10 += 2.886751294e-01f * v; + s11 += 2.886751294e-01f * v; + } + } + + { + float v = src[1 * src_stride]; + if (v != 0.0f) + { + s0 += 4.047556818e-01f * v; + s1 += 3.771722317e-01f * v; + s2 += 3.238851428e-01f * v; + s3 += 2.485257983e-01f * v; + s4 += 1.562298536e-01f * v; + s5 += 5.328707024e-02f * v; + s6 += -5.328710750e-02f * v; + s7 += -1.562298536e-01f * v; + s8 += -2.485258281e-01f * v; + s9 += -3.238851428e-01f * v; + s10 += -3.771722913e-01f * v; + s11 += -4.047556818e-01f * v; + } + } + + { + float v = src[2 * src_stride]; + if (v != 0.0f) + { + s0 += 3.943375647e-01f * v; + s1 += 2.886751294e-01f * v; + s2 += 1.056623980e-01f * v; + s3 += -1.056624874e-01f * v; + s4 += -2.886751294e-01f * v; + s5 += -3.943375945e-01f * v; + s6 += -3.943375647e-01f * v; + s7 += -2.886751592e-01f * v; + s8 += -1.056624129e-01f * v; + s9 += 1.056624204e-01f * v; + s10 += 2.886752486e-01f * v; + s11 += 3.943375647e-01f * v; + } + } + + { + float v = src[3 * src_stride]; + if (v != 0.0f) + { + s0 += 3.771722317e-01f * v; + s1 += 1.562298536e-01f * v; + s2 += -1.562298536e-01f * v; + s3 += -3.771722913e-01f * v; + s4 += -3.771722317e-01f * v; + s5 += -1.562297344e-01f * v; + s6 += 1.562299281e-01f * v; + s7 += 3.771722615e-01f * v; + s8 += 3.771722019e-01f * v; + s9 += 1.562297940e-01f * v; + s10 += -1.562300622e-01f * v; + s11 += -3.771722317e-01f * v; + } + } + + { + float v = src[4 * src_stride]; + if (v != 0.0f) + { + s0 += 3.535533845e-01f * v; + s2 += -3.535534441e-01f * v; + s3 += -3.535533547e-01f * v; + s5 += 3.535534739e-01f * v; + s6 += 3.535533845e-01f * v; + s8 += -3.535534143e-01f * v; + s9 += -3.535534143e-01f * v; + s11 += 3.535532951e-01f * v; + } + } + + { + float v = src[5 * src_stride]; + if (v != 0.0f) + { + s0 += 3.238851428e-01f * v; + s1 += -1.562298536e-01f * v; + s2 += -4.047556818e-01f * v; + s3 += -5.328698456e-02f * v; + s4 += 3.771722615e-01f * v; + s5 += 2.485257536e-01f * v; + s6 += -2.485258281e-01f * v; + s7 += -3.771722317e-01f * v; + s8 += 5.328687653e-02f * v; + s9 += 4.047557116e-01f * v; + s10 += 1.562295407e-01f * v; + s11 += -3.238854110e-01f * v; + } + } + + { + float v = src[6 * src_stride]; + if (v != 0.0f) + { + s0 += 2.886751294e-01f * v; + s1 += -2.886751294e-01f * v; + s2 += -2.886751592e-01f * v; + s3 += 2.886752486e-01f * v; + s4 += 2.886749804e-01f * v; + s5 += -2.886753380e-01f * v; + s6 += -2.886750400e-01f * v; + s7 += 2.886751592e-01f * v; + s8 += 2.886749506e-01f * v; + s9 += -2.886752486e-01f * v; + s10 += -2.886748612e-01f * v; + s11 += 2.886750698e-01f * v; + } + } + + { + float v = src[7 * src_stride]; + if (v != 0.0f) + { + s0 += 2.485257983e-01f * v; + s1 += -3.771722913e-01f * v; + s2 += -5.328698456e-02f * v; + s3 += 4.047556818e-01f * v; + s4 += -1.562300622e-01f * v; + s5 += -3.238852322e-01f * v; + s6 += 3.238853514e-01f * v; + s7 += 1.562295407e-01f * v; + s8 += -4.047557414e-01f * v; + s9 += 5.328752100e-02f * v; + s10 += 3.771720827e-01f * v; + s11 += -2.485256344e-01f * v; + } + } + + { + float v = src[8 * src_stride]; + if (v != 0.0f) + { + s0 += 2.041241378e-01f * v; + s1 += -4.082483053e-01f * v; + s2 += 2.041243017e-01f * v; + s3 += 2.041239887e-01f * v; + s4 += -4.082483053e-01f * v; + s5 += 2.041243464e-01f * v; + s6 += 2.041241080e-01f * v; + s7 += -4.082483053e-01f * v; + s8 += 2.041242570e-01f * v; + s9 += 2.041241974e-01f * v; + s10 += -4.082483053e-01f * v; + s11 += 2.041237950e-01f * v; + } + } + + { + float v = src[9 * src_stride]; + if (v != 0.0f) + { + s0 += 1.562298536e-01f * v; + s1 += -3.771722317e-01f * v; + s2 += 3.771722615e-01f * v; + s3 += -1.562300622e-01f * v; + s4 += -1.562296748e-01f * v; + s5 += 3.771723211e-01f * v; + s6 += -3.771723509e-01f * v; + s7 += 1.562300622e-01f * v; + s8 += 1.562293023e-01f * v; + s9 += -3.771721721e-01f * v; + s10 += 3.771724999e-01f * v; + s11 += -1.562300622e-01f * v; + } + } + + { + float v = src[10 * src_stride]; + if (v != 0.0f) + { + s0 += 1.056623980e-01f * v; + s1 += -2.886751592e-01f * v; + s2 += 3.943375647e-01f * v; + s3 += -3.943376541e-01f * v; + s4 += 2.886751592e-01f * v; + s5 += -1.056626216e-01f * v; + s6 += -1.056624576e-01f * v; + s7 += 2.886750400e-01f * v; + s8 += -3.943376839e-01f * v; + s9 += 3.943377137e-01f * v; + s10 += -2.886756361e-01f * v; + s11 += 1.056632623e-01f * v; + } + } + + { + float v = src[11 * src_stride]; + if (v != 0.0f) + { + s0 += 5.328707024e-02f * v; + s1 += -1.562297344e-01f * v; + s2 += 2.485257536e-01f * v; + s3 += -3.238852322e-01f * v; + s4 += 3.771723211e-01f * v; + s5 += -4.047556818e-01f * v; + s6 += 4.047556818e-01f * v; + s7 += -3.771722913e-01f * v; + s8 += 3.238852024e-01f * v; + s9 += -2.485264540e-01f * v; + s10 += 1.562305540e-01f * v; + s11 += -5.328702182e-02f * v; + } + } + + dst[0 * dst_stride] = s0; + dst[1 * dst_stride] = s1; + dst[2 * dst_stride] = s2; + dst[3 * dst_stride] = s3; + dst[4 * dst_stride] = s4; + dst[5 * dst_stride] = s5; + dst[6 * dst_stride] = s6; + dst[7 * dst_stride] = s7; + dst[8 * dst_stride] = s8; + dst[9 * dst_stride] = s9; + dst[10 * dst_stride] = s10; + dst[11 * dst_stride] = s11; +} diff --git a/external/basis_universal/transcoder/basisu_transcoder.cpp b/external/basis_universal/transcoder/basisu_transcoder.cpp index f55d34e1d4..a511d8d697 100644 --- a/external/basis_universal/transcoder/basisu_transcoder.cpp +++ b/external/basis_universal/transcoder/basisu_transcoder.cpp @@ -1,5 +1,5 @@ // basisu_transcoder.cpp -// Copyright (C) 2019-2024 Binomial LLC. All Rights Reserved. +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -107,6 +107,10 @@ #define BASISD_SUPPORT_ASTC 1 #endif +#ifndef BASISD_SUPPORT_XUASTC +#define BASISD_SUPPORT_XUASTC 1 +#endif + // Note that if BASISD_SUPPORT_ATC is enabled, BASISD_SUPPORT_DXT5A should also be enabled for alpha support. #ifndef BASISD_SUPPORT_ATC #define BASISD_SUPPORT_ATC 1 @@ -161,6 +165,7 @@ #define BASISD_WRITE_NEW_ETC2_EAC_R11_TABLES 0 #ifndef BASISD_ENABLE_DEBUG_FLAGS + // DO NOT CHECK IN #define BASISD_ENABLE_DEBUG_FLAGS 0 #endif @@ -177,6 +182,14 @@ using namespace basist::astc_6x6_hdr; #endif +#if BASISD_IS_BIG_ENDIAN +const uint32_t BASISD_COLOR_RGBA_A_MASK = 0x000000FF; +const uint32_t BASISD_COLOR_RGBA_RGB_MASK = ~BASISD_COLOR_RGBA_A_MASK; +#else +const uint32_t BASISD_COLOR_RGBA_A_MASK = 0xFF000000; +const uint32_t BASISD_COLOR_RGBA_RGB_MASK = ~BASISD_COLOR_RGBA_A_MASK; +#endif + namespace basisu { bool g_debug_printf; @@ -188,7 +201,7 @@ namespace basisu void debug_printf(const char* pFmt, ...) { -#if BASISU_FORCE_DEVEL_MESSAGES +#if BASISU_FORCE_DEVEL_MESSAGES g_debug_printf = true; #endif if (g_debug_printf) @@ -202,7 +215,7 @@ namespace basisu void debug_puts(const char* p) { -#if BASISU_FORCE_DEVEL_MESSAGES +#if BASISU_FORCE_DEVEL_MESSAGES g_debug_printf = true; #endif if (g_debug_printf) @@ -235,6 +248,14 @@ namespace basist g_debug_flags = f; #endif } + + // Used by arith encoder/decoder + namespace arith_fastbits_f32 + { + bool g_initialized; + float g_lut_edge[TABLE_SIZE + 1]; // samples at m = 1 + i/TABLE_SIZE (for linear) + + } // namespace arith_fastbits_f32 inline uint16_t byteswap_uint16(uint16_t v) { @@ -312,6 +333,8 @@ namespace basist res[i] = basisu::lerp(a[i], b[i], s); return res; } + + inline float norm() const { return dot(*this); } }; uint16_t crc16(const void* r, size_t size, uint16_t crc) @@ -329,6 +352,62 @@ namespace basist return static_cast(~crc); } + uint32_t hash_hsieh(const uint8_t* pBuf, size_t len) + { + if (!pBuf || !len) + return 0; + + uint32_t h = static_cast(len); + + const uint32_t bytes_left = len & 3; + len >>= 2; + + while (len--) + { + const uint16_t* pWords = reinterpret_cast(pBuf); + + h += pWords[0]; + + const uint32_t t = (pWords[1] << 11) ^ h; + h = (h << 16) ^ t; + + pBuf += sizeof(uint32_t); + + h += h >> 11; + } + + switch (bytes_left) + { + case 1: + h += *reinterpret_cast(pBuf); + h ^= h << 10; + h += h >> 1; + break; + case 2: + h += *reinterpret_cast(pBuf); + h ^= h << 11; + h += h >> 17; + break; + case 3: + h += *reinterpret_cast(pBuf); + h ^= h << 16; + h ^= (static_cast(pBuf[sizeof(uint16_t)])) << 18; + h += h >> 11; + break; + default: + break; + } + + h ^= h << 3; + h += h >> 5; + h ^= h << 4; + h += h >> 17; + h ^= h << 25; + h += h >> 6; + + return h; + } + struct vec4F { float c[4]; @@ -338,7 +417,7 @@ namespace basist float operator[] (uint32_t index) const { assert(index < 4); return c[index]; } float& operator[] (uint32_t index) { assert(index < 4); return c[index]; } }; - + enum etc_constants { cETC1BytesPerBlock = 8U, @@ -411,14 +490,14 @@ namespace basist //const uint8_t g_etc1_to_selector_index[cETC1SelectorValues] = { 2, 3, 1, 0 }; const uint8_t g_selector_index_to_etc1[cETC1SelectorValues] = { 3, 2, 0, 1 }; - + static const uint8_t g_etc_5_to_8[32] = { 0, 8, 16, 24, 33, 41, 49, 57, 66, 74, 82, 90, 99, 107, 115, 123, 132, 140, 148, 156, 165, 173, 181, 189, 198, 206, 214, 222, 231, 239, 247, 255 }; struct decoder_etc_block { // big endian uint64: // bit ofs: 56 48 40 32 24 16 8 0 - // byte ofs: b0, b1, b2, b3, b4, b5, b6, b7 + // byte ofs: b0, b1, b2, b3, b4, b5, b6, b7 union { uint64_t m_uint64; @@ -686,7 +765,7 @@ namespace basist { return (m_bytes[3] & 2) != 0; } - + inline uint32_t get_inten_table(uint32_t subblock_id) const { assert(subblock_id < 2); @@ -701,7 +780,7 @@ namespace basist const uint32_t b = get_byte_bits(cETC1DeltaColor3BBitOffset, 3); return static_cast(b | (g << 3U) | (r << 6U)); } - + void get_block_colors(color32* pBlock_colors, uint32_t subblock_index) const { color32 b; @@ -819,7 +898,7 @@ namespace basist g = c.g; b = c.b; } - + static void unpack_color5(color32& result, uint16_t packed_color5, bool scaled) { result = unpack_color5(packed_color5, scaled, 255); @@ -948,7 +1027,7 @@ namespace basist static void get_block_color5_r(const color32& base_color5, uint32_t inten_table, uint32_t index, uint32_t &r) { assert(index < 4); - + uint32_t br = (base_color5.r << 3) | (base_color5.r >> 2); const int* pInten_table = g_etc1_inten_tables[inten_table]; @@ -1124,7 +1203,7 @@ namespace basist { 1, 2, 2, 2 }, { 1, 2, 3, 3 }, }; - + static uint8_t g_etc1_to_dxt1_selector_mappings_raw_dxt1_256[NUM_ETC1_TO_DXT1_SELECTOR_MAPPINGS][256]; static uint8_t g_etc1_to_dxt1_selector_mappings_raw_dxt1_inv_256[NUM_ETC1_TO_DXT1_SELECTOR_MAPPINGS][256]; @@ -1515,9 +1594,9 @@ namespace basist return best_err; } #endif // BASISD_WRITE_NEW_ETC2_EAC_A8_TABLES - + static -#if !BASISD_WRITE_NEW_ETC2_EAC_A8_TABLES +#if !BASISD_WRITE_NEW_ETC2_EAC_A8_TABLES const #endif etc1_g_to_eac_conversion s_etc1_g_to_etc2_a8[32 * 8][NUM_ETC2_EAC_SELECTOR_RANGES] = @@ -2006,7 +2085,7 @@ namespace basist void uastc_init(); #endif -#if BASISD_SUPPORT_UASTC_HDR +#if BASISD_SUPPORT_UASTC_HDR namespace astc_6x6_hdr { static void init_quantize_tables(); @@ -2021,19 +2100,24 @@ namespace basist } #endif - static bool g_transcoder_initialized; + namespace astc_ldr_t + { + void init_transcoding_tables(); + } + static bool g_transcoder_initialized; + // Library global initialization. Requires ~9 milliseconds when compiled and executed natively on a Core i7 2.2 GHz. // If this is too slow, these computed tables can easilky be moved to be compiled in. void basisu_transcoder_init() { if (g_transcoder_initialized) { - BASISU_DEVEL_ERROR("basisu_transcoder::basisu_transcoder_init: Called more than once\n"); + BASISU_DEVEL_ERROR("basisu_transcoder::basisu_transcoder_init: Called more than once\n"); return; } - - BASISU_DEVEL_ERROR("basisu_transcoder::basisu_transcoder_init: Initializing (this is not an error)\n"); + + BASISU_DEVEL_ERROR("basisu_transcoder::basisu_transcoder_init: Initializing (this is not an error)\n"); #if BASISD_SUPPORT_UASTC uastc_init(); @@ -2041,7 +2125,8 @@ namespace basist #if BASISD_SUPPORT_UASTC_HDR // TODO: Examine this, optimize for startup time/mem utilization. - astc_helpers::init_tables(false); + // XUASTC LDR decompressors need the rank tables + astc_helpers::init_tables(); astc_hdr_core_init(); #endif @@ -2049,7 +2134,7 @@ namespace basist #if BASISD_SUPPORT_ASTC transcoder_init_astc(); #endif - + #if BASISD_WRITE_NEW_ASTC_TABLES create_etc1_to_astc_conversion_table_0_47(); create_etc1_to_astc_conversion_table_0_255(); @@ -2159,11 +2244,25 @@ namespace basist astc_6x6_hdr::init_quantize_tables(); fast_encode_bc6h_init(); #endif - + #if BASISD_SUPPORT_BC7_MODE5 bc7_mode_5_encoder::encode_bc7_mode5_init(); #endif +#if BASISD_SUPPORT_XUASTC + // TODO: XUASTC support macro + astc_ldr_t::init(); + + astc_ldr_t::init_transcoding_tables(); + + // Used by arith encoder/decoder + arith_fastbits_f32::init(); + + // Used by astc ldr transcoding + bc7f::init(); + etc1f::init(); +#endif + g_transcoder_initialized = true; } @@ -2315,7 +2414,7 @@ namespace basist std::swap(l, h); pSelectors_xlat_256 = &g_etc1_to_dxt1_selector_mappings_raw_dxt1_inv_256[best_mapping][0]; } - + pDst_block->set_low_color(static_cast(l)); pDst_block->set_high_color(static_cast(h)); @@ -2475,7 +2574,7 @@ namespace basist fxt1_block* pBlock = static_cast(pDst); // CC_MIXED is basically DXT1 with different encoding tricks. - // So transcode ETC1S to DXT1, then transcode that to FXT1 which is easy and nearly lossless. + // So transcode ETC1S to DXT1, then transcode that to FXT1 which is easy and nearly lossless. // (It's not completely lossless because FXT1 rounds in its color lerps while DXT1 doesn't, but it should be good enough.) dxt1_block blk; convert_etc1s_to_dxt1(&blk, pEndpoints, pSelectors, false); @@ -2488,7 +2587,7 @@ namespace basist uint32_t g0 = color0.g & 1; uint32_t g1 = color1.g & 1; - + color0.g >>= 1; color1.g >>= 1; @@ -2496,7 +2595,7 @@ namespace basist blk.m_selectors[1] = conv_dxt1_to_fxt1_sels(blk.m_selectors[1]); blk.m_selectors[2] = conv_dxt1_to_fxt1_sels(blk.m_selectors[2]); blk.m_selectors[3] = conv_dxt1_to_fxt1_sels(blk.m_selectors[3]); - + if ((blk.get_selector(0, 0) >> 1) != (g0 ^ g1)) { std::swap(color0, color1); @@ -2510,7 +2609,7 @@ namespace basist if (fxt1_subblock == 0) { - pBlock->m_hi.m_mode = 1; + pBlock->m_hi.m_mode = 1; pBlock->m_hi.m_alpha = 0; pBlock->m_hi.m_glsb = g1 | (g1 << 1); pBlock->m_hi.m_r0 = color0.r; @@ -2831,7 +2930,7 @@ namespace basist { uint32_t r; decoder_etc_block::get_block_color5_r(base_color, inten_table, low_selector, r); - + pDst_block->set_low_alpha(r); pDst_block->set_high_alpha(r); pDst_block->m_selectors[0] = 0; @@ -2914,7 +3013,7 @@ namespace basist static const uint8_t g_pvrtc_4[16] = { 0,16,33,49,66,82,99,115,140,156,173,189,206,222,239,255 }; static const uint8_t g_pvrtc_3[8] = { 0,33,74,107,148,181,222,255 }; static const uint8_t g_pvrtc_alpha[9] = { 0,34,68,102,136,170,204,238,255 }; - + static const uint8_t g_pvrtc_5_floor[256] = { 0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3, @@ -2938,7 +3037,7 @@ namespace basist 24,24,24,24,24,24,24,25,25,25,25,25,25,25,25,26,26,26,26,26,26,26,26,27,27,27,27,27,27,27,27,28, 28,28,28,28,28,28,28,28,29,29,29,29,29,29,29,29,30,30,30,30,30,30,30,30,31,31,31,31,31,31,31,31 }; - + static const uint8_t g_pvrtc_4_floor[256] = { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, @@ -2962,7 +3061,7 @@ namespace basist 12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,14, 14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15 }; - + static const uint8_t g_pvrtc_3_floor[256] = { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, @@ -2986,7 +3085,7 @@ namespace basist 6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,7, 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7 }; - + static const uint8_t g_pvrtc_alpha_floor[256] = { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, @@ -3093,10 +3192,10 @@ namespace basist } assert((r < 32) && (g < 32) && (b < 32) && (a < 16)); - + return color32(r, g, b, a); } - + inline color32 get_endpoint_8888(uint32_t endpoint_index) const { assert(endpoint_index < 2); @@ -3143,7 +3242,7 @@ namespace basist a = g_pvrtc_alpha[a]; } - + return color32(r, g, b, a); } @@ -3152,7 +3251,7 @@ namespace basist color32 c(get_endpoint_8888(endpoint_index)); return c.r + c.g + c.b + c.a; } - + inline uint32_t get_opaque_endpoint_l0() const { uint32_t packed = m_endpoints & 0xFFFE; @@ -3267,7 +3366,7 @@ namespace basist else m_endpoints = (m_endpoints & 0xFFFF0000U) | packed; } - + // opaque endpoints: 554 or 555 // transparent endpoints: 3443 or 3444 inline void set_endpoint_raw(uint32_t endpoint_index, const color32& c, bool opaque_endpoint) @@ -3320,7 +3419,7 @@ namespace basist else m_endpoints = (m_endpoints & 0xFFFF0000U) | packed; } - + inline void set_endpoint_floor(uint32_t endpoint_index, const color32& c) { assert(endpoint_index < 2); @@ -3430,12 +3529,6 @@ namespace basist }; #endif - struct pvrtc1_temp_block - { - decoder_etc_block m_etc1_block; - uint32_t m_pvrtc_endpoints; - }; - static inline uint32_t get_opaque_endpoint_l0(uint32_t endpoints) { uint32_t packed = endpoints; @@ -3545,7 +3638,7 @@ namespace basist for (int ey = 0; ey < 3; ey++) { - int by = y + ey - 1; + int by = y + ey - 1; const uint32_t* pE = &pPVRTC_endpoints[(by & y_mask) * num_blocks_x]; @@ -3553,7 +3646,7 @@ namespace basist for (int ex = 0; ex < 3; ex++) { - int bx = 0 + ex - 1; + int bx = 0 + ex - 1; const uint32_t e = pE[bx & x_mask]; @@ -3702,8 +3795,8 @@ namespace basist } static void fixup_pvrtc1_4_modulation_rgba( - const decoder_etc_block* pETC_Blocks, - const uint32_t* pPVRTC_endpoints, + const decoder_etc_block* pETC_Blocks, + const uint32_t* pPVRTC_endpoints, void* pDst_blocks, uint32_t num_blocks_x, uint32_t num_blocks_y, void *pAlpha_blocks, const endpoint* pEndpoints, const selector* pSelectors) { @@ -3726,7 +3819,7 @@ namespace basist for (int ey = 0; ey < 3; ey++) { - int by = y + ey - 1; + int by = y + ey - 1; const uint32_t* pE = &pPVRTC_endpoints[(by & y_mask) * num_blocks_x]; @@ -3734,7 +3827,7 @@ namespace basist for (int ex = 0; ex < 3; ex++) { - int bx = 0 + ex - 1; + int bx = 0 + ex - 1; const uint32_t e = pE[bx & x_mask]; @@ -3748,13 +3841,13 @@ namespace basist for (int x = 0; x < static_cast(num_blocks_x); x++, block_index++) { const decoder_etc_block& src_block = pETC_Blocks[block_index]; - + const uint16_t* pSrc_alpha_block = reinterpret_cast(static_cast(pAlpha_blocks) + x + (y * num_blocks_x)); const endpoint* pAlpha_endpoints = &pEndpoints[pSrc_alpha_block[0]]; const selector* pAlpha_selectors = &pSelectors[pSrc_alpha_block[1]]; - + const uint32_t x_swizzle = (g_pvrtc_swizzle_table[x >> 8] << 17) | (g_pvrtc_swizzle_table[x & 0xFF] << 1); - + uint32_t swizzled = x_swizzle | y_swizzle; if (num_blocks_x != num_blocks_y) { @@ -3897,7 +3990,7 @@ namespace basist const uint32_t NUM_ETC1_TO_BC7_M5_SELECTOR_RANGES = sizeof(g_etc1_to_bc7_m5_selector_ranges) / sizeof(g_etc1_to_bc7_m5_selector_ranges[0]); static uint32_t g_etc1_to_bc7_m5_selector_range_index[4][4]; - + const uint32_t NUM_ETC1_TO_BC7_M5_SELECTOR_MAPPINGS = 10; static const uint8_t g_etc1_to_bc7_m5_selector_mappings[NUM_ETC1_TO_BC7_M5_SELECTOR_MAPPINGS][4] = { @@ -3919,11 +4012,11 @@ namespace basist uint8_t m_hi; uint16_t m_err; }; - + static const etc1_to_bc7_m5_solution g_etc1_to_bc7_m5_color[32 * 8 * NUM_ETC1_TO_BC7_M5_SELECTOR_MAPPINGS * NUM_ETC1_TO_BC7_M5_SELECTOR_RANGES] = { #include "basisu_transcoder_tables_bc7_m5_color.inc" }; - + static dxt_selector_range g_etc1_to_bc7_m5a_selector_ranges[] = { { 0, 3 }, @@ -3948,7 +4041,7 @@ namespace basist { #include "basisu_transcoder_tables_bc7_m5_alpha.inc" }; - + static inline uint32_t set_block_bits(uint8_t* pBytes, uint32_t val, uint32_t num_bits, uint32_t cur_ofs) { assert(num_bits < 32); @@ -3973,83 +4066,6 @@ namespace basist return cur_ofs; } - struct bc7_mode_5 - { - union - { - struct - { - uint64_t m_mode : 6; - uint64_t m_rot : 2; - - uint64_t m_r0 : 7; - uint64_t m_r1 : 7; - uint64_t m_g0 : 7; - uint64_t m_g1 : 7; - uint64_t m_b0 : 7; - uint64_t m_b1 : 7; - uint64_t m_a0 : 8; - uint64_t m_a1_0 : 6; - - } m_lo; - - uint64_t m_lo_bits; - }; - - union - { - struct - { - uint64_t m_a1_1 : 2; - - // bit 2 - uint64_t m_c00 : 1; - uint64_t m_c10 : 2; - uint64_t m_c20 : 2; - uint64_t m_c30 : 2; - - uint64_t m_c01 : 2; - uint64_t m_c11 : 2; - uint64_t m_c21 : 2; - uint64_t m_c31 : 2; - - uint64_t m_c02 : 2; - uint64_t m_c12 : 2; - uint64_t m_c22 : 2; - uint64_t m_c32 : 2; - - uint64_t m_c03 : 2; - uint64_t m_c13 : 2; - uint64_t m_c23 : 2; - uint64_t m_c33 : 2; - - // bit 33 - uint64_t m_a00 : 1; - uint64_t m_a10 : 2; - uint64_t m_a20 : 2; - uint64_t m_a30 : 2; - - uint64_t m_a01 : 2; - uint64_t m_a11 : 2; - uint64_t m_a21 : 2; - uint64_t m_a31 : 2; - - uint64_t m_a02 : 2; - uint64_t m_a12 : 2; - uint64_t m_a22 : 2; - uint64_t m_a32 : 2; - - uint64_t m_a03 : 2; - uint64_t m_a13 : 2; - uint64_t m_a23 : 2; - uint64_t m_a33 : 2; - - } m_hi; - - uint64_t m_hi_bits; - }; - }; - #if BASISD_WRITE_NEW_BC7_MODE5_TABLES static void create_etc1_to_bc7_m5_color_conversion_table() { @@ -4095,7 +4111,7 @@ namespace basist int err = block_colors[s].g - colors[g_etc1_to_bc7_m5_selector_mappings[m][s]]; int err_scale = 1; - // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor + // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor // the low/high selectors which are clamping to either 0 or 255. if (((inten == 7) && (low_selector == 0) && (high_selector == 3)) && ((s == 0) || (s == 3))) err_scale = 5; @@ -4174,7 +4190,7 @@ namespace basist int mapping_err = block_colors[s].g - colors[k]; mapping_err *= mapping_err; - // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor + // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor // the low/high selectors which are clamping to either 0 or 255. if (((inten == 7) && (low_selector == 0) && (high_selector == 3)) && ((s == 0) || (s == 3))) mapping_err *= 5; @@ -4185,7 +4201,7 @@ namespace basist best_k = k; } } // k - + total_err += best_mapping_err; output_selectors |= (best_k << (s * 2)); } // s @@ -4200,7 +4216,7 @@ namespace basist } // lo } // hi - + fprintf(pFile, "{%u,%u,%u},", best_lo, best_hi, best_output_selectors); n++; if ((n & 31) == 31) @@ -4239,7 +4255,7 @@ namespace basist {127,104},{126,105},{126,106},{127,106},{127,107},{126,108},{125,109},{127,109},{126,110},{126,111},{127,111},{127,112},{126,113},{126,114},{127,114},{127,115}, {126,116},{126,117},{127,117},{127,118},{126,119},{126,120},{127,120},{127,121},{126,122},{126,123},{127,123},{127,124},{126,125},{126,126},{127,126},{127,127} }; - + static void transcoder_init_bc7_mode5() { #if 0 @@ -4267,9 +4283,9 @@ namespace basist } } // hi - + } // lo - + printf("{%u,%u},", g_bc7_m5_equals_1[i].m_hi, g_bc7_m5_equals_1[i].m_lo); if ((i & 15) == 15) printf("\n"); } @@ -4293,7 +4309,7 @@ namespace basist static void convert_etc1s_to_bc7_m5_color(void* pDst, const endpoint* pEndpoints, const selector* pSelector) { bc7_mode_5* pDst_block = static_cast(pDst); - + // First ensure the block is cleared to all 0's static_cast(pDst)[0] = 0; static_cast(pDst)[1] = 0; @@ -4419,7 +4435,7 @@ namespace basist pDst_block->m_lo.m_r1 = pTable_r[best_mapping].m_lo; pDst_block->m_lo.m_g1 = pTable_g[best_mapping].m_lo; pDst_block->m_lo.m_b1 = pTable_b[best_mapping].m_lo; - + s_inv = 3; } else @@ -4440,7 +4456,7 @@ namespace basist for (uint32_t x = 0; x < 4; x++) { const uint32_t s = pSelector->get_selector(x, y); - + const uint32_t os = pSelectors_xlat[s] ^ s_inv; output_bits |= (os << output_bit_ofs); @@ -4470,7 +4486,7 @@ namespace basist pDst_block->m_lo.m_a0 = r; pDst_block->m_lo.m_a1_0 = r & 63; pDst_block->m_hi.m_a1_1 = r >> 6; - + return; } else if (pSelector->m_num_unique_selectors == 2) @@ -4520,7 +4536,7 @@ namespace basist } const uint32_t selector_range_table = g_etc1_to_bc7_m5a_selector_range_index[low_selector][high_selector]; - + const etc1_g_to_bc7_m5a_conversion* pTable = &g_etc1_g_to_bc7_m5a[inten_table * (32 * NUM_ETC1_TO_BC7_M5A_SELECTOR_RANGES) + base_color_r * NUM_ETC1_TO_BC7_M5A_SELECTOR_RANGES + selector_range_table]; pDst_block->m_lo.m_a0 = pTable->m_lo; @@ -4561,10 +4577,12 @@ namespace basist set_block_bits((uint8_t*)pDst, output_bits, 31, 97); } +#if 0 static inline vec3F rgb_to_ycocg(const vec3F& rgb) { return vec3F(rgb.dot(vec3F(0.25f, 0.5f, 0.25f)), rgb.dot(vec3F(0.5f, 0.0f, -0.5f)), rgb.dot(vec3F(-0.25f, 0.5f, -0.25f))); } +#endif static inline vec2F rgb_to_cocg(const vec3F& rgb) { @@ -4576,11 +4594,14 @@ namespace basist return vec3F(ycocg.dot(vec3F(1.0f, 1.0f, -1.0f)), ycocg.dot(vec3F(1.0f, 0.0f, 1.0f)), ycocg.dot(vec3F(1.0f, -1.0f, -1.0f))); } +#if 0 static inline vec3F color32_to_vec3F(const color32& c) { return vec3F(c.r, c.g, c.b); } +#endif +#if 0 static inline vec3F color5_to_ycocg(const endpoint& e) { const int r = (e.m_color5[0] << 3) | (e.m_color5[0] >> 2); @@ -4588,6 +4609,7 @@ namespace basist const int b = (e.m_color5[2] << 3) | (e.m_color5[2] >> 2); return rgb_to_ycocg(vec3F((float)r, (float)g, (float)b)); } +#endif static inline vec2F color5_to_cocg(const endpoint& e) { @@ -4620,7 +4642,7 @@ namespace basist const bool hq_bc7_mode_5_encoder_mode = false; const int CHROMA_THRESH = 10; - + uint32_t total_filtered_blocks = 0; BASISU_NOTE_UNUSED(total_filtered_blocks); @@ -4629,7 +4651,7 @@ namespace basist for (int bx = 0; bx < (int)num_blocks_x; bx++) { vec2F center_cocg(color5_to_cocg(pEndpoints[decoded_endpoints(bx, by)])); - + //bool filter_flag = false; for (int dy = -1; dy <= 1; dy++) { @@ -4667,7 +4689,7 @@ namespace basist total_filtered_blocks++; bc7_mode_5* pDst_block = (bc7_mode_5*)(static_cast(pDst_blocks) + (bx + by * output_row_pitch_in_blocks_or_pixels) * sizeof(bc7_mode_5)); - + //memset(pDst_block, 0x80, 16); int lr = bc7_7_to_8(pDst_block->m_lo.m_r0); @@ -4691,7 +4713,7 @@ namespace basist float block_y_vals[16]; // [y][x] float y_sum = 0.0f, y_sum_sq = 0.0f; - + for (uint32_t i = 0; i < 16; i++) { uint32_t sel = sel_bits & (i ? 3 : 1); @@ -4700,7 +4722,7 @@ namespace basist block_y_vals[i] = y; y_sum += y; y_sum_sq += y * y; - + } // i const float S = 1.0f / 16.0f; @@ -4723,14 +4745,14 @@ namespace basist const float fy = ((float)((bpy + 2) & 3) + .5f) * (1.0f / 4.0f); const int ubx = bx + ((bpx - 2) >> 2); - + vec2F a(get_endpoint_cocg_clamped(ubx, uby, decoded_endpoints, pEndpoints)); vec2F b(get_endpoint_cocg_clamped(ubx + 1, uby, decoded_endpoints, pEndpoints)); vec2F c(get_endpoint_cocg_clamped(ubx, uby + 1, decoded_endpoints, pEndpoints)); vec2F d(get_endpoint_cocg_clamped(ubx + 1, uby + 1, decoded_endpoints, pEndpoints)); assert((fx >= 0) && (fx <= 1.0f) && (fy >= 0) && (fy <= 1.0f)); - + // TODO: Could merge this into 4 muls on each corner by weights vec2F ab = vec2F::lerp(a, b, fx); vec2F cd = vec2F::lerp(c, d, fx); @@ -4747,7 +4769,7 @@ namespace basist } // y bc7_mode_5_encoder::encode_bc7_mode_5_block(pDst_block, block_to_pack, hq_bc7_mode_5_encoder_mode); - + } // bx } // by @@ -5136,7 +5158,7 @@ namespace basist } #endif // BASISD_SUPPORT_ETC2_EAC_RG11 -// ASTC + // ASTC struct etc1_to_astc_solution { uint8_t m_lo; @@ -5183,7 +5205,7 @@ namespace basist // The best selector mapping to use given a base base+inten table and used selector range for converting grayscale data. static uint8_t g_etc1_to_astc_best_grayscale_mapping[32][8][NUM_ETC1_TO_ASTC_SELECTOR_RANGES]; - + #if BASISD_SUPPORT_ASTC_HIGHER_OPAQUE_QUALITY static const etc1_to_astc_solution g_etc1_to_astc_0_255[32 * 8 * NUM_ETC1_TO_ASTC_SELECTOR_MAPPINGS * NUM_ETC1_TO_ASTC_SELECTOR_RANGES] = { #include "basisu_transcoder_tables_astc_0_255.inc" @@ -5248,7 +5270,7 @@ namespace basist int err = block_colors[s].g - colors[g_etc1_to_astc_selector_mappings[m][s]]; int err_scale = 1; - // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor + // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor // the low/high selectors which are clamping to either 0 or 255. if (((inten == 7) && (low_selector == 0) && (high_selector == 3)) && ((s == 0) || (s == 3))) err_scale = 8; @@ -5269,7 +5291,7 @@ namespace basist mapping_best_high[m] = best_hi; mapping_best_err[m] = best_err; highest_best_err = basisu::maximum(highest_best_err, best_err); - + } // m for (uint32_t m = 0; m < NUM_ETC1_TO_ASTC_SELECTOR_MAPPINGS; m++) @@ -5345,7 +5367,7 @@ namespace basist { int err = block_colors[s].g - colors[g_etc1_to_astc_selector_mappings[m][s]]; - // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor + // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor // the low/high selectors which are clamping to either 0 or 255. int err_scale = 1; if (((inten == 7) && (low_selector == 0) && (high_selector == 3)) && ((s == 0) || (s == 3))) @@ -5374,9 +5396,9 @@ namespace basist uint64_t err = mapping_best_err[m]; err = basisu::minimum(err, 0xFFFF); - + fprintf(pFile, "{%u,%u,%u},", mapping_best_low[m], mapping_best_high[m], (uint32_t)err); - + n++; if ((n & 31) == 31) fprintf(pFile, "\n"); @@ -5459,14 +5481,14 @@ namespace basist struct astc_block_params { // 2 groups of 5, but only a max of 8 are used (RRGGBBAA00) - uint8_t m_endpoints[10]; + uint8_t m_endpoints[10]; uint8_t m_weights[32]; }; - - // Packs a single format ASTC block using Color Endpoint Mode 12 (LDR RGBA direct), endpoint BISE range 13, 2-bit weights (range 2). + + // Packs a single format ASTC block using Color Endpoint Mode 12 (LDR RGBA direct), endpoint BISE range 13, 2-bit weights (range 2). // We're always going to output blocks containing alpha, even if the input doesn't have alpha, for simplicity. // Each block always has 4x4 weights, uses range 13 BISE encoding on the endpoints (0-47), and each weight ranges from 0-3. This encoding should be roughly equal in quality vs. BC1 for color. - // 8 total endpoints, stored as RGBA LH LH LH LH order, each ranging from 0-47. + // 8 total endpoints, stored as RGBA LH LH LH LH order, each ranging from 0-47. // Note the input [0,47] endpoint values are not linear - they are encoded as outlined in the ASTC spec: // https://www.khronos.org/registry/DataFormat/specs/1.2/dataformat.1.2.html#astc-endpoint-unquantization // 32 total weights, stored as 16 CA CA, each ranging from 0-3. @@ -5488,7 +5510,7 @@ namespace basist astc_encode_trits(pOutput, pBlock->m_endpoints + 5, bit_pos, 4); // Pack 32 2-bit weights, which are stored from the top down into the block in opposite bit order. - + for (uint32_t i = 0; i < 32; i++) { static const uint8_t s_reverse_bits[4] = { 0, 2, 1, 3 }; @@ -5497,7 +5519,7 @@ namespace basist } } - // CEM mode 12 (LDR RGBA Direct), 8-bit endpoints, 1-bit weights + // CEM mode 12 (LDR RGBA Direct), 8-bit endpoints, 1-bit weights // This ASTC mode is basically block truncation coding (BTC) using 1-bit weights and 8-bit/component endpoints - very convenient. static void astc_pack_block_cem_12_weight_range0(uint32_t* pOutput, const astc_block_params* pBlock) { @@ -5535,7 +5557,7 @@ namespace basist // https://www.khronos.org/registry/DataFormat/specs/1.2/dataformat.1.2.html#_block_mode pBytes[0] = 0x42; pBytes[1] = 0x84; pBytes[2] = 0x00; pBytes[3] = 0x00; pBytes[4] = 0x00; pBytes[5] = 0x00; pBytes[6] = 0x00; pBytes[7] = 0xc0; - + pOutput[2] = 0; pOutput[3] = 0; @@ -5561,7 +5583,7 @@ namespace basist // Write constant block mode, color component selector, number of partitions, color endpoint mode // https://www.khronos.org/registry/DataFormat/specs/1.2/dataformat.1.2.html#_block_mode pBytes[0] = 0x42; pBytes[1] = 0x00; pBytes[2] = 0x01; pBytes[3] = 0x00; - + pOutput[1] = 0; pOutput[2] = 0; pOutput[3] = 0; @@ -5589,7 +5611,7 @@ namespace basist { uint8_t m_lo, m_hi; } g_astc_single_color_encoding_1[256]; - + static void transcoder_init_astc() { for (uint32_t base_color = 0; base_color < 32; base_color++) @@ -5667,7 +5689,7 @@ namespace basist g_ise_to_unquant[bit | (trit << 4)] = unq; } } - + // Compute table used for optimal single color encoding. for (int i = 0; i < 256; i++) { @@ -5682,9 +5704,9 @@ namespace basist int l = lo_v | (lo_v << 8); int h = hi_v | (hi_v << 8); - + int v = ((l * (64 - 21) + (h * 21) + 32) / 64) >> 8; - + int e = abs(v - i); if (e < lowest_e) @@ -5706,7 +5728,7 @@ namespace basist for (int lo = 0; lo < 48; lo++) { const int lo_v = g_ise_to_unquant[lo]; - + int e = abs(lo_v - i); if (e < lowest_e) @@ -5721,7 +5743,7 @@ namespace basist // Converts opaque or color+alpha ETC1S block to ASTC 4x4. // This function tries to use the best ASTC mode given the block's actual contents. - static void convert_etc1s_to_astc_4x4(void* pDst_block, const endpoint* pEndpoints, const selector* pSelector, + static void convert_etc1s_to_astc_4x4(void* pDst_block, const endpoint* pEndpoints, const selector* pSelector, bool transcode_alpha, const endpoint *pEndpoint_codebook, const selector *pSelector_codebook) { astc_block_params blk; @@ -5765,7 +5787,7 @@ namespace basist // See https://www.khronos.org/registry/DataFormat/specs/1.2/dataformat.1.2.html#astc-void-extent-blocks uint32_t r, g, b; decoder_etc_block::get_block_color5(base_color, inten_table, low_selector, r, g, b); - + uint32_t* pOutput = static_cast(pDst_block); uint8_t* pBytes = reinterpret_cast(pDst_block); @@ -5785,7 +5807,7 @@ namespace basist } else if ((pSelector->m_num_unique_selectors <= 2) && (num_unique_alpha_selectors <= 2)) { - // Both color and alpha use <= 2 unique selectors each. + // Both color and alpha use <= 2 unique selectors each. // Use block truncation coding, which is lossless with ASTC (8-bit endpoints, 1-bit weights). color32 block_colors[4]; decoder_etc_block::get_block_colors5(block_colors, base_color, inten_table); @@ -5832,7 +5854,7 @@ namespace basist { uint32_t s = alpha_selectors.get_selector(x, y); s = (s == alpha_high_selector) ? 1 : 0; - + blk.m_weights[(x + y * 4) * 2 + 1] = static_cast(s); } // x } // y @@ -5865,12 +5887,12 @@ namespace basist return; } - + // Either alpha and/or color use > 2 unique selectors each, so we must do something more complex. - + #if BASISD_SUPPORT_ASTC_HIGHER_OPAQUE_QUALITY // The optional higher quality modes use 8-bits endpoints vs. [0,47] endpoints. - + // If the block's base color is grayscale, all pixels are grayscale, so encode the block as Luminance+Alpha. if ((base_color.r == base_color.g) && (base_color.r == base_color.b)) { @@ -5904,7 +5926,7 @@ namespace basist { // Convert ETC1S alpha const uint32_t alpha_selector_range_table = g_etc1_to_astc_selector_range_index[alpha_low_selector][alpha_high_selector]; - + //[32][8][RANGES][MAPPING] const etc1_to_astc_solution* pTable_g = &g_etc1_to_astc_0_255[(alpha_inten_table * 32 + alpha_base_color.g) * (NUM_ETC1_TO_ASTC_SELECTOR_RANGES * NUM_ETC1_TO_ASTC_SELECTOR_MAPPINGS) + alpha_selector_range_table * NUM_ETC1_TO_ASTC_SELECTOR_MAPPINGS]; @@ -5912,7 +5934,7 @@ namespace basist blk.m_endpoints[2] = pTable_g[best_mapping].m_lo; blk.m_endpoints[3] = pTable_g[best_mapping].m_hi; - + const uint8_t* pSelectors_xlat = &g_etc1_to_astc_selector_mappings[best_mapping][0]; for (uint32_t y = 0; y < 4; y++) @@ -5956,10 +5978,10 @@ namespace basist { // Convert ETC1S alpha const uint32_t selector_range_table = g_etc1_to_astc_selector_range_index[low_selector][high_selector]; - + //[32][8][RANGES][MAPPING] const etc1_to_astc_solution* pTable_g = &g_etc1_to_astc_0_255[(inten_table * 32 + base_color.g) * (NUM_ETC1_TO_ASTC_SELECTOR_RANGES * NUM_ETC1_TO_ASTC_SELECTOR_MAPPINGS) + selector_range_table * NUM_ETC1_TO_ASTC_SELECTOR_MAPPINGS]; - + const uint32_t best_mapping = g_etc1_to_astc_best_grayscale_mapping_0_255[base_color.g][inten_table][selector_range_table]; blk.m_endpoints[0] = pTable_g[best_mapping].m_lo; @@ -6101,7 +6123,7 @@ namespace basist { // Convert ETC1S alpha const uint32_t alpha_selector_range_table = g_etc1_to_astc_selector_range_index[alpha_low_selector][alpha_high_selector]; - + //[32][8][RANGES][MAPPING] const etc1_to_astc_solution* pTable_g = &g_etc1_to_astc[(alpha_inten_table * 32 + alpha_base_color.g) * (NUM_ETC1_TO_ASTC_SELECTOR_RANGES * NUM_ETC1_TO_ASTC_SELECTOR_MAPPINGS) + alpha_selector_range_table * NUM_ETC1_TO_ASTC_SELECTOR_MAPPINGS]; @@ -6145,7 +6167,7 @@ namespace basist const uint32_t r = block_colors[low_selector].r; const uint32_t g = block_colors[low_selector].g; const uint32_t b = block_colors[low_selector].b; - + blk.m_endpoints[0] = g_astc_single_color_encoding_1[r].m_lo; blk.m_endpoints[1] = g_astc_single_color_encoding_1[r].m_hi; @@ -6247,7 +6269,7 @@ namespace basist blk.m_endpoints[4] = pTable_b[best_mapping].m_lo; blk.m_endpoints[5] = pTable_b[best_mapping].m_hi; - + int s0 = g_ise_to_unquant[blk.m_endpoints[0]] + g_ise_to_unquant[blk.m_endpoints[2]] + g_ise_to_unquant[blk.m_endpoints[4]]; int s1 = g_ise_to_unquant[blk.m_endpoints[1]] + g_ise_to_unquant[blk.m_endpoints[3]] + g_ise_to_unquant[blk.m_endpoints[5]]; bool invert = false; @@ -6412,8 +6434,8 @@ namespace basist static void transcoder_init_atc() { prepare_atc_single_color_table(g_pvrtc2_match45_equals_1, 16, 32, 1); - prepare_atc_single_color_table(g_atc_match55_equals_1, 32, 32, 1); - prepare_atc_single_color_table(g_atc_match56_equals_1, 32, 64, 1); + prepare_atc_single_color_table(g_atc_match55_equals_1, 32, 32, 1); + prepare_atc_single_color_table(g_atc_match56_equals_1, 32, 64, 1); prepare_atc_single_color_table(g_pvrtc2_match4, 1, 16, 3); prepare_atc_single_color_table(g_atc_match5, 1, 32, 3); @@ -6467,7 +6489,7 @@ namespace basist pBlock->set_low_color(g_atc_match55_equals_1[r].m_lo, g_atc_match56_equals_1[g].m_lo, g_atc_match55_equals_1[b].m_lo); pBlock->set_high_color(g_atc_match55_equals_1[r].m_hi, g_atc_match56_equals_1[g].m_hi, g_atc_match55_equals_1[b].m_hi); - + pBlock->m_sels[0] = 0x55; pBlock->m_sels[1] = 0x55; pBlock->m_sels[2] = 0x55; @@ -6602,7 +6624,7 @@ namespace basist int err = block_colors[s].g - colors[g_etc1s_to_atc_selector_mappings[m][s]]; int err_scale = 1; - // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor + // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor // the low/high selectors which are clamping to either 0 or 255. if (((inten == 7) && (low_selector == 0) && (high_selector == 3)) && ((s == 0) || (s == 3))) err_scale = 5; @@ -6676,7 +6698,7 @@ namespace basist int err = block_colors[s].g - colors[g_etc1s_to_atc_selector_mappings[m][s]]; int err_scale = 1; - // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor + // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor // the low/high selectors which are clamping to either 0 or 255. if (((inten == 7) && (low_selector == 0) && (high_selector == 3)) && ((s == 0) || (s == 3))) err_scale = 5; @@ -6706,7 +6728,7 @@ namespace basist } // inten fclose(pFile); - + // PVRTC2 45 fopen_s(&pFile, "basisu_transcoder_tables_pvrtc2_45.inc", "w"); @@ -6751,7 +6773,7 @@ namespace basist int err = block_colors[s].g - colors[g_etc1s_to_atc_selector_mappings[m][s]]; int err_scale = 1; - // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor + // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor // the low/high selectors which are clamping to either 0 or 255. if (((inten == 7) && (low_selector == 0) && (high_selector == 3)) && ((s == 0) || (s == 3))) err_scale = 5; @@ -6828,7 +6850,7 @@ namespace basist int err = block_colors[s].g - colors[g_etc1s_to_atc_selector_mappings[m][s]]; int err_scale = 1; - // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor + // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor // the low/high selectors which are clamping to either 0 or 255. if (((inten == 7) && (low_selector == 0) && (high_selector == 3)) && ((s == 0) || (s == 3))) err_scale = 5; @@ -6905,7 +6927,7 @@ namespace basist int err = block_colors[s].g - colors[g_etc1s_to_atc_selector_mappings[m][s]]; int err_scale = 1; - // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor + // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor // the low/high selectors which are clamping to either 0 or 255. if (((inten == 7) && (low_selector == 0) && (high_selector == 3)) && ((s == 0) || (s == 3))) err_scale = 5; @@ -6982,7 +7004,7 @@ namespace basist int err = block_colors[s].g - colors[g_etc1s_to_atc_selector_mappings[m][s]]; int err_scale = 1; - // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor + // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor // the low/high selectors which are clamping to either 0 or 255. if (((inten == 7) && (low_selector == 0) && (high_selector == 3)) && ((s == 0) || (s == 3))) err_scale = 5; @@ -7110,12 +7132,12 @@ namespace basist { uint8_t m_l, m_h; } g_pvrtc2_trans_match44[256]; - + static struct { uint8_t m_l, m_h; } g_pvrtc2_alpha_match33[256]; - + static struct { uint8_t m_l, m_h; @@ -7125,7 +7147,7 @@ namespace basist { uint8_t m_l, m_h; } g_pvrtc2_alpha_match33_3[256]; - + // PVRTC2 can be forced to look like a slightly weaker variant of ATC/BC1, so that's what we do here for simplicity. static void convert_etc1s_to_pvrtc2_rgb(void* pDst, const endpoint* pEndpoints, const selector* pSelector) { @@ -7237,7 +7259,7 @@ namespace basist pBlock->m_modulation[3] = (uint8_t)sels3; } } - + static inline vec4F* vec4F_set_scalar(vec4F* pV, float x) { pV->c[0] = x; pV->c[1] = x; pV->c[2] = x; pV->c[3] = x; return pV; } static inline vec4F* vec4F_set(vec4F* pV, float x, float y, float z, float w) { pV->c[0] = x; pV->c[1] = y; pV->c[2] = z; pV->c[3] = w; return pV; } static inline vec4F* vec4F_saturate_in_place(vec4F* pV) { pV->c[0] = saturate(pV->c[0]); pV->c[1] = saturate(pV->c[1]); pV->c[2] = saturate(pV->c[2]); pV->c[3] = saturate(pV->c[3]); return pV; } @@ -7255,9 +7277,9 @@ namespace basist } static inline int sq(int x) { return x * x; } - - // PVRTC2 is a slightly borked format for alpha: In Non-Interpolated mode, the way AlphaB8 is expanded from 4 to 8 bits means it can never be 0. - // This is actually very bad, because on 100% transparent blocks which have non-trivial color pixels, part of the color channel will leak into alpha! + + // PVRTC2 is a slightly borked format for alpha: In Non-Interpolated mode, the way AlphaB8 is expanded from 4 to 8 bits means it can never be 0. + // This is actually very bad, because on 100% transparent blocks which have non-trivial color pixels, part of the color channel will leak into alpha! // And there's nothing straightforward we can do because using the other modes is too expensive/complex. I can see why Apple didn't adopt it. static void convert_etc1s_to_pvrtc2_rgba(void* pDst, const endpoint* pEndpoints, const selector* pSelector, const endpoint* pEndpoint_codebook, const selector* pSelector_codebook) { @@ -7312,13 +7334,13 @@ namespace basist const uint32_t high_selector = pSelector->m_hi_selector; const int num_unique_color_selectors = pSelector->m_num_unique_selectors; - + // We need to reencode the block at the pixel level, unfortunately, from two ETC1S planes. // Do 4D incremental PCA, project all pixels to this hyperline, then quantize to packed endpoints and compute the modulation values. const int br = (base_color.r << 3) | (base_color.r >> 2); const int bg = (base_color.g << 3) | (base_color.g >> 2); const int bb = (base_color.b << 3) | (base_color.b >> 2); - + color32 block_cols[4]; for (uint32_t i = 0; i < 4; i++) { @@ -7347,14 +7369,14 @@ namespace basist decoder_etc_block::get_block_color5(base_color, inten_table, low_selector, r, g, b); // Mod 0 - uint32_t lr0 = (r * 15 + 128) / 255, lg0 = (g * 15 + 128) / 255, lb0 = (b * 7 + 128) / 255; + uint32_t lr0 = (r * 15 + 128) / 255, lg0 = (g * 15 + 128) / 255, lb0 = (b * 7 + 128) / 255; uint32_t la0 = g_pvrtc2_alpha_match33_0[constant_alpha_val].m_l; uint32_t cr0 = (lr0 << 1) | (lr0 >> 3); uint32_t cg0 = (lg0 << 1) | (lg0 >> 3); uint32_t cb0 = (lb0 << 2) | (lb0 >> 1); uint32_t ca0 = (la0 << 1); - + cr0 = (cr0 << 3) | (cr0 >> 2); cg0 = (cg0 << 3) | (cg0 >> 2); cb0 = (cb0 << 3) | (cb0 >> 2); @@ -7383,14 +7405,14 @@ namespace basist uint32_t cg3 = (lg3 << 1) | (lg3 >> 3); uint32_t cb3 = (lb3 << 1) | (lb3 >> 3); uint32_t ca3 = (la3 << 1) | 1; - + cr3 = (cr3 << 3) | (cr3 >> 2); cg3 = (cg3 << 3) | (cg3 >> 2); cb3 = (cb3 << 3) | (cb3 >> 2); ca3 = (ca3 << 4) | ca3; uint32_t err3 = sq(cr3 - r) + sq(cg3 - g) + sq(cb3 - b) + sq(ca3 - constant_alpha_val) * 2; - + // Mod 1 uint32_t lr1 = g_pvrtc2_trans_match44[r].m_l, lg1 = g_pvrtc2_trans_match44[g].m_l, lb1 = g_pvrtc2_trans_match34[b].m_l; uint32_t hr1 = g_pvrtc2_trans_match44[r].m_h, hg1 = g_pvrtc2_trans_match44[g].m_h, hb1 = g_pvrtc2_trans_match34[b].m_h; @@ -7465,7 +7487,7 @@ namespace basist // It's a solid color block. uint32_t low_a = block_cols[alpha_selectors.m_lo_selector].a; uint32_t high_a = block_cols[alpha_selectors.m_hi_selector].a; - + const float S = 1.0f / 255.0f; vec4F_set(&minColor, block_cols[low_selector].r * S, block_cols[low_selector].g * S, block_cols[low_selector].b * S, low_a * S); vec4F_set(&maxColor, block_cols[low_selector].r * S, block_cols[low_selector].g * S, block_cols[low_selector].b * S, high_a * S); @@ -7477,7 +7499,7 @@ namespace basist vec4F_set(&minColor, block_cols[low_selector].r * S, block_cols[low_selector].g * S, block_cols[low_selector].b * S, constant_alpha_val * S); vec4F_set(&maxColor, block_cols[high_selector].r * S, block_cols[high_selector].g * S, block_cols[high_selector].b * S, constant_alpha_val * S); } - // See if any of the block colors got clamped - if so the principle axis got distorted (it's no longer just the ETC1S luma axis). + // See if any of the block colors got clamped - if so the principle axis got distorted (it's no longer just the ETC1S luma axis). // To keep quality up we need to use full 4D PCA in this case. else if ((block_cols[low_selector].c[0] == 0) || (block_cols[high_selector].c[0] == 255) || (block_cols[low_selector].c[1] == 0) || (block_cols[high_selector].c[1] == 255) || @@ -7528,7 +7550,7 @@ namespace basist } vec4F_normalize_in_place(&axis); - + if (vec4F_dot(&axis, &axis) < .5f) vec4F_set_scalar(&axis, .5f); @@ -7628,10 +7650,10 @@ namespace basist // 4433 4443 color32 trialMinColor, trialMaxColor; - + trialMinColor.set_clamped((int)(minColor.c[0] * 15.0f + .5f), (int)(minColor.c[1] * 15.0f + .5f), (int)(minColor.c[2] * 7.0f + .5f), (int)(minColor.c[3] * 7.0f + .5f)); trialMaxColor.set_clamped((int)(maxColor.c[0] * 15.0f + .5f), (int)(maxColor.c[1] * 15.0f + .5f), (int)(maxColor.c[2] * 15.0f + .5f), (int)(maxColor.c[3] * 7.0f + .5f)); - + pBlock->set_trans_low_color(trialMinColor.r, trialMinColor.g, trialMinColor.b, trialMinColor.a); pBlock->set_trans_high_color(trialMaxColor.r, trialMaxColor.g, trialMaxColor.b, trialMaxColor.a); @@ -7704,7 +7726,7 @@ namespace basist } } } - + static void transcoder_init_pvrtc2() { for (uint32_t v = 0; v < 256; v++) @@ -7810,7 +7832,7 @@ namespace basist g_pvrtc2_trans_match34[v].m_l = (uint8_t)best_l; g_pvrtc2_trans_match34[v].m_h = (uint8_t)best_h; } - + for (uint32_t v = 0; v < 256; v++) { int best_l = 0, best_h = 0, lowest_err = INT_MAX; @@ -7844,12 +7866,12 @@ namespace basist #endif // BASISD_SUPPORT_PVRTC2 //------------------------------------------------------------------------------------------------ - + // BC7 mode 5 RGB encoder #if BASISD_SUPPORT_BC7_MODE5 namespace bc7_mode_5_encoder - { + { static float g_mode5_rgba_midpoints[128]; void encode_bc7_mode5_init() @@ -8124,10 +8146,10 @@ namespace basist } int block_max_var = basisu::maximum(icov[0], icov[3], icov[5]); // not divided by 16, i.e. scaled by 16 - + // TODO: Tune this const int32_t SIMPLE_BLOCK_THRESH = 10 * 16; - + if ((!hq_mode) && (block_max_var < SIMPLE_BLOCK_THRESH)) { const int L = 16, H = 239; @@ -8168,7 +8190,7 @@ namespace basist saxis_g = (int)(alt_xg * m); saxis_b = (int)(alt_xb * m); } - + saxis_r = (int)((uint32_t)saxis_r << 4U); saxis_g = (int)((uint32_t)saxis_g << 4U); saxis_b = (int)((uint32_t)saxis_b << 4U); @@ -8320,7 +8342,7 @@ namespace basist sym_codec.stop(); m_local_selectors.resize(num_selectors); - + if (!sym_codec.init(pSelectors_data, selectors_data_size)) { BASISU_DEVEL_ERROR("basisu_lowlevel_etc1s_transcoder::decode_palettes: fail 5\n"); @@ -8345,7 +8367,7 @@ namespace basist BASISU_DEVEL_ERROR("basisu_lowlevel_etc1s_transcoder::decode_palettes: hybrid global selector codebooks are unsupported\n"); return false; } - + const bool used_raw_encoding = (sym_codec.get_bits(1) == 1); if (used_raw_encoding) @@ -8526,7 +8548,7 @@ namespace basist if (!output_rows_in_pixels) output_rows_in_pixels = orig_height; } - + basisu::vector* pPrev_frame_indices = nullptr; if (is_video) { @@ -8554,12 +8576,12 @@ namespace basist } approx_move_to_front selector_history_buf(m_selector_history_buf_size); - + uint32_t cur_selector_rle_count = 0; decoder_etc_block block; memset(&block, 0, sizeof(block)); - + //block.set_flip_bit(true); // Setting the flip bit to false to be compatible with the Khronos KDFS. block.set_flip_bit(false); @@ -8595,7 +8617,7 @@ namespace basist if (!endpoints.size() || !selectors.size()) { BASISU_DEVEL_ERROR("basisu_lowlevel_etc1s_transcoder::transcode_slice: global codebooks must be unpacked first\n"); - + if (pPVRTC_work_mem) free(pPVRTC_work_mem); @@ -8606,7 +8628,7 @@ namespace basist const uint32_t SELECTOR_HISTORY_BUF_RLE_SYMBOL_INDEX = m_selector_history_buf_size + SELECTOR_HISTORY_BUF_FIRST_SYMBOL_INDEX; #if BASISD_SUPPORT_BC7_MODE5 - const bool bc7_chroma_filtering = ((decode_flags & cDecodeFlagsNoETC1SChromaFiltering) == 0) && + const bool bc7_chroma_filtering = ((decode_flags & cDecodeFlagsNoETC1SChromaFiltering) == 0) && ((fmt == block_format::cBC7_M5_COLOR) || (fmt == block_format::cBC7)); basisu::vector2D decoded_endpoints; @@ -8831,7 +8853,7 @@ namespace basist case block_format::cETC1: { decoder_etc_block* pDst_block = reinterpret_cast(static_cast(pDst_blocks) + (block_x + block_y * output_row_pitch_in_blocks_or_pixels) * output_block_or_pixel_stride_in_bytes); - + block.set_base5_color(decoder_etc_block::pack_color5(pEndpoints->m_color5, false)); block.set_inten_table(0, pEndpoints->m_inten5); block.set_inten_table(1, pEndpoints->m_inten5); @@ -8882,7 +8904,7 @@ namespace basist const uint32_t low_selector = pSelector->m_lo_selector; const uint32_t high_selector = pSelector->m_hi_selector; - // Get block's RGB bounding box + // Get block's RGB bounding box color32 block_colors[2]; decoder_etc_block::get_block_colors5_bounds(block_colors, base_color, inten_table, low_selector, high_selector); @@ -8898,7 +8920,7 @@ namespace basist pPVRTC_endpoints[block_x + block_y * num_blocks_x] = temp.m_endpoints; #else assert(0); -#endif +#endif break; } @@ -8906,7 +8928,7 @@ namespace basist { #if BASISD_SUPPORT_PVRTC1 assert(pAlpha_blocks); - + block.set_base5_color(decoder_etc_block::pack_color5(pEndpoints->m_color5, false)); block.set_inten_table(0, pEndpoints->m_inten5); block.set_inten_table(1, pEndpoints->m_inten5); @@ -8914,7 +8936,7 @@ namespace basist ((decoder_etc_block*)pPVRTC_work_mem)[block_x + block_y * num_blocks_x] = block; - // Get block's RGBA bounding box + // Get block's RGBA bounding box const color32& base_color = pEndpoints->m_color5; const uint32_t inten_table = pEndpoints->m_inten5; const uint32_t low_selector = pSelector->m_lo_selector; @@ -8949,7 +8971,7 @@ namespace basist pPVRTC_endpoints[block_x + block_y * num_blocks_x] = temp.m_endpoints; #else assert(0); -#endif +#endif break; } @@ -8990,7 +9012,7 @@ namespace basist #endif break; } - case block_format::cASTC_4x4: + case block_format::cASTC_LDR_4x4: { #if BASISD_SUPPORT_ASTC void* pDst_block = static_cast(pDst_blocks) + (block_x + block_y * output_row_pitch_in_blocks_or_pixels) * output_block_or_pixel_stride_in_bytes; @@ -9039,7 +9061,7 @@ namespace basist assert(transcode_alpha); void* pDst_block = static_cast(pDst_blocks) + (block_x + block_y * output_row_pitch_in_blocks_or_pixels) * output_block_or_pixel_stride_in_bytes; - + convert_etc1s_to_pvrtc2_rgba(pDst_block, pEndpoints, pSelector, &endpoints[0], &selectors[0]); #endif break; @@ -9055,10 +9077,10 @@ namespace basist { assert(sizeof(uint32_t) == output_block_or_pixel_stride_in_bytes); uint8_t* pDst_pixels = static_cast(pDst_blocks) + (block_x * 4 + block_y * 4 * output_row_pitch_in_blocks_or_pixels) * sizeof(uint32_t); - + const uint32_t max_x = basisu::minimum(4, (int)output_row_pitch_in_blocks_or_pixels - (int)block_x * 4); const uint32_t max_y = basisu::minimum(4, (int)output_rows_in_pixels - (int)block_y * 4); - + int colors[4]; decoder_etc_block::get_block_colors5_g(colors, pEndpoints->m_color5, pEndpoints->m_inten5); @@ -9072,7 +9094,7 @@ namespace basist pDst_pixels[3+4] = static_cast(colors[(s >> 2) & 3]); pDst_pixels[3+8] = static_cast(colors[(s >> 4) & 3]); pDst_pixels[3+12] = static_cast(colors[(s >> 6) & 3]); - + pDst_pixels += output_row_pitch_in_blocks_or_pixels * sizeof(uint32_t); } } @@ -9101,7 +9123,7 @@ namespace basist color32 colors[4]; decoder_etc_block::get_block_colors5(colors, pEndpoints->m_color5, pEndpoints->m_inten5); - + for (uint32_t y = 0; y < max_y; y++) { const uint32_t s = pSelector->m_selectors[y]; @@ -9222,7 +9244,7 @@ namespace basist cur = byteswap_uint16(cur); cur = (cur & 0xF) | packed_colors[(s >> (x * 2)) & 3]; - + if (BASISD_IS_BIG_ENDIAN) cur = byteswap_uint16(cur); @@ -9322,7 +9344,7 @@ namespace basist if (endpoint_pred_repeat_count != 0) { BASISU_DEVEL_ERROR("basisu_lowlevel_etc1s_transcoder::transcode_slice: endpoint_pred_repeat_count != 0. The file is corrupted or this is a bug\n"); - + if (pPVRTC_work_mem) free(pPVRTC_work_mem); @@ -9353,22 +9375,19 @@ namespace basist } bool basis_validate_output_buffer_size( - basis_tex_format source_format, transcoder_texture_format target_format, uint32_t output_blocks_buf_size_in_blocks_or_pixels, uint32_t orig_width, uint32_t orig_height, uint32_t output_row_pitch_in_blocks_or_pixels, uint32_t output_rows_in_pixels) { - BASISU_NOTE_UNUSED(source_format); - if (basis_transcoder_format_is_uncompressed(target_format)) { // Assume the output buffer is orig_width by orig_height if (!output_row_pitch_in_blocks_or_pixels) output_row_pitch_in_blocks_or_pixels = orig_width; - if (!output_rows_in_pixels) + if (!output_rows_in_pixels) output_rows_in_pixels = orig_height; // Now make sure the output buffer is large enough, or we'll overwrite memory. @@ -9380,11 +9399,12 @@ namespace basist } else { + // Take into account the destination format's block width/height. const uint32_t dst_block_width = basis_get_block_width(target_format); const uint32_t dst_block_height = basis_get_block_height(target_format); //const uint32_t bytes_per_block = basis_get_bytes_per_block_or_pixel(target_format); - - // Take into account the destination format's block width/height. + + // Compute how many blocks should be in the output. const uint32_t num_dst_blocks_x = (orig_width + dst_block_width - 1) / dst_block_width; const uint32_t num_dst_blocks_y = (orig_height + dst_block_height - 1) / dst_block_height; const uint32_t total_dst_blocks = num_dst_blocks_x * num_dst_blocks_y; @@ -9402,7 +9422,7 @@ namespace basist return true; } - + uint32_t basis_compute_transcoded_image_size_in_bytes(transcoder_texture_format target_format, uint32_t orig_width, uint32_t orig_height) { assert(orig_width && orig_height); @@ -9418,7 +9438,7 @@ namespace basist const uint32_t bytes_per_slice = bytes_per_line * orig_height; return bytes_per_slice; } - + // Compressed formats are 2D arrays of blocks. const uint32_t bytes_per_block = basis_get_bytes_per_block_or_pixel(target_format); @@ -9489,12 +9509,12 @@ namespace basist // Switch to PVRTC1 RGB if the input doesn't have alpha. target_format = transcoder_texture_format::cTFPVRTC1_4_RGB; } - + const bool transcode_alpha_data_to_opaque_formats = (decode_flags & cDecodeFlagsTranscodeAlphaDataToOpaqueFormats) != 0; const uint32_t bytes_per_block_or_pixel = basis_get_bytes_per_block_or_pixel(target_format); const uint32_t total_slice_blocks = num_blocks_x * num_blocks_y; - - if (!basis_validate_output_buffer_size(basis_tex_format::cETC1S, target_format, output_blocks_buf_size_in_blocks_or_pixels, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, output_rows_in_pixels)) + + if (!basis_validate_output_buffer_size(target_format, output_blocks_buf_size_in_blocks_or_pixels, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, output_rows_in_pixels)) { BASISU_DEVEL_ERROR("basisu_lowlevel_etc1s_transcoder::transcode_image: output buffer size too small\n"); return false; @@ -9520,7 +9540,7 @@ namespace basist { //status = transcode_slice(pData, data_size, slice_index_to_decode, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cETC1, bytes_per_block_or_pixel, decode_flags, output_row_pitch_in_blocks_or_pixels, pState); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pData, data_len, block_format::cETC1, bytes_per_block_or_pixel, false, is_video, is_alpha_slice, level_index, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, false, nullptr, output_rows_in_pixels, decode_flags); - + if (!status) { BASISU_DEVEL_ERROR("basisu_lowlevel_etc1s_transcoder::transcode_image: transcode_slice() to ETC1 failed\n"); @@ -9645,7 +9665,7 @@ namespace basist if (basis_file_has_alpha_slices) { - // First decode the alpha data + // First decode the alpha data //status = transcode_slice(pData, data_size, slice_index + 1, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cETC2_EAC_A8, 16, decode_flags, output_row_pitch_in_blocks_or_pixels, pState); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + alpha_offset, alpha_length, block_format::cETC2_EAC_A8, bytes_per_block_or_pixel, false, is_video, true, level_index, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, false, nullptr, output_rows_in_pixels, decode_flags); } @@ -9683,8 +9703,8 @@ namespace basist return false; #else assert(bytes_per_block_or_pixel == 16); - - // First decode the alpha data + + // First decode the alpha data if (basis_file_has_alpha_slices) { //status = transcode_slice(pData, data_size, slice_index + 1, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cBC4, 16, decode_flags, output_row_pitch_in_blocks_or_pixels, pState); @@ -9754,7 +9774,7 @@ namespace basist break; #endif } - case transcoder_texture_format::cTFASTC_4x4_RGBA: + case transcoder_texture_format::cTFASTC_LDR_4x4_RGBA: { #if !BASISD_SUPPORT_ASTC BASISU_DEVEL_ERROR("basisu_lowlevel_etc1s_transcoder::transcode_image: ASTC unsupported\n"); @@ -9771,13 +9791,13 @@ namespace basist { // Now decode the color data and transcode to ASTC. The transcoder function will read the alpha selector data from the output texture as it converts and // transcode both the alpha and color data at the same time to ASTC. - //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cASTC_4x4, 16, decode_flags | cDecodeFlagsOutputHasAlphaIndices, output_row_pitch_in_blocks_or_pixels, pState); - status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + rgb_offset, rgb_length, block_format::cASTC_4x4, bytes_per_block_or_pixel, false, is_video, false, level_index, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, true, nullptr, output_rows_in_pixels, decode_flags); + //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cASTC_LDR_4x4, 16, decode_flags | cDecodeFlagsOutputHasAlphaIndices, output_row_pitch_in_blocks_or_pixels, pState); + status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + rgb_offset, rgb_length, block_format::cASTC_LDR_4x4, bytes_per_block_or_pixel, false, is_video, false, level_index, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, true, nullptr, output_rows_in_pixels, decode_flags); } } else - //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cASTC_4x4, 16, decode_flags, output_row_pitch_in_blocks_or_pixels, pState); - status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + rgb_offset, rgb_length, block_format::cASTC_4x4, bytes_per_block_or_pixel, false, is_video, false, level_index, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, false, nullptr, output_rows_in_pixels, decode_flags); + //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cASTC_LDR_4x4, 16, decode_flags, output_row_pitch_in_blocks_or_pixels, pState); + status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + rgb_offset, rgb_length, block_format::cASTC_LDR_4x4, bytes_per_block_or_pixel, false, is_video, false, level_index, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, false, nullptr, output_rows_in_pixels, decode_flags); if (!status) { @@ -9813,7 +9833,7 @@ namespace basist #else assert(bytes_per_block_or_pixel == 16); - // First decode the alpha data + // First decode the alpha data if (basis_file_has_alpha_slices) { //status = transcode_slice(pData, data_size, slice_index + 1, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cBC4, 16, decode_flags, output_row_pitch_in_blocks_or_pixels, pState); @@ -9873,7 +9893,7 @@ namespace basist } else { - // Now decode the color data and transcode to PVRTC2 RGBA. + // Now decode the color data and transcode to PVRTC2 RGBA. //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cPVRTC2_4_RGBA, bytes_per_block_or_pixel, decode_flags | cDecodeFlagsOutputHasAlphaIndices, output_row_pitch_in_blocks_or_pixels, pState); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + rgb_offset, rgb_length, block_format::cPVRTC2_4_RGBA, bytes_per_block_or_pixel, false, is_video, false, level_index, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, true, nullptr, output_rows_in_pixels, decode_flags); } @@ -9894,7 +9914,7 @@ namespace basist { // Raw 32bpp pixels, decoded in the usual raster order (NOT block order) into an image in memory. - // First decode the alpha data + // First decode the alpha data if (basis_file_has_alpha_slices) //status = transcode_slice(pData, data_size, slice_index + 1, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cA32, sizeof(uint32_t), decode_flags, output_row_pitch_in_blocks_or_pixels, pState, nullptr, output_rows_in_pixels); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + alpha_offset, alpha_length, block_format::cA32, sizeof(uint32_t), false, is_video, true, level_index, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, false, nullptr, output_rows_in_pixels, decode_flags); @@ -9935,7 +9955,7 @@ namespace basist { // Raw 16bpp pixels, decoded in the usual raster order (NOT block order) into an image in memory. - // First decode the alpha data + // First decode the alpha data if (basis_file_has_alpha_slices) //status = transcode_slice(pData, data_size, slice_index + 1, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cRGBA4444_ALPHA, sizeof(uint16_t), decode_flags, output_row_pitch_in_blocks_or_pixels, pState, nullptr, output_rows_in_pixels); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + alpha_offset, alpha_length, block_format::cRGBA4444_ALPHA, sizeof(uint16_t), false, is_video, true, level_index, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, false, nullptr, output_rows_in_pixels, decode_flags); @@ -10039,14 +10059,15 @@ namespace basist } //------------------------------------------------------------------------------------------------ - + // UASTC LDR 4x4 transcoder + //------------------------------------------------------------------------------------------------ basisu_lowlevel_uastc_ldr_4x4_transcoder::basisu_lowlevel_uastc_ldr_4x4_transcoder() { } bool basisu_lowlevel_uastc_ldr_4x4_transcoder::transcode_slice( void* pDst_blocks, uint32_t num_blocks_x, uint32_t num_blocks_y, const uint8_t* pImage_data, uint32_t image_data_size, block_format fmt, - uint32_t output_block_or_pixel_stride_in_bytes, bool bc1_allow_threecolor_blocks, bool has_alpha, + uint32_t output_block_or_pixel_stride_in_bytes, bool bc1_allow_threecolor_blocks, bool has_alpha, const uint32_t orig_width, const uint32_t orig_height, uint32_t output_row_pitch_in_blocks_or_pixels, basisu_transcoder_state* pState, uint32_t output_rows_in_pixels, int channel0, int channel1, uint32_t decode_flags) { @@ -10107,7 +10128,7 @@ namespace basist for (uint32_t block_y = 0; block_y < num_blocks_y; ++block_y) { void* pDst_block = (uint8_t*)pDst_blocks + block_y * output_row_pitch_in_blocks_or_pixels * output_block_or_pixel_stride_in_bytes; - + for (uint32_t block_x = 0; block_x < num_blocks_x; ++block_x, ++pSource_block, pDst_block = (uint8_t *)pDst_block + output_block_or_pixel_stride_in_bytes) { switch (fmt) @@ -10143,7 +10164,7 @@ namespace basist } case block_format::cBC4: { - if (channel0 < 0) + if (channel0 < 0) channel0 = 0; status = transcode_uastc_to_bc4(*pSource_block, pDst_block, high_quality, channel0); break; @@ -10163,7 +10184,7 @@ namespace basist status = transcode_uastc_to_bc7(*pSource_block, pDst_block); break; } - case block_format::cASTC_4x4: + case block_format::cASTC_LDR_4x4: { status = transcode_uastc_to_astc(*pSource_block, pDst_block); break; @@ -10306,7 +10327,7 @@ namespace basist return false; #endif } - + bool basisu_lowlevel_uastc_ldr_4x4_transcoder::transcode_image( transcoder_texture_format target_format, void* pOutput_blocks, uint32_t output_blocks_buf_size_in_blocks_or_pixels, @@ -10328,7 +10349,7 @@ namespace basist { BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_ldr_4x4_transcoder::transcode_image: source data buffer too small\n"); return false; - } + } if ((target_format == transcoder_texture_format::cTFPVRTC1_4_RGB) || (target_format == transcoder_texture_format::cTFPVRTC1_4_RGBA)) { @@ -10350,12 +10371,12 @@ namespace basist const uint32_t bytes_per_block_or_pixel = basis_get_bytes_per_block_or_pixel(target_format); //const uint32_t total_slice_blocks = num_blocks_x * num_blocks_y; - if (!basis_validate_output_buffer_size(basis_tex_format::cUASTC4x4, target_format, output_blocks_buf_size_in_blocks_or_pixels, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, output_rows_in_pixels)) + if (!basis_validate_output_buffer_size(target_format, output_blocks_buf_size_in_blocks_or_pixels, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, output_rows_in_pixels)) { BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_ldr_4x4_transcoder::transcode_image: output buffer size too small\n"); return false; } - + bool status = false; // UASTC4x4 @@ -10363,10 +10384,9 @@ namespace basist { case transcoder_texture_format::cTFETC1_RGB: { - //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cETC1, bytes_per_block_or_pixel, decode_flags, output_row_pitch_in_blocks_or_pixels, pState); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cETC1, bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, channel0, channel1, decode_flags); - + if (!status) { BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_ldr_4x4_transcoder::transcode_image: transcode_slice() to ETC1 failed\n"); @@ -10375,7 +10395,6 @@ namespace basist } case transcoder_texture_format::cTFETC2_RGBA: { - //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cETC2_RGBA, bytes_per_block_or_pixel, decode_flags, output_row_pitch_in_blocks_or_pixels, pState); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cETC2_RGBA, bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, channel0, channel1, decode_flags); if (!status) @@ -10387,7 +10406,6 @@ namespace basist case transcoder_texture_format::cTFBC1_RGB: { // TODO: ETC1S allows BC1 from alpha channel. That doesn't seem actually useful, though. - //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cBC1, bytes_per_block_or_pixel, decode_flags, output_row_pitch_in_blocks_or_pixels, pState); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cBC1, bytes_per_block_or_pixel, true, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, channel0, channel1, decode_flags); if (!status) @@ -10398,7 +10416,6 @@ namespace basist } case transcoder_texture_format::cTFBC3_RGBA: { - //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cBC3, bytes_per_block_or_pixel, decode_flags, output_row_pitch_in_blocks_or_pixels, pState); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cBC3, bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, channel0, channel1, decode_flags); if (!status) @@ -10409,9 +10426,6 @@ namespace basist } case transcoder_texture_format::cTFBC4_R: { - //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cBC4, bytes_per_block_or_pixel, decode_flags, output_row_pitch_in_blocks_or_pixels, pState, - // nullptr, 0, - // ((has_alpha) && (transcode_alpha_data_to_opaque_formats)) ? 3 : 0); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cBC4, bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, ((has_alpha) && (transcode_alpha_data_to_opaque_formats)) ? 3 : 0, -1, decode_flags); @@ -10423,9 +10437,6 @@ namespace basist } case transcoder_texture_format::cTFBC5_RG: { - //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cBC5, bytes_per_block_or_pixel, decode_flags, output_row_pitch_in_blocks_or_pixels, pState, - // nullptr, 0, - // 0, 3); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cBC5, bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, 0, 3, decode_flags); @@ -10438,7 +10449,6 @@ namespace basist case transcoder_texture_format::cTFBC7_RGBA: case transcoder_texture_format::cTFBC7_ALT: { - //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cBC7, bytes_per_block_or_pixel, decode_flags, output_row_pitch_in_blocks_or_pixels, pState); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cBC7, bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, -1, -1, decode_flags); if (!status) @@ -10449,7 +10459,6 @@ namespace basist } case transcoder_texture_format::cTFPVRTC1_4_RGB: { - //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cPVRTC1_4_RGB, bytes_per_block_or_pixel, decode_flags, output_row_pitch_in_blocks_or_pixels, pState); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cPVRTC1_4_RGB, bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, -1, -1, decode_flags); if (!status) @@ -10460,7 +10469,6 @@ namespace basist } case transcoder_texture_format::cTFPVRTC1_4_RGBA: { - //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cPVRTC1_4_RGBA, bytes_per_block_or_pixel, decode_flags, output_row_pitch_in_blocks_or_pixels, pState); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cPVRTC1_4_RGBA, bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, -1, -1, decode_flags); if (!status) @@ -10469,10 +10477,9 @@ namespace basist } break; } - case transcoder_texture_format::cTFASTC_4x4_RGBA: + case transcoder_texture_format::cTFASTC_LDR_4x4_RGBA: { - //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cASTC_4x4, bytes_per_block_or_pixel, decode_flags, output_row_pitch_in_blocks_or_pixels, pState); - status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cASTC_4x4, + status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cASTC_LDR_4x4, bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, -1, -1, decode_flags); if (!status) { @@ -10483,29 +10490,26 @@ namespace basist case transcoder_texture_format::cTFATC_RGB: case transcoder_texture_format::cTFATC_RGBA: { - BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_ldr_4x4_transcoder::transcode_image: UASTC->ATC currently unsupported\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_ldr_4x4_transcoder::transcode_image: UASTC LDR 4x4->ATC currently unsupported\n"); return false; } case transcoder_texture_format::cTFFXT1_RGB: { - BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_ldr_4x4_transcoder::transcode_image: UASTC->FXT1 currently unsupported\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_ldr_4x4_transcoder::transcode_image: UASTC LDR 4x4->FXT1 currently unsupported\n"); return false; } case transcoder_texture_format::cTFPVRTC2_4_RGB: { - BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_ldr_4x4_transcoder::transcode_image: UASTC->PVRTC2 currently unsupported\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_ldr_4x4_transcoder::transcode_image: UASTC LDR 4x4->PVRTC2 currently unsupported\n"); return false; } case transcoder_texture_format::cTFPVRTC2_4_RGBA: { - BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_ldr_4x4_transcoder::transcode_image: UASTC->PVRTC2 currently unsupported\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_ldr_4x4_transcoder::transcode_image: UASTC LDR 4x4->PVRTC2 currently unsupported\n"); return false; } case transcoder_texture_format::cTFETC2_EAC_R11: { - //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cETC2_EAC_R11, bytes_per_block_or_pixel, decode_flags, output_row_pitch_in_blocks_or_pixels, pState, - // nullptr, 0, - // ((has_alpha) && (transcode_alpha_data_to_opaque_formats)) ? 3 : 0); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cETC2_EAC_R11, bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, ((has_alpha) && (transcode_alpha_data_to_opaque_formats)) ? 3 : 0, -1, decode_flags); @@ -10517,9 +10521,6 @@ namespace basist } case transcoder_texture_format::cTFETC2_EAC_RG11: { - //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cETC2_EAC_RG11, bytes_per_block_or_pixel, decode_flags, output_row_pitch_in_blocks_or_pixels, pState, - // nullptr, 0, - // 0, 3); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cETC2_EAC_RG11, bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, 0, 3, decode_flags); @@ -10531,7 +10532,6 @@ namespace basist } case transcoder_texture_format::cTFRGBA32: { - //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cRGBA32, bytes_per_block_or_pixel, decode_flags, output_row_pitch_in_blocks_or_pixels, pState); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cRGBA32, bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, -1, -1, decode_flags); if (!status) @@ -10542,7 +10542,6 @@ namespace basist } case transcoder_texture_format::cTFRGB565: { - //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cRGB565, bytes_per_block_or_pixel, decode_flags, output_row_pitch_in_blocks_or_pixels, pState); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cRGB565, bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, -1, -1, decode_flags); if (!status) @@ -10553,7 +10552,6 @@ namespace basist } case transcoder_texture_format::cTFBGR565: { - //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cBGR565, bytes_per_block_or_pixel, decode_flags, output_row_pitch_in_blocks_or_pixels, pState); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cBGR565, bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, -1, -1, decode_flags); if (!status) @@ -10564,7 +10562,6 @@ namespace basist } case transcoder_texture_format::cTFRGBA4444: { - //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cRGBA4444, bytes_per_block_or_pixel, decode_flags, output_row_pitch_in_blocks_or_pixels, pState); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cRGBA4444, bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, -1, -1, decode_flags); if (!status) @@ -10583,9 +10580,10 @@ namespace basist return status; } - + + //------------------------------------------------------------------------------------------------ + // UASTC HDR 4x4 transcoding //------------------------------------------------------------------------------------------------ - // UASTC HDR 4x4 basisu_lowlevel_uastc_hdr_4x4_transcoder::basisu_lowlevel_uastc_hdr_4x4_transcoder() { @@ -10593,7 +10591,7 @@ namespace basist bool basisu_lowlevel_uastc_hdr_4x4_transcoder::transcode_slice( void* pDst_blocks, uint32_t num_blocks_x, uint32_t num_blocks_y, const uint8_t* pImage_data, uint32_t image_data_size, block_format fmt, - uint32_t output_block_or_pixel_stride_in_bytes, bool bc1_allow_threecolor_blocks, bool has_alpha, + uint32_t output_block_or_pixel_stride_in_bytes, bool bc1_allow_threecolor_blocks, bool has_alpha, const uint32_t orig_width, const uint32_t orig_height, uint32_t output_row_pitch_in_blocks_or_pixels, basisu_transcoder_state* pState, uint32_t output_rows_in_pixels, int channel0, int channel1, uint32_t decode_flags) { @@ -10642,7 +10640,7 @@ namespace basist bool status = false; // TODO: Optimize pure memcpy() case. - + for (uint32_t block_y = 0; block_y < num_blocks_y; ++block_y) { void* pDst_block = (uint8_t*)pDst_blocks + block_y * output_row_pitch_in_blocks_or_pixels * output_block_or_pixel_stride_in_bytes; @@ -10677,7 +10675,7 @@ namespace basist uint32_t blk_texels[4][4]; status = astc_helpers::decode_block(log_blk, blk_texels, 4, 4, astc_helpers::cDecodeModeRGB9E5); - + if (status) { const uint32_t max_x = basisu::minimum(4, (int)output_row_pitch_in_blocks_or_pixels - (int)block_x * 4); @@ -10691,7 +10689,7 @@ namespace basist } // y } } - + break; } case block_format::cRGBA_HALF: @@ -10703,7 +10701,7 @@ namespace basist half_float* pDst_pixels = reinterpret_cast( static_cast(pDst_blocks) + (block_x * 4 + block_y * 4 * output_row_pitch_in_blocks_or_pixels) * sizeof(half_float) * 4 ); - + half_float blk_texels[4][4][4]; status = astc_helpers::decode_block(log_blk, blk_texels, 4, 4, astc_helpers::cDecodeModeHDR16); @@ -10769,7 +10767,7 @@ namespace basist if (!status) { - BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_4x4_transcoder::transcode_slice: Transcoder failed to unpack a UASTC HDR block - this is a bug, or the data was corrupted\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_4x4_transcoder::transcode_slice: Transcoder failed to unpack a UASTC HDR block - this is a bug, or the data was corrupted\n"); return false; } @@ -10825,7 +10823,7 @@ namespace basist const uint32_t bytes_per_block_or_pixel = basis_get_bytes_per_block_or_pixel(target_format); //const uint32_t total_slice_blocks = num_blocks_x * num_blocks_y; - if (!basis_validate_output_buffer_size(basis_tex_format::cUASTC_HDR_4x4, target_format, output_blocks_buf_size_in_blocks_or_pixels, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, output_rows_in_pixels)) + if (!basis_validate_output_buffer_size(target_format, output_blocks_buf_size_in_blocks_or_pixels, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, output_rows_in_pixels)) { BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_4x4_transcoder::transcode_image: output buffer size too small\n"); return false; @@ -10962,7 +10960,7 @@ namespace basist assert(((orig_width + 5) / 6) == num_blocks_x); assert(((orig_height + 5) / 6) == num_blocks_y); - + if (fmt == block_format::cBC6H) { const uint32_t num_dst_blocks_x = (orig_width + 3) / 4; @@ -10987,7 +10985,7 @@ namespace basist fast_bc6h_params bc6h_enc_params; const bool hq_flag = (decode_flags & cDecodeFlagsHighQuality) != 0; bc6h_enc_params.m_max_2subset_pats_to_try = hq_flag ? 1 : 0; - + for (uint32_t src_block_y = 0; src_block_y < num_blocks_y; src_block_y += 2) { const uint32_t num_inner_blocks_y = basisu::minimum(2, num_blocks_y - src_block_y); @@ -11003,7 +11001,7 @@ namespace basist const astc_blk* pS = pSource_block + (src_block_y + iy) * num_blocks_x + (src_block_x + ix); half_float blk_texels[6][6][4]; - + astc_helpers::log_astc_block log_blk; status = astc_helpers::unpack_block(pS, log_blk, 6, 6); if (!status) @@ -11011,7 +11009,7 @@ namespace basist BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_transcoder::transcode_slice: Transcoder failed to unpack a ASTC HDR block - this is a bug, or the data was corrupted\n"); return false; } - + status = astc_helpers::decode_block(log_blk, blk_texels, 6, 6, astc_helpers::cDecodeModeHDR16); if (!status) { @@ -11026,14 +11024,14 @@ namespace basist unpacked_blocks[iy * 6 + y][ix * 6 + x][0] = blk_texels[y][x][0]; unpacked_blocks[iy * 6 + y][ix * 6 + x][1] = blk_texels[y][x][1]; unpacked_blocks[iy * 6 + y][ix * 6 + x][2] = blk_texels[y][x][2]; - + } // x } // y } // ix } // iy - + const uint32_t dst_x = src_block_x * 6; assert((dst_x & 3) == 0); const uint32_t dst_block_x = dst_x >> 2; @@ -11066,10 +11064,10 @@ namespace basist src_pixels[y][x][0] = unpacked_blocks[src_pixel_y][src_pixel_x][0]; src_pixels[y][x][1] = unpacked_blocks[src_pixel_y][src_pixel_x][1]; src_pixels[y][x][2] = unpacked_blocks[src_pixel_y][src_pixel_x][2]; - + } // x } // y - + astc_6x6_hdr::fast_encode_bc6h(&src_pixels[0][0][0], pDst_block, bc6h_enc_params); } // dx @@ -11078,7 +11076,7 @@ namespace basist } // block_x } // block_y - + status = true; } else @@ -11204,7 +11202,7 @@ namespace basist if (!status) { - BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_transcoder::transcode_slice: Transcoder failed to unpack a ASTC HDR block - this is a bug, or the data was corrupted\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_transcoder::transcode_slice: Transcoder failed to unpack a ASTC HDR block - this is a bug, or the data was corrupted\n"); return false; } @@ -11261,7 +11259,7 @@ namespace basist const uint32_t bytes_per_block_or_pixel = basis_get_bytes_per_block_or_pixel(target_format); //const uint32_t total_slice_blocks = num_blocks_x * num_blocks_y; - if (!basis_validate_output_buffer_size(basis_tex_format::cASTC_HDR_6x6, target_format, output_blocks_buf_size_in_blocks_or_pixels, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, output_rows_in_pixels)) + if (!basis_validate_output_buffer_size(target_format, output_blocks_buf_size_in_blocks_or_pixels, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, output_rows_in_pixels)) { BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_transcoder::transcode_image: output buffer size too small\n"); return false; @@ -11334,14 +11332,14 @@ namespace basist } //------------------------------------------------------------------------------------------------ - // ASTC 6x6 HDR intermediate + // UASTC 6x6 HDR intermediate - basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder() + basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder() { } // num_blocks_x/num_blocks_y are source 6x6 blocks - bool basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_slice( + bool basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_slice( void* pDst_blocks, uint32_t num_blocks_x, uint32_t num_blocks_y, const uint8_t* pImage_data, uint32_t image_data_size, block_format fmt, uint32_t output_block_or_pixel_stride_in_bytes, bool bc1_allow_threecolor_blocks, bool has_alpha, const uint32_t orig_width, const uint32_t orig_height, uint32_t output_row_pitch_in_blocks_or_pixels, @@ -11359,7 +11357,7 @@ namespace basist assert(g_transcoder_initialized); if (!g_transcoder_initialized) { - BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_slice: Transcoder not globally initialized.\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_slice: Transcoder not globally initialized.\n"); return false; } @@ -11372,14 +11370,14 @@ namespace basist bool dec_status = astc_6x6_hdr::decode_6x6_hdr(pImage_data, image_data_size, decoded_blocks, dec_width, dec_height); if (!dec_status) { - BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_slice: decode_6x6_hdr() failed.\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_slice: decode_6x6_hdr() failed.\n"); return false; } if ((dec_width != orig_width) || (dec_height != orig_height) || (decoded_blocks.get_width() != num_blocks_x) || (decoded_blocks.get_height() != num_blocks_y)) { - BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_slice: unexpected decoded width/height\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_slice: unexpected decoded width/height\n"); return false; } @@ -11422,20 +11420,20 @@ namespace basist } else if (output_row_pitch_in_blocks_or_pixels < num_dst_blocks_x) { - BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_slice: output_row_pitch_in_blocks_or_pixels is too low\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_slice: output_row_pitch_in_blocks_or_pixels is too low\n"); return false; } if (output_block_or_pixel_stride_in_bytes != sizeof(bc6h_block)) { - BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_slice: invalid output_block_or_pixel_stride_in_bytes\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_slice: invalid output_block_or_pixel_stride_in_bytes\n"); return false; } fast_bc6h_params bc6h_enc_params; const bool hq_flag = (decode_flags & cDecodeFlagsHighQuality) != 0; bc6h_enc_params.m_max_2subset_pats_to_try = hq_flag ? 1 : 0; - + for (uint32_t src_block_y = 0; src_block_y < num_blocks_y; src_block_y += 2) { const uint32_t num_inner_blocks_y = basisu::minimum(2, num_blocks_y - src_block_y); @@ -11456,14 +11454,14 @@ namespace basist status = astc_helpers::unpack_block(pS, log_blk, 6, 6); if (!status) { - BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_slice: Transcoder failed to unpack a ASTC HDR block - this is a bug, or the data was corrupted\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_slice: Transcoder failed to unpack a ASTC HDR block - this is a bug, or the data was corrupted\n"); return false; } status = astc_helpers::decode_block(log_blk, blk_texels, 6, 6, astc_helpers::cDecodeModeHDR16); if (!status) { - BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_slice: Transcoder failed to unpack a ASTC HDR block - this is a bug, or the data was corrupted\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_slice: Transcoder failed to unpack a ASTC HDR block - this is a bug, or the data was corrupted\n"); return false; } @@ -11516,7 +11514,7 @@ namespace basist } // x } // y - + astc_6x6_hdr::fast_encode_bc6h(&src_pixels[0][0][0], pDst_block, bc6h_enc_params); } // dx @@ -11651,7 +11649,7 @@ namespace basist if (!status) { - BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_slice: Transcoder failed to unpack a ASTC HDR block - this is a bug, or the data was corrupted\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_slice: Transcoder failed to unpack a ASTC HDR block - this is a bug, or the data was corrupted\n"); return false; } @@ -11662,7 +11660,7 @@ namespace basist return true; #else - BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_slice: ASTC HDR is unsupported\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_slice: ASTC HDR is unsupported\n"); BASISU_NOTE_UNUSED(decode_flags); BASISU_NOTE_UNUSED(channel0); @@ -11681,7 +11679,7 @@ namespace basist #endif } - bool basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_image( + bool basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_image( transcoder_texture_format target_format, void* pOutput_blocks, uint32_t output_blocks_buf_size_in_blocks_or_pixels, const uint8_t* pCompressed_data, uint32_t compressed_data_length, @@ -11701,16 +11699,16 @@ namespace basist if (((uint64_t)slice_offset + slice_length) > (uint64_t)compressed_data_length) { - BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_image: source data buffer too small\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_image: source data buffer too small\n"); return false; } const uint32_t bytes_per_block_or_pixel = basis_get_bytes_per_block_or_pixel(target_format); //const uint32_t total_slice_blocks = num_blocks_x * num_blocks_y; - if (!basis_validate_output_buffer_size(basis_tex_format::cASTC_HDR_6x6, target_format, output_blocks_buf_size_in_blocks_or_pixels, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, output_rows_in_pixels)) + if (!basis_validate_output_buffer_size(target_format, output_blocks_buf_size_in_blocks_or_pixels, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, output_rows_in_pixels)) { - BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_image: output buffer size too small\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_image: output buffer size too small\n"); return false; } @@ -11725,7 +11723,7 @@ namespace basist if (!status) { - BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_image: transcode_slice() to ASTC_HDR failed\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_image: transcode_slice() to ASTC_HDR failed\n"); } break; } @@ -11735,7 +11733,7 @@ namespace basist bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, channel0, channel1, decode_flags); if (!status) { - BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_image: transcode_slice() to BC6H failed\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_image: transcode_slice() to BC6H failed\n"); } break; } @@ -11745,7 +11743,7 @@ namespace basist bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, channel0, channel1, decode_flags); if (!status) { - BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_image: transcode_slice() to RGB_HALF failed\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_image: transcode_slice() to RGB_HALF failed\n"); } break; } @@ -11755,7 +11753,7 @@ namespace basist bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, channel0, channel1, decode_flags); if (!status) { - BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_image: transcode_slice() to RGBA_HALF failed\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_image: transcode_slice() to RGBA_HALF failed\n"); } break; } @@ -11765,14 +11763,14 @@ namespace basist bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, channel0, channel1 , decode_flags); if (!status) { - BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_image: transcode_slice() to RGBA_HALF failed\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_image: transcode_slice() to RGBA_HALF failed\n"); } break; } default: { assert(0); - BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_image: Invalid format\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_image: Invalid format\n"); break; } } @@ -11781,7 +11779,7 @@ namespace basist } //------------------------------------------------------------------------------------------------ - + basisu_transcoder::basisu_transcoder() : m_ready_to_transcode(false) { @@ -11809,7 +11807,7 @@ namespace basist return false; } } -#endif +#endif return true; } @@ -11896,7 +11894,7 @@ namespace basist return false; } } - + // This flag dates back to pre-Basis Universal, when .basis supported full ETC1 too. if ((pHeader->m_flags & cBASISHeaderFlagETC1S) == 0) { @@ -11912,7 +11910,7 @@ namespace basist return false; } } - + if ((pHeader->m_slice_desc_file_ofs >= data_size) || ((data_size - pHeader->m_slice_desc_file_ofs) < (sizeof(basis_slice_desc) * pHeader->m_total_slices)) ) @@ -12028,20 +12026,20 @@ namespace basist image_info.m_image_index = image_index; image_info.m_total_levels = total_levels; - + image_info.m_alpha_flag = false; // For ETC1S, if anything has alpha all images have alpha. For UASTC, we only report alpha when the image actually has alpha. if (pHeader->m_tex_format == (int)basis_tex_format::cETC1S) - image_info.m_alpha_flag = (pHeader->m_flags & cBASISHeaderFlagHasAlphaSlices) != 0; + image_info.m_alpha_flag = (pHeader->m_flags & cBASISHeaderFlagHasAlphaSlices) != 0; else image_info.m_alpha_flag = (slice_desc.m_flags & cSliceDescFlagsHasAlpha) != 0; image_info.m_iframe_flag = (slice_desc.m_flags & cSliceDescFlagsFrameIsIFrame) != 0; - + const uint32_t block_width = basis_tex_format_get_block_width((basis_tex_format)((uint32_t)pHeader->m_tex_format)); const uint32_t block_height = basis_tex_format_get_block_height((basis_tex_format)((uint32_t)pHeader->m_tex_format)); - + image_info.m_width = slice_desc.m_num_blocks_x * block_width; image_info.m_height = slice_desc.m_num_blocks_y * block_height; image_info.m_orig_width = slice_desc.m_orig_width; @@ -12161,13 +12159,13 @@ namespace basist image_info.m_image_index = image_index; image_info.m_level_index = level_index; - + // For ETC1S, if anything has alpha all images have alpha. For UASTC, we only report alpha when the image actually has alpha. if (pHeader->m_tex_format == (int)basis_tex_format::cETC1S) image_info.m_alpha_flag = (pHeader->m_flags & cBASISHeaderFlagHasAlphaSlices) != 0; else image_info.m_alpha_flag = (slice_desc.m_flags & cSliceDescFlagsHasAlpha) != 0; - + const uint32_t block_width = basis_tex_format_get_block_width((basis_tex_format)((uint32_t)pHeader->m_tex_format)); const uint32_t block_height = basis_tex_format_get_block_height((basis_tex_format)((uint32_t)pHeader->m_tex_format)); @@ -12230,8 +12228,9 @@ namespace basist file_info.m_tex_format = static_cast(static_cast(pHeader->m_tex_format)); file_info.m_etc1s = (pHeader->m_tex_format == (int)basis_tex_format::cETC1S); - + file_info.m_y_flipped = (pHeader->m_flags & cBASISHeaderFlagYFlipped) != 0; + file_info.m_srgb = (pHeader->m_flags & cBASISHeaderFlagSRGB) != 0; file_info.m_has_alpha_slices = (pHeader->m_flags & cBASISHeaderFlagHasAlphaSlices) != 0; const uint32_t total_slices = pHeader->m_total_slices; @@ -12302,7 +12301,7 @@ namespace basist return true; } - + bool basisu_transcoder::start_transcoding(const void* pData, uint32_t data_size) { if (!validate_header_quick(pData, data_size)) @@ -12410,7 +12409,7 @@ namespace basist m_lowlevel_etc1s_decoder.clear(); } } - + m_ready_to_transcode = true; return true; @@ -12421,7 +12420,7 @@ namespace basist m_lowlevel_etc1s_decoder.clear(); m_ready_to_transcode = false; - + return true; } @@ -12459,6 +12458,8 @@ namespace basist const basis_slice_desc& slice_desc = reinterpret_cast(pDataU8 + pHeader->m_slice_desc_file_ofs)[slice_index]; + const uint32_t dst_block_width = get_block_width(fmt), dst_block_height = get_block_height(fmt); + if (basis_block_format_is_uncompressed(fmt)) { // Assume the output buffer is orig_width by orig_height @@ -12487,33 +12488,23 @@ namespace basist return false; } } - else if (fmt == block_format::cASTC_HDR_6x6) - { - const uint32_t num_blocks_6x6_x = (slice_desc.m_orig_width + 5) / 6; - const uint32_t num_blocks_6x6_y = (slice_desc.m_orig_height + 5) / 6; - const uint32_t total_blocks_6x6 = num_blocks_6x6_x * num_blocks_6x6_y; - - if (output_blocks_buf_size_in_blocks_or_pixels < total_blocks_6x6) - { - BASISU_DEVEL_ERROR("basisu_transcoder::transcode_slice: output_blocks_buf_size_in_blocks_or_pixels < total_blocks_6x6\n"); - return false; - } - } else { - // must be a 4x4 pixel block format - const uint32_t num_blocks_4x4_x = (slice_desc.m_orig_width + 3) / 4; - const uint32_t num_blocks_4x4_y = (slice_desc.m_orig_height + 3) / 4; - const uint32_t total_4x4_blocks = num_blocks_4x4_x * num_blocks_4x4_y; + const uint32_t dst_num_blocks_x = (slice_desc.m_orig_width + dst_block_width - 1) / dst_block_width; + const uint32_t dst_num_blocks_y = (slice_desc.m_orig_height + dst_block_height - 1) / dst_block_height; + const uint32_t dst_total_blocks = dst_num_blocks_x * dst_num_blocks_y; - if (output_blocks_buf_size_in_blocks_or_pixels < total_4x4_blocks) + if (output_blocks_buf_size_in_blocks_or_pixels < dst_total_blocks) { BASISU_DEVEL_ERROR("basisu_transcoder::transcode_slice: output_blocks_buf_size_in_blocks_or_pixels < total_blocks\n"); return false; } } + + const bool is_xuastc_ldr = basis_tex_format_is_xuastc_ldr((basis_tex_format)(uint32_t)pHeader->m_tex_format); + const bool is_astc_ldr = basis_tex_format_is_astc_ldr((basis_tex_format)(uint32_t)pHeader->m_tex_format); - if ((pHeader->m_tex_format == (uint32_t)basis_tex_format::cETC1S) || (pHeader->m_tex_format == (uint32_t)basis_tex_format::cUASTC4x4)) + if ((pHeader->m_tex_format == (uint32_t)basis_tex_format::cETC1S) || (pHeader->m_tex_format == (uint32_t)basis_tex_format::cUASTC_LDR_4x4) || is_xuastc_ldr || is_astc_ldr) { if ((fmt == block_format::cPVRTC1_4_RGB) || (fmt == block_format::cPVRTC1_4_RGBA)) { @@ -12538,16 +12529,18 @@ namespace basist BASISU_DEVEL_ERROR("basisu_transcoder::transcode_slice: invalid slice_desc.m_file_size, or passed in buffer too small\n"); return false; } - + if (pHeader->m_tex_format == (int)basis_tex_format::cASTC_HDR_6x6) { + // ASTC HDR 6x6 return m_lowlevel_astc_6x6_hdr_decoder.transcode_slice(pOutput_blocks, slice_desc.m_num_blocks_x, slice_desc.m_num_blocks_y, pDataU8 + slice_desc.m_file_ofs, slice_desc.m_file_size, fmt, output_block_or_pixel_stride_in_bytes, (decode_flags & cDecodeFlagsBC1ForbidThreeColorBlocks) == 0, *pHeader, slice_desc, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, channel0, channel1, decode_flags); } - else if (pHeader->m_tex_format == (int)basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE) + else if (pHeader->m_tex_format == (int)basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE) { + // UASTC HDR 6x6 return m_lowlevel_astc_6x6_hdr_intermediate_decoder.transcode_slice(pOutput_blocks, slice_desc.m_num_blocks_x, slice_desc.m_num_blocks_y, pDataU8 + slice_desc.m_file_ofs, slice_desc.m_file_size, fmt, output_block_or_pixel_stride_in_bytes, (decode_flags & cDecodeFlagsBC1ForbidThreeColorBlocks) == 0, *pHeader, slice_desc, output_row_pitch_in_blocks_or_pixels, pState, @@ -12555,21 +12548,34 @@ namespace basist } else if (pHeader->m_tex_format == (int)basis_tex_format::cUASTC_HDR_4x4) { + // UASTC HDR 4x4 return m_lowlevel_uastc_4x4_hdr_decoder.transcode_slice(pOutput_blocks, slice_desc.m_num_blocks_x, slice_desc.m_num_blocks_y, pDataU8 + slice_desc.m_file_ofs, slice_desc.m_file_size, fmt, output_block_or_pixel_stride_in_bytes, (decode_flags & cDecodeFlagsBC1ForbidThreeColorBlocks) == 0, *pHeader, slice_desc, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, channel0, channel1, decode_flags); } - else if (pHeader->m_tex_format == (int)basis_tex_format::cUASTC4x4) + else if (pHeader->m_tex_format == (int)basis_tex_format::cUASTC_LDR_4x4) { - return m_lowlevel_uastc_decoder.transcode_slice(pOutput_blocks, slice_desc.m_num_blocks_x, slice_desc.m_num_blocks_y, + // UASTC LDR 4x4 + return m_lowlevel_uastc_ldr_4x4_decoder.transcode_slice(pOutput_blocks, slice_desc.m_num_blocks_x, slice_desc.m_num_blocks_y, pDataU8 + slice_desc.m_file_ofs, slice_desc.m_file_size, fmt, output_block_or_pixel_stride_in_bytes, (decode_flags & cDecodeFlagsBC1ForbidThreeColorBlocks) == 0, *pHeader, slice_desc, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, channel0, channel1, decode_flags); } - else + else if ((is_xuastc_ldr) || (is_astc_ldr)) { - return m_lowlevel_etc1s_decoder.transcode_slice(pOutput_blocks, slice_desc.m_num_blocks_x, slice_desc.m_num_blocks_y, + // XUASTC LDR 4x4-12x12 or ASTC LDR 4x4-12x12 + const bool use_astc_srgb_decode_profile = (pHeader->m_flags & cBASISHeaderFlagSRGB) != 0; + + return m_lowlevel_xuastc_ldr_decoder.transcode_slice((basis_tex_format)(uint32_t)pHeader->m_tex_format, use_astc_srgb_decode_profile, pOutput_blocks, slice_desc.m_num_blocks_x, slice_desc.m_num_blocks_y, + pDataU8 + slice_desc.m_file_ofs, slice_desc.m_file_size, + fmt, output_block_or_pixel_stride_in_bytes, (decode_flags & cDecodeFlagsBC1ForbidThreeColorBlocks) == 0, *pHeader, slice_desc, output_row_pitch_in_blocks_or_pixels, pState, + output_rows_in_pixels, channel0, channel1, decode_flags); + } + else + { + // must be ETC1S + return m_lowlevel_etc1s_decoder.transcode_slice(pOutput_blocks, slice_desc.m_num_blocks_x, slice_desc.m_num_blocks_y, pDataU8 + slice_desc.m_file_ofs, slice_desc.m_file_size, fmt, output_block_or_pixel_stride_in_bytes, (decode_flags & cDecodeFlagsBC1ForbidThreeColorBlocks) == 0, *pHeader, slice_desc, output_row_pitch_in_blocks_or_pixels, pState, (decode_flags & cDecodeFlagsOutputHasAlphaIndices) != 0, pAlpha_blocks, output_rows_in_pixels); @@ -12647,7 +12653,7 @@ namespace basist if (!output_row_pitch_in_blocks_or_pixels) output_row_pitch_in_blocks_or_pixels = num_blocks_x; - + if ((fmt == block_format::cETC2_EAC_A8) || (fmt == block_format::cETC2_EAC_R11)) { #if BASISD_SUPPORT_ETC2_EAC_A8 @@ -12733,7 +12739,7 @@ namespace basist if (slice_index < 0) { BASISU_DEVEL_ERROR("basisu_transcoder::transcode_image_level: failed finding slice index\n"); - // Unable to find the requested image/level + // Unable to find the requested image/level return false; } @@ -12742,7 +12748,7 @@ namespace basist // Switch to PVRTC1 RGB if the input doesn't have alpha. fmt = transcoder_texture_format::cTFPVRTC1_4_RGB; } - + if (pHeader->m_tex_format == (int)basis_tex_format::cETC1S) { if (pSlice_descs[slice_index].m_flags & cSliceDescFlagsHasAlpha) @@ -12779,10 +12785,13 @@ namespace basist } } } - + bool status = false; - if ((pHeader->m_tex_format == (int)basis_tex_format::cETC1S) || (pHeader->m_tex_format == (int)basis_tex_format::cUASTC4x4)) + const bool is_xuastc_ldr = basis_tex_format_is_xuastc_ldr((basis_tex_format)(uint32_t)pHeader->m_tex_format); + const bool is_astc_ldr = basis_tex_format_is_astc_ldr((basis_tex_format)(uint32_t)pHeader->m_tex_format); + + if ((pHeader->m_tex_format == (int)basis_tex_format::cETC1S) || (pHeader->m_tex_format == (int)basis_tex_format::cUASTC_LDR_4x4) || is_xuastc_ldr || is_astc_ldr) { // Only do this on 4x4 LDR formats that supports transcoding to PVRTC1. const uint32_t total_slice_blocks = pSlice_descs[slice_index].m_num_blocks_x * pSlice_descs[slice_index].m_num_blocks_y; @@ -12790,28 +12799,28 @@ namespace basist if (((fmt == transcoder_texture_format::cTFPVRTC1_4_RGB) || (fmt == transcoder_texture_format::cTFPVRTC1_4_RGBA)) && (output_blocks_buf_size_in_blocks_or_pixels > total_slice_blocks)) { // The transcoder doesn't write beyond total_slice_blocks, so we need to clear the rest ourselves. - // For GL usage, PVRTC1 4bpp image size is (max(width, 8)* max(height, 8) * 4 + 7) / 8. + // For GL usage, PVRTC1 4bpp image size is (max(width, 8)* max(height, 8) * 4 + 7) / 8. // However, for KTX and internally in Basis this formula isn't used, it's just ((width+3)/4) * ((height+3)/4) * bytes_per_block_or_pixel. This is all the transcoder actually writes to memory. memset(static_cast(pOutput_blocks) + total_slice_blocks * bytes_per_block_or_pixel, 0, (output_blocks_buf_size_in_blocks_or_pixels - total_slice_blocks) * bytes_per_block_or_pixel); } } - + if (pHeader->m_tex_format == (int)basis_tex_format::cASTC_HDR_6x6) { + // ASTC HDR 6x6 const basis_slice_desc* pSlice_desc = &pSlice_descs[slice_index]; - // Use the container independent image transcode method. status = m_lowlevel_astc_6x6_hdr_decoder.transcode_image(fmt, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, (const uint8_t*)pData, data_size, pSlice_desc->m_num_blocks_x, pSlice_desc->m_num_blocks_y, pSlice_desc->m_orig_width, pSlice_desc->m_orig_height, pSlice_desc->m_level_index, pSlice_desc->m_file_ofs, pSlice_desc->m_file_size, decode_flags, basis_file_has_alpha_slices, pHeader->m_tex_type == cBASISTexTypeVideoFrames, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels); } - else if (pHeader->m_tex_format == (int)basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE) + else if (pHeader->m_tex_format == (int)basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE) { + // UASTC HDR 6x6 const basis_slice_desc* pSlice_desc = &pSlice_descs[slice_index]; - // Use the container independent image transcode method. status = m_lowlevel_astc_6x6_hdr_intermediate_decoder.transcode_image(fmt, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, (const uint8_t*)pData, data_size, pSlice_desc->m_num_blocks_x, pSlice_desc->m_num_blocks_y, pSlice_desc->m_orig_width, pSlice_desc->m_orig_height, pSlice_desc->m_level_index, @@ -12820,28 +12829,50 @@ namespace basist } else if (pHeader->m_tex_format == (int)basis_tex_format::cUASTC_HDR_4x4) { + // UASTC HDR 4x4 const basis_slice_desc* pSlice_desc = &pSlice_descs[slice_index]; - // Use the container independent image transcode method. status = m_lowlevel_uastc_4x4_hdr_decoder.transcode_image(fmt, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, (const uint8_t*)pData, data_size, pSlice_desc->m_num_blocks_x, pSlice_desc->m_num_blocks_y, pSlice_desc->m_orig_width, pSlice_desc->m_orig_height, pSlice_desc->m_level_index, pSlice_desc->m_file_ofs, pSlice_desc->m_file_size, decode_flags, basis_file_has_alpha_slices, pHeader->m_tex_type == cBASISTexTypeVideoFrames, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels); } - else if (pHeader->m_tex_format == (int)basis_tex_format::cUASTC4x4) + else if (pHeader->m_tex_format == (int)basis_tex_format::cUASTC_LDR_4x4) { + // UASTC LDR 4x4 const basis_slice_desc* pSlice_desc = &pSlice_descs[slice_index]; - // Use the container independent image transcode method. - status = m_lowlevel_uastc_decoder.transcode_image(fmt, + status = m_lowlevel_uastc_ldr_4x4_decoder.transcode_image(fmt, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, (const uint8_t*)pData, data_size, pSlice_desc->m_num_blocks_x, pSlice_desc->m_num_blocks_y, pSlice_desc->m_orig_width, pSlice_desc->m_orig_height, pSlice_desc->m_level_index, pSlice_desc->m_file_ofs, pSlice_desc->m_file_size, decode_flags, basis_file_has_alpha_slices, pHeader->m_tex_type == cBASISTexTypeVideoFrames, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels); } - else + else if (is_xuastc_ldr || is_astc_ldr) + { + // XUASTC LDR 4x4-12x12 or ASTC LDR 4x4-12x12 + const basis_slice_desc* pSlice_desc = &pSlice_descs[slice_index]; + + const bool use_astc_srgb_decode_profile = (pHeader->m_flags & cBASISHeaderFlagSRGB) != 0; + + status = m_lowlevel_xuastc_ldr_decoder.transcode_image((basis_tex_format)(uint32_t)pHeader->m_tex_format, use_astc_srgb_decode_profile, fmt, + pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, + (const uint8_t*)pData, data_size, pSlice_desc->m_num_blocks_x, pSlice_desc->m_num_blocks_y, pSlice_desc->m_orig_width, pSlice_desc->m_orig_height, pSlice_desc->m_level_index, + pSlice_desc->m_file_ofs, pSlice_desc->m_file_size, + decode_flags, basis_file_has_alpha_slices, pHeader->m_tex_type == cBASISTexTypeVideoFrames, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels); + } + else { + // ETC1S + + // sanity check + if (pHeader->m_tex_format != (uint32_t)basis_tex_format::cETC1S) + { + BASISU_DEVEL_ERROR("basisu_transcoder::transcode_image_level: unsupported texture format\n"); + return false; + } + // ETC1S const basis_slice_desc* pSlice_desc = &pSlice_descs[slice_index]; const basis_slice_desc* pAlpha_slice_desc = basis_file_has_alpha_slices ? &pSlice_descs[slice_index + 1] : nullptr; @@ -12866,14 +12897,14 @@ namespace basist decode_flags, basis_file_has_alpha_slices, pHeader->m_tex_type == cBASISTexTypeVideoFrames, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels); } // if (pHeader->m_tex_format == (int)basis_tex_format::cUASTC4x4) - + if (!status) { BASISU_DEVEL_ERROR("basisu_transcoder::transcode_image_level: Returning false\n"); } else { - //BASISU_DEVEL_ERROR("basisu_transcoder::transcode_image_level: Returning true\n"); + //BASISU_DEVEL_ERROR("basisu_transcoder::transcode_image_level: Returning true\n"); } return status; @@ -12899,7 +12930,22 @@ namespace basist case transcoder_texture_format::cTFETC2_RGBA: case transcoder_texture_format::cTFBC3_RGBA: case transcoder_texture_format::cTFBC5_RG: - case transcoder_texture_format::cTFASTC_4x4_RGBA: + + case transcoder_texture_format::cTFASTC_LDR_4x4_RGBA: + case transcoder_texture_format::cTFASTC_LDR_5x4_RGBA: + case transcoder_texture_format::cTFASTC_LDR_5x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_6x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_6x6_RGBA: + case transcoder_texture_format::cTFASTC_LDR_8x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_8x6_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x6_RGBA: + case transcoder_texture_format::cTFASTC_LDR_8x8_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x8_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x10_RGBA: + case transcoder_texture_format::cTFASTC_LDR_12x10_RGBA: + case transcoder_texture_format::cTFASTC_LDR_12x12_RGBA: + case transcoder_texture_format::cTFASTC_HDR_4x4_RGBA: case transcoder_texture_format::cTFASTC_HDR_6x6_RGBA: case transcoder_texture_format::cTFATC_RGBA: @@ -12939,7 +12985,7 @@ namespace basist case transcoder_texture_format::cTFETC2_RGBA: return "ETC2_RGBA"; case transcoder_texture_format::cTFBC3_RGBA: return "BC3_RGBA"; case transcoder_texture_format::cTFBC5_RG: return "BC5_RG"; - case transcoder_texture_format::cTFASTC_4x4_RGBA: return "ASTC_RGBA"; + case transcoder_texture_format::cTFASTC_HDR_4x4_RGBA: return "ASTC_HDR_4X4_RGBA"; case transcoder_texture_format::cTFASTC_HDR_6x6_RGBA: return "ASTC_HDR_6X6_RGBA"; case transcoder_texture_format::cTFATC_RGB: return "ATC_RGB"; @@ -12957,6 +13003,22 @@ namespace basist case transcoder_texture_format::cTFETC2_EAC_R11: return "ETC2_EAC_R11"; case transcoder_texture_format::cTFETC2_EAC_RG11: return "ETC2_EAC_RG11"; case transcoder_texture_format::cTFBC6H: return "BC6H"; + + case transcoder_texture_format::cTFASTC_LDR_4x4_RGBA: return "ASTC_LDR_4X4_RGBA"; + case transcoder_texture_format::cTFASTC_LDR_5x4_RGBA: return "ASTC_LDR_5X4_RGBA"; + case transcoder_texture_format::cTFASTC_LDR_5x5_RGBA: return "ASTC_LDR_5X5_RGBA"; + case transcoder_texture_format::cTFASTC_LDR_6x5_RGBA: return "ASTC_LDR_6X5_RGBA"; + case transcoder_texture_format::cTFASTC_LDR_6x6_RGBA: return "ASTC_LDR_6X6_RGBA"; + case transcoder_texture_format::cTFASTC_LDR_8x5_RGBA: return "ASTC_LDR_8X5_RGBA"; + case transcoder_texture_format::cTFASTC_LDR_8x6_RGBA: return "ASTC_LDR_8X6_RGBA"; + case transcoder_texture_format::cTFASTC_LDR_10x5_RGBA: return "ASTC_LDR_10X5_RGBA"; + case transcoder_texture_format::cTFASTC_LDR_10x6_RGBA: return "ASTC_LDR_10X6_RGBA"; + case transcoder_texture_format::cTFASTC_LDR_8x8_RGBA: return "ASTC_LDR_8X8_RGBA"; + case transcoder_texture_format::cTFASTC_LDR_10x8_RGBA: return "ASTC_LDR_10X8_RGBA"; + case transcoder_texture_format::cTFASTC_LDR_10x10_RGBA: return "ASTC_LDR_10X10_RGBA"; + case transcoder_texture_format::cTFASTC_LDR_12x10_RGBA: return "ASTC_LDR_12X10_RGBA"; + case transcoder_texture_format::cTFASTC_LDR_12x12_RGBA: return "ASTC_LDR_12X12_RGBA"; + default: assert(0); BASISU_DEVEL_ERROR("basis_get_basisu_texture_format: Invalid fmt\n"); @@ -12965,6 +13027,51 @@ namespace basist return ""; } + const char* basis_get_tex_format_name(basis_tex_format fmt) + { + switch (fmt) + { + case basis_tex_format::cETC1S: return "ETC1S"; break; + case basis_tex_format::cUASTC_LDR_4x4: return "UASTC LDR 4x4"; break; + case basis_tex_format::cUASTC_HDR_4x4: return "UASTC_HDR_4x4"; break; + case basis_tex_format::cASTC_HDR_6x6: return "ASTC_HDR_6x6"; break; + case basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE: return "UASTC_HDR_6x6"; break; + case basis_tex_format::cXUASTC_LDR_4x4: return "XUASTC LDR 4x4"; break; + case basis_tex_format::cXUASTC_LDR_5x4: return "XUASTC LDR 5x4"; break; + case basis_tex_format::cXUASTC_LDR_5x5: return "XUASTC LDR 5x5"; break; + case basis_tex_format::cXUASTC_LDR_6x5: return "XUASTC LDR 6x5"; break; + case basis_tex_format::cXUASTC_LDR_6x6: return "XUASTC LDR 6x6"; break; + case basis_tex_format::cXUASTC_LDR_8x5: return "XUASTC LDR 8x5"; break; + case basis_tex_format::cXUASTC_LDR_8x6: return "XUASTC LDR 8x6"; break; + case basis_tex_format::cXUASTC_LDR_10x5: return "XUASTC LDR 10x5"; break; + case basis_tex_format::cXUASTC_LDR_10x6: return "XUASTC LDR 10x6"; break; + case basis_tex_format::cXUASTC_LDR_8x8: return "XUASTC LDR 8x8"; break; + case basis_tex_format::cXUASTC_LDR_10x8: return "XUASTC LDR 10x8"; break; + case basis_tex_format::cXUASTC_LDR_10x10: return "XUASTC LDR 10x10"; break; + case basis_tex_format::cXUASTC_LDR_12x10: return "XUASTC LDR 12x10"; break; + case basis_tex_format::cXUASTC_LDR_12x12: return "XUASTC LDR 12x12"; break; + case basis_tex_format::cASTC_LDR_4x4: return "ASTC LDR 4x4"; break; + case basis_tex_format::cASTC_LDR_5x4: return "ASTC LDR 5x4"; break; + case basis_tex_format::cASTC_LDR_5x5: return "ASTC LDR 5x5"; break; + case basis_tex_format::cASTC_LDR_6x5: return "ASTC LDR 6x5"; break; + case basis_tex_format::cASTC_LDR_6x6: return "ASTC LDR 6x6"; break; + case basis_tex_format::cASTC_LDR_8x5: return "ASTC LDR 8x5"; break; + case basis_tex_format::cASTC_LDR_8x6: return "ASTC LDR 8x6"; break; + case basis_tex_format::cASTC_LDR_10x5: return "ASTC LDR 10x5"; break; + case basis_tex_format::cASTC_LDR_10x6: return "ASTC LDR 10x6"; break; + case basis_tex_format::cASTC_LDR_8x8: return "ASTC LDR 8x8"; break; + case basis_tex_format::cASTC_LDR_10x8: return "ASTC LDR 10x8"; break; + case basis_tex_format::cASTC_LDR_10x10: return "ASTC LDR 10x10"; break; + case basis_tex_format::cASTC_LDR_12x10: return "ASTC LDR 12x10"; break; + case basis_tex_format::cASTC_LDR_12x12: return "ASTC LDR 12x12"; break; + default: + assert(0); + BASISU_DEVEL_ERROR("basis_get_tex_format_name: Invalid parameter\n"); + break; + } + return ""; + } + const char* basis_get_block_format_name(block_format fmt) { switch (fmt) @@ -12976,7 +13083,22 @@ namespace basist case block_format::cBC7: return "BC7"; case block_format::cETC2_RGBA: return "ETC2_RGBA"; case block_format::cBC3: return "BC3"; - case block_format::cASTC_4x4: return "ASTC_4x4"; + + case block_format::cASTC_LDR_4x4: return "ASTC_LDR_4x4"; + case block_format::cASTC_LDR_5x4: return "ASTC_LDR_5x4"; + case block_format::cASTC_LDR_5x5: return "ASTC_LDR_5x5"; + case block_format::cASTC_LDR_6x5: return "ASTC_LDR_6x5"; + case block_format::cASTC_LDR_6x6: return "ASTC_LDR_6x6"; + case block_format::cASTC_LDR_8x5: return "ASTC_LDR_8x5"; + case block_format::cASTC_LDR_8x6: return "ASTC_LDR_8x6"; + case block_format::cASTC_LDR_10x5: return "ASTC_LDR_10x5"; + case block_format::cASTC_LDR_10x6: return "ASTC_LDR_10x6"; + case block_format::cASTC_LDR_8x8: return "ASTC_LDR_8x8"; + case block_format::cASTC_LDR_10x8: return "ASTC_LDR_10x8"; + case block_format::cASTC_LDR_10x10: return "ASTC_LDR_10x10"; + case block_format::cASTC_LDR_12x10: return "ASTC_LDR_12x10"; + case block_format::cASTC_LDR_12x12: return "ASTC_LDR_12x12"; + case block_format::cATC_RGB: return "ATC_RGB"; case block_format::cRGBA32: return "RGBA32"; case block_format::cRGB565: return "RGB565"; @@ -13027,7 +13149,22 @@ namespace basist { case transcoder_texture_format::cTFETC2_RGBA: case transcoder_texture_format::cTFBC3_RGBA: - case transcoder_texture_format::cTFASTC_4x4_RGBA: + + case transcoder_texture_format::cTFASTC_LDR_4x4_RGBA: + case transcoder_texture_format::cTFASTC_LDR_5x4_RGBA: + case transcoder_texture_format::cTFASTC_LDR_5x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_6x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_6x6_RGBA: + case transcoder_texture_format::cTFASTC_LDR_8x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_8x6_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x6_RGBA: + case transcoder_texture_format::cTFASTC_LDR_8x8_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x8_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x10_RGBA: + case transcoder_texture_format::cTFASTC_LDR_12x10_RGBA: + case transcoder_texture_format::cTFASTC_LDR_12x12_RGBA: + case transcoder_texture_format::cTFASTC_HDR_4x4_RGBA: // technically this ASTC HDR format supports alpha, but we currently don't exploit that in our encoders case transcoder_texture_format::cTFASTC_HDR_6x6_RGBA: // technically this ASTC HDR format supports alpha, but we currently don't exploit that in our encoders case transcoder_texture_format::cTFBC7_RGBA: @@ -13062,6 +13199,33 @@ namespace basist return false; } + bool basis_is_transcoder_texture_format_astc(transcoder_texture_format fmt) + { + switch (fmt) + { + case transcoder_texture_format::cTFASTC_LDR_4x4_RGBA: + case transcoder_texture_format::cTFASTC_LDR_5x4_RGBA: + case transcoder_texture_format::cTFASTC_LDR_5x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_6x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_6x6_RGBA: + case transcoder_texture_format::cTFASTC_LDR_8x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_8x6_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x6_RGBA: + case transcoder_texture_format::cTFASTC_LDR_8x8_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x8_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x10_RGBA: + case transcoder_texture_format::cTFASTC_LDR_12x10_RGBA: + case transcoder_texture_format::cTFASTC_LDR_12x12_RGBA: + case transcoder_texture_format::cTFASTC_HDR_4x4_RGBA: + case transcoder_texture_format::cTFASTC_HDR_6x6_RGBA: + return true; + default: + break; + } + return false; + } + basisu::texture_format basis_get_basisu_texture_format(transcoder_texture_format fmt) { switch (fmt) @@ -13076,7 +13240,22 @@ namespace basist case transcoder_texture_format::cTFETC2_RGBA: return basisu::texture_format::cETC2_RGBA; case transcoder_texture_format::cTFBC3_RGBA: return basisu::texture_format::cBC3; case transcoder_texture_format::cTFBC5_RG: return basisu::texture_format::cBC5; - case transcoder_texture_format::cTFASTC_4x4_RGBA: return basisu::texture_format::cASTC_LDR_4x4; + + case transcoder_texture_format::cTFASTC_LDR_4x4_RGBA: return basisu::texture_format::cASTC_LDR_4x4; + case transcoder_texture_format::cTFASTC_LDR_5x4_RGBA: return basisu::texture_format::cASTC_LDR_5x4; + case transcoder_texture_format::cTFASTC_LDR_5x5_RGBA: return basisu::texture_format::cASTC_LDR_5x5; + case transcoder_texture_format::cTFASTC_LDR_6x5_RGBA: return basisu::texture_format::cASTC_LDR_6x5; + case transcoder_texture_format::cTFASTC_LDR_6x6_RGBA: return basisu::texture_format::cASTC_LDR_6x6; + case transcoder_texture_format::cTFASTC_LDR_8x5_RGBA: return basisu::texture_format::cASTC_LDR_8x5; + case transcoder_texture_format::cTFASTC_LDR_8x6_RGBA: return basisu::texture_format::cASTC_LDR_8x6; + case transcoder_texture_format::cTFASTC_LDR_10x5_RGBA: return basisu::texture_format::cASTC_LDR_10x5; + case transcoder_texture_format::cTFASTC_LDR_10x6_RGBA: return basisu::texture_format::cASTC_LDR_10x6; + case transcoder_texture_format::cTFASTC_LDR_8x8_RGBA: return basisu::texture_format::cASTC_LDR_8x8; + case transcoder_texture_format::cTFASTC_LDR_10x8_RGBA: return basisu::texture_format::cASTC_LDR_10x8; + case transcoder_texture_format::cTFASTC_LDR_10x10_RGBA: return basisu::texture_format::cASTC_LDR_10x10; + case transcoder_texture_format::cTFASTC_LDR_12x10_RGBA: return basisu::texture_format::cASTC_LDR_12x10; + case transcoder_texture_format::cTFASTC_LDR_12x12_RGBA: return basisu::texture_format::cASTC_LDR_12x12; + case transcoder_texture_format::cTFASTC_HDR_4x4_RGBA: return basisu::texture_format::cASTC_HDR_4x4; case transcoder_texture_format::cTFASTC_HDR_6x6_RGBA: return basisu::texture_format::cASTC_HDR_6x6; case transcoder_texture_format::cTFBC6H: return basisu::texture_format::cBC6HUnsigned; @@ -13142,14 +13321,14 @@ namespace basist } return false; } - + uint32_t basis_get_uncompressed_bytes_per_pixel(transcoder_texture_format fmt) { switch (fmt) { case transcoder_texture_format::cTFRGBA32: case transcoder_texture_format::cTFRGB_9E5: - return sizeof(uint32_t); + return sizeof(uint32_t); case transcoder_texture_format::cTFRGB565: case transcoder_texture_format::cTFBGR565: case transcoder_texture_format::cTFRGBA4444: @@ -13163,27 +13342,57 @@ namespace basist } return 0; } - - uint32_t basis_get_block_width(transcoder_texture_format tex_type) + + uint32_t basis_get_block_width(transcoder_texture_format fmt) { - switch (tex_type) + switch (fmt) { case transcoder_texture_format::cTFFXT1_RGB: return 8; case transcoder_texture_format::cTFASTC_HDR_6x6_RGBA: return 6; + + case transcoder_texture_format::cTFASTC_LDR_5x4_RGBA: return 5; + case transcoder_texture_format::cTFASTC_LDR_5x5_RGBA: return 5; + case transcoder_texture_format::cTFASTC_LDR_6x5_RGBA: return 6; + case transcoder_texture_format::cTFASTC_LDR_6x6_RGBA: return 6; + case transcoder_texture_format::cTFASTC_LDR_8x5_RGBA: return 8; + case transcoder_texture_format::cTFASTC_LDR_8x6_RGBA: return 8; + case transcoder_texture_format::cTFASTC_LDR_10x5_RGBA: return 10; + case transcoder_texture_format::cTFASTC_LDR_10x6_RGBA: return 10; + case transcoder_texture_format::cTFASTC_LDR_8x8_RGBA: return 8; + case transcoder_texture_format::cTFASTC_LDR_10x8_RGBA: return 10; + case transcoder_texture_format::cTFASTC_LDR_10x10_RGBA: return 10; + case transcoder_texture_format::cTFASTC_LDR_12x10_RGBA: return 12; + case transcoder_texture_format::cTFASTC_LDR_12x12_RGBA: return 12; + default: break; } return 4; } - uint32_t basis_get_block_height(transcoder_texture_format tex_type) + uint32_t basis_get_block_height(transcoder_texture_format fmt) { - switch (tex_type) + switch (fmt) { case transcoder_texture_format::cTFASTC_HDR_6x6_RGBA: return 6; + + case transcoder_texture_format::cTFASTC_LDR_5x5_RGBA: return 5; + case transcoder_texture_format::cTFASTC_LDR_6x5_RGBA: return 5; + case transcoder_texture_format::cTFASTC_LDR_6x6_RGBA: return 6; + case transcoder_texture_format::cTFASTC_LDR_8x5_RGBA: return 5; + case transcoder_texture_format::cTFASTC_LDR_8x6_RGBA: return 6; + case transcoder_texture_format::cTFASTC_LDR_10x5_RGBA: return 5; + case transcoder_texture_format::cTFASTC_LDR_10x6_RGBA: return 6; + case transcoder_texture_format::cTFASTC_LDR_8x8_RGBA: return 8; + case transcoder_texture_format::cTFASTC_LDR_10x8_RGBA: return 8; + case transcoder_texture_format::cTFASTC_LDR_10x10_RGBA: return 10; + case transcoder_texture_format::cTFASTC_LDR_12x10_RGBA: return 10; + case transcoder_texture_format::cTFASTC_LDR_12x12_RGBA: return 12; + + default: break; } @@ -13195,8 +13404,36 @@ namespace basist switch (fmt) { case basis_tex_format::cASTC_HDR_6x6: - case basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE: + case basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE: return 6; + case basis_tex_format::cXUASTC_LDR_4x4: return 4; + case basis_tex_format::cXUASTC_LDR_5x4: return 5; + case basis_tex_format::cXUASTC_LDR_5x5: return 5; + case basis_tex_format::cXUASTC_LDR_6x5: return 6; + case basis_tex_format::cXUASTC_LDR_6x6: return 6; + case basis_tex_format::cXUASTC_LDR_8x5: return 8; + case basis_tex_format::cXUASTC_LDR_8x6: return 8; + case basis_tex_format::cXUASTC_LDR_10x5: return 10; + case basis_tex_format::cXUASTC_LDR_10x6: return 10; + case basis_tex_format::cXUASTC_LDR_8x8: return 8; + case basis_tex_format::cXUASTC_LDR_10x8: return 10; + case basis_tex_format::cXUASTC_LDR_10x10: return 10; + case basis_tex_format::cXUASTC_LDR_12x10: return 12; + case basis_tex_format::cXUASTC_LDR_12x12: return 12; + case basis_tex_format::cASTC_LDR_4x4: return 4; + case basis_tex_format::cASTC_LDR_5x4: return 5; + case basis_tex_format::cASTC_LDR_5x5: return 5; + case basis_tex_format::cASTC_LDR_6x5: return 6; + case basis_tex_format::cASTC_LDR_6x6: return 6; + case basis_tex_format::cASTC_LDR_8x5: return 8; + case basis_tex_format::cASTC_LDR_8x6: return 8; + case basis_tex_format::cASTC_LDR_10x5: return 10; + case basis_tex_format::cASTC_LDR_10x6: return 10; + case basis_tex_format::cASTC_LDR_8x8: return 8; + case basis_tex_format::cASTC_LDR_10x8: return 10; + case basis_tex_format::cASTC_LDR_10x10: return 10; + case basis_tex_format::cASTC_LDR_12x10: return 12; + case basis_tex_format::cASTC_LDR_12x12: return 12; default: break; } @@ -13208,8 +13445,36 @@ namespace basist switch (fmt) { case basis_tex_format::cASTC_HDR_6x6: - case basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE: + case basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE: return 6; + case basis_tex_format::cXUASTC_LDR_4x4: return 4; + case basis_tex_format::cXUASTC_LDR_5x4: return 4; + case basis_tex_format::cXUASTC_LDR_5x5: return 5; + case basis_tex_format::cXUASTC_LDR_6x5: return 5; + case basis_tex_format::cXUASTC_LDR_6x6: return 6; + case basis_tex_format::cXUASTC_LDR_8x5: return 5; + case basis_tex_format::cXUASTC_LDR_8x6: return 6; + case basis_tex_format::cXUASTC_LDR_10x5: return 5; + case basis_tex_format::cXUASTC_LDR_10x6: return 6; + case basis_tex_format::cXUASTC_LDR_8x8: return 8; + case basis_tex_format::cXUASTC_LDR_10x8: return 8; + case basis_tex_format::cXUASTC_LDR_10x10: return 10; + case basis_tex_format::cXUASTC_LDR_12x10: return 10; + case basis_tex_format::cXUASTC_LDR_12x12: return 12; + case basis_tex_format::cASTC_LDR_4x4: return 4; + case basis_tex_format::cASTC_LDR_5x4: return 4; + case basis_tex_format::cASTC_LDR_5x5: return 5; + case basis_tex_format::cASTC_LDR_6x5: return 5; + case basis_tex_format::cASTC_LDR_6x6: return 6; + case basis_tex_format::cASTC_LDR_8x5: return 5; + case basis_tex_format::cASTC_LDR_8x6: return 6; + case basis_tex_format::cASTC_LDR_10x5: return 5; + case basis_tex_format::cASTC_LDR_10x6: return 6; + case basis_tex_format::cASTC_LDR_8x8: return 8; + case basis_tex_format::cASTC_LDR_10x8: return 8; + case basis_tex_format::cASTC_LDR_10x10: return 10; + case basis_tex_format::cASTC_LDR_12x10: return 10; + case basis_tex_format::cASTC_LDR_12x12: return 12; default: break; } @@ -13222,7 +13487,7 @@ namespace basist { case basis_tex_format::cUASTC_HDR_4x4: case basis_tex_format::cASTC_HDR_6x6: - case basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE: + case basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE: return true; default: break; @@ -13230,9 +13495,136 @@ namespace basist return false; } + // Given a basis_tex_format (mode or codec), return the corresponding ASTC texture_format with the proper block size from 4x4-12x12. + basisu::texture_format basis_get_texture_format_from_xuastc_or_astc_ldr_basis_tex_format(basis_tex_format fmt) + { + switch (fmt) + { + case basis_tex_format::cXUASTC_LDR_4x4: + case basis_tex_format::cASTC_LDR_4x4: + return basisu::texture_format::cASTC_LDR_4x4; + case basis_tex_format::cXUASTC_LDR_5x4: + case basis_tex_format::cASTC_LDR_5x4: + return basisu::texture_format::cASTC_LDR_5x4; + case basis_tex_format::cXUASTC_LDR_5x5: + case basis_tex_format::cASTC_LDR_5x5: + return basisu::texture_format::cASTC_LDR_5x5; + case basis_tex_format::cXUASTC_LDR_6x5: + case basis_tex_format::cASTC_LDR_6x5: + return basisu::texture_format::cASTC_LDR_6x5; + case basis_tex_format::cXUASTC_LDR_6x6: + case basis_tex_format::cASTC_LDR_6x6: + return basisu::texture_format::cASTC_LDR_6x6; + case basis_tex_format::cXUASTC_LDR_8x5: + case basis_tex_format::cASTC_LDR_8x5: + return basisu::texture_format::cASTC_LDR_8x5; + case basis_tex_format::cXUASTC_LDR_8x6: + case basis_tex_format::cASTC_LDR_8x6: + return basisu::texture_format::cASTC_LDR_8x6; + case basis_tex_format::cXUASTC_LDR_10x5: + case basis_tex_format::cASTC_LDR_10x5: + return basisu::texture_format::cASTC_LDR_10x5; + case basis_tex_format::cXUASTC_LDR_10x6: + case basis_tex_format::cASTC_LDR_10x6: + return basisu::texture_format::cASTC_LDR_10x6; + case basis_tex_format::cXUASTC_LDR_8x8: + case basis_tex_format::cASTC_LDR_8x8: + return basisu::texture_format::cASTC_LDR_8x8; + case basis_tex_format::cXUASTC_LDR_10x8: + case basis_tex_format::cASTC_LDR_10x8: + return basisu::texture_format::cASTC_LDR_10x8; + case basis_tex_format::cXUASTC_LDR_10x10: + case basis_tex_format::cASTC_LDR_10x10: + return basisu::texture_format::cASTC_LDR_10x10; + case basis_tex_format::cXUASTC_LDR_12x10: + case basis_tex_format::cASTC_LDR_12x10: + return basisu::texture_format::cASTC_LDR_12x10; + case basis_tex_format::cXUASTC_LDR_12x12: + case basis_tex_format::cASTC_LDR_12x12: + return basisu::texture_format::cASTC_LDR_12x12; + default: + assert(0); + return basisu::texture_format::cInvalidTextureFormat; + } + } + + // Given any basis_tex_format (mode or codec), return the corresponding transcoder_texture_format with the proper ASTC block size from 4x4-12x12. + transcoder_texture_format basis_get_transcoder_texture_format_from_xuastc_or_astc_ldr_basis_tex_format(basis_tex_format fmt) + { + switch (fmt) + { + // XUASTC 4x4-12x12 and ASTC 4x4-12x12 + case basis_tex_format::cXUASTC_LDR_4x4: + case basis_tex_format::cASTC_LDR_4x4: + return transcoder_texture_format::cTFASTC_LDR_4x4_RGBA; + case basis_tex_format::cXUASTC_LDR_5x4: + case basis_tex_format::cASTC_LDR_5x4: + return transcoder_texture_format::cTFASTC_LDR_5x4_RGBA; + case basis_tex_format::cXUASTC_LDR_5x5: + case basis_tex_format::cASTC_LDR_5x5: + return transcoder_texture_format::cTFASTC_LDR_5x5_RGBA; + case basis_tex_format::cXUASTC_LDR_6x5: + case basis_tex_format::cASTC_LDR_6x5: + return transcoder_texture_format::cTFASTC_LDR_6x5_RGBA; + case basis_tex_format::cXUASTC_LDR_6x6: + case basis_tex_format::cASTC_LDR_6x6: + return transcoder_texture_format::cTFASTC_LDR_6x6_RGBA; + case basis_tex_format::cXUASTC_LDR_8x5: + case basis_tex_format::cASTC_LDR_8x5: + return transcoder_texture_format::cTFASTC_LDR_8x5_RGBA; + case basis_tex_format::cXUASTC_LDR_8x6: + case basis_tex_format::cASTC_LDR_8x6: + return transcoder_texture_format::cTFASTC_LDR_8x6_RGBA; + case basis_tex_format::cXUASTC_LDR_10x5: + case basis_tex_format::cASTC_LDR_10x5: + return transcoder_texture_format::cTFASTC_LDR_10x5_RGBA; + case basis_tex_format::cXUASTC_LDR_10x6: + case basis_tex_format::cASTC_LDR_10x6: + return transcoder_texture_format::cTFASTC_LDR_10x6_RGBA; + case basis_tex_format::cXUASTC_LDR_8x8: + case basis_tex_format::cASTC_LDR_8x8: + return transcoder_texture_format::cTFASTC_LDR_8x8_RGBA; + case basis_tex_format::cXUASTC_LDR_10x8: + case basis_tex_format::cASTC_LDR_10x8: + return transcoder_texture_format::cTFASTC_LDR_10x8_RGBA; + case basis_tex_format::cXUASTC_LDR_10x10: + case basis_tex_format::cASTC_LDR_10x10: + return transcoder_texture_format::cTFASTC_LDR_10x10_RGBA; + case basis_tex_format::cXUASTC_LDR_12x10: + case basis_tex_format::cASTC_LDR_12x10: + return transcoder_texture_format::cTFASTC_LDR_12x10_RGBA; + case basis_tex_format::cXUASTC_LDR_12x12: + case basis_tex_format::cASTC_LDR_12x12: + return transcoder_texture_format::cTFASTC_LDR_12x12_RGBA; + + // ETC1S/UASTC LDR 4x4 + case basis_tex_format::cETC1S: + case basis_tex_format::cUASTC_LDR_4x4: + return transcoder_texture_format::cTFASTC_LDR_4x4_RGBA; + + // HDR formats + case basis_tex_format::cUASTC_HDR_4x4: + return transcoder_texture_format::cTFASTC_HDR_4x4_RGBA; + + case basis_tex_format::cASTC_HDR_6x6: + case basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE: + return transcoder_texture_format::cTFASTC_HDR_6x6_RGBA; + + default: + assert(0); + return transcoder_texture_format::cTFASTC_LDR_4x4_RGBA; + } + } + + transcoder_texture_format basis_get_transcoder_texture_format_from_basis_tex_format(basis_tex_format fmt) + { + return basis_get_transcoder_texture_format_from_xuastc_or_astc_ldr_basis_tex_format(fmt); + } + + // For a given basis_tex_format (mode or codec), is the specified transcoder_texture_format supported? bool basis_is_format_supported(transcoder_texture_format tex_type, basis_tex_format fmt) { - if ((fmt == basis_tex_format::cASTC_HDR_6x6) || (fmt == basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE)) + if ((fmt == basis_tex_format::cASTC_HDR_6x6) || (fmt == basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE)) { // RDO UASTC HDR 6x6, or our custom intermediate format #if BASISD_SUPPORT_UASTC_HDR @@ -13266,37 +13658,182 @@ namespace basist } #endif } - else if (fmt == basis_tex_format::cUASTC4x4) + else if (fmt == basis_tex_format::cUASTC_LDR_4x4) { // UASTC LDR 4x4 #if BASISD_SUPPORT_UASTC + // IMPORTANT : This is defined as the formats which DON'T support UASTC LDR 4x4 transcoding. switch (tex_type) { - // These niche formats aren't currently supported for UASTC - everything else is. + // These niche formats aren't currently supported for UASTC LDR 4x4 - everything else is. case transcoder_texture_format::cTFPVRTC2_4_RGB: case transcoder_texture_format::cTFPVRTC2_4_RGBA: case transcoder_texture_format::cTFATC_RGB: case transcoder_texture_format::cTFATC_RGBA: case transcoder_texture_format::cTFFXT1_RGB: - // UASTC LDR doesn't support transcoding to HDR formats + // UASTC LDR 4x4 doesn't support transcoding to HDR formats case transcoder_texture_format::cTFASTC_HDR_4x4_RGBA: case transcoder_texture_format::cTFASTC_HDR_6x6_RGBA: case transcoder_texture_format::cTFBC6H: case transcoder_texture_format::cTFRGBA_HALF: case transcoder_texture_format::cTFRGB_HALF: case transcoder_texture_format::cTFRGB_9E5: + case transcoder_texture_format::cTFASTC_LDR_5x4_RGBA: + case transcoder_texture_format::cTFASTC_LDR_5x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_6x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_6x6_RGBA: + case transcoder_texture_format::cTFASTC_LDR_8x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_8x6_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x6_RGBA: + case transcoder_texture_format::cTFASTC_LDR_8x8_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x8_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x10_RGBA: + case transcoder_texture_format::cTFASTC_LDR_12x10_RGBA: + case transcoder_texture_format::cTFASTC_LDR_12x12_RGBA: return false; default: return true; } #endif } + else if ( (basis_tex_format_is_xuastc_ldr(fmt)) || (basis_tex_format_is_astc_ldr(fmt)) ) + { + // XUASTC LDR 4x4-12x12 or ASTC LDR 4x4-12x12 + switch (tex_type) + { + case transcoder_texture_format::cTFBC1_RGB: + case transcoder_texture_format::cTFBC3_RGBA: + case transcoder_texture_format::cTFBC4_R: + case transcoder_texture_format::cTFBC5_RG: + case transcoder_texture_format::cTFBC7_RGBA: + case transcoder_texture_format::cTFETC1_RGB: + case transcoder_texture_format::cTFETC2_RGBA: + case transcoder_texture_format::cTFETC2_EAC_R11: + case transcoder_texture_format::cTFETC2_EAC_RG11: + case transcoder_texture_format::cTFPVRTC1_4_RGB: + case transcoder_texture_format::cTFPVRTC1_4_RGBA: + // Uncompressed formats + case transcoder_texture_format::cTFRGBA32: + case transcoder_texture_format::cTFRGB565: + case transcoder_texture_format::cTFBGR565: + case transcoder_texture_format::cTFRGBA4444: + return true; + default: + break; + } + + // Ensure they're using the block size for ASTC LDR that matches the XUASTC format's block size. + switch (fmt) + { + case basis_tex_format::cXUASTC_LDR_4x4: + case basis_tex_format::cASTC_LDR_4x4: + { + if (tex_type == transcoder_texture_format::cTFASTC_LDR_4x4_RGBA) + return true; + break; + } + case basis_tex_format::cXUASTC_LDR_5x4: + case basis_tex_format::cASTC_LDR_5x4: + { + if (tex_type == transcoder_texture_format::cTFASTC_LDR_5x4_RGBA) + return true; + break; + } + case basis_tex_format::cXUASTC_LDR_5x5: + case basis_tex_format::cASTC_LDR_5x5: + { + if (tex_type == transcoder_texture_format::cTFASTC_LDR_5x5_RGBA) + return true; + break; + } + case basis_tex_format::cXUASTC_LDR_6x5: + case basis_tex_format::cASTC_LDR_6x5: + { + if (tex_type == transcoder_texture_format::cTFASTC_LDR_6x5_RGBA) + return true; + break; + } + case basis_tex_format::cXUASTC_LDR_6x6: + case basis_tex_format::cASTC_LDR_6x6: + { + if (tex_type == transcoder_texture_format::cTFASTC_LDR_6x6_RGBA) + return true; + break; + } + case basis_tex_format::cXUASTC_LDR_8x5: + case basis_tex_format::cASTC_LDR_8x5: + { + if (tex_type == transcoder_texture_format::cTFASTC_LDR_8x5_RGBA) + return true; + break; + } + case basis_tex_format::cXUASTC_LDR_8x6: + case basis_tex_format::cASTC_LDR_8x6: + { + if (tex_type == transcoder_texture_format::cTFASTC_LDR_8x6_RGBA) + return true; + break; + } + case basis_tex_format::cXUASTC_LDR_10x5: + case basis_tex_format::cASTC_LDR_10x5: + { + if (tex_type == transcoder_texture_format::cTFASTC_LDR_10x5_RGBA) + return true; + break; + } + case basis_tex_format::cXUASTC_LDR_10x6: + case basis_tex_format::cASTC_LDR_10x6: + { + if (tex_type == transcoder_texture_format::cTFASTC_LDR_10x6_RGBA) + return true; + break; + } + case basis_tex_format::cXUASTC_LDR_8x8: + case basis_tex_format::cASTC_LDR_8x8: + { + if (tex_type == transcoder_texture_format::cTFASTC_LDR_8x8_RGBA) + return true; + break; + } + case basis_tex_format::cXUASTC_LDR_10x8: + case basis_tex_format::cASTC_LDR_10x8: + { + if (tex_type == transcoder_texture_format::cTFASTC_LDR_10x8_RGBA) + return true; + break; + } + case basis_tex_format::cXUASTC_LDR_10x10: + case basis_tex_format::cASTC_LDR_10x10: + { + if (tex_type == transcoder_texture_format::cTFASTC_LDR_10x10_RGBA) + return true; + break; + } + case basis_tex_format::cXUASTC_LDR_12x10: + case basis_tex_format::cASTC_LDR_12x10: + { + if (tex_type == transcoder_texture_format::cTFASTC_LDR_12x10_RGBA) + return true; + break; + } + case basis_tex_format::cXUASTC_LDR_12x12: + case basis_tex_format::cASTC_LDR_12x12: + { + if (tex_type == transcoder_texture_format::cTFASTC_LDR_12x12_RGBA) + return true; + break; + } + default: + break; + } + } else { // ETC1S switch (tex_type) { - // ETC1 and uncompressed are always supported. + // ETC1 and uncompressed are always supported. case transcoder_texture_format::cTFETC1_RGB: case transcoder_texture_format::cTFRGBA32: case transcoder_texture_format::cTFRGB565: @@ -13330,8 +13867,8 @@ namespace basist case transcoder_texture_format::cTFETC2_RGBA: return true; #endif -#if BASISD_SUPPORT_ASTC - case transcoder_texture_format::cTFASTC_4x4_RGBA: +#if BASISD_SUPPORT_ASTC + case transcoder_texture_format::cTFASTC_LDR_4x4_RGBA: return true; #endif #if BASISD_SUPPORT_ATC @@ -13361,9 +13898,9 @@ namespace basist return false; } - // ------------------------------------------------------------------------------------------------------ + // ------------------------------------------------------------------------------------------------------ // UASTC LDR 4x4 - // ------------------------------------------------------------------------------------------------------ + // ------------------------------------------------------------------------------------------------------ #if BASISD_SUPPORT_UASTC const astc_bc7_common_partition2_desc g_astc_bc7_common_partitions2[TOTAL_ASTC_BC7_COMMON_PARTITIONS2] = @@ -14088,7 +14625,7 @@ namespace basist if (group_size) { - // Range has trits or quints - pack each group of 5 or 3 values + // Range has trits or quints - pack each group of 5 or 3 values const int total_groups = (group_size == 5) ? ((num_vals + 4) / 5) : ((num_vals + 2) / 3); for (int group_index = 0; group_index < total_groups; group_index++) @@ -14380,7 +14917,7 @@ namespace basist bool unpack_uastc(const uastc_block& blk, unpacked_uastc_block& unpacked, bool blue_contract_check, bool read_hints) { //memset(&unpacked, 0, sizeof(unpacked)); - + #if 0 uint8_t table[128]; memset(table, 0xFF, sizeof(table)); @@ -14436,7 +14973,7 @@ namespace basist return true; } - + if (read_hints) { if (g_uastc_mode_has_bc1_hint0[mode]) @@ -14469,7 +15006,7 @@ namespace basist } else bit_ofs += g_uastc_mode_total_hint_bits[mode]; - + uint32_t subsets = 1; switch (mode) { @@ -14682,7 +15219,7 @@ namespace basist { // All other modes have <= 64 weight bits. uint64_t bits; - + // Read the weight bits if ((BASISD_IS_BIG_ENDIAN) || (!BASISD_USE_UNALIGNED_WORD_READS)) bits = read_bits64(blk.m_bytes, bit_ofs, basisu::minimum(64, 128 - (int)bit_ofs)); @@ -14690,31 +15227,31 @@ namespace basist { bits = blk.m_dwords[2]; bits |= (((uint64_t)blk.m_dwords[3]) << 32U); - + if (bit_ofs >= 64U) bits >>= (bit_ofs - 64U); else { assert(bit_ofs >= 56U); - + uint32_t bits_needed = 64U - bit_ofs; bits <<= bits_needed; bits |= (blk.m_bytes[7] >> (8U - bits_needed)); } } - + bit_ofs = 0; const uint32_t mask = (1U << weight_bits) - 1U; const uint32_t anchor_mask = (1U << (weight_bits - 1U)) - 1U; - + if (total_planes == 2) { // Dual plane modes always have a single subset, and the first 2 weights are anchors. unpacked.m_astc.m_weights[0] = (uint8_t)((uint32_t)(bits >> bit_ofs) & anchor_mask); bit_ofs += (weight_bits - 1); - + unpacked.m_astc.m_weights[1] = (uint8_t)((uint32_t)(bits >> bit_ofs) & anchor_mask); bit_ofs += (weight_bits - 1); @@ -14732,7 +15269,7 @@ namespace basist if (weight_bits == 4) { assert(bit_ofs == 0); - + // Specialize the most common case: 4-bit weights. unpacked.m_astc.m_weights[0] = (uint8_t)((uint32_t)(bits) & 7); unpacked.m_astc.m_weights[1] = (uint8_t)((uint32_t)(bits >> 3) & 15); @@ -14991,7 +15528,7 @@ namespace basist return unpack_uastc(unpacked_blk, pPixels, srgb); } - // Determines the best shared pbits to use to encode xl/xh + // Determines the best shared pbit to use to encode xl/xh static void determine_shared_pbits( uint32_t total_comps, uint32_t comp_bits, float xl[4], float xh[4], color_quad_u8& bestMinColor, color_quad_u8& bestMaxColor, uint32_t best_pbits[2]) @@ -15195,6 +15732,8 @@ namespace basist } uint32_t best_pbits[2]; + basisu::clear_obj(best_pbits); + color_quad_u8 bestMinColor, bestMaxColor; determine_unique_pbits((total_comps == 2) ? 4 : total_comps, 7, xl, xh, bestMinColor, bestMaxColor, best_pbits); @@ -15278,7 +15817,7 @@ namespace basist } case 2: { - // 2. DualPlane: 0, WeightRange : 5 (8), Subsets : 2, EndpointRange : 8 (16) - BC7 MODE1 + // 2. DualPlane: 0, WeightRange : 5 (8), Subsets : 2, EndpointRange : 8 (16) - BC7 MODE1 dst_blk.m_mode = 1; dst_blk.m_partition = g_astc_bc7_common_partitions2[unpacked_src_blk.m_common_pattern].m_bc7; @@ -15505,6 +16044,7 @@ namespace basist case UASTC_MODE_INDEX_SOLID_COLOR: { // Void-Extent: Solid Color RGBA (BC7 MODE5 or MODE6) + // TODO: Why prefer mode 6 here? Mode 5 is lossless. const color32& solid_color = unpacked_src_blk.m_solid_color; uint32_t best_err0 = g_bc7_mode_6_optimal_endpoints[solid_color.r][0].m_error + g_bc7_mode_6_optimal_endpoints[solid_color.g][0].m_error + @@ -16217,7 +16757,7 @@ namespace basist bool flip = pack_etc1_y_estimate_flipped(&block_y[0][0], upper_avg, lower_avg, left_avg, right_avg); // non-flipped: | | - // vs. + // vs. // flipped: -- // -- @@ -16828,7 +17368,7 @@ namespace basist static const uint8_t s_uastc2_to_bc1[4] = { 0, 2, 3, 1 }; static const uint8_t s_uastc1_to_bc1[2] = { 0, 1 }; const uint8_t* s_uastc_to_bc1_weights[6] = { nullptr, s_uastc1_to_bc1, s_uastc2_to_bc1, s_uastc3_to_bc1, s_uastc4_to_bc1, s_uastc5_to_bc1 }; - + void encode_bc4(void* pDst, const uint8_t* pPixels, uint32_t stride) { uint32_t min0_v, max0_v, min1_v, max1_v,min2_v, max2_v, min3_v, max3_v; @@ -16916,7 +17456,7 @@ namespace basist a2 |= (s_tran2[(v2 >= t0) + (v2 >= t1) + (v2 >= t2) + (v2 >= t3) + (v2 >= t4) + (v2 >= t5) + (v2 >= t6)] << 12U); a3 |= (s_tran3[(v3 >= t0) + (v3 >= t1) + (v3 >= t2) + (v3 >= t3) + (v3 >= t4) + (v3 >= t5) + (v3 >= t6)] << 12U); } - + { const int v0 = pPixels[8 * stride] * 14 + bias; const int v1 = pPixels[9 * stride] * 14 + bias; @@ -16940,7 +17480,7 @@ namespace basist } const uint64_t f = a0 | a1 | a2 | a3; - + pDst_bytes[2] = (uint8_t)f; pDst_bytes[3] = (uint8_t)(f >> 8U); pDst_bytes[4] = (uint8_t)(f >> 16U); @@ -16963,7 +17503,7 @@ namespace basist int dots[4]; for (uint32_t i = 0; i < 4; i++) dots[i] = (int)block_r[i] * ar + (int)block_g[i] * ag + (int)block_b[i] * ab; - + int t0 = dots[0] + dots[1], t1 = dots[1] + dots[2], t2 = dots[2] + dots[3]; ar *= 2; ag *= 2; ab *= 2; @@ -16972,7 +17512,7 @@ namespace basist { const int d = pSrc_pixels[i].r * ar + pSrc_pixels[i].g * ag + pSrc_pixels[i].b * ab; static const uint8_t s_sels[4] = { 3, 2, 1, 0 }; - + // Rounding matters here! // d <= t0: <=, not <, to the later LS step "sees" a wider range of selectors. It matters for quality. sels[i] = s_sels[(d <= t0) + (d < t1) + (d < t2)]; @@ -17013,11 +17553,11 @@ namespace basist sels[i+3] = s_sels[(d3 <= t0) + (d3 < t1) + (d3 < t2)]; } } - + static bool compute_least_squares_endpoints_rgb(const color32* pColors, const uint8_t* pSelectors, vec3F* pXl, vec3F* pXh) { // Derived from bc7enc16's LS function. - // Least squares using normal equations: http://www.cs.cornell.edu/~bindel/class/cs3220-s12/notes/lec10.pdf + // Least squares using normal equations: http://www.cs.cornell.edu/~bindel/class/cs3220-s12/notes/lec10.pdf // I did this in matrix form first, expanded out all the ops, then optimized it a bit. uint32_t uq00_r = 0, uq10_r = 0, ut_r = 0, uq00_g = 0, uq10_g = 0, ut_g = 0, uq00_b = 0, uq10_b = 0, ut_b = 0; @@ -17091,7 +17631,7 @@ namespace basist return true; } - void encode_bc1_solid_block(void* pDst, uint32_t fr, uint32_t fg, uint32_t fb) + void encode_bc1_solid_block(void* pDst, uint32_t fr, uint32_t fg, uint32_t fb) { dxt1_block* pDst_block = static_cast(pDst); @@ -17143,19 +17683,19 @@ namespace basist { const color32* pSrc_pixels = (const color32*)pPixels; dxt1_block* pDst_block = static_cast(pDst); - + int avg_r = -1, avg_g = 0, avg_b = 0; int lr = 0, lg = 0, lb = 0, hr = 0, hg = 0, hb = 0; uint8_t sels[16]; - + const bool use_sels = (flags & cEncodeBC1UseSelectors) != 0; if (use_sels) { // Caller is jamming in their own selectors for us to try. const uint32_t s = pDst_block->m_selectors[0] | (pDst_block->m_selectors[1] << 8) | (pDst_block->m_selectors[2] << 16) | (pDst_block->m_selectors[3] << 24); - + static const uint8_t s_sel_tran[4] = { 0, 3, 1, 2 }; - + for (uint32_t i = 0; i < 16; i++) sels[i] = s_sel_tran[(s >> (i * 2)) & 3]; } @@ -17167,13 +17707,13 @@ namespace basist for (j = 1; j < 16; j++) if ((pSrc_pixels[j].r != fr) || (pSrc_pixels[j].g != fg) || (pSrc_pixels[j].b != fb)) break; - + if (j == 16) { encode_bc1_solid_block(pDst, fr, fg, fb); return; } - + // Select 2 colors along the principle axis. (There must be a faster/simpler way.) int total_r = fr, total_g = fg, total_b = fb; int max_r = fr, max_g = fg, max_b = fb; @@ -17207,7 +17747,7 @@ namespace basist float cov[6]; for (uint32_t i = 0; i < 6; i++) cov[i] = static_cast(icov[i])* (1.0f / 255.0f); - + #if 0 // Seems silly to use full PCA to choose 2 colors. The diff in avg. PSNR between using PCA vs. not is small (~.025 difference). // TODO: Try 2 or 3 different normalized diagonal vectors, choose the one that results in the largest dot delta @@ -17239,7 +17779,7 @@ namespace basist saxis_b = (int)(xb * m); } #endif - + int low_dot = INT_MAX, high_dot = INT_MIN, low_c = 0, high_c = 0; for (uint32_t i = 0; i < 16; i++) { @@ -17263,7 +17803,7 @@ namespace basist hr = to_5(pSrc_pixels[high_c].r); hg = to_6(pSrc_pixels[high_c].g); hb = to_5(pSrc_pixels[high_c].b); - + bc1_find_sels(pSrc_pixels, lr, lg, lb, hr, hg, hb, sels); } // if (use_sels) @@ -17310,13 +17850,13 @@ namespace basist hg = basisu::clamp((int)((xh.c[1]) * (63.0f / 255.0f) + .5f), 0, 63); hb = basisu::clamp((int)((xh.c[2]) * (31.0f / 255.0f) + .5f), 0, 31); } - + bc1_find_sels(pSrc_pixels, lr, lg, lb, hr, hg, hb, sels); } uint32_t lc16 = dxt1_block::pack_unscaled_color(lr, lg, lb); uint32_t hc16 = dxt1_block::pack_unscaled_color(hr, hg, hb); - + // Always forbid 3 color blocks if (lc16 == hc16) { @@ -17368,7 +17908,7 @@ namespace basist pDst_block->m_selectors[3] = (uint8_t)(packed_sels >> 24) ^ invert_mask; } } - + void encode_bc1_alt(void* pDst, const uint8_t* pPixels, uint32_t flags) { const color32* pSrc_pixels = (const color32*)pPixels; @@ -17417,8 +17957,8 @@ namespace basist min_r = basisu::minimum(min_r, r); min_g = basisu::minimum(min_g, g); min_b = basisu::minimum(min_b, b); total_r += r; total_g += g; total_b += b; } - - if (grayscale_flag) + + if (grayscale_flag) { // Grayscale blocks are a common enough case to specialize. if ((max_r - min_r) < 2) @@ -17735,7 +18275,7 @@ namespace basist // Always forbid 3 color blocks uint16_t lc16 = (uint16_t)b.get_low_color(); uint16_t hc16 = (uint16_t)b.get_high_color(); - + uint8_t mask = 0; // Make l > h @@ -17965,7 +18505,7 @@ namespace basist blk.m_base = static_cast(a); blk.m_table = 13; blk.m_multiplier = 0; - + memcpy(blk.m_selectors, g_etc2_eac_a8_sel4, sizeof(g_etc2_eac_a8_sel4)); return; @@ -18655,7 +19195,7 @@ namespace basist if (!unpack_uastc(pSrc_blocks[x + y * num_blocks_x], block_pixels, false)) return false; - // Get block's RGB bounding box + // Get block's RGB bounding box color32 low_color(255, 255, 255, 255), high_color(0, 0, 0, 0); if (from_alpha) @@ -18714,7 +19254,7 @@ namespace basist if (!unpack_uastc(pSrc_blocks[x + y * num_blocks_x], block_pixels, false)) return false; - // Get block's RGBA bounding box + // Get block's RGBA bounding box color32 low_color(255, 255, 255, 255), high_color(0, 0, 0, 0); for (uint32_t i = 0; i < 16; i++) @@ -18830,9 +19370,9 @@ namespace basist #endif // #if BASISD_SUPPORT_UASTC -// ------------------------------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------------------------------ // KTX2 -// ------------------------------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------------------------------ #if BASISD_SUPPORT_KTX2 const uint8_t g_ktx2_file_identifier[12] = { 0xAB, 0x4B, 0x54, 0x58, 0x20, 0x32, 0x30, 0xBB, 0x0D, 0x0A, 0x1A, 0x0A }; @@ -18854,8 +19394,8 @@ namespace basist m_key_values.clear(); memset((void *)&m_etc1s_header, 0, sizeof(m_etc1s_header)); m_etc1s_image_descs.clear(); - m_astc_6x6_intermediate_image_descs.clear(); - + m_slice_offset_len_descs.clear(); + m_format = basist::basis_tex_format::cETC1S; m_dfd_color_model = 0; @@ -18867,14 +19407,19 @@ namespace basist m_dfd_chan1 = KTX2_DF_CHANNEL_UASTC_RGB; m_etc1s_transcoder.clear(); - + m_def_transcoder_state.clear(); - + m_has_alpha = false; m_is_video = false; m_ldr_hdr_upconversion_nit_multiplier = 0.0f; } + static bool is_vk_format_astc_ldr(uint32_t fmt) + { + return (fmt >= KTX2_FORMAT_ASTC_4x4_UNORM_BLOCK) && (fmt <= KTX2_FORMAT_ASTC_12x12_SRGB_BLOCK); + } + bool ktx2_transcoder::init(const void* pData, uint32_t data_size) { clear(); @@ -18904,9 +19449,10 @@ namespace basist memcpy((void *)&m_header, pData, sizeof(m_header)); // Check for supported VK formats. We may also need to parse the DFD. - if ((m_header.m_vk_format != KTX2_VK_FORMAT_UNDEFINED) && - (m_header.m_vk_format != basist::KTX2_FORMAT_ASTC_4x4_SFLOAT_BLOCK) && - (m_header.m_vk_format != basist::KTX2_FORMAT_ASTC_6x6_SFLOAT_BLOCK)) + if ((m_header.m_vk_format != KTX2_VK_FORMAT_UNDEFINED) && + (m_header.m_vk_format != basist::KTX2_FORMAT_ASTC_4x4_SFLOAT_BLOCK) && + (m_header.m_vk_format != basist::KTX2_FORMAT_ASTC_6x6_SFLOAT_BLOCK) && + !is_vk_format_astc_ldr(m_header.m_vk_format)) { BASISU_DEVEL_ERROR("ktx2_transcoder::init: KTX2 file must be in ETC1S or UASTC LDR/HDR format\n"); return false; @@ -18943,7 +19489,7 @@ namespace basist return false; } } - + // 3.7 levelCount: "levelCount=0 is allowed, except for block-compressed formats" if (m_header.m_level_count < 1) { @@ -18966,14 +19512,6 @@ namespace basist if (m_header.m_supercompression_scheme == KTX2_SS_BASISLZ) { -#if 0 - if (m_header.m_sgd_byte_length <= sizeof(ktx2_etc1s_global_data_header)) - { - BASISU_DEVEL_ERROR("ktx2_transcoder::init: Supercompression global data is too small\n"); - return false; - } -#endif - if (m_header.m_sgd_byte_offset.get_uint64() < sizeof(ktx2_header)) { BASISU_DEVEL_ERROR("ktx2_transcoder::init: Supercompression global data offset is too low\n"); @@ -19002,7 +19540,7 @@ namespace basist } memcpy((void *)&m_levels[0], m_pData + sizeof(ktx2_header), level_index_size_in_bytes); - + // Sanity check the level offsets and byte sizes for (uint32_t i = 0; i < m_levels.size(); i++) { @@ -19022,9 +19560,9 @@ namespace basist BASISU_DEVEL_ERROR("ktx2_transcoder::init: Invalid level offset and/or length\n"); return false; } - + const uint64_t MAX_SANE_LEVEL_UNCOMP_SIZE = 2048ULL * 1024ULL * 1024ULL; - + if (m_levels[i].m_uncompressed_byte_length.get_uint64() >= MAX_SANE_LEVEL_UNCOMP_SIZE) { BASISU_DEVEL_ERROR("ktx2_transcoder::init: Invalid level offset (too large)\n"); @@ -19061,7 +19599,7 @@ namespace basist BASISU_DEVEL_ERROR("ktx2_transcoder::init: Invalid DFD offset and/or length\n"); return false; } - + const uint8_t* pDFD = m_pData + m_header.m_dfd_byte_offset; if (!m_dfd.try_resize(m_header.m_dfd_byte_length)) @@ -19071,17 +19609,16 @@ namespace basist } memcpy(m_dfd.data(), pDFD, m_header.m_dfd_byte_length); - - // This is all hard coded for only ETC1S and UASTC. + uint32_t dfd_total_size = basisu::read_le_dword(pDFD); - + // 3.10.3: Sanity check if (dfd_total_size != m_header.m_dfd_byte_length) { BASISU_DEVEL_ERROR("ktx2_transcoder::init: DFD size validation failed (1)\n"); return false; } - + // 3.10.3: More sanity checking if (m_header.m_kvd_byte_length) { @@ -19094,12 +19631,16 @@ namespace basist const uint32_t dfd_bits = basisu::read_le_dword(pDFD + 3 * sizeof(uint32_t)); const uint32_t sample_channel0 = basisu::read_le_dword(pDFD + 7 * sizeof(uint32_t)); - + const uint32_t texel_block_dimensions = basisu::read_le_dword(pDFD + 4 * sizeof(uint32_t)); + m_dfd_color_model = dfd_bits & 255; m_dfd_color_prims = (ktx2_df_color_primaries)((dfd_bits >> 8) & 255); m_dfd_transfer_func = (dfd_bits >> 16) & 255; m_dfd_flags = (dfd_bits >> 24) & 255; + const uint32_t block_width = (texel_block_dimensions & 0xFF) + 1; + const uint32_t block_height = ((texel_block_dimensions >> 8) & 0xFF) + 1; + // See 3.10.1.Restrictions if ((m_dfd_transfer_func != KTX2_KHR_DF_TRANSFER_LINEAR) && (m_dfd_transfer_func != KTX2_KHR_DF_TRANSFER_SRGB)) { @@ -19107,7 +19648,59 @@ namespace basist return false; } - if (m_dfd_color_model == KTX2_KDF_DF_MODEL_ETC1S) + if (is_vk_format_astc_ldr(m_header.m_vk_format)) + { + // ASTC LDR 4x4-12x12 + // We usually read the DFD and decide the format from there. This decides off the VK format. + if (m_dfd_color_model != KTX2_KDF_DF_MODEL_ASTC) + { + BASISU_DEVEL_ERROR("ktx2_transcoder::init: Invalid DFD color model (expected ASTC)\n"); + return false; + } + + uint32_t vk_fmt = m_header.m_vk_format; + const bool is_srgb_fmt = (vk_fmt & 1) == 0; + + if (is_srgb_fmt) + vk_fmt--; + + switch (vk_fmt) + { + case KTX2_FORMAT_ASTC_4x4_UNORM_BLOCK: m_format = basis_tex_format::cASTC_LDR_4x4; break; + case KTX2_FORMAT_ASTC_5x4_UNORM_BLOCK: m_format = basis_tex_format::cASTC_LDR_5x4; break; + case KTX2_FORMAT_ASTC_5x5_UNORM_BLOCK: m_format = basis_tex_format::cASTC_LDR_5x5; break; + case KTX2_FORMAT_ASTC_6x5_UNORM_BLOCK: m_format = basis_tex_format::cASTC_LDR_6x5; break; + case KTX2_FORMAT_ASTC_6x6_UNORM_BLOCK: m_format = basis_tex_format::cASTC_LDR_6x6; break; + case KTX2_FORMAT_ASTC_8x5_UNORM_BLOCK: m_format = basis_tex_format::cASTC_LDR_8x5; break; + case KTX2_FORMAT_ASTC_8x6_UNORM_BLOCK: m_format = basis_tex_format::cASTC_LDR_8x6; break; + case KTX2_FORMAT_ASTC_8x8_UNORM_BLOCK: m_format = basis_tex_format::cASTC_LDR_8x8; break; + case KTX2_FORMAT_ASTC_10x5_UNORM_BLOCK: m_format = basis_tex_format::cASTC_LDR_10x5; break; + case KTX2_FORMAT_ASTC_10x6_UNORM_BLOCK: m_format = basis_tex_format::cASTC_LDR_10x6; break; + case KTX2_FORMAT_ASTC_10x8_UNORM_BLOCK: m_format = basis_tex_format::cASTC_LDR_10x8; break; + case KTX2_FORMAT_ASTC_10x10_UNORM_BLOCK: m_format = basis_tex_format::cASTC_LDR_10x10; break; + case KTX2_FORMAT_ASTC_12x10_UNORM_BLOCK: m_format = basis_tex_format::cASTC_LDR_12x10; break; + case KTX2_FORMAT_ASTC_12x12_UNORM_BLOCK: m_format = basis_tex_format::cASTC_LDR_12x12; break; + default: + assert(0); + return false; + } + + // Sanity check the vkformat's astc block size vs. the DFD's. + uint32_t actual_block_width = 0, actual_block_height = 0; + get_basis_tex_format_block_size(m_format, actual_block_width, actual_block_height); + if ((actual_block_width != block_width) || (actual_block_height != block_height)) + { + BASISU_DEVEL_ERROR("ktx2_transcoder::init: vkFormat's ASTC block size is not in sync with the DFD's block dimensions\n"); + return false; + } + + m_dfd_samples = 1; + m_dfd_chan0 = (ktx2_df_channel_id)((sample_channel0 >> 24) & 15); + + // We're assuming "DATA" means RGBA so it has alpha. + m_has_alpha = (m_dfd_chan0 == KTX2_DF_CHANNEL_UASTC_RGBA) || (m_dfd_chan0 == KTX2_DF_CHANNEL_UASTC_RRRG); + } + else if (m_dfd_color_model == KTX2_KDF_DF_MODEL_ETC1S) { if (m_header.m_vk_format != basist::KTX2_VK_FORMAT_UNDEFINED) { @@ -19116,11 +19709,11 @@ namespace basist } m_format = basist::basis_tex_format::cETC1S; - + // 3.10.2: "Whether the image has 1 or 2 slices can be determined from the DFD's sample count." // If m_has_alpha is true it may be 2-channel RRRG or 4-channel RGBA, but we let the caller deal with that. m_has_alpha = (m_header.m_dfd_byte_length == 60); - + m_dfd_samples = m_has_alpha ? 2 : 1; m_dfd_chan0 = (ktx2_df_channel_id)((sample_channel0 >> 24) & 15); @@ -19138,11 +19731,11 @@ namespace basist return false; } - m_format = basist::basis_tex_format::cUASTC4x4; + m_format = basist::basis_tex_format::cUASTC_LDR_4x4; m_dfd_samples = 1; m_dfd_chan0 = (ktx2_df_channel_id)((sample_channel0 >> 24) & 15); - + // We're assuming "DATA" means RGBA so it has alpha. m_has_alpha = (m_dfd_chan0 == KTX2_DF_CHANNEL_UASTC_RGBA) || (m_dfd_chan0 == KTX2_DF_CHANNEL_UASTC_RRRG); } @@ -19180,7 +19773,7 @@ namespace basist m_has_alpha = false; } - else if (m_dfd_color_model == KTX2_KDF_DF_MODEL_ASTC_HDR_6X6_INTERMEDIATE) + else if (m_dfd_color_model == KTX2_KDF_DF_MODEL_UASTC_HDR_6X6_INTERMEDIATE) { // Custom variable block size ASTC HDR 6x6 texture data. if (m_header.m_vk_format != basist::KTX2_VK_FORMAT_UNDEFINED) @@ -19189,20 +19782,59 @@ namespace basist return false; } - m_format = basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE; + m_format = basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE; m_dfd_samples = 1; m_dfd_chan0 = (ktx2_df_channel_id)((sample_channel0 >> 24) & 15); m_has_alpha = false; } + else if (m_dfd_color_model == KTX2_KDF_DF_MODEL_XUASTC_LDR_INTERMEDIATE) + { + if (m_header.m_vk_format != basist::KTX2_VK_FORMAT_UNDEFINED) + { + BASISU_DEVEL_ERROR("ktx2_transcoder::init: Invalid header vkFormat\n"); + return false; + } + + // Extract ASTC block dimensions from texel_block_dimensions, validate, select basis_tex_format. + m_format = basist::basis_tex_format::cETC1S; // bogus value to start + +#define BUT_BLOCK_SIZE(x, y, t) if ((block_width == (x)) && (block_height == (y))) { m_format = (t); } + BUT_BLOCK_SIZE(4, 4, basis_tex_format::cXUASTC_LDR_4x4); + BUT_BLOCK_SIZE(5, 4, basis_tex_format::cXUASTC_LDR_5x4); + BUT_BLOCK_SIZE(5, 5, basis_tex_format::cXUASTC_LDR_5x5); + BUT_BLOCK_SIZE(6, 5, basis_tex_format::cXUASTC_LDR_6x5); + BUT_BLOCK_SIZE(6, 6, basis_tex_format::cXUASTC_LDR_6x6); + BUT_BLOCK_SIZE(8, 5, basis_tex_format::cXUASTC_LDR_8x5); + BUT_BLOCK_SIZE(8, 6, basis_tex_format::cXUASTC_LDR_8x6); + BUT_BLOCK_SIZE(10, 5, basis_tex_format::cXUASTC_LDR_10x5); + BUT_BLOCK_SIZE(10, 6, basis_tex_format::cXUASTC_LDR_10x6); + BUT_BLOCK_SIZE(8, 8, basis_tex_format::cXUASTC_LDR_8x8); + BUT_BLOCK_SIZE(10, 8, basis_tex_format::cXUASTC_LDR_10x8); + BUT_BLOCK_SIZE(10, 10, basis_tex_format::cXUASTC_LDR_10x10); + BUT_BLOCK_SIZE(12, 10, basis_tex_format::cXUASTC_LDR_12x10); + BUT_BLOCK_SIZE(12, 12, basis_tex_format::cXUASTC_LDR_12x12); +#undef BUT_BLOCK_SIZE + + if (m_format == basist::basis_tex_format::cETC1S) + { + BASISU_DEVEL_ERROR("ktx2_transcoder::init: Unsupported XUASTC LDR block dimensions (not valid ASTC)\n"); + return false; + } + + m_dfd_samples = 1; + m_dfd_chan0 = (ktx2_df_channel_id)((sample_channel0 >> 24) & 15); + + m_has_alpha = (m_dfd_chan0 == KTX2_DF_CHANNEL_UASTC_RGBA) || (m_dfd_chan0 == KTX2_DF_CHANNEL_UASTC_RRRG); + } else { // Unsupported DFD color model. BASISU_DEVEL_ERROR("ktx2_transcoder::init: Unsupported DFD color model\n"); return false; } - + if (!read_key_values()) { BASISU_DEVEL_ERROR("ktx2_transcoder::init: read_key_values() failed\n"); @@ -19261,7 +19893,7 @@ namespace basist return nullptr; } - + bool ktx2_transcoder::start_transcoding() { if (!m_pData) @@ -19270,14 +19902,14 @@ namespace basist return false; } - if (m_header.m_supercompression_scheme == KTX2_SS_BASISLZ) + if (m_header.m_supercompression_scheme == KTX2_SS_BASISLZ) { if (m_format == basis_tex_format::cETC1S) { // Check if we've already decompressed the ETC1S global data. If so don't unpack it again. if (!m_etc1s_transcoder.get_endpoints().empty()) return true; - + if (!decompress_etc1s_global_data()) { BASISU_DEVEL_ERROR("ktx2_transcoder::start_transcoding: decompress_etc1s_global_data() failed\n"); @@ -19301,14 +19933,15 @@ namespace basist } } } - else if (m_format == basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE) + else if ( (m_format == basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE) || basis_tex_format_is_xuastc_ldr(m_format) ) { - if (m_astc_6x6_intermediate_image_descs.size()) + // UASTC HDR 6x6 and XUASTC LDR 4x4-12x12 require an array of slice offset/len structs to determine where the compressed data starts for each independent compressed texture slice. + if (m_slice_offset_len_descs.size()) return true; - if (!read_astc_6x6_hdr_intermediate_global_data()) + if (!read_slice_offset_len_global_data()) { - BASISU_DEVEL_ERROR("ktx2_transcoder::start_transcoding: read_astc_6x6_hdr_intermediate_global_data() failed\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::start_transcoding: read_slice_offset_len_global_data() failed\n"); return false; } } @@ -19356,7 +19989,7 @@ namespace basist BASISU_DEVEL_ERROR("ktx2_transcoder::get_image_level_info: layer_index >= maximum(m_header.m_layer_count, 1)\n"); return false; } - + const uint32_t level_width = basisu::maximum(m_header.m_pixel_width >> level_index, 1); const uint32_t level_height = basisu::maximum(m_header.m_pixel_height >> level_index, 1); @@ -19380,7 +20013,7 @@ namespace basist level_info.m_total_blocks = num_blocks_x * num_blocks_y; level_info.m_alpha_flag = m_has_alpha; level_info.m_iframe_flag = false; - + if (m_etc1s_image_descs.size()) { const uint32_t etc1s_image_index = @@ -19393,9 +20026,9 @@ namespace basist return true; } - + bool ktx2_transcoder::transcode_image_level( - uint32_t level_index, uint32_t layer_index, uint32_t face_index, + uint32_t level_index, uint32_t layer_index, uint32_t face_index, void* pOutput_blocks, uint32_t output_blocks_buf_size_in_blocks_or_pixels, basist::transcoder_texture_format fmt, uint32_t decode_flags, uint32_t output_row_pitch_in_blocks_or_pixels, uint32_t output_rows_in_pixels, int channel0, int channel1, @@ -19403,16 +20036,16 @@ namespace basist { if (!m_pData) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: Must call init() first\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: Must call init() first\n"); return false; } if (!pState) pState = &m_def_transcoder_state; - + if (level_index >= m_levels.size()) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: level_index >= m_levels.size()\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: level_index >= m_levels.size()\n"); return false; } @@ -19420,34 +20053,34 @@ namespace basist { if (face_index >= 6) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: face_index >= 6\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: face_index >= 6\n"); return false; } } else if (face_index != 0) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: face_index != 0\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: face_index != 0\n"); return false; } if (layer_index >= basisu::maximum(m_header.m_layer_count, 1)) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: layer_index >= maximum(m_header.m_layer_count, 1)\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: layer_index >= maximum(m_header.m_layer_count, 1)\n"); return false; } const uint8_t* pComp_level_data = m_pData + m_levels[level_index].m_byte_offset.get_uint64(); uint64_t comp_level_data_size = m_levels[level_index].m_byte_length.get_uint64(); - + const uint8_t* pUncomp_level_data = pComp_level_data; uint64_t uncomp_level_data_size = comp_level_data_size; if (uncomp_level_data_size > UINT32_MAX) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: uncomp_level_data_size > UINT32_MAX\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: uncomp_level_data_size > UINT32_MAX\n"); return false; } - + if (m_header.m_supercompression_scheme == KTX2_SS_ZSTANDARD) { // Check if we've already decompressed this level's supercompressed data. @@ -19456,7 +20089,7 @@ namespace basist // Uncompress the entire level's supercompressed data. if (!decompress_level_data(level_index, pState->m_level_uncomp_data)) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: decompress_level_data() failed\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: decompress_level_data() failed\n"); return false; } pState->m_uncomp_data_level_index = level_index; @@ -19465,18 +20098,19 @@ namespace basist pUncomp_level_data = pState->m_level_uncomp_data.data(); uncomp_level_data_size = pState->m_level_uncomp_data.size(); } - + const uint32_t level_width = basisu::maximum(m_header.m_pixel_width >> level_index, 1); const uint32_t level_height = basisu::maximum(m_header.m_pixel_height >> level_index, 1); const uint32_t num_blocks4_x = (level_width + 3) >> 2; const uint32_t num_blocks4_y = (level_height + 3) >> 2; - + if (m_format == basist::basis_tex_format::cETC1S) { + // ETC1S // Ensure start_transcoding() was called. if (m_etc1s_transcoder.get_endpoints().empty()) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: must call start_transcoding() first\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: must call start_transcoding() first\n"); return false; } @@ -19484,12 +20118,11 @@ namespace basist (level_index * basisu::maximum(m_header.m_layer_count, 1) * m_header.m_face_count) + layer_index * m_header.m_face_count + face_index; - + // Sanity check if (etc1s_image_index >= m_etc1s_image_descs.size()) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: etc1s_image_index >= m_etc1s_image_descs.size()\n"); - assert(0); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: etc1s_image_index >= m_etc1s_image_descs.size()\n"); return false; } @@ -19504,15 +20137,16 @@ namespace basist decode_flags, m_has_alpha, m_is_video, output_row_pitch_in_blocks_or_pixels, &pState->m_transcoder_state, output_rows_in_pixels)) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: ETC1S transcode_image() failed, this is either a bug or the file is corrupted/invalid\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: ETC1S transcode_image() failed, this is either a bug or the file is corrupted/invalid\n"); return false; } } - else if (m_format == basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE) + else if (m_format == basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE) { - if (!m_astc_6x6_intermediate_image_descs.size()) + // UASTC HDR 6x6 + if (!m_slice_offset_len_descs.size()) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: must call start_transcoding() first\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: must call start_transcoding() first\n"); return false; } @@ -19525,102 +20159,219 @@ namespace basist face_index; // Sanity check - if (image_index >= m_astc_6x6_intermediate_image_descs.size()) + if (image_index >= m_slice_offset_len_descs.size()) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: Invalid image_index\n"); - assert(0); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: Invalid image_index\n"); return false; } - const ktx2_astc_hdr_6x6_intermediate_image_desc& image_desc = m_astc_6x6_intermediate_image_descs[image_index]; - + const ktx2_slice_offset_len_desc& image_desc = m_slice_offset_len_descs[image_index]; + if (!m_astc_hdr_6x6_intermediate_transcoder.transcode_image(fmt, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, m_pData, m_data_size, num_blocks6_x, num_blocks6_y, level_width, level_height, level_index, - m_levels[level_index].m_byte_offset.get_uint64() + image_desc.m_rgb_slice_byte_offset, image_desc.m_rgb_slice_byte_length, + m_levels[level_index].m_byte_offset.get_uint64() + image_desc.m_slice_byte_offset, image_desc.m_slice_byte_length, decode_flags, m_has_alpha, m_is_video, output_row_pitch_in_blocks_or_pixels, nullptr, output_rows_in_pixels, channel0, channel1)) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: ASTC 6x6 HDR transcode_image() failed, this is either a bug or the file is corrupted/invalid\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: ASTC 6x6 HDR transcode_image() failed, this is either a bug or the file is corrupted/invalid\n"); return false; } } else if (m_format == basist::basis_tex_format::cASTC_HDR_6x6) { + // plain ASTC HDR 6x6 const uint32_t num_blocks6_x = (level_width + 5) / 6; const uint32_t num_blocks6_y = (level_height + 5) / 6; // Compute length and offset to uncompressed 2D UASTC texture data, given the face/layer indices. assert(uncomp_level_data_size == m_levels[level_index].m_uncompressed_byte_length.get_uint64()); - const uint32_t total_2D_image_size = num_blocks6_x * num_blocks6_y * sizeof(astc_helpers::astc_block); - const uint32_t uncomp_ofs = (layer_index * m_header.m_face_count + face_index) * total_2D_image_size; + const uint64_t total_2D_image_size = (uint64_t)num_blocks6_x * num_blocks6_y * sizeof(astc_helpers::astc_block); + + const uint64_t uncomp_ofs = (layer_index * m_header.m_face_count + face_index) * total_2D_image_size; + + if ((total_2D_image_size > UINT32_MAX) || ((size_t)uncomp_ofs != uncomp_ofs)) + { + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: size too large\n"); + return false; + } // Sanity checks if (uncomp_ofs >= uncomp_level_data_size) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: uncomp_ofs >= total_2D_image_size\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: uncomp_ofs >= total_2D_image_size\n"); return false; } if ((uncomp_level_data_size - uncomp_ofs) < total_2D_image_size) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: (uncomp_level_data_size - uncomp_ofs) < total_2D_image_size\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: (uncomp_level_data_size - uncomp_ofs) < total_2D_image_size\n"); return false; } + assert(total_2D_image_size <= UINT32_MAX); + if (!m_astc_hdr_6x6_transcoder.transcode_image(fmt, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, - (const uint8_t*)pUncomp_level_data + uncomp_ofs, (uint32_t)total_2D_image_size, num_blocks6_x, num_blocks6_y, level_width, level_height, level_index, + (const uint8_t*)pUncomp_level_data + (size_t)uncomp_ofs, (uint32_t)total_2D_image_size, num_blocks6_x, num_blocks6_y, level_width, level_height, level_index, + 0, (uint32_t)total_2D_image_size, + decode_flags, m_has_alpha, m_is_video, output_row_pitch_in_blocks_or_pixels, nullptr, output_rows_in_pixels, channel0, channel1)) + { + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: ASTC 6x6 HDR transcode_image() failed, this is either a bug or the file is corrupted/invalid\n"); + return false; + } + } + else if (basis_tex_format_is_astc_ldr(m_format)) + { + // ASTC LDR 4x4-12x12 + const uint32_t block_width = get_block_width(), block_height = get_block_height(); + + const uint32_t num_blocks_x = (level_width + block_width - 1) / block_width; + const uint32_t num_blocks_y = (level_height + block_height - 1) / block_height; + + //assert(uncomp_level_data_size == m_levels[level_index].m_uncompressed_byte_length.get_uint64()); + if (uncomp_level_data_size != m_levels[level_index].m_uncompressed_byte_length.get_uint64()) + { + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: m_uncompressed_byte_length is invalid\n"); + return false; + } + + const uint64_t total_2D_image_size = (uint64_t)num_blocks_x * num_blocks_y * sizeof(astc_helpers::astc_block); + + const uint64_t uncomp_ofs = (layer_index * m_header.m_face_count + face_index) * total_2D_image_size; + + if ((total_2D_image_size > UINT32_MAX) || ((size_t)uncomp_ofs != uncomp_ofs)) + { + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: size too large\n"); + return false; + } + + // Sanity checks + if (uncomp_ofs >= uncomp_level_data_size) + { + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: uncomp_ofs >= total_2D_image_size\n"); + return false; + } + + if ((uncomp_level_data_size - uncomp_ofs) < total_2D_image_size) + { + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: (uncomp_level_data_size - uncomp_ofs) < total_2D_image_size\n"); + return false; + } + + assert(total_2D_image_size <= UINT32_MAX); + + // if the header's vkformat is odd, it's linear, even is sRGB + const bool uses_astc_src_decode_profile = ((uint32_t)m_header.m_vk_format & 1) == 0; + + if (!m_xuastc_ldr_transcoder.transcode_image(m_format, uses_astc_src_decode_profile, fmt, + pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, + (const uint8_t*)pUncomp_level_data + (size_t)uncomp_ofs, (uint32_t)total_2D_image_size, num_blocks_x, num_blocks_y, level_width, level_height, level_index, 0, (uint32_t)total_2D_image_size, decode_flags, m_has_alpha, m_is_video, output_row_pitch_in_blocks_or_pixels, nullptr, output_rows_in_pixels, channel0, channel1)) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: ASTC 6x6 HDR transcode_image() failed, this is either a bug or the file is corrupted/invalid\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: ASTC 6x6 HDR transcode_image() failed, this is either a bug or the file is corrupted/invalid\n"); + return false; + } + + } + else if (basis_tex_format_is_xuastc_ldr(m_format)) + { + // XUASTC LDR 4x4-12x12 + if (!m_slice_offset_len_descs.size()) + { + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: must call start_transcoding() first\n"); + return false; + } + + const uint32_t block_width = get_block_width(), block_height = get_block_height(); + + const uint32_t num_blocks_x = (level_width + block_width - 1) / block_width; + const uint32_t num_blocks_y = (level_height + block_height - 1) / block_height; + + const uint32_t image_index = + (level_index * basisu::maximum(m_header.m_layer_count, 1) * m_header.m_face_count) + + layer_index * m_header.m_face_count + + face_index; + + // Sanity check + if (image_index >= m_slice_offset_len_descs.size()) + { + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: Invalid image_index\n"); + return false; + } + + const ktx2_slice_offset_len_desc& image_desc = m_slice_offset_len_descs[image_index]; + + // XUASTC LDR has its own tiny header at the start of the compressed data with this profile bit, so it'll use that for decoding if needed. + bool uses_astc_src_decode_profile = true; + + if (!m_xuastc_ldr_transcoder.transcode_image(m_format, uses_astc_src_decode_profile, fmt, + pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, + m_pData, m_data_size, num_blocks_x, num_blocks_y, level_width, level_height, level_index, + m_levels[level_index].m_byte_offset.get_uint64() + image_desc.m_slice_byte_offset, image_desc.m_slice_byte_length, + decode_flags, m_has_alpha, m_is_video, output_row_pitch_in_blocks_or_pixels, nullptr, output_rows_in_pixels, channel0, channel1)) + { + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: XUASTC LDR transcode_image() failed, this is either a bug or the file is corrupted/invalid\n"); return false; } + } - else if ((m_format == basist::basis_tex_format::cUASTC4x4) || + else if ((m_format == basist::basis_tex_format::cUASTC_LDR_4x4) || (m_format == basist::basis_tex_format::cUASTC_HDR_4x4)) { + // UASTC LDR 4x4 and UASTC HDR 4x4 + // Compute length and offset to uncompressed 2D UASTC texture data, given the face/layer indices. assert(uncomp_level_data_size == m_levels[level_index].m_uncompressed_byte_length.get_uint64()); - const uint32_t total_2D_image_size = num_blocks4_x * num_blocks4_y * KTX2_UASTC_BLOCK_SIZE; + const uint64_t total_2D_image_size = (uint64_t)num_blocks4_x * num_blocks4_y * KTX2_UASTC_BLOCK_SIZE; + + const uint64_t uncomp_ofs = (layer_index * m_header.m_face_count + face_index) * total_2D_image_size; - const uint32_t uncomp_ofs = (layer_index * m_header.m_face_count + face_index) * total_2D_image_size; + if ((total_2D_image_size > UINT32_MAX) || ((size_t)uncomp_ofs != uncomp_ofs)) + { + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: size too large\n"); + return false; + } // Sanity checks if (uncomp_ofs >= uncomp_level_data_size) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: uncomp_ofs >= total_2D_image_size\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: uncomp_ofs >= total_2D_image_size\n"); return false; } if ((uncomp_level_data_size - uncomp_ofs) < total_2D_image_size) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: (uncomp_level_data_size - uncomp_ofs) < total_2D_image_size\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: (uncomp_level_data_size - uncomp_ofs) < total_2D_image_size\n"); return false; } + assert(total_2D_image_size <= UINT32_MAX); + if (m_format == basist::basis_tex_format::cUASTC_HDR_4x4) { + // UASTC HDR 4x4 if (!m_uastc_hdr_transcoder.transcode_image(fmt, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, (const uint8_t*)pUncomp_level_data + uncomp_ofs, (uint32_t)total_2D_image_size, num_blocks4_x, num_blocks4_y, level_width, level_height, level_index, 0, (uint32_t)total_2D_image_size, decode_flags, m_has_alpha, m_is_video, output_row_pitch_in_blocks_or_pixels, nullptr, output_rows_in_pixels, channel0, channel1)) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: UASTC HDR transcode_image() failed, this is either a bug or the file is corrupted/invalid\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: UASTC HDR transcode_image() failed, this is either a bug or the file is corrupted/invalid\n"); return false; } } else { - if (!m_uastc_transcoder.transcode_image(fmt, + // UASTC LDR 4x4 + if (!m_uastc_ldr_transcoder.transcode_image(fmt, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, (const uint8_t*)pUncomp_level_data + uncomp_ofs, (uint32_t)total_2D_image_size, num_blocks4_x, num_blocks4_y, level_width, level_height, level_index, 0, (uint32_t)total_2D_image_size, decode_flags, m_has_alpha, m_is_video, output_row_pitch_in_blocks_or_pixels, nullptr, output_rows_in_pixels, channel0, channel1)) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: UASTC transcode_image() failed, this is either a bug or the file is corrupted/invalid\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: UASTC transcode_image() failed, this is either a bug or the file is corrupted/invalid\n"); return false; } } @@ -19628,19 +20379,19 @@ namespace basist else { // Shouldn't get here. - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: Internal error\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: Internal error\n"); assert(0); return false; } return true; } - + bool ktx2_transcoder::decompress_level_data(uint32_t level_index, basisu::uint8_vec& uncomp_data) { const uint8_t* pComp_data = m_levels[level_index].m_byte_offset.get_uint64() + m_pData; const uint64_t comp_size = m_levels[level_index].m_byte_length.get_uint64(); - + const uint64_t uncomp_size = m_levels[level_index].m_uncompressed_byte_length.get_uint64(); if (((size_t)comp_size) != comp_size) @@ -19659,7 +20410,7 @@ namespace basist BASISU_DEVEL_ERROR("ktx2_transcoder::decompress_level_data: Out of memory\n"); return false; } - + if (m_header.m_supercompression_scheme == KTX2_SS_ZSTANDARD) { #if BASISD_SUPPORT_KTX2_ZSTD @@ -19675,6 +20426,8 @@ namespace basist return false; } #else + BASISU_NOTE_UNUSED(pComp_data); + BASISU_DEVEL_ERROR("ktx2_transcoder::decompress_level_data: File uses Zstd supercompression, but Zstd support was not enabled at compile time (BASISD_SUPPORT_KTX2_ZSTD is 0)\n"); return false; #endif @@ -19683,38 +20436,38 @@ namespace basist return true; } - bool ktx2_transcoder::read_astc_6x6_hdr_intermediate_global_data() + bool ktx2_transcoder::read_slice_offset_len_global_data() { const uint32_t image_count = basisu::maximum(m_header.m_layer_count, 1) * m_header.m_face_count * m_header.m_level_count; assert(image_count); const uint8_t* pSrc = m_pData + m_header.m_sgd_byte_offset.get_uint64(); - if (m_header.m_sgd_byte_length.get_uint64() != image_count * sizeof(ktx2_astc_hdr_6x6_intermediate_image_desc)) + if (m_header.m_sgd_byte_length.get_uint64() != image_count * sizeof(ktx2_slice_offset_len_desc)) { - BASISU_DEVEL_ERROR("ktx2_transcoder::decompress_astc_6x6_hdr_intermediate_global_data: Invalid global data length\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::read_slice_offset_len_global_data: Invalid global data length\n"); return false; } - m_astc_6x6_intermediate_image_descs.resize(image_count); - - memcpy((void *)m_astc_6x6_intermediate_image_descs.data(), pSrc, sizeof(ktx2_astc_hdr_6x6_intermediate_image_desc) * image_count); + m_slice_offset_len_descs.resize(image_count); + + memcpy((void *)m_slice_offset_len_descs.data(), pSrc, sizeof(ktx2_slice_offset_len_desc) * image_count); // Sanity check the image descs for (uint32_t i = 0; i < image_count; i++) { // transcode_image() will validate the slice offsets/lengths before transcoding. - if (!m_astc_6x6_intermediate_image_descs[i].m_rgb_slice_byte_length) + if (!m_slice_offset_len_descs[i].m_slice_byte_length) { - BASISU_DEVEL_ERROR("ktx2_transcoder::decompress_astc_6x6_hdr_intermediate_global_data: image descs sanity check failed (1)\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::read_slice_offset_len_global_data: image descs sanity check failed (1)\n"); return false; } } return true; } - + bool ktx2_transcoder::decompress_etc1s_global_data() { // Note: we don't actually support 3D textures in here yet @@ -19753,13 +20506,13 @@ namespace basist BASISU_DEVEL_ERROR("ktx2_transcoder::decompress_etc1s_global_data: SGD byte length is too small, file is invalid or corrupted\n"); return false; } - + if (!m_etc1s_image_descs.try_resize(image_count)) { BASISU_DEVEL_ERROR("ktx2_transcoder::decompress_etc1s_global_data: Out of memory\n"); return false; } - + memcpy((void *)m_etc1s_image_descs.data(), pSrc, sizeof(ktx2_etc1s_image_desc) * image_count); pSrc += sizeof(ktx2_etc1s_image_desc) * image_count; @@ -19793,7 +20546,7 @@ namespace basist BASISU_DEVEL_ERROR("ktx2_transcoder::decompress_etc1s_global_data: decode_tables() failed, file is invalid or corrupted\n"); return false; } - + if (!m_etc1s_transcoder.decode_palettes( m_etc1s_header.m_endpoint_count, pEndpoint_data, m_etc1s_header.m_endpoints_byte_length, m_etc1s_header.m_selector_count, pSelector_data, m_etc1s_header.m_selectors_byte_length)) @@ -19801,7 +20554,7 @@ namespace basist BASISU_DEVEL_ERROR("ktx2_transcoder::decompress_etc1s_global_data: decode_palettes() failed, file is likely corrupted\n"); return false; } - + return true; } @@ -19842,7 +20595,7 @@ namespace basist while (src_left > sizeof(uint32_t)) { uint32_t l = basisu::read_le_dword(pSrc); - + pSrc += sizeof(uint32_t); src_left -= sizeof(uint32_t); @@ -19863,7 +20616,7 @@ namespace basist BASISU_DEVEL_ERROR("ktx2_transcoder::read_key_values: Out of memory\n"); return false; } - + basisu::uint8_vec& key_data = m_key_values.back().m_key; basisu::uint8_vec& value_data = m_key_values.back().m_value; @@ -19892,7 +20645,7 @@ namespace basist BASISU_DEVEL_ERROR("ktx2_transcoder::read_key_values: Out of memory\n"); return false; } - + if (!value_data.try_resize(l)) { BASISU_DEVEL_ERROR("ktx2_transcoder::read_key_values: Out of memory\n"); @@ -19928,7 +20681,7 @@ namespace basist return true; } - + #endif // BASISD_SUPPORT_KTX2 bool basisu_transcoder_supports_ktx2() @@ -19995,10 +20748,10 @@ namespace basist basist::half_float result = (basist::half_float)((s << 15) | (e << 10) | m); return result; } - + //------------------------------------------------------------------------------------------------ // HDR support - // + // // Originally from bc6h_enc.cpp // BC6H decoder fuzzed vs. DirectXTex's for unsigned/signed @@ -20027,7 +20780,7 @@ namespace basist const bc6h_bit_layout g_bc6h_bit_layouts[NUM_BC6H_MODES][MAX_BC6H_LAYOUT_INDEX] = { // comp_index, subset*2+lh_index, last_bit, first_bit - //------------------------ mode 0: 2 subsets, Weight bits: 46 bits, Endpoint bits: 75 bits (10.555, 10.555, 10.555), delta + //------------------------ mode 0: 2 subsets, Weight bits: 46 bits, Endpoint bits: 75 bits (10.555, 10.555, 10.555), delta { { 1, 2, 4, -1 }, { 2, 2, 4, -1 }, { 2, 3, 4, -1 }, { 0, 0, 9, 0 }, { 1, 0, 9, 0 }, { 2, 0, 9, 0 }, { 0, 1, 4, 0 }, { 1, 3, 4, -1 }, { 1, 2, 3, 0 }, { 1, 1, 4, 0 }, { 2, 3, 0, -1 }, { 1, 3, 3, 0 }, { 2, 1, 4, 0 }, { 2, 3, 1, -1 }, { 2, 2, 3, 0 }, { 0, 2, 4, 0 }, { 2, 3, 2, -1 }, { 0, 3, 4, 0 }, { 2, 3, 3, -1 }, { 3, -1, 4, 0 }, {-1, 0, 0, 0} }, @@ -20078,7 +20831,7 @@ namespace basist { { 0, 0, 9, 0 },{ 1, 0, 9, 0 },{ 2, 0, 9, 0 },{ 0, 1, 3, 0 },{ 0, 0, 10, 15 },{ 1, 1, 3, 0 },{ 1, 0, 10, 15 },{ 2, 1, 3, 0 },{ 2, 0, 10, 15 }, {-1, 0, 0, 0} } }; - // The same as the first 32 2-subset patterns in BC7. + // The same as the first 32 2-subset patterns in BC7. // Bit 7 is a flag indicating that the weight uses 1 less bit than usual. const uint8_t g_bc6h_2subset_patterns[TOTAL_BC6H_PARTITION_PATTERNS][4][4] = // [pat][y][x] { @@ -20102,7 +20855,7 @@ namespace basist const uint8_t g_bc6h_weight3[8] = { 0, 9, 18, 27, 37, 46, 55, 64 }; const uint8_t g_bc6h_weight4[16] = { 0, 4, 9, 13, 17, 21, 26, 30, 34, 38, 43, 47, 51, 55, 60, 64 }; - + static inline void write_bits(uint64_t val, uint32_t num_bits, uint32_t& bit_pos, uint64_t& l, uint64_t& h) { assert((num_bits) && (num_bits < 64) && (bit_pos < 128)); @@ -20231,15 +20984,18 @@ namespace basist return unq; } #endif - + // 6,7,8,9,10,11,12 const uint32_t BC6H_BLOG_TAB_MIN = 6; const uint32_t BC6H_BLOG_TAB_MAX = 12; //const uint32_t BC6H_BLOG_TAB_NUM = BC6H_BLOG_TAB_MAX - BC6H_BLOG_TAB_MIN + 1; - + // Handles 16, or 6-12 bits. Others assert. static inline uint32_t half_to_blog_tab(half_float h, uint32_t num_bits) { + BASISU_NOTE_UNUSED(BC6H_BLOG_TAB_MIN); + BASISU_NOTE_UNUSED(BC6H_BLOG_TAB_MAX); + assert(h <= MAX_BC6H_HALF_FLOAT_AS_UINT); assert((num_bits == 16) || ((num_bits >= BC6H_BLOG_TAB_MIN) && (num_bits <= BC6H_BLOG_TAB_MAX))); @@ -20255,7 +21011,7 @@ namespace basist else { assert((num_bits >= BC6H_BLOG_TAB_MIN) && (num_bits <= BC6H_BLOG_TAB_MAX)); - + // Note: This used to be done using a table lookup, but it required ~224KB of tables. This isn't quite as accurate, but the error is very slight (+-1 half values as ints). return bc6h_half_to_blog(h, num_bits); } @@ -20385,7 +21141,7 @@ namespace basist log_blk.m_mode = mode; pack_bc6h_block(*pPacked_block, log_blk); - + return; } @@ -20785,13 +21541,13 @@ namespace basist half_float endpoints[3][2]; endpoints[0][0] = pColor[0]; endpoints[0][1] = pColor[0]; - + endpoints[1][0] = pColor[1]; endpoints[1][1] = pColor[1]; endpoints[2][0] = pColor[2]; endpoints[2][1] = pColor[2]; - + bc6h_enc_block_1subset_4bit_weights(pPacked_block, endpoints, weights); return true; @@ -20828,7 +21584,7 @@ namespace basist static inline int astc_hdr_sign_extend(int src, int num_src_bits) { - assert(basisu::in_range(num_src_bits, 2, 31)); + assert(basisu::is_in_range(num_src_bits, 2, 31)); const bool negative = (src & (1 << (num_src_bits - 1))) != 0; if (negative) @@ -21069,7 +21825,7 @@ namespace basist if (ohm & 0x12) vb1 |= (x3 << 7); const int shamt = (mode >> 1) ^ 3; - + va = (uint32_t)va << shamt; vb0 = (uint32_t)vb0 << shamt; vb1 = (uint32_t)vb1 << shamt; @@ -21139,7 +21895,7 @@ namespace basist { assert(g_astc_hdr_core_initialized); assert((best_blk.m_weight_ise_range >= 1) && (best_blk.m_weight_ise_range <= 8)); - + if (best_blk.m_weight_ise_range == 5) { // Use 3-bit BC6H weights which are a perfect match for 3-bit ASTC weights, but encode 1-subset as 2 equal subsets @@ -21221,7 +21977,7 @@ namespace basist assert(g_astc_hdr_core_initialized); assert(best_blk.m_num_partitions == 2); assert(common_part_index < basist::TOTAL_ASTC_BC6H_COMMON_PARTITIONS2); - + half_float bc6h_endpoints[2][3][2]; // [subset][comp][lh_index] // UASTC HDR checks @@ -21230,7 +21986,7 @@ namespace basist return false; if ((best_blk.m_color_endpoint_modes[0] != 7) && (best_blk.m_color_endpoint_modes[0] != 11)) return false; - + if (best_blk.m_color_endpoint_modes[0] == 7) { if (!(((best_blk.m_weight_ise_range == 1) && (best_blk.m_endpoint_ise_range == 20)) || @@ -21355,7 +22111,7 @@ namespace basist assert(0); return false; } - + if (log_blk.m_solid_color_flag_ldr) { // Don't support LDR solid colors. @@ -21371,7 +22127,7 @@ namespace basist // Only support 4x4 grid sizes if ((log_blk.m_grid_width != 4) || (log_blk.m_grid_height != 4)) return false; - + // Don't support dual plane encoding if (log_blk.m_dual_plane) return false; @@ -21379,11 +22135,11 @@ namespace basist if (log_blk.m_num_partitions == 1) { // Handle 1 partition (or subset) - + // UASTC HDR checks if ((log_blk.m_weight_ise_range < 1) || (log_blk.m_weight_ise_range > 8)) return false; - + int e[2][3]; bool success; @@ -21429,7 +22185,7 @@ namespace basist for (uint32_t i = 0; i < 2; i++) if (is_half_inf_or_nan(h_e[0][i]) || is_half_inf_or_nan(h_e[1][i]) || is_half_inf_or_nan(h_e[2][i])) return false; - + // Transcode to bc6h if (!transcode_bc6h_1subset(h_e, log_blk, dst_blk)) return false; @@ -21442,7 +22198,7 @@ namespace basist return false; assert(common_bc7_pat_index < (int)basist::TOTAL_ASTC_BC6H_COMMON_PARTITIONS2); - + if (!transcode_bc6h_2subsets(common_bc7_pat_index, log_blk, dst_blk)) return false; } @@ -21454,7 +22210,7 @@ namespace basist return true; } - + // ASTC 6x6 support namespace astc_6x6_hdr { @@ -21582,7 +22338,7 @@ namespace basist // 3x3 { false, 7, 3, 3, 3, astc_helpers::BISE_64_LEVELS, astc_helpers::BISE_8_LEVELS, astc_helpers::BISE_64_LEVELS, astc_helpers::BISE_8_LEVELS, 0, 0 }, - // 6x4 + // 6x4 { false, 7, 3, 6, 4, astc_helpers::BISE_64_LEVELS, astc_helpers::BISE_2_LEVELS, astc_helpers::BISE_64_LEVELS, astc_helpers::BISE_2_LEVELS, BASIST_HDR_6X6_LEVEL2, 0 }, { false, 7, 3, 4, 6, astc_helpers::BISE_64_LEVELS, astc_helpers::BISE_2_LEVELS, astc_helpers::BISE_64_LEVELS, astc_helpers::BISE_2_LEVELS, BASIST_HDR_6X6_LEVEL2, 0 }, @@ -21610,7 +22366,7 @@ namespace basist { false, 7, 3, 5, 4, astc_helpers::BISE_40_LEVELS, astc_helpers::BISE_3_LEVELS, astc_helpers::BISE_40_LEVELS, astc_helpers::BISE_3_LEVELS, 0, 0 }, { false, 7, 3, 4, 5, astc_helpers::BISE_40_LEVELS, astc_helpers::BISE_3_LEVELS, astc_helpers::BISE_40_LEVELS, astc_helpers::BISE_3_LEVELS, 0, 0 }, }; - + const reuse_xy_delta g_reuse_xy_deltas[NUM_REUSE_XY_DELTAS] = { { -1, 0 }, { -2, 0 }, { -3, 0 }, { -4, 0 }, @@ -21656,27 +22412,27 @@ namespace basist int bit = get_bit(src_val, src_bit); dst |= (bit << dst_bit); } - - // Valid for weight ISE ranges 12-192 levels. Preserves upper 2 or 3 bits post-quantization. - static uint8_t g_quantize_tables_preserve2[astc_helpers::TOTAL_ISE_RANGES - 1][256]; - static uint8_t g_quantize_tables_preserve3[astc_helpers::TOTAL_ISE_RANGES - 1][256]; + + // Valid for weight ISE ranges 6-192 or 8-192 levels. Preserves upper 2 or 3 bits post-quantization. + uint8_t g_quantize_tables_preserve2[21 - 1][256]; // astc_helpers::TOTAL_ISE_RANGES=21, valid for >= BISE_6_LEVELS + uint8_t g_quantize_tables_preserve3[21 - 1][256]; // valid for >= BISE_8_LEVELS const uint32_t g_part2_unique_index_to_seed[NUM_UNIQUE_PARTITIONS2] = { - 86, 959, 936, 476, 1007, 672, 447, 423, 488, 422, 273, 65, 267, 786, 585, 195, 108, 731, 878, 812, 264, 125, 868, 581, 258, 390, 549, 872, 661, 352, 645, 543, 988, - 906, 903, 616, 482, 529, 3, 286, 272, 303, 151, 504, 498, 260, 79, 66, 608, 769, 305, 610, 1014, 967, 835, 789, 7, 951, 691, 15, 763, 976, 438, 314, 601, 673, 177, - 252, 615, 436, 220, 899, 623, 433, 674, 278, 797, 107, 847, 114, 470, 760, 821, 490, 329, 945, 387, 471, 225, 172, 83, 418, 966, 439, 316, 247, 43, 343, 625, 798, - 1, 61, 73, 307, 136, 474, 42, 664, 1013, 249, 389, 227, 374, 121, 48, 538, 226, 309, 554, 802, 834, 335, 495, 10, 955, 461, 293, 508, 153, 101, 63, 139, 31, 687, - 132, 174, 324, 545, 289, 39, 178, 594, 963, 854, 222, 323, 998, 964, 598, 475, 720, 1019, 983, 91, 703, 614, 394, 612, 281, 207, 930, 758, 586, 128, 517, 426, 306, - 168, 713, 36, 458, 876, 368, 780, 5, 9, 214, 109, 553, 726, 175, 103, 753, 684, 44, 665, 53, 500, 367, 611, 119, 732, 639, 326, 203, 156, 686, 910, 255, 62, 392, 591, - 112, 88, 213, 19, 1022, 478, 90, 486, 799, 702, 730, 414, 99, 1008, 142, 886, 373, 216, 69, 393, 299, 648, 415, 822, 912, 110, 567, 550, 693, 2, 138, 59, 271, 562, 295, - 714, 719, 199, 893, 831, 1006, 662, 235, 262, 78, 51, 902, 298, 190, 169, 583, 347, 890, 958, 909, 49, 987, 696, 633, 480, 50, 764, 826, 1023, 1016, 437, 891, 774, 257, - 724, 791, 526, 593, 690, 638, 858, 895, 794, 995, 130, 87, 877, 819, 318, 649, 376, 211, 284, 937, 370, 688, 229, 994, 115, 842, 60, 521, 95, 694, 804, 146, 754, 487, 55, - 17, 770, 450, 223, 4, 137, 911, 236, 683, 523, 47, 181, 24, 270, 602, 736, 11, 355, 148, 351, 762, 1009, 16, 210, 619, 805, 874, 807, 887, 403, 999, 810, 27, 402, 551, 135, - 778, 33, 409, 993, 71, 363, 159, 183, 77, 596, 670, 380, 968, 811, 404, 348, 539, 158, 578, 196, 621, 68, 530, 193, 100, 167, 919, 353, 366, 327, 643, 948, 518, 756, 801, 558, - 28, 705, 116, 94, 898, 453, 622, 647, 231, 445, 652, 230, 191, 277, 292, 254, 198, 766, 386, 232, 29, 70, 942, 740, 291, 607, 411, 496, 839, 8, 675, 319, 742, 21, 547, 627, 716, - 663, 23, 914, 631, 595, 499, 685, 950, 510, 54, 587, 432, 45, 646, 25, 122, 947, 171, 862, 441, 808, 722, 14, 74, 658, 129, 266, 1001, 534, 395, 527, 250, 206, 237, 67, 897, 634, - 572, 569, 533, 37, 341, 89, 463, 419, 75, 134, 283, 943, 519, 362, 144, 681, 407, 954, 131, 455, 934, 46, 513, 339, 194, 361, 606, 852, 546, 655, 1015, 147, 506, 240, 56, 836, 76, + 86, 959, 936, 476, 1007, 672, 447, 423, 488, 422, 273, 65, 267, 786, 585, 195, 108, 731, 878, 812, 264, 125, 868, 581, 258, 390, 549, 872, 661, 352, 645, 543, 988, + 906, 903, 616, 482, 529, 3, 286, 272, 303, 151, 504, 498, 260, 79, 66, 608, 769, 305, 610, 1014, 967, 835, 789, 7, 951, 691, 15, 763, 976, 438, 314, 601, 673, 177, + 252, 615, 436, 220, 899, 623, 433, 674, 278, 797, 107, 847, 114, 470, 760, 821, 490, 329, 945, 387, 471, 225, 172, 83, 418, 966, 439, 316, 247, 43, 343, 625, 798, + 1, 61, 73, 307, 136, 474, 42, 664, 1013, 249, 389, 227, 374, 121, 48, 538, 226, 309, 554, 802, 834, 335, 495, 10, 955, 461, 293, 508, 153, 101, 63, 139, 31, 687, + 132, 174, 324, 545, 289, 39, 178, 594, 963, 854, 222, 323, 998, 964, 598, 475, 720, 1019, 983, 91, 703, 614, 394, 612, 281, 207, 930, 758, 586, 128, 517, 426, 306, + 168, 713, 36, 458, 876, 368, 780, 5, 9, 214, 109, 553, 726, 175, 103, 753, 684, 44, 665, 53, 500, 367, 611, 119, 732, 639, 326, 203, 156, 686, 910, 255, 62, 392, 591, + 112, 88, 213, 19, 1022, 478, 90, 486, 799, 702, 730, 414, 99, 1008, 142, 886, 373, 216, 69, 393, 299, 648, 415, 822, 912, 110, 567, 550, 693, 2, 138, 59, 271, 562, 295, + 714, 719, 199, 893, 831, 1006, 662, 235, 262, 78, 51, 902, 298, 190, 169, 583, 347, 890, 958, 909, 49, 987, 696, 633, 480, 50, 764, 826, 1023, 1016, 437, 891, 774, 257, + 724, 791, 526, 593, 690, 638, 858, 895, 794, 995, 130, 87, 877, 819, 318, 649, 376, 211, 284, 937, 370, 688, 229, 994, 115, 842, 60, 521, 95, 694, 804, 146, 754, 487, 55, + 17, 770, 450, 223, 4, 137, 911, 236, 683, 523, 47, 181, 24, 270, 602, 736, 11, 355, 148, 351, 762, 1009, 16, 210, 619, 805, 874, 807, 887, 403, 999, 810, 27, 402, 551, 135, + 778, 33, 409, 993, 71, 363, 159, 183, 77, 596, 670, 380, 968, 811, 404, 348, 539, 158, 578, 196, 621, 68, 530, 193, 100, 167, 919, 353, 366, 327, 643, 948, 518, 756, 801, 558, + 28, 705, 116, 94, 898, 453, 622, 647, 231, 445, 652, 230, 191, 277, 292, 254, 198, 766, 386, 232, 29, 70, 942, 740, 291, 607, 411, 496, 839, 8, 675, 319, 742, 21, 547, 627, 716, + 663, 23, 914, 631, 595, 499, 685, 950, 510, 54, 587, 432, 45, 646, 25, 122, 947, 171, 862, 441, 808, 722, 14, 74, 658, 129, 266, 1001, 534, 395, 527, 250, 206, 237, 67, 897, 634, + 572, 569, 533, 37, 341, 89, 463, 419, 75, 134, 283, 943, 519, 362, 144, 681, 407, 954, 131, 455, 934, 46, 513, 339, 194, 361, 606, 852, 546, 655, 1015, 147, 506, 240, 56, 836, 76, 98, 600, 430, 388, 980, 695, 817, 279, 58, 215, 149, 170, 531, 870, 18, 727, 154, 26, 938, 929, 302, 697, 452, 218, 700, 524, 828, 751, 869, 217, 440, 354 }; @@ -21707,7 +22463,9 @@ namespace basist static void init_quantize_tables() { - for (uint32_t ise_range = astc_helpers::BISE_192_LEVELS; ise_range >= astc_helpers::BISE_12_LEVELS; ise_range--) + // 9/15/2025 changed lower range for LDR + // for (uint32_t ise_range = astc_helpers::BISE_192_LEVELS; ise_range >= astc_helpers::BISE_12_LEVELS; ise_range--) + for (uint32_t ise_range = astc_helpers::BISE_192_LEVELS; ise_range >= astc_helpers::BISE_6_LEVELS; ise_range--) { const uint32_t num_levels = astc_helpers::get_ise_levels(ise_range); const auto& ise_to_val_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(ise_range).m_ISE_to_val; @@ -21739,6 +22497,7 @@ namespace basist g_quantize_tables_preserve2[ise_range][desired_val] = (uint8_t)best_ise_val; } + if (ise_range >= astc_helpers::BISE_8_LEVELS) { uint32_t best_err = UINT32_MAX; int best_ise_val = -1; @@ -21894,7 +22653,7 @@ namespace basist #endif } - void copy_weight_grid(bool dual_plane, uint32_t grid_x, uint32_t grid_y, const uint8_t* transcode_weights, astc_helpers::log_astc_block& decomp_blk) + void copy_weight_grid(bool dual_plane, uint32_t grid_x, uint32_t grid_y, const uint8_t* transcode_weights, astc_helpers::log_astc_block& decomp_blk, bool orig_behavior) { assert(decomp_blk.m_weight_ise_range >= astc_helpers::BISE_2_LEVELS); assert(decomp_blk.m_weight_ise_range <= astc_helpers::BISE_32_LEVELS); @@ -21913,11 +22672,12 @@ namespace basist compute_upsample_weights(4, 4, 2, 2, weights); - for (uint32_t y = 0; y < 4; y++) + for (uint32_t dy = 0; dy < 4; dy++) { - for (uint32_t x = 0; x < 4; x++) + for (uint32_t dx = 0; dx < 4; dx++) { - const astc_helpers::weighted_sample& sample = weights[x + y * 4]; + const astc_helpers::weighted_sample& sample = weights[dx + dy * 4]; + const int sx = sample.m_src_x, sy = sample.m_src_y; uint32_t total_weight = 8; @@ -21928,7 +22688,19 @@ namespace basist if (!sample.m_weights[yo][xo]) continue; - total_weight += dequant_weight[transcode_weights[basisu::in_bounds((x + xo) + (y + yo) * grid_x, 0, grid_x * grid_y)]] * sample.m_weights[yo][xo]; + // 10/17/2025 - bugfix. Orig release would always sample the 1st or 2nd weight here. Minor issue - encoder would have detected it, hurting R-D performance a tiny bit but not encoding/decoding correctness. + // However, this fix does cause decoding divergence from original encodes. The divergence seems minor and can only happen at higher lambdas. + if (orig_behavior) + { + // Original, incorrect, but ultimately harmless behavior. + total_weight += dequant_weight[transcode_weights[basisu::is_in_bounds((dx + xo) + (dy + yo) * grid_x, 0, grid_x * grid_y)]] * sample.m_weights[yo][xo]; + } + else + { + // Correct behavior. + assert(basisu::is_in_bounds((sx + xo) + (sy + yo) * grid_x, 0, grid_x * grid_y)); + total_weight += dequant_weight[transcode_weights[(sx + xo) + (sy + yo) * grid_x]] * sample.m_weights[yo][xo]; + } } // x } // y @@ -21936,7 +22708,7 @@ namespace basist assert(total_weight <= 64); - decomp_blk.m_weights[x + y * 4] = quant_weight[total_weight]; + decomp_blk.m_weights[dx + dy * 4] = quant_weight[total_weight]; } } } @@ -22063,7 +22835,7 @@ namespace basist assert((cem == 7) || (cem == 11)); return (cem == 11) ? basist::NUM_MODE11_ENDPOINTS : basist::NUM_MODE7_ENDPOINTS; } - + const uint32_t g_bc6h_weights4[16] = { 0, 4, 9, 13, 17, 21, 26, 30, 34, 38, 43, 47, 51, 55, 60, 64 }; #if 0 @@ -22147,7 +22919,7 @@ namespace basist float res = (float)fast_float_to_half_no_clamp_neg_nan_or_inf(fabsf(f)) * ((f < 0.0f) ? -1.0f : 1.0f); return res; } - + // Supports positive and denormals only. No NaN or Inf. static BASISU_FORCE_INLINE float fast_half_to_float_pos_not_inf_or_nan(basist::half_float h) { @@ -22188,7 +22960,7 @@ namespace basist int max_r, int max_g, int max_b, int64_t block_max_var) { BASISU_NOTE_UNUSED(block_max_var); - + float fmin_r = fast_half_to_float_pos_not_inf_or_nan((basist::half_float)min_r); float fmin_g = fast_half_to_float_pos_not_inf_or_nan((basist::half_float)min_g); float fmin_b = fast_half_to_float_pos_not_inf_or_nan((basist::half_float)min_b); @@ -22225,12 +22997,12 @@ namespace basist pWeights[i] = (uint8_t)basisu::clamp((int)(w * frr + lr), 0, 15); } } - + static double assign_weights_4( const vec3F* pFloat_pixels, const float* pPixel_scales, uint8_t* pWeights, int min_r, int min_g, int min_b, - int max_r, int max_g, int max_b, int64_t block_max_var, bool try_2subsets_flag, + int max_r, int max_g, int max_b, int64_t block_max_var, bool try_2subsets_flag, const fast_bc6h_params& params) { float cr[16], cg[16], cb[16]; @@ -22348,7 +23120,7 @@ namespace basist pWeights[i] = (uint8_t)best_idx; - // Giesen's MRSSE (Mean Relative Sum of Squared Errors). + // Giesen's MRSSE (Mean Relative Sum of Squared Errors). // Our ASTC HDR encoder uses slightly slower approx. MSLE, and it's too late/risky to eval the difference vs. MRSSE on the larger ASTC HDR blocks. float err = basisu::squaref(qr - cr[best_idx]) + basisu::squaref(qg - cg[best_idx]) + basisu::squaref(qb - cb[best_idx]); total_err += err * pPixel_scales[i]; @@ -22448,7 +23220,7 @@ namespace basist float best_error = basisu::squaref(subset_cr[subset][0] - qr) + basisu::squaref(subset_cg[subset][0] - qg) + basisu::squaref(subset_cb[subset][0] - qb); uint32_t best_idx = 0; - + for (uint32_t j = 1; j < 8; j++) { float e = basisu::squaref(subset_cr[subset][j] - qr) + basisu::squaref(subset_cg[subset][j] - qg) + basisu::squaref(subset_cb[subset][j] - qb); @@ -22521,7 +23293,7 @@ namespace basist static basist::vec4F g_bc6h_ls_weights_3[8]; static basist::vec4F g_bc6h_ls_weights_4[16]; - + const uint32_t BC6H_NUM_PATS = 32; static uint32_t g_bc6h_pats2[BC6H_NUM_PATS]; @@ -22587,7 +23359,7 @@ namespace basist static void bc6h_quant_endpoints( uint32_t min_hr, uint32_t min_hg, uint32_t min_hb, uint32_t max_hr, uint32_t max_hg, uint32_t max_hb, - uint32_t& min_r, uint32_t& min_g, uint32_t& min_b, uint32_t& max_r, uint32_t& max_g, uint32_t& max_b, + uint32_t& min_r, uint32_t& min_g, uint32_t& min_b, uint32_t& max_r, uint32_t& max_g, uint32_t& max_b, int bits) { min_r = basist::bc6h_half_to_blog((basist::half_float)min_hr, bits); @@ -22613,7 +23385,7 @@ namespace basist max_hb = bc6h_convert_to_half(bc6h_dequantize(max_bb, bits)); } - static BASISU_FORCE_INLINE int popcount32(uint32_t x) + static BASISU_FORCE_INLINE int popcount32(uint32_t x) { #if defined(__EMSCRIPTEN__) || defined(__clang__) || defined(__GNUC__) return __builtin_popcount(x); @@ -22621,7 +23393,7 @@ namespace basist return __popcnt(x); #else int count = 0; - while (x) + while (x) { x &= (x - 1); ++count; @@ -22634,23 +23406,23 @@ namespace basist { return (x >= 0.0f) ? (int)(x + 0.5f) : (int)(x - 0.5f); } - + static void fast_encode_bc6h_2subsets_pattern( uint32_t best_pat_index, uint32_t best_pat_bits, const basist::half_float* pPixels, const vec3F* pFloat_pixels, const float* pPixel_scales, double& cur_error, basist::bc6h_logical_block& log_blk, int64_t block_max_var, - int mean_r, int mean_g, int mean_b, + int mean_r, int mean_g, int mean_b, const fast_bc6h_params& params) { BASISU_NOTE_UNUSED(block_max_var); - + uint32_t subset_means[2][3] = { { 0 } }; for (uint32_t i = 0; i < 16; i++) { const uint32_t subset_index = (best_pat_bits >> i) & 1; const uint32_t r = pPixels[i * 3 + 0], g = pPixels[i * 3 + 1], b = pPixels[i * 3 + 2]; - + subset_means[subset_index][0] += r; subset_means[subset_index][1] += g; subset_means[subset_index][2] += b; @@ -22703,7 +23475,7 @@ namespace basist subset_axis[subset_index].set(axis_r, axis_g, axis_b); } // s - + float subset_min_dot[2] = { basisu::BIG_FLOAT_VAL, basisu::BIG_FLOAT_VAL }; float subset_max_dot[2] = { -basisu::BIG_FLOAT_VAL, -basisu::BIG_FLOAT_VAL }; int subset_min_idx[2] = { 0 }, subset_max_idx[2] = { 0 }; @@ -22820,7 +23592,7 @@ namespace basist if (params.m_num_diff_endpoint_modes_to_try) { // ordered from largest base bits to least - static const int s_bc6h_mode_order2[2] = { 5, 1 }; + static const int s_bc6h_mode_order2[2] = { 5, 1 }; static const int s_bc6h_mode_order4[4] = { 0, 5, 7, 1 }; static const int s_bc6h_mode_order9[9] = { 2, 3, 4, 0, 5, 6, 7, 8, 1 }; @@ -22848,7 +23620,7 @@ namespace basist BASISU_NOTE_UNUSED(base_bitmask); const uint32_t num_delta_bits[3] = { g_bc6h_mode_sig_bits[mode][1], g_bc6h_mode_sig_bits[mode][2], g_bc6h_mode_sig_bits[mode][3] }; - const int delta_bitmasks[3] = { (1 << num_delta_bits[0]) - 1, (1 << num_delta_bits[1]) - 1, (1 << num_delta_bits[2]) - 1 }; + //const int delta_bitmasks[3] = { (1 << num_delta_bits[0]) - 1, (1 << num_delta_bits[1]) - 1, (1 << num_delta_bits[2]) - 1 }; for (uint32_t subset_index = 0; subset_index < 2; subset_index++) { @@ -22928,10 +23700,10 @@ namespace basist trial_log_blk.m_mode = bc6h_mode_index; trial_log_blk.m_partition_pattern = best_pat_index; - + memcpy(trial_log_blk.m_endpoints, abs_blog_endpoints, sizeof(trial_log_blk.m_endpoints)); memcpy(trial_log_blk.m_weights, trial_weights, 16); - + if (trial_log_blk.m_weights[0] & 4) { for (uint32_t c = 0; c < 3; c++) @@ -22958,7 +23730,7 @@ namespace basist trial_log_blk.m_weights[i] = 7 - trial_log_blk.m_weights[i]; } } - + if (bc6h_mode_index != BC6H_2SUBSET_ABS_ENDPOINT_MODE) { const uint32_t num_delta_bits[3] = { g_bc6h_mode_sig_bits[bc6h_mode_index][1], g_bc6h_mode_sig_bits[bc6h_mode_index][2], g_bc6h_mode_sig_bits[bc6h_mode_index][3] }; @@ -22999,7 +23771,7 @@ namespace basist const basist::half_float* pPixels, const vec3F* pFloat_pixels, const float* pPixel_scales, double& cur_error, basist::bc6h_logical_block& log_blk, int64_t block_max_var, - int mean_r, int mean_g, int mean_b, float block_axis_r, float block_axis_g, float block_axis_b, + int mean_r, int mean_g, int mean_b, float block_axis_r, float block_axis_g, float block_axis_b, const fast_bc6h_params& params) { assert((params.m_max_2subset_pats_to_try > 0) && (params.m_max_2subset_pats_to_try <= BC6H_NUM_PATS)); @@ -23020,7 +23792,7 @@ namespace basist } return; } - + uint32_t desired_pat_bits = 0; for (uint32_t i = 0; i < 16; i++) { @@ -23098,13 +23870,13 @@ namespace basist uint32_t omin_r = UINT32_MAX, omin_g = UINT32_MAX, omin_b = UINT32_MAX; uint32_t omax_r = 0, omax_g = 0, omax_b = 0; uint32_t total_r = 0, total_g = 0, total_b = 0; - + for (uint32_t i = 0; i < 16; i++) { uint32_t r = pPixels[i * 3 + 0]; uint32_t g = pPixels[i * 3 + 1]; uint32_t b = pPixels[i * 3 + 2]; - + total_r += r; total_g += g; total_b += b; @@ -23129,13 +23901,13 @@ namespace basist log_blk.m_endpoints[2][0] = basist::bc6h_half_to_blog16((basist::half_float)omin_b); log_blk.m_endpoints[2][1] = 0; - + log_blk.m_mode = 13; pack_bc6h_block(*pBlock, log_blk); return; } - + uint32_t min_r, min_g, min_b, max_r, max_g, max_b; int mean_r = (total_r + 8) / 16; @@ -23157,9 +23929,9 @@ namespace basist icov[4] += g * b; icov[5] += b * b; } - + int64_t block_max_var = basisu::maximum(icov[0], icov[3], icov[5]); // not divided by 16, i.e. scaled by 16 - + if (block_max_var < (FAST_BC6H_STD_DEV_THRESH * FAST_BC6H_STD_DEV_THRESH * 16)) { // Simple block @@ -23254,7 +24026,7 @@ namespace basist uint32_t min_idx = 0, max_idx = 0; float min_dot = basisu::BIG_FLOAT_VAL, max_dot = -basisu::BIG_FLOAT_VAL; - + for (uint32_t i = 0; i < 16; i++) { float r = (float)pPixels[i * 3 + 0]; @@ -23295,7 +24067,7 @@ namespace basist bc6h_quant_dequant_endpoints(min_r, min_g, min_b, max_r, max_g, max_b, 10); cur_err = assign_weights_4(float_pixels, pixel_scales, log_blk.m_weights, min_r, min_g, min_b, max_r, max_g, max_b, block_max_var, try_2subsets, params); - + const uint32_t MAX_LS_PASSES = params.m_hq_ls ? 2 : 1; for (uint32_t pass = 0; pass < MAX_LS_PASSES; pass++) { @@ -23371,7 +24143,7 @@ namespace basist min_b = trial_min_b; max_b = trial_max_b; - + memcpy(log_blk.m_weights, trial_weights, 16); } else @@ -23413,7 +24185,7 @@ namespace basist std::swap(log_blk.m_endpoints[c][0], log_blk.m_endpoints[c][1]); } } - + if ((params.m_max_2subset_pats_to_try > 0) && ((try_2subsets) && (block_max_var > (FAST_BC6H_COMPLEX_STD_DEV_THRESH * FAST_BC6H_COMPLEX_STD_DEV_THRESH * 16)))) { fast_encode_bc6h_2subsets(pPixels, float_pixels, pixel_scales, cur_err, log_blk, block_max_var, mean_r, mean_g, mean_b, axis_r, axis_g, axis_b, params); @@ -23439,7 +24211,12 @@ namespace basist if (!decoder.init(pComp_data, comp_data_size)) return false; - if (decoder.get_bits(16) != 0xABCD) + bool orig_behavior = false; + + uint32_t hdr_sig = decoder.get_bits(16); + if (hdr_sig == UASTC_6x6_HDR_SIG0) + orig_behavior = true; + else if (hdr_sig != UASTC_6x6_HDR_SIG1) return false; width = decoder.get_bits(16); @@ -23621,7 +24398,7 @@ namespace basist uint8_t transcode_weights[MAX_BLOCK_W * MAX_BLOCK_H * 2]; requantize_astc_weights(total_grid_weights, log_blk.m_weights, log_blk.m_weight_ise_range, transcode_weights, decomp_blk.m_weight_ise_range); - copy_weight_grid(log_blk.m_dual_plane, log_blk.m_grid_width, log_blk.m_grid_height, transcode_weights, decomp_blk); + copy_weight_grid(log_blk.m_dual_plane, log_blk.m_grid_width, log_blk.m_grid_height, transcode_weights, decomp_blk, orig_behavior); #else assert(log_blk.m_user_mode < TOTAL_BLOCK_MODE_DECS); const block_mode_desc& bmd = g_block_mode_descs[(uint32_t)log_blk.m_user_mode]; @@ -23655,7 +24432,7 @@ namespace basist uint8_t transcode_weights[BLOCK_W * BLOCK_H * 2]; requantize_astc_weights(total_grid_weights, log_blk.m_weights, log_blk.m_weight_ise_range, transcode_weights, bmd.m_transcode_weight_ise_range); - copy_weight_grid(bmd.m_dp, bmd.m_grid_x, bmd.m_grid_y, transcode_weights, decomp_blk); + copy_weight_grid(bmd.m_dp, bmd.m_grid_x, bmd.m_grid_y, transcode_weights, decomp_blk, orig_behavior); #endif status = astc_helpers::pack_astc_block(phys_blk, decomp_blk); if (!status) @@ -23741,7 +24518,7 @@ namespace basist uint8_t transcode_weights[BLOCK_W * BLOCK_H * 2]; requantize_astc_weights(total_grid_weights, log_blk.m_weights, bmd.m_weight_ise_range, transcode_weights, bmd.m_transcode_weight_ise_range); - copy_weight_grid(bmd.m_dp, bmd.m_grid_x, bmd.m_grid_y, transcode_weights, decomp_blk); + copy_weight_grid(bmd.m_dp, bmd.m_grid_x, bmd.m_grid_y, transcode_weights, decomp_blk, orig_behavior); status = astc_helpers::pack_astc_block(phys_blk, decomp_blk); if (!status) @@ -23839,7 +24616,7 @@ namespace basist uint8_t transcode_weights[BLOCK_W * BLOCK_H * 2]; requantize_astc_weights(total_grid_weights, log_blk.m_weights, bmd.m_weight_ise_range, transcode_weights, bmd.m_transcode_weight_ise_range); - copy_weight_grid(bmd.m_dp, bmd.m_grid_x, bmd.m_grid_y, transcode_weights, decomp_blk); + copy_weight_grid(bmd.m_dp, bmd.m_grid_x, bmd.m_grid_y, transcode_weights, decomp_blk, orig_behavior); status = astc_helpers::pack_astc_block(phys_blk, decomp_blk); if (!status) @@ -23923,7 +24700,7 @@ namespace basist uint8_t transcode_weights[BLOCK_W * BLOCK_H * 2]; requantize_astc_weights(total_grid_weights, log_blk.m_weights, bmd.m_weight_ise_range, transcode_weights, bmd.m_transcode_weight_ise_range); - copy_weight_grid(bmd.m_dp, bmd.m_grid_x, bmd.m_grid_y, transcode_weights, decomp_blk); + copy_weight_grid(bmd.m_dp, bmd.m_grid_x, bmd.m_grid_y, transcode_weights, decomp_blk, orig_behavior); status = astc_helpers::pack_astc_block(phys_blk, decomp_blk); if (!status) @@ -23971,4 +24748,17026 @@ namespace basist #endif // BASISD_SUPPORT_UASTC_HDR -} // namespace basist +#if BASISD_SUPPORT_XUASTC +namespace astc_ldr_t +{ + bool g_initialized; + astc_block_grid_data_hash_t g_astc_block_grid_data_hash; + + // Used for quickly bumping up or down quantized, 2 complement+shifted base+offset delta values without disturbing the MSB. + static basisu::vector g_base_ofs_nudges[astc_helpers::BISE_256_LEVELS + 1][2]; // [endpoint_ise_range][pos=0, neg=1] + + const int s_unique_ldr_index_to_astc_cem[6] = + { + astc_helpers::CEM_LDR_LUM_DIRECT, + astc_helpers::CEM_LDR_LUM_ALPHA_DIRECT, + astc_helpers::CEM_LDR_RGB_BASE_SCALE, + astc_helpers::CEM_LDR_RGB_DIRECT, + astc_helpers::CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A, + astc_helpers::CEM_LDR_RGBA_DIRECT + }; + + static void compute_base_ofs_requantize_tabs() + { + for (uint32_t e_ise_range = astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE; e_ise_range <= astc_helpers::LAST_VALID_ENDPOINT_ISE_RANGE; e_ise_range++) + { + const uint32_t num_levels = astc_helpers::get_ise_levels(e_ise_range); + + for (uint32_t pos_or_neg = 0; pos_or_neg < 2; pos_or_neg++) + { + g_base_ofs_nudges[e_ise_range][pos_or_neg].resize(num_levels); + + const int delta = pos_or_neg ? -1 : 1; + + for (uint32_t cur_ise = 0; cur_ise < num_levels; cur_ise++) + { + int cur_dequant = astc_helpers::g_dequant_tables.get_endpoint_tab(e_ise_range).m_ISE_to_val[cur_ise]; + + int cur_a = cur_dequant, cur_b = 0; + astc_helpers::bit_transfer_signed_dec(cur_a, cur_b); + + int best_err = INT_MAX; + uint32_t best_trial_ise = 0; + + for (uint32_t trial_ise = 0; trial_ise < num_levels; trial_ise++) + { + int trial_dequant = astc_helpers::g_dequant_tables.get_endpoint_tab(e_ise_range).m_ISE_to_val[trial_ise]; + + int trial_a = trial_dequant, trial_b = 0; + astc_helpers::bit_transfer_signed_dec(trial_a, trial_b); + + // ensure the transferred bit hasn't changed + if (cur_b != trial_b) + continue; + + // skip if the decoded delta hasn't changed at all + if (trial_a == cur_a) + continue; + + // do they want to nudge neg or pos + if (delta < 0) + { + // neg nudge, but trial delta is higher + if (trial_a > cur_a) + continue; + } + else + { + // pos nudge, but trial delta is lower + if (trial_a < cur_a) + continue; + } + + int e = basisu::iabs(trial_a - cur_a); + if (e < best_err) + { + best_err = e; + best_trial_ise = trial_ise; + } + } // trial_ise + + if (best_err == INT_MAX) + { + //fmt_printf("Failed nudge: eise:{}, delta: {}, curise:{}, cura:{}, curb:{}\n", e_ise_range, delta, cur_ise, cur_a, cur_b); + + // Failed to nudge, leave it unchanged + best_trial_ise = cur_ise; + } + + g_base_ofs_nudges[e_ise_range][pos_or_neg][cur_ise] = (uint8_t)best_trial_ise; + + } // cur_ise + + } // pos_or_neg + + } // e_ise_range + } + + void init() + { + if (g_initialized) + return; + + g_initialized = true; + + init_astc_block_grid_data_hash(); + + compute_base_ofs_requantize_tabs(); + } + + color_rgba blue_contract_enc(color_rgba orig, bool& did_clamp, int encoded_b) + { + color_rgba enc; + + int tr = orig.r * 2 - encoded_b; + int tg = orig.g * 2 - encoded_b; + if ((tr < 0) || (tr > 255) || (tg < 0) || (tg > 255)) + did_clamp = true; + + enc.r = (uint8_t)basisu::clamp(tr, 0, 255); + enc.g = (uint8_t)basisu::clamp(tg, 0, 255); + enc.b = (uint8_t)orig.b; + enc.a = orig.a; + return enc; + } + + color_rgba blue_contract_dec(int enc_r, int enc_g, int enc_b, int enc_a) + { + color_rgba dec; + dec.r = (uint8_t)((enc_r + enc_b) >> 1); + dec.g = (uint8_t)((enc_g + enc_b) >> 1); + dec.b = (uint8_t)enc_b; + dec.a = (uint8_t)enc_a; + return dec; + } + + static inline int quant_preserve2(uint32_t ise_range, uint32_t v) + { + if (ise_range == astc_helpers::BISE_256_LEVELS) + return v; + + assert(ise_range >= astc_helpers::BISE_6_LEVELS); + + return basist::astc_6x6_hdr::g_quantize_tables_preserve2[ise_range][v]; + } + + //---------------------------------------------------------------------------------- + // Requantize endpoints, but preserves blue contraction and base+ofs bits as much as possible. + + // Blue contraction should be preserved almost always if quantizing down, except with base+ofs (extremely to incredibly rare). + // endpoints never swapped for base+ofs + // NOTE: Cannot use any floating point math for determinism across compilers. + bool requantize_ise_endpoints(uint32_t cem, + uint32_t src_ise_endpoint_range, const uint8_t* pSrc_endpoints, + uint32_t dst_ise_endpoint_range, uint8_t* pDst_endpoints) + { + if (!astc_helpers::is_cem_ldr(cem)) + { + assert(0); + return false; + } + + const uint32_t num_endpoint_vals = astc_helpers::get_num_cem_values(cem); + assert(num_endpoint_vals <= astc_helpers::MAX_CEM_ENDPOINT_VALS); + + if (src_ise_endpoint_range == dst_ise_endpoint_range) + { + memcpy(pDst_endpoints, pSrc_endpoints, num_endpoint_vals); + return true; + } + + uint8_t dequantized_src_vals_temp[astc_helpers::MAX_CEM_ENDPOINT_VALS]; + const uint8_t* pDequantized_src_vals = pSrc_endpoints; + + if (src_ise_endpoint_range != astc_helpers::BISE_256_LEVELS) + { + const auto& dequant_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(src_ise_endpoint_range).m_ISE_to_val; + + for (uint32_t i = 0; i < num_endpoint_vals; i++) + dequantized_src_vals_temp[i] = dequant_tab[pSrc_endpoints[i]]; + + pDequantized_src_vals = dequantized_src_vals_temp; + } + + if (dst_ise_endpoint_range == astc_helpers::BISE_256_LEVELS) + { + memcpy(pDst_endpoints, pDequantized_src_vals, num_endpoint_vals); + return true; + } + + const auto& dst_quant_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(dst_ise_endpoint_range).m_val_to_ise; + + if ((cem == astc_helpers::CEM_LDR_RGB_BASE_PLUS_OFFSET) || (cem == astc_helpers::CEM_LDR_RGBA_BASE_PLUS_OFFSET)) + { + const auto& dst_dequant_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(dst_ise_endpoint_range).m_ISE_to_val; + + for (uint32_t i = 0; i < num_endpoint_vals; i++) + { + // preserve v1,v3,v5,v7, which have 2 MSB's that need to be preserved during requant + if (i & 1) + pDst_endpoints[i] = (uint8_t)quant_preserve2(dst_ise_endpoint_range, pDequantized_src_vals[i]); + else + pDst_endpoints[i] = dst_quant_tab[pDequantized_src_vals[i]]; + } + +#ifdef _DEBUG + { + const auto& src_dequant_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(src_ise_endpoint_range).m_ISE_to_val; + + // ensure MSB's did not change + for (uint32_t i = 0; i < num_endpoint_vals; i++) + { + int src_v = src_dequant_tab[pSrc_endpoints[i]]; + int dst_v = dst_dequant_tab[pDst_endpoints[i]]; + assert((src_v & 128) == (dst_v & 128)); + + if (i & 1) + { + assert((src_v & 64) == (dst_v & 64)); + } + } + } +#endif + + const bool src_used_blue_contract = astc_helpers::used_blue_contraction(cem, pSrc_endpoints, src_ise_endpoint_range); + // src delta sum was < 0 if it used blue contraction, >= 0 if it did NOT + + int v0 = dst_dequant_tab[pDst_endpoints[0]], v1 = dst_dequant_tab[pDst_endpoints[1]]; + int v2 = dst_dequant_tab[pDst_endpoints[2]], v3 = dst_dequant_tab[pDst_endpoints[3]]; + int v4 = dst_dequant_tab[pDst_endpoints[4]], v5 = dst_dequant_tab[pDst_endpoints[5]]; + + astc_helpers::bit_transfer_signed_dec(v1, v0); + astc_helpers::bit_transfer_signed_dec(v3, v2); + astc_helpers::bit_transfer_signed_dec(v5, v4); + + int s = v1 + v3 + v5; + bool quant_used_blue_contraction = (s < 0); + + // Kind of a dumb algorithm, but it only tries 2-3 times in random testing. + //const uint32_t MAX_TRIES = 10; + const uint32_t MAX_TRIES = 5; + + uint32_t tries = 0; + + if (src_used_blue_contract != quant_used_blue_contraction) + { + int nudge_delta = quant_used_blue_contraction ? 1 : -1; + + uint32_t cur_c_rover = 2; // b first + + for (tries = 0; tries < MAX_TRIES; tries++) + { + for (uint32_t j = 0; j < 3; j++) + { + const uint32_t i = (cur_c_rover + j) % 3; + + // This will either nudge the delta, or fail because it's either at the [-32,31] limit or it can't go further in the desired delta direction due to quantization limits + uint32_t new_ise_v = g_base_ofs_nudges[dst_ise_endpoint_range][(nudge_delta < 0) ? 1 : 0][pDst_endpoints[1 + i * 2]]; + + if (new_ise_v != pDst_endpoints[1 + i * 2]) + { + // It changed, so a successful nudge, but the base MSB should be preserved + pDst_endpoints[1 + i * 2] = (uint8_t)new_ise_v; + break; + } + } + + v0 = dst_dequant_tab[pDst_endpoints[0]], v1 = dst_dequant_tab[pDst_endpoints[1]]; + v2 = dst_dequant_tab[pDst_endpoints[2]], v3 = dst_dequant_tab[pDst_endpoints[3]]; + v4 = dst_dequant_tab[pDst_endpoints[4]], v5 = dst_dequant_tab[pDst_endpoints[5]]; + + astc_helpers::bit_transfer_signed_dec(v1, v0); + astc_helpers::bit_transfer_signed_dec(v3, v2); + astc_helpers::bit_transfer_signed_dec(v5, v4); + + s = v1 + v3 + v5; + quant_used_blue_contraction = (s < 0); + + if (src_used_blue_contract == quant_used_blue_contraction) + break; + + ++cur_c_rover; + + } // tries + } + + if (tries < MAX_TRIES) + { + assert(astc_helpers::used_blue_contraction(cem, pDst_endpoints, dst_ise_endpoint_range) == astc_helpers::used_blue_contraction(cem, pSrc_endpoints, src_ise_endpoint_range)); + } + else + { + // It failed to adjust, ultimately harmless as we have RGB(A) direct anyway (and at this likely very low quant level, it won't matter). + // TODO: We could try more adjustments, but this seems extremely unlikely to be worth the trouble after random testing. + +#if BASISU_ASTC_LDR_DEBUG_MSGS + static bool s_msg_printed = false; + if (!s_msg_printed) + fmt_debug_printf("requantize_ise_endpoints: blue contraction enforcement failed\n"); +#endif + } + +#ifdef _DEBUG + { + const auto& src_dequant_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(src_ise_endpoint_range).m_ISE_to_val; + + // ensure MSB's did not change + for (uint32_t i = 0; i < num_endpoint_vals; i++) + { + int src_v = src_dequant_tab[pSrc_endpoints[i]]; + int dst_v = dst_dequant_tab[pDst_endpoints[i]]; + assert((src_v & 128) == (dst_v & 128)); + } + } +#endif + } + else if ((cem == astc_helpers::CEM_LDR_RGB_DIRECT) || (cem == astc_helpers::CEM_LDR_RGBA_DIRECT)) + { + const auto& dst_dequant_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(dst_ise_endpoint_range).m_ISE_to_val; + + // See if the original colors were blue contracted + uint32_t s0 = pDequantized_src_vals[0] + pDequantized_src_vals[2] + pDequantized_src_vals[4]; + uint32_t s1 = pDequantized_src_vals[1] + pDequantized_src_vals[3] + pDequantized_src_vals[5]; + + const bool orig_used_blue_contract = s1 < s0; + + for (uint32_t i = 0; i < num_endpoint_vals; i++) + pDst_endpoints[i] = dst_quant_tab[pDequantized_src_vals[i]]; + + uint32_t dequant_s0 = dst_dequant_tab[pDst_endpoints[0]] + dst_dequant_tab[pDst_endpoints[2]] + dst_dequant_tab[pDst_endpoints[4]]; + uint32_t dequant_s1 = dst_dequant_tab[pDst_endpoints[1]] + dst_dequant_tab[pDst_endpoints[3]] + dst_dequant_tab[pDst_endpoints[5]]; + + const bool quant_used_blue_contract = dequant_s1 < dequant_s0; + + if (orig_used_blue_contract != quant_used_blue_contract) + { + if (dequant_s0 == dequant_s1) + { + assert(orig_used_blue_contract); + assert(!quant_used_blue_contract); + + // swapping won't work because sums are equal, so force dst to use blue contraction by nudgling a component + // original s1=requant_s0 + + if (dequant_s1) + { + // decrease s1 + for (uint32_t i = 0; i < 3; i++) + { + uint32_t new_ise_v = astc_helpers::apply_delta_to_bise_endpoint_val(dst_ise_endpoint_range, pDst_endpoints[1 + i * 2], -1); + if (new_ise_v != pDst_endpoints[1 + i * 2]) + { + pDst_endpoints[1 + i * 2] = (uint8_t)new_ise_v; + break; + } + } + } + else + { + // both are 0, increase s0 + for (uint32_t i = 0; i < 3; i++) + { + uint32_t new_ise_val = astc_helpers::apply_delta_to_bise_endpoint_val(dst_ise_endpoint_range, pDst_endpoints[i * 2], 1); + if (new_ise_val != pDst_endpoints[i * 2]) + { + pDst_endpoints[i * 2] = (uint8_t)new_ise_val; + break; + } + } + } + } + else + { + std::swap(pDst_endpoints[0], pDst_endpoints[1]); + std::swap(pDst_endpoints[2], pDst_endpoints[3]); + std::swap(pDst_endpoints[4], pDst_endpoints[5]); + + if (cem == astc_helpers::CEM_LDR_RGBA_DIRECT) + std::swap(pDst_endpoints[6], pDst_endpoints[7]); + } + } + + assert(astc_helpers::used_blue_contraction(cem, pDst_endpoints, dst_ise_endpoint_range) == astc_helpers::used_blue_contraction(cem, pSrc_endpoints, src_ise_endpoint_range)); + } + else + { + for (uint32_t i = 0; i < num_endpoint_vals; i++) + pDst_endpoints[i] = dst_quant_tab[pDequantized_src_vals[i]]; + +#ifdef _DEBUG + { + const auto& src_dequant_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(src_ise_endpoint_range).m_ISE_to_val; + const auto& dst_dequant_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(dst_ise_endpoint_range).m_ISE_to_val; + + // ensure MSB's did not change + for (uint32_t i = 0; i < num_endpoint_vals; i++) + { + int src_v = src_dequant_tab[pSrc_endpoints[i]]; + int dst_v = dst_dequant_tab[pDst_endpoints[i]]; + assert((src_v & 128) == (dst_v & 128)); + } + } +#endif + } + + return true; + } + + // First packs base+ofs to ise20 (always enforcing blue contraction), then quantizes down (preserving blue contraction whenever possible, which might possibly not be in extreme quantizations). + // NOTE: Cannot use any floating point math for determinism across compilers. + bool pack_base_offset( + uint32_t cem_index, uint32_t dst_ise_endpoint_range, uint8_t* pPacked_endpoints, + const color_rgba& l, const color_rgba& h, + bool use_blue_contraction, bool auto_disable_blue_contraction_if_clamped, + bool& blue_contraction_clamped_flag, bool& base_ofs_clamped_flag, bool& endpoints_swapped) + { + blue_contraction_clamped_flag = false; + base_ofs_clamped_flag = false; + endpoints_swapped = false; + + if ((cem_index != astc_helpers::CEM_LDR_RGB_BASE_PLUS_OFFSET) && (cem_index != astc_helpers::CEM_LDR_RGBA_BASE_PLUS_OFFSET)) + { + assert(0); + return false; + } + + color_rgba pack_l(l), pack_h(h); + + if (use_blue_contraction) + { + color_rgba enc_l(blue_contract_enc(pack_l, blue_contraction_clamped_flag, pack_l.b)); + color_rgba enc_h(blue_contract_enc(pack_h, blue_contraction_clamped_flag, pack_h.b)); + + if ((blue_contraction_clamped_flag) && (auto_disable_blue_contraction_if_clamped)) + { + use_blue_contraction = false; + } + else + { + pack_h = enc_l; + pack_l = enc_h; + + endpoints_swapped = true; + } + } + + int dr = 0, dg = 0, db = 0, da = 0; + bool pack_uses_blue_contraction = false; + int low_clamp = -32; + + // first 2 passes try with swapping by asymmetric clamping, then next 2 try with symmetric clamping, should always succeed + for (uint32_t pass = 0; pass < 4; pass++) + { + // Take previous CEM's values and try to encode to base+offset as best we can, it may clamp + int orig_dr = pack_h.r - pack_l.r, orig_dg = pack_h.g - pack_l.g, orig_db = pack_h.b - pack_l.b, orig_da = pack_h.a - pack_l.a; + + base_ofs_clamped_flag = false; + + dr = basisu::clamp(orig_dr, low_clamp, 31); + if (dr != orig_dr) base_ofs_clamped_flag = true; + + dg = basisu::clamp(orig_dg, low_clamp, 31); + if (dg != orig_dg) base_ofs_clamped_flag = true; + + db = basisu::clamp(orig_db, low_clamp, 31); + if (db != orig_db) base_ofs_clamped_flag = true; + + da = basisu::clamp(orig_da, low_clamp, 31); + if (da != orig_da) base_ofs_clamped_flag = true; + + int s = dr + dg + db; + + pack_uses_blue_contraction = s < 0; + + if (pack_uses_blue_contraction == use_blue_contraction) + break; + + if (s == 0) + { + assert(!pack_uses_blue_contraction); + assert(use_blue_contraction); + + // !pack_uses_blue_contraction here, sum=0, so force sum negative + if (db > -32) + db--; + else if (dr > -32) + dr--; + else if (dg > -32) + dg--; + else + { + // they can't be all -32 (or negative), otherwise sum couldn't be 0 + assert(0); + } + + assert((dr + dg + db) < 0); + + pack_uses_blue_contraction = true; + + break; + } + + if (pass == 3) + { + // theoretically unreachable + assert(0); + break; + } + + if (pass == 1) + { + // Try 2 more swap passes, but enforce a symmetric clamp range - this *should* work. + low_clamp = -31; + } + + std::swap(pack_l, pack_h); + endpoints_swapped = !endpoints_swapped; + + } // pass + + int v0 = pack_l.r, v2 = pack_l.g, v4 = pack_l.b; + int v1 = dr, v3 = dg, v5 = db; + + // lossless at 8-bits + astc_helpers::bit_transfer_signed_enc(v1, v0); + astc_helpers::bit_transfer_signed_enc(v3, v2); + astc_helpers::bit_transfer_signed_enc(v5, v4); + + int v6 = 0, v7 = 0; + if (astc_helpers::does_cem_have_alpha(cem_index)) + { + v6 = pack_l.a; + v7 = da; + + astc_helpers::bit_transfer_signed_enc(v7, v6); + } + + uint8_t new_endpoints8[astc_helpers::MAX_CEM_ENDPOINT_VALS]; + + new_endpoints8[0] = (uint8_t)v0; + new_endpoints8[1] = (uint8_t)v1; + + new_endpoints8[2] = (uint8_t)v2; + new_endpoints8[3] = (uint8_t)v3; + + new_endpoints8[4] = (uint8_t)v4; + new_endpoints8[5] = (uint8_t)v5; + + if (astc_helpers::does_cem_have_alpha(cem_index)) + { + new_endpoints8[6] = (uint8_t)v6; + new_endpoints8[7] = (uint8_t)v7; + } + + // This should always succeed. + assert(astc_helpers::used_blue_contraction(cem_index, new_endpoints8, astc_helpers::BISE_256_LEVELS) == use_blue_contraction); + + // requant predicted 256 level endpoints to current endpoint quant level, this will nearly always (if not always) succeed + bool status = requantize_ise_endpoints( + cem_index, astc_helpers::BISE_256_LEVELS, new_endpoints8, + dst_ise_endpoint_range, pPacked_endpoints); + + // can't assert because requant to a very low quant level could have failed to preserve blue contraction (in practice, super rare, perhaps impossible - still determining) + //assert(astc_helpers::used_blue_contraction(cem_index, pPredicted_endpoints, cur_blk.m_endpoint_ise_range) == pack_uses_blue_contraction); + + return status; + } + + // converts a previous block's endpoints, using any supported LDR CEM/endpoint quant level, into a new CEM/endpoint quant level + // used for prediction or *potentially* coding purposes + // will return num_dst_endpoint_vals residuals in cur_blk's endpoint level quant + // NOTE: Cannot use any floating point math for determinism across compilers. + bool convert_endpoints_across_cems( + uint32_t prev_cem, uint32_t prev_endpoint_ise_range, const uint8_t* pPrev_endpoints, + uint32_t dst_cem, uint32_t dst_endpoint_ise_range, uint8_t* pDst_endpoints, + bool always_repack, + bool use_blue_contraction, bool auto_disable_blue_contraction_if_clamped, + bool& blue_contraction_clamped_flag, bool& base_ofs_clamped_flag) + { + blue_contraction_clamped_flag = false; + base_ofs_clamped_flag = false; + + const uint32_t num_dst_endpoint_vals = astc_helpers::get_num_cem_values(dst_cem); + + const auto& dst_quant_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(dst_endpoint_ise_range).m_val_to_ise; + const auto& dst_dequant_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(dst_endpoint_ise_range).m_ISE_to_val; + + if ((prev_cem == dst_cem) && (!always_repack)) + { + // CEM's are precisely equal + // Requantize prev block's endpoints into the current block's quant levels, and we're done + return requantize_ise_endpoints( + prev_cem, prev_endpoint_ise_range, pPrev_endpoints, + dst_endpoint_ise_range, pDst_endpoints); + } + + // CEM's cannot be precisely equal now, compute base CEM's (removing alpha from consideration) + if (!always_repack) + { + // this path preserves the original's blue contraction status + const uint32_t prev_base_cem = astc_helpers::get_base_cem_without_alpha(prev_cem); + const uint32_t dst_base_cem = astc_helpers::get_base_cem_without_alpha(dst_cem); + + // prev cem has alpha, cur cem doesn't, but otherwise modes identical, so it's being stripped + if ((prev_base_cem == dst_base_cem) && (!astc_helpers::does_cem_have_alpha(dst_cem))) + { + assert(astc_helpers::does_cem_have_alpha(prev_cem)); + assert(astc_helpers::get_num_cem_values(prev_base_cem) == num_dst_endpoint_vals); + + // Requantize prev block's endpoints into the current block's quant levels, but ignore the alpha values (which are always the last 2 entries) + return requantize_ise_endpoints( + prev_base_cem, prev_endpoint_ise_range, pPrev_endpoints, + dst_endpoint_ise_range, pDst_endpoints); + } + + // if prev cem doesn't have alpha, but the current cem does, but otherwise modes are identical, so add sane alpha (both 255) and hope for best in the prediction + if ((prev_base_cem == dst_base_cem) && (astc_helpers::does_cem_have_alpha(dst_cem))) + { + assert(!astc_helpers::does_cem_have_alpha(prev_base_cem)); + + // requant previous endpoints to current endpoint quant level + bool status = requantize_ise_endpoints( + prev_base_cem, prev_endpoint_ise_range, pPrev_endpoints, + dst_endpoint_ise_range, pDst_endpoints); + + if (!status) + return false; + + // just plug in 255 to both alphas + const int ise_a_val = dst_quant_tab[255]; + + switch (dst_cem) + { + case astc_helpers::CEM_LDR_LUM_ALPHA_DIRECT: + { + assert(num_dst_endpoint_vals == 4); + pDst_endpoints[2] = (uint8_t)ise_a_val; + pDst_endpoints[3] = (uint8_t)ise_a_val; + break; + } + case astc_helpers::CEM_LDR_RGBA_DIRECT: + { + assert(num_dst_endpoint_vals == 8); + pDst_endpoints[6] = (uint8_t)ise_a_val; + pDst_endpoints[7] = (uint8_t)ise_a_val; + break; + } + case astc_helpers::CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A: + { + assert(num_dst_endpoint_vals == 6); + pDst_endpoints[4] = (uint8_t)ise_a_val; + pDst_endpoints[5] = (uint8_t)ise_a_val; + break; + } + case astc_helpers::CEM_LDR_RGBA_BASE_PLUS_OFFSET: + { + assert(num_dst_endpoint_vals == 8); + + // alphas should decode to 255, 255 + pDst_endpoints[6] = (uint8_t)ise_a_val; // base + pDst_endpoints[7] = (uint8_t)dst_quant_tab[128]; // offset, top bit should be preserved + break; + } + default: + assert(0); + break; + } + + return true; + } + + } // !always_repack + + // Here the CEM's are not even in the same class. + // Do something reasonable to convert to the current block's CEM encoding and hope the residual distribution is reasonable. + + // fully decode the endpoints, undoing any quant, blue contraction or bit transfers + color_rgba prev_l, prev_h; + decode_endpoints(prev_cem, pPrev_endpoints, prev_endpoint_ise_range, prev_l, prev_h); + + uint8_t new_endpoints8[astc_helpers::MAX_CEM_ENDPOINT_VALS] = { 0 }; + + // now pack the endpoints to the desired CEM + switch (dst_cem) + { + case astc_helpers::CEM_LDR_LUM_DIRECT: + case astc_helpers::CEM_LDR_LUM_ALPHA_DIRECT: + { + // Take previous endpoints and convert to luma/alpha low/high. + new_endpoints8[0] = (prev_l.r + prev_l.g + prev_l.b + 1) / 3; + new_endpoints8[1] = (prev_h.r + prev_h.g + prev_h.b + 1) / 3; + + if (dst_cem == astc_helpers::CEM_LDR_LUM_ALPHA_DIRECT) + { + new_endpoints8[2] = prev_l.a; + new_endpoints8[3] = prev_h.a; + } + + // ensure L new_endpoints8[1]) + { + std::swap(new_endpoints8[0], new_endpoints8[1]); + std::swap(new_endpoints8[2], new_endpoints8[3]); + } + } + + // requant predicted 256 level endpoints to current endpoint quant level + return requantize_ise_endpoints(dst_cem, astc_helpers::BISE_256_LEVELS, new_endpoints8, dst_endpoint_ise_range, pDst_endpoints); + } + + case astc_helpers::CEM_LDR_RGB_DIRECT: + case astc_helpers::CEM_LDR_RGBA_DIRECT: + { + // Take previous endpoints and convert to rgb(a) direct, explictly preserving previous ordering (to preserve the previous's endpoints usage of blue contraction, if it used it). + new_endpoints8[0] = prev_l.r; + new_endpoints8[1] = prev_h.r; + + new_endpoints8[2] = prev_l.g; + new_endpoints8[3] = prev_h.g; + + new_endpoints8[4] = prev_l.b; + new_endpoints8[5] = prev_h.b; + + if (dst_cem == astc_helpers::CEM_LDR_RGBA_DIRECT) + { + new_endpoints8[6] = prev_l.a; + new_endpoints8[7] = prev_h.a; + } + + if (use_blue_contraction) + { + color_rgba enc_l(blue_contract_enc(prev_l, blue_contraction_clamped_flag, dst_dequant_tab[dst_quant_tab[prev_l.b]])); + color_rgba enc_h(blue_contract_enc(prev_h, blue_contraction_clamped_flag, dst_dequant_tab[dst_quant_tab[prev_h.b]])); + + if ((auto_disable_blue_contraction_if_clamped) && (blue_contraction_clamped_flag)) + { + use_blue_contraction = false; + } + else + { + new_endpoints8[0] = enc_h.r; + new_endpoints8[1] = enc_l.r; + + new_endpoints8[2] = enc_h.g; + new_endpoints8[3] = enc_l.g; + + new_endpoints8[4] = enc_h.b; + new_endpoints8[5] = enc_l.b; + + if (dst_cem == astc_helpers::CEM_LDR_RGBA_DIRECT) + { + new_endpoints8[6] = prev_h.a; + new_endpoints8[7] = prev_l.a; + } + } + } + + uint32_t s0 = new_endpoints8[0] + new_endpoints8[2] + new_endpoints8[4]; + uint32_t s1 = new_endpoints8[1] + new_endpoints8[3] + new_endpoints8[5]; + bool pack_used_blue_contraction = s1 < s0; + + if (pack_used_blue_contraction != use_blue_contraction) + { + if (s0 == s1) + { + assert(!pack_used_blue_contraction); + + // swapping won't work because sums are equal, so force dst to use blue contraction by nudgling a component + // require s1 (hc.r + hc.g + hc.b)) + { + std::swap(lc, hc); + } + } + + new_endpoints8[0] = hc.r; + new_endpoints8[1] = hc.g; + new_endpoints8[2] = hc.b; + +#if 0 + // TODO: remove FP here + vec3F lf((float)lc.r, (float)lc.g, (float)lc.b); + vec3F hf((float)hc.r, (float)hc.g, (float)hc.b); + + const float MAX_S = 255.0f / 256.0f; + + float scale = MAX_S; + + float d = lf.dot(hf); + float nrm = hf.norm(); + if (nrm > 0.0f) + scale = d / nrm; + + scale = basisu::clamp(scale, 0.0f, MAX_S); + + new_endpoints8[3] = (uint8_t)basisu::clamp((int)std::round(scale * 256.0f), 0, 255); // explictly not 255.0f, but 256.0f, decoder divides scale by 256.0f +#endif + + { + int id = (lc.r * hc.r) + (lc.g * hc.g) + (lc.b * hc.b); + int inrm = (hc.r * hc.r) + (hc.g * hc.g) + (hc.b * hc.b); + + const int IMAX_S = (1024 * 255) / 256; + + int iscale = IMAX_S; + if (inrm > 0) + iscale = (id * 1024) / inrm; + + iscale = basisu::clamp(iscale, 0, IMAX_S); + + iscale = (iscale + 2) >> 2; + iscale = basisu::clamp(iscale, 0, 255); + + //assert(basisu::iabs(new_endpoints8[3] - iscale) <= 1); + new_endpoints8[3] = static_cast(iscale); + } + + if (dst_cem == astc_helpers::CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A) + { + new_endpoints8[4] = lc.a; + new_endpoints8[5] = hc.a; + + if ((prev_cem != astc_helpers::CEM_LDR_RGB_BASE_SCALE) && + (prev_cem != astc_helpers::CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A)) + { + // ensure val4 < val5 for proper lerping on single plane (correlated alpha) + if (new_endpoints8[4] > new_endpoints8[5]) + std::swap(new_endpoints8[4], new_endpoints8[5]); + } + } + + // requant predicted 256 level endpoints to current endpoint quant level + return requantize_ise_endpoints(dst_cem, astc_helpers::BISE_256_LEVELS, new_endpoints8, dst_endpoint_ise_range, pDst_endpoints); + } + case astc_helpers::CEM_LDR_RGB_BASE_PLUS_OFFSET: + case astc_helpers::CEM_LDR_RGBA_BASE_PLUS_OFFSET: + { + bool endpoints_swapped = false; + + return pack_base_offset(dst_cem, dst_endpoint_ise_range, pDst_endpoints, prev_l, prev_h, use_blue_contraction, auto_disable_blue_contraction_if_clamped, + blue_contraction_clamped_flag, base_ofs_clamped_flag, endpoints_swapped); + } + default: + { + assert(0); + return false; + } + } + + return true; + } + + // Assumes ise 20 (256 levels) + void decode_endpoints_ise20(uint32_t cem_index, const uint8_t* pEndpoint_vals, color32& l, color32& h) + { + assert(astc_helpers::is_cem_ldr(cem_index)); + + int ldr_endpoints[4][2]; + astc_helpers::decode_endpoint(cem_index, ldr_endpoints, pEndpoint_vals); + + for (uint32_t c = 0; c < 4; c++) + { + assert((ldr_endpoints[c][0] >= 0) && (ldr_endpoints[c][0] <= 255)); + assert((ldr_endpoints[c][1] >= 0) && (ldr_endpoints[c][1] <= 255)); + + l[c] = (uint8_t)ldr_endpoints[c][0]; + h[c] = (uint8_t)ldr_endpoints[c][1]; + } + } + + void decode_endpoints(uint32_t cem_index, const uint8_t* pEndpoint_vals, uint32_t endpoint_ise_index, color32& l, color32& h, float* pScale) + { + const uint32_t total_endpoint_vals = astc_helpers::get_num_cem_values(cem_index); + + const auto& endpoint_dequant_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(endpoint_ise_index).m_ISE_to_val; + + uint8_t dequantized_endpoints[astc_helpers::MAX_CEM_ENDPOINT_VALS]; + for (uint32_t i = 0; i < total_endpoint_vals; i++) + dequantized_endpoints[i] = endpoint_dequant_tab[pEndpoint_vals[i]]; + + decode_endpoints_ise20(cem_index, dequantized_endpoints, l, h); + + if ((pScale) && ((cem_index == astc_helpers::CEM_LDR_RGB_BASE_SCALE) || (cem_index == astc_helpers::CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A))) + { + *pScale = (float)dequantized_endpoints[3] * (1.0f / 256.0f); + } + } + + // Assumes ise 20 (256 levels) + void decode_endpoints_ise20(uint32_t cem_index, const uint8_t* pEndpoint_vals, color_rgba& l, color_rgba& h) + { + assert(astc_helpers::is_cem_ldr(cem_index)); + + int ldr_endpoints[4][2]; + astc_helpers::decode_endpoint(cem_index, ldr_endpoints, pEndpoint_vals); + + for (uint32_t c = 0; c < 4; c++) + { + assert((ldr_endpoints[c][0] >= 0) && (ldr_endpoints[c][0] <= 255)); + assert((ldr_endpoints[c][1] >= 0) && (ldr_endpoints[c][1] <= 255)); + + l[c] = (uint8_t)ldr_endpoints[c][0]; + h[c] = (uint8_t)ldr_endpoints[c][1]; + } + } + + void decode_endpoints(uint32_t cem_index, const uint8_t* pEndpoint_vals, uint32_t endpoint_ise_index, color_rgba& l, color_rgba& h, float* pScale) + { + const uint32_t total_endpoint_vals = astc_helpers::get_num_cem_values(cem_index); + + const auto& endpoint_dequant_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(endpoint_ise_index).m_ISE_to_val; + + uint8_t dequantized_endpoints[astc_helpers::MAX_CEM_ENDPOINT_VALS]; + for (uint32_t i = 0; i < total_endpoint_vals; i++) + dequantized_endpoints[i] = endpoint_dequant_tab[pEndpoint_vals[i]]; + + decode_endpoints_ise20(cem_index, dequantized_endpoints, l, h); + + if ((pScale) && ((cem_index == astc_helpers::CEM_LDR_RGB_BASE_SCALE) || (cem_index == astc_helpers::CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A))) + { + *pScale = (float)dequantized_endpoints[3] * (1.0f / 256.0f); + } + } + + // TODO: Duplicated in astc_hdr + void compute_upsample_matrix(basisu::vector2D& upsample_matrix, uint32_t block_width, uint32_t block_height, uint32_t grid_width, uint32_t grid_height) + { + assert((block_width >= 2) && (block_width <= astc_helpers::MAX_BLOCK_DIM)); + assert((block_height >= 2) && (block_height <= astc_helpers::MAX_BLOCK_DIM)); + assert((grid_width >= 2) && (grid_width <= block_width)); + assert((grid_height >= 2) && (grid_height <= block_height)); + + const uint32_t num_block_samples = block_width * block_height; + const uint32_t num_grid_samples = grid_width * grid_height; + + astc_helpers::weighted_sample samples[astc_helpers::MAX_BLOCK_DIM * astc_helpers::MAX_BLOCK_DIM]; + basisu::clear_obj(samples); + + astc_helpers::compute_upsample_weights(block_width, block_height, grid_width, grid_height, samples); + + // Compute upsample matrix: output num_block_samples (rows), input num_grid_samples (cols) + upsample_matrix.resize_rows_cols(num_block_samples, num_grid_samples); + + basisu::vector weights(num_grid_samples); + + // compute which source sample(s) contribute to it. + for (uint32_t d = 0; d < num_block_samples; d++) + { + const astc_helpers::weighted_sample& ws = samples[d]; + + weights.set_all(0.0f); + + for (uint32_t y = 0; y < 2; y++) + { + for (uint32_t x = 0; x < 2; x++) + { + float w = ws.m_weights[y][x] * (1.0f / 16.0f); + if (!w) + continue; + + assert((ws.m_src_x + x) < grid_width); + assert((ws.m_src_y + y) < grid_height); + + assert(weights[(ws.m_src_x + x) + (ws.m_src_y + y) * grid_width] == 0.0f); + weights[(ws.m_src_x + x) + (ws.m_src_y + y) * grid_width] = w; + } // x + } // y + + for (uint32_t i = 0; i < num_grid_samples; i++) + upsample_matrix.at_row_col(d, i) = weights[i]; + + } // d + } + + // TODO: Only needed by ASTC LDR encoder + void compute_adjoint_downsample_matrix(basisu::vector& downsample_matrix, uint32_t block_width, uint32_t block_height, uint32_t grid_width, uint32_t grid_height) + { + assert((block_width >= 2) && (block_width <= astc_helpers::MAX_BLOCK_DIM)); + assert((block_height >= 2) && (block_height <= astc_helpers::MAX_BLOCK_DIM)); + assert((grid_width >= 2) && (grid_width <= block_width)); + assert((grid_height >= 2) && (grid_height <= block_height)); + + const uint32_t num_block_samples = block_width * block_height; + const uint32_t num_grid_samples = grid_width * grid_height; + + // Compute upsample matrix: output num_block_samples (rows), input num_grid_samples (cols) + basisu::vector2D upsample_matrix; + compute_upsample_matrix(upsample_matrix, block_width, block_height, grid_width, grid_height); + + basisu::vector Dinv(num_grid_samples); + for (uint32_t j = 0; j < num_grid_samples; j++) + { + float sum = 0.0f; + + for (uint32_t i = 0; i < num_block_samples; i++) + sum += upsample_matrix.at_row_col(i, j); + + if (sum > 0.0f) + Dinv[j] = 1.0f / sum; + } + + // Create downsample matrix: num_grid_samples rows, num_block_samples cols + downsample_matrix.resize(num_grid_samples * num_block_samples); + downsample_matrix.set_all(0.0f); + + for (uint32_t j = 0; j < num_grid_samples; ++j) + for (uint32_t i = 0; i < num_block_samples; ++i) + downsample_matrix[j * num_block_samples + i] = Dinv[j] * upsample_matrix.at_row_col(i, j); + } + + const astc_block_grid_data* find_astc_block_grid_data(uint32_t block_width, uint32_t block_height, uint32_t grid_width, uint32_t grid_height) + { + auto find_res(g_astc_block_grid_data_hash.find(astc_block_grid_config(block_width, block_height, grid_width, grid_height))); + assert(find_res != g_astc_block_grid_data_hash.end()); + + return &find_res->second; + } + + void init_astc_block_grid_data_hash() + { + if (g_astc_block_grid_data_hash.size()) + return; + + g_astc_block_grid_data_hash.clear(); + g_astc_block_grid_data_hash.reserve(384); + + // TODO: Iterate over all valid block sizes more efficiently + for (uint32_t block_h = 4; block_h <= 12; block_h++) + { + for (uint32_t block_w = 4; block_w <= 12; block_w++) + { + if (!astc_helpers::is_valid_block_size(block_w, block_h)) + continue; + + for (uint32_t grid_h = 2; grid_h <= block_h; grid_h++) + { + for (uint32_t grid_w = 2; grid_w <= block_w; grid_w++) + { + const int bw = block_w, bh = block_h; + const int gw = grid_w, gh = grid_h; + + const int num_texels = bw * bh; + const int num_weights = gw * gh; + + basisu::vector2D upsample_matrix; + compute_upsample_matrix(upsample_matrix, bw, bh, gw, gh); + + float accum = 0.0f; + for (int t = 0; t < num_texels; ++t) + { + float row_sum_sq = 0.0f; + const float* row = &(upsample_matrix.get_ptr())[t * num_weights]; + + for (int i = 0; i < num_weights; ++i) + { + float w = row[i]; + row_sum_sq += w * w; + } + + accum += row_sum_sq; + } + + // estimate of MSE weight quantization reduction due to bilinear weight grid upsampling + // TODO: Gamma is used during encoding now, not transcoding. + const float weight_gamma = accum / (float)num_texels; + + astc_block_grid_data grid_data(weight_gamma); + grid_data.m_upsample_matrix = upsample_matrix; + + basisu::vector& downsample_matrix = grid_data.m_downsample_matrix; + compute_adjoint_downsample_matrix(downsample_matrix, bw, bh, gw, gh); + + auto res = g_astc_block_grid_data_hash.insert(astc_block_grid_config(bw, bh, gw, gh), grid_data); + assert(res.second); + BASISU_NOTE_UNUSED(res); + + } // grid_w + } // grid_h + + } // block_h + + } // block_w + } + +#include "basisu_idct.h" + +#if 0 + typedef void (*idct_1d_func_ptr)(const float* src, int src_stride, float* dst, int dst_stride); + + const idct_1d_func_ptr g_idct_1d_func_ptrs[11] = + { + idct_1d_2, + idct_1d_3, + idct_1d_4, + + idct_1d_5, + idct_1d_6, + idct_1d_7, + idct_1d_8, + + idct_1d_9, + idct_1d_10, + idct_1d_11, + idct_1d_12, + }; +#endif + + static inline void idct_2d(const float* pSrc, float* pDst, uint32_t num_rows, uint32_t num_cols) + { + assert((num_rows >= 2) && (num_rows <= 12)); + assert((num_cols >= 2) && (num_cols <= 12)); + + float temp[12 * 12]; + + // IDCT cols from src to temp + +#if 0 + // This works but uses slow WASM indirect calls + + const idct_1d_func_ptr pCol_xform = g_idct_1d_func_ptrs[num_rows - 2]; + for (uint32_t c = 0; c < num_cols; c++) + { + (*pCol_xform)(pSrc + c, num_cols, temp + c, num_cols); + } +#else + switch (num_rows) + { + case 2: + for (uint32_t c = 0; c < num_cols; c++) + idct_1d_2(pSrc + c, num_cols, temp + c, num_cols); + break; + case 3: + for (uint32_t c = 0; c < num_cols; c++) + idct_1d_3(pSrc + c, num_cols, temp + c, num_cols); + break; + case 4: + for (uint32_t c = 0; c < num_cols; c++) + idct_1d_4(pSrc + c, num_cols, temp + c, num_cols); + break; + case 5: + for (uint32_t c = 0; c < num_cols; c++) + idct_1d_5(pSrc + c, num_cols, temp + c, num_cols); + break; + case 6: + for (uint32_t c = 0; c < num_cols; c++) + idct_1d_6(pSrc + c, num_cols, temp + c, num_cols); + break; + case 7: + for (uint32_t c = 0; c < num_cols; c++) + idct_1d_7(pSrc + c, num_cols, temp + c, num_cols); + break; + case 8: + for (uint32_t c = 0; c < num_cols; c++) + idct_1d_8(pSrc + c, num_cols, temp + c, num_cols); + break; + case 9: + for (uint32_t c = 0; c < num_cols; c++) + idct_1d_9(pSrc + c, num_cols, temp + c, num_cols); + break; + case 10: + for (uint32_t c = 0; c < num_cols; c++) + idct_1d_10(pSrc + c, num_cols, temp + c, num_cols); + break; + case 11: + for (uint32_t c = 0; c < num_cols; c++) + idct_1d_11(pSrc + c, num_cols, temp + c, num_cols); + break; + case 12: + default: + for (uint32_t c = 0; c < num_cols; c++) + idct_1d_12(pSrc + c, num_cols, temp + c, num_cols); + break; + } +#endif + + // IDCT rows from temp to dst + +#if 0 + // This works but uses slow WASM indirect calls + const idct_1d_func_ptr pRow_xform = g_idct_1d_func_ptrs[num_cols - 2]; + + float* pTemp_row = temp; + float* pDst_row = pDst; + for (uint32_t r = 0; r < num_rows; r++) + { + (*pRow_xform)(pTemp_row, 1, pDst_row, 1); + pTemp_row += num_cols; + pDst_row += num_cols; + } +#else + float* pTemp_row = temp; + float* pDst_row = pDst; + switch (num_cols) + { + case 2: + for (uint32_t r = 0; r < num_rows; r++, pTemp_row += num_cols, pDst_row += num_cols) + idct_1d_2(pTemp_row, 1, pDst_row, 1); + break; + case 3: + for (uint32_t r = 0; r < num_rows; r++, pTemp_row += num_cols, pDst_row += num_cols) + idct_1d_3(pTemp_row, 1, pDst_row, 1); + break; + case 4: + for (uint32_t r = 0; r < num_rows; r++, pTemp_row += num_cols, pDst_row += num_cols) + idct_1d_4(pTemp_row, 1, pDst_row, 1); + break; + case 5: + for (uint32_t r = 0; r < num_rows; r++, pTemp_row += num_cols, pDst_row += num_cols) + idct_1d_5(pTemp_row, 1, pDst_row, 1); + break; + case 6: + for (uint32_t r = 0; r < num_rows; r++, pTemp_row += num_cols, pDst_row += num_cols) + idct_1d_6(pTemp_row, 1, pDst_row, 1); + break; + case 7: + for (uint32_t r = 0; r < num_rows; r++, pTemp_row += num_cols, pDst_row += num_cols) + idct_1d_7(pTemp_row, 1, pDst_row, 1); + break; + case 8: + for (uint32_t r = 0; r < num_rows; r++, pTemp_row += num_cols, pDst_row += num_cols) + idct_1d_8(pTemp_row, 1, pDst_row, 1); + break; + case 9: + for (uint32_t r = 0; r < num_rows; r++, pTemp_row += num_cols, pDst_row += num_cols) + idct_1d_9(pTemp_row, 1, pDst_row, 1); + break; + case 10: + for (uint32_t r = 0; r < num_rows; r++, pTemp_row += num_cols, pDst_row += num_cols) + idct_1d_10(pTemp_row, 1, pDst_row, 1); + break; + case 11: + for (uint32_t r = 0; r < num_rows; r++, pTemp_row += num_cols, pDst_row += num_cols) + idct_1d_11(pTemp_row, 1, pDst_row, 1); + break; + case 12: + default: + for (uint32_t r = 0; r < num_rows; r++, pTemp_row += num_cols, pDst_row += num_cols) + idct_1d_12(pTemp_row, 1, pDst_row, 1); + break; + } +#endif + } + + bool dct2f::init(uint32_t rows, uint32_t cols) + { + if ((rows < 2u) || (rows > cMaxSize) || + (cols < 2u) || (cols > cMaxSize)) + { + assert(0); + return false; + } + + m_rows = rows; + m_cols = cols; + + m_c_col.assign(m_rows * m_rows, 0.0f); + m_c_row.assign(m_cols * m_cols, 0.0f); + m_a_col.assign(m_rows, 0.0f); + m_a_row.assign(m_cols, 0.0f); + + const float pi = 3.14159265358979323846f; + + // alpha scaling + const float inv_m = 1.0f / static_cast(m_rows); + m_a_col[0] = sqrtf(inv_m); + for (uint32_t u = 1; u < m_rows; ++u) + m_a_col[u] = sqrtf(2.0f * inv_m); + + const float inv_n = 1.0f / static_cast(m_cols); + m_a_row[0] = sqrtf(inv_n); + for (uint32_t v = 1; v < m_cols; ++v) + m_a_row[v] = sqrtf(2.0f * inv_n); + + // cos tables + for (uint32_t u = 0; u < m_rows; ++u) + { + for (uint32_t x = 0; x < m_rows; ++x) + { + float angle = (pi * static_cast((2 * x + 1) * u)) / (2.0f * static_cast(m_rows)); + m_c_col[u * m_rows + x] = cosf(angle); + } + } + + for (uint32_t v = 0; v < m_cols; ++v) + { + for (uint32_t y = 0; y < m_cols; ++y) + { + float angle = (pi * static_cast((2 * y + 1) * v)) / (2.0f * static_cast(m_cols)); + m_c_row[v * m_cols + y] = cosf(angle); + } + } + + return true; + } + + void dct2f::forward(const float* pSrc, float* pDst, fvec& work) const + { + forward(pSrc, m_cols, pDst, m_cols, work); + } + + void dct2f::inverse(const float* pSrc, float* pDst, fvec& work) const + { + inverse(pSrc, m_cols, pDst, m_cols, work); + } + + void dct2f::inverse_check(const float* pSrc, float* pDst, fvec& work) const + { + inverse_check(pSrc, m_cols, pDst, m_cols, work); + } + + void dct2f::forward(const float* pSrc, uint32_t src_stride, + float* pDst, uint32_t dst_stride, fvec& work) const + { + assert(m_rows && m_cols); + work.resize(m_rows * m_cols); + + const uint32_t m = m_rows, n = m_cols; + + float* pWork = &work[0]; + + // horizontal + for (uint32_t x = 0; x < m; ++x) + { + const float* pRowIn = pSrc + x * src_stride; + float* pRowT = pWork + x * n; + for (uint32_t v = 0; v < n; ++v) + { + const float* pCv = &m_c_row[v * n]; + float s = 0.0f; + for (uint32_t y = 0; y < n; ++y) + { + s += pRowIn[y] * pCv[y]; + } + pRowT[v] = s * m_a_row[v]; + } + } + + // vertical + for (uint32_t v = 0; v < n; ++v) + { + for (uint32_t u = 0; u < m; ++u) + { + const float* pCu = &m_c_col[u * m]; + float s = 0.0f; + for (uint32_t x = 0; x < m; ++x) + { + s += pWork[x * n + v] * pCu[x]; + } + pDst[u * dst_stride + v] = s * m_a_col[u]; + } + } + } + + // src_stride/dst_stride must be m_cols + void dct2f::inverse(const float* pSrc, uint32_t src_stride, + float* pDst, uint32_t dst_stride, fvec& work) const + { + BASISU_NOTE_UNUSED(src_stride); + BASISU_NOTE_UNUSED(dst_stride); + +#if 0 + assert(m_rows && m_cols); + work.resize(m_rows * m_cols); + + const uint32_t m = m_rows, n = m_cols; + float* pWork = &work[0]; + + // vertical + for (uint32_t v = 0; v < n; ++v) // cols + { + float sums[cMaxSize] = { 0 }; + + for (uint32_t u = 0; u < m; ++u) // rows + { + if (((const uint32_t*)pSrc)[u * src_stride + v] == 0) + continue; + + float yU = pSrc[u * src_stride + v]; // most coeffs will be 0 + //if (yU == 0.0f) + // continue; + + yU *= m_a_col[u]; + + for (uint32_t x = 0; x < m; ++x) + { + const float cU = m_c_col[u * m + x]; + sums[x] += yU * cU; + } // x + + } // u + + for (uint32_t x = 0; x < m; ++x) + pWork[x * n + v] = sums[x]; + + } // v + + // horizontal + for (uint32_t x = 0; x < m; ++x) // rows + { + const float* pRowT = pWork + x * n; + float* pRowOut = pDst + x * dst_stride; + + for (uint32_t y = 0; y < n; ++y) // cols + { + float s = 0.0f; + for (uint32_t v = 0; v < n; ++v) // cols + { + const float cV = m_c_row[v * n + y]; + s += (pRowT[v] * m_a_row[v]) * cV; + } + pRowOut[y] = s; + } + } +#else + BASISU_NOTE_UNUSED(work); + assert(src_stride == m_cols); + assert(dst_stride == m_cols); + idct_2d(pSrc, pDst, m_rows, m_cols); +#endif + } + + void dct2f::inverse_check(const float* pSrc, uint32_t src_stride, + float* pDst, uint32_t dst_stride, fvec& work) const + { + assert(m_rows && m_cols); + work.resize(m_rows * m_cols); + + const uint32_t m = m_rows, n = m_cols; + float* pWork = &work[0]; + + // vertical + for (uint32_t v = 0; v < n; ++v) + { + for (uint32_t x = 0; x < m; ++x) + { + float s = 0.0f; + for (uint32_t u = 0; u < m; ++u) + { + const float yU = pSrc[u * src_stride + v]; + const float cU = m_c_col[u * m + x]; + s += (yU * m_a_col[u]) * cU; + } + pWork[x * n + v] = s; + } + } + + // horizontal + for (uint32_t x = 0; x < m; ++x) // rows + { + const float* pRowT = pWork + x * n; + float* pRowOut = pDst + x * dst_stride; + + for (uint32_t y = 0; y < n; ++y) // cols + { + float s = 0.0f; + for (uint32_t v = 0; v < n; ++v) // cols + { + const float cV = m_c_row[v * n + y]; + s += (pRowT[v] * m_a_row[v]) * cV; + } + pRowOut[y] = s; + } + } + } + + static int* generate_zigzag_order(int width, int height) + { + assert((width > 0) && (height > 0)); + + const int total = width * height; + int* pOrder = (int*)malloc(total * sizeof(int)); + if (!pOrder) + return nullptr; + + int idx = 0; + for (int s = 0; s < (width + height - 1); ++s) + { + // Start x at max(0, s - height + 1), end at min(s, width - 1) + const int x_start = (s < height) ? 0 : (s - height + 1); + const int x_end = (s < width) ? s : (width - 1); + + // Diagonal size + const int diag_size = x_end - x_start + 1; + int* pDiag = (int*)malloc(diag_size * sizeof(int)); + if (!pDiag) + { + free(pOrder); + return nullptr; + } + + int j = 0; + for (int x = x_start; x <= x_end; ++x) + { + int y = s - x; + assert(j < diag_size); + pDiag[j++] = x + y * width; + } + + // Reverse if s is odd (alternate direction) + if ((s & 1) == 1) + { + for (int k = diag_size - 1; k >= 0; --k) + { + assert(idx < total); + pOrder[idx++] = pDiag[k]; + } + } + else + { + for (int k = 0; k < diag_size; ++k) + { + assert(idx < total); + pOrder[idx++] = pDiag[k]; + } + } + + free(pDiag); + } + + return pOrder; + } + + static const int g_baseline_jpeg_y[8][8] = + { + // DC element modified so bilinear fetches near (0,0) grab a smaller quant table value, protecting most important LF coefficients + { 4, 11, 10, 16, 24, 40, 51, 61 }, + { 12, 12, 14, 19, 26, 58, 60, 55 }, + { 14, 13, 16, 24, 40, 57, 69, 56 }, + { 14, 17, 22, 29, 51, 87, 80, 62 }, + { 18, 22, 37, 56, 68,109,103, 77 }, + { 24, 35, 55, 64, 81,104,113, 92 }, + { 49, 64, 78, 87,103,121,120,101 }, + { 72, 92, 95, 98,112,100,103, 99 } + }; + + // centers at (0,0) + static inline float sample_jpeg_quant(const int Q8[8][8], float i, float j) + { + i = basisu::minimum(basisu::maximum(i, 0.0f), 7.0f); + j = basisu::minimum(basisu::maximum(j, 0.0f), 7.0f); + int i0 = (int)floorf(i), j0 = (int)floorf(j); + int i1 = basisu::minimum(i0 + 1, 7), j1 = basisu::minimum(j0 + 1, 7); + float ti = i - i0, tj = j - j0; + float a = (1 - ti) * Q8[j0][i0] + ti * Q8[j0][i1]; + float b = (1 - ti) * Q8[j1][i0] + ti * Q8[j1][i1]; + return (1 - tj) * a + tj * b; + } + + void grid_weight_dct::init(uint32_t block_width, uint32_t block_height) + { + m_block_width = block_width; + m_block_height = block_height; + + for (uint32_t grid_height = 2; grid_height <= block_height; grid_height++) + { + for (uint32_t grid_width = 2; grid_width <= block_width; grid_width++) + { + // Check if this is a valid ASTC weight grid dimension + if ((grid_width * grid_height) > astc_helpers::MAX_GRID_WEIGHTS) + continue; + + auto ins_res = m_grid_dim_key_vals.insert(grid_dim_key(grid_width, grid_height), grid_dim_value()); + auto& val = ins_res.first->second; + + val.m_dct.init(grid_height, grid_width); + + int* pZigZag = generate_zigzag_order(grid_width, grid_height); + + basisu::int_vec v(grid_width * grid_height); + memcpy(v.data(), pZigZag, sizeof(int) * grid_width * grid_height); + + free(pZigZag); + + val.m_zigzag.swap(v); + + } // w + } // h + } + + // This can used FP as it only impacts the final decoded weights (not future blocks) + bool grid_weight_dct::decode_block_weights( + float q, uint32_t plane_index, // plane of weights to decode and IDCT from stream + astc_helpers::log_astc_block& log_blk, // must be initialized except for the plane weights which are decoded + basist::bitwise_decoder* pDec, + const astc_block_grid_data* pGrid_data, // grid data for this grid size + block_stats* pS, + fvec& dct_work, + const dct_syms* pSyms) const + { + const uint32_t grid_width = log_blk.m_grid_width, grid_height = log_blk.m_grid_height; + const uint32_t total_grid_samples = grid_width * grid_height; + const uint32_t num_planes = log_blk.m_dual_plane ? 2 : 1; + + //const auto& dequant_tab = astc_helpers::g_dequant_tables.get_weight_tab(log_blk.m_weight_ise_range).m_ISE_to_val; + const auto& quant_tab = astc_helpers::g_dequant_tables.get_weight_tab(log_blk.m_weight_ise_range).m_val_to_ise; + + auto grid_dim_vals_iter = m_grid_dim_key_vals.find(grid_dim_key(grid_width, grid_height)); + + if (grid_dim_vals_iter == m_grid_dim_key_vals.end()) + { + // Invalid grid dimension for this block size + assert(0); + return false; + } + + auto& grid_dim_vals = grid_dim_vals_iter->second; + + const float span_len = get_max_span_len(log_blk, plane_index); + + const float level_scale = compute_level_scale(q, span_len, pGrid_data->m_weight_gamma, grid_width, grid_height, log_blk.m_weight_ise_range); + + float scaled_weight_coding_scale = SCALED_WEIGHT_BASE_CODING_SCALE; + if (log_blk.m_weight_ise_range <= astc_helpers::BISE_8_LEVELS) + scaled_weight_coding_scale = 1.0f / 8.0f; + + float mean_weight = 0; + + if (pDec) + mean_weight = (float)pDec->decode_truncated_binary((uint32_t)(64.0f * scaled_weight_coding_scale) + 1) / (float)scaled_weight_coding_scale; + else if (pSyms) + mean_weight = (float)pSyms->m_dc_sym / (float)scaled_weight_coding_scale; + else + { + assert(0); + return false; + } + + if (pS) + { + pS->m_mean_weight = mean_weight; + pS->m_total_coded_acs = 0; + pS->m_max_ac_coeff = 0; + } + + float dct_weights[astc_helpers::MAX_BLOCK_PIXELS]; + + const auto& zigzag = grid_dim_vals.m_zigzag; + + basisu::clear_obj(dct_weights); + + sample_quant_table_state quant_state; + quant_state.init(q, m_block_width, m_block_height, level_scale); + + if (pDec) + { + for (uint32_t zig_idx = 1; zig_idx < total_grid_samples; zig_idx++) + { + uint32_t run_len = pDec->decode_rice(m_zero_run); + + if ((run_len + zig_idx) > total_grid_samples) + return false; + + zig_idx += run_len; + + if (zig_idx >= total_grid_samples) + break; + + int sign = pDec->get_bits(1); + + int coeff = pDec->decode_rice(m_coeff); + + if (sign) + coeff = -coeff; + + int dct_idx = zigzag[zig_idx]; + + const uint32_t y = (uint32_t)dct_idx / grid_width; + const uint32_t x = (uint32_t)dct_idx % grid_width; + + //const int quant = dct_quant_tab[dct_idx]; + const int quant = sample_quant_table(quant_state, x, y); + //assert(quant == sample_quant_table(quant_state, x, y)); + + dct_weights[dct_idx] = dequant_deadzone(coeff, quant, DEADZONE_ALPHA, x, y); + + if (pS) + { + ++pS->m_total_coded_acs; + pS->m_max_ac_coeff = basisu::maximum(pS->m_max_ac_coeff, basisu::iabs(coeff)); + } + } + } + else + { + uint32_t zig_idx = 1; + uint32_t coeff_ofs = 0; + while (coeff_ofs < pSyms->m_coeffs.size()) + { + const uint32_t run_len = pSyms->m_coeffs[coeff_ofs].m_num_zeros; + const int coeff = pSyms->m_coeffs[coeff_ofs].m_coeff; + coeff_ofs++; + + if ((run_len + zig_idx) > total_grid_samples) + return false; + + zig_idx += run_len; + + if (zig_idx >= total_grid_samples) + break; + + assert(coeff != INT_MAX); + + int dct_idx = zigzag[zig_idx]; + + const uint32_t y = (uint32_t)dct_idx / grid_width; + const uint32_t x = (uint32_t)dct_idx % grid_width; + + //const int quant = dct_quant_tab[dct_idx]; + const int quant = sample_quant_table(quant_state, x, y); + //assert(quant == sample_quant_table(quant_state, x, y)); + + dct_weights[dct_idx] = dequant_deadzone(coeff, quant, DEADZONE_ALPHA, x, y); + + if (pS) + { + ++pS->m_total_coded_acs; + pS->m_max_ac_coeff = basisu::maximum(pS->m_max_ac_coeff, basisu::iabs(coeff)); + } + + zig_idx++; + } + } + + float idct_weights[astc_helpers::MAX_BLOCK_PIXELS]; + + grid_dim_vals.m_dct.inverse(dct_weights, idct_weights, dct_work); + +#if defined(_DEBUG) || defined(DEBUG) + // Sanity check IDCT vs. less optimized variant + // Also quant table sanity check vs. sample_quant_table(). + { + float idct_weights_temp[astc_helpers::MAX_BLOCK_PIXELS]; + grid_dim_vals.m_dct.inverse_check(dct_weights, idct_weights_temp, dct_work); + + int dct_quant_tab[astc_helpers::MAX_BLOCK_PIXELS]; + compute_quant_table(q, grid_width, grid_height, level_scale, dct_quant_tab); + + for (uint32_t i = 0; i < grid_width * grid_height; i++) + { + assert(basisu::equal_tol(idct_weights[i], idct_weights_temp[i], .00125f)); + + assert(!i || (dct_quant_tab[i] == sample_quant_table(quant_state, i % grid_width, i / grid_width))); + } + } +#endif + + // Compute final grid weights + for (uint32_t y = 0; y < grid_height; y++) + for (uint32_t x = 0; x < grid_width; x++) + log_blk.m_weights[(x + y * grid_width) * num_planes + plane_index] = quant_tab[basisu::clamp(fast_roundf_int(mean_weight + idct_weights[x + y * grid_width]), 0, 64)]; + + return true; + } + + // results of calling scale_quant_steps() for each # of ASTC weight levels + static const float g_scale_quant_steps[12] = { 1.51333141f, 1.41198814f, 1.35588217f, 1.31743157f, 1.28835952f, 1.24573100f, 1.21481407f, 1.19067919f, 1.15431654f, 1.12734985f, 1.10601568f, 1.07348967f }; + + // Adaptive quantization + float grid_weight_dct::compute_level_scale(float q, float span_len, float weight_gamma, uint32_t grid_width, uint32_t grid_height, uint32_t weight_ise_range) const + { + BASISU_NOTE_UNUSED(weight_gamma); + BASISU_NOTE_UNUSED(grid_width); + BASISU_NOTE_UNUSED(grid_height); + + assert((weight_ise_range >= astc_helpers::BISE_2_LEVELS) && (weight_ise_range <= astc_helpers::BISE_32_LEVELS)); + + // Standard JPEG quality factor calcs + // TODO: Precompute this once + float level_scale; + q = basisu::clamp(q, 1.0f, 100.0f); + if (q < 50.0f) + level_scale = 5000.0f / q; + else + level_scale = 200.0f - 2.0f * q; + + level_scale *= (1.0f / 100.0f); // because JPEG's quant table is scaled by 100 + + //const float span_floor = 28.0f; + const float span_floor = 14.0f; + //const float adaptive_factor = 255.0f / maximum(span_len, span_floor); + // 64.0 = dynamic range adjustment (JPEG uses 255) + // divide by span len to adjustment adaptive low/high values per-block (JPEG always uses effective span=0-255) + // actually (64/255) * 255/max(span_len, span_floor) + float adaptive_factor = 64.0f / basisu::maximum(span_len, span_floor); + + // input signal scalar quantization noise will be distributed between multiple AC coefficients - compensate by adaptively adjusting the quant step size + float weight_quant_adaptive_factor = g_scale_quant_steps[weight_ise_range]; + adaptive_factor *= weight_quant_adaptive_factor; + + // sanity + assert(fabs(weight_quant_adaptive_factor - scale_quant_steps(astc_helpers::get_ise_levels(weight_ise_range))) < .000125f); + + // Adjust for ASTC weight grid bilinear upsampling using precomputed constants depending on the weight grid dims (usually .5-1.0, smaller grids=lower weights) + // This compensates for weight quant error being smoothed out due to bilinear. + // It's unclear if this is actually useful, and looks worse on smaller weight grids. + //level_scale *= adaptive_factor / sqrtf(weight_gamma); // weight_gamma is power domain, not amplitude + + // (Adaptive quant) + level_scale *= adaptive_factor; + + // The higher the level_scale, the more quantized DCT coefficients will be and vice versa. + + return level_scale; + } + + int grid_weight_dct::sample_quant_table(sample_quant_table_state& state, uint32_t x, uint32_t y) const + { + assert(x || y); + + if (state.m_q >= 100.0f) + return 1; + + float ny = float(y); + float ry = ny * state.m_sy; + + float nx = float(x); + float rx = nx * state.m_sx; + + assert(x || y); + + // sample from the JPEG baseline luma 8x8 DCT quant matrix + // this is an approximation (we could do an actual desired radians per spatial sample search vs. each of the 8x8 basis vectors to find the best, most conservative mapping), + // but for 4x4 and 6x6 block sizes it's reasonable enough and simple/fast + // at 4x4, the lowest frequencies are slightly more heavily quantized than we would want (but the quant table entries near DC are so similar it's doubtful it matters much if at all) + //float base = sample_jpeg_quant(g_baseline_jpeg_y, rx, ry); + + float base; + { + float i = rx, j = ry; + assert((i >= 0.0f) && (j >= 0.0f)); + + i = basisu::minimum(i, 7.0f); + j = basisu::minimum(j, 7.0f); + + int i0 = (int)(i), j0 = (int)(j); + int i1 = basisu::minimum(i0 + 1, 7), j1 = basisu::minimum(j0 + 1, 7); + + float ti = i - i0, tj = j - j0; + float a = (1 - ti) * g_baseline_jpeg_y[j0][i0] + ti * g_baseline_jpeg_y[j0][i1]; + float b = (1 - ti) * g_baseline_jpeg_y[j1][i0] + ti * g_baseline_jpeg_y[j1][i1]; + + base = (1 - tj) * a + tj * b; + } + + int quant_scale = (int)(base * state.m_level_scale + 0.5f); + + quant_scale = basisu::maximum(1, quant_scale); + + return quant_scale; + } + + void grid_weight_dct::compute_quant_table(float q, + uint32_t grid_width, uint32_t grid_height, + float level_scale, int* dct_quant_tab) const + { + assert(q > 0.0f); + + dct_quant_tab[0] = 1; + + if (q >= 100.0f) + { + for (uint32_t y = 0; y < grid_height; y++) + for (uint32_t x = 0; x < grid_width; x++) + if (x || y) + dct_quant_tab[x + y * grid_width] = 1; + return; + } + + const int Bx = m_block_width, By = m_block_height; + + const float sx = (float)8.0f / (float)Bx; + const float sy = (float)8.0f / (float)By; + + for (uint32_t y = 0; y < grid_height; y++) + { + float ny = float(y); + float ry = ny * sy; + + for (uint32_t x = y ? 0 : 1; x < grid_width; x++) + { + int quant_scale = 0; + + assert(x || y); + + float nx = float(x); + float rx = nx * sx; + + // sample from the JPEG baseline luma 8x8 DCT quant matrix + // this is an approximation (we could do an actual desired radians per spatial sample search vs. each of the 8x8 basis vectors to find the best, most conservative mapping), + // but for 4x4 and 6x6 block sizes it's reasonable enough and simple/fast + // at 4x4, the lowest frequencies are slightly more heavily quantized than we would want (but the quant table entries near DC are so similar it's doubtful it matters much if at all) + float base = sample_jpeg_quant(g_baseline_jpeg_y, rx, ry); + + //quant_scale = (int)std::floor(base * level_scale + 0.5f); + quant_scale = (int)(base * level_scale + 0.5f); + assert(quant_scale == (int)std::floor(base * level_scale + 0.5f)); + + quant_scale = basisu::maximum(1, quant_scale); + + dct_quant_tab[x + y * grid_width] = quant_scale; + } // x + } // y + } + + // Needed by AQ + float grid_weight_dct::get_max_span_len(const astc_helpers::log_astc_block& log_blk, uint32_t plane_index) const + { + float span_len = 0.0f; + + if (log_blk.m_dual_plane) + { + color32 l, h; + decode_endpoints(log_blk.m_color_endpoint_modes[0], log_blk.m_endpoints, log_blk.m_endpoint_ise_range, l, h); + + for (uint32_t c = 0; c < 4; c++) + { + if (plane_index == 1) + { + if (c == log_blk.m_color_component_selector) + { + span_len += basisu::squaref((float)h[c] - (float)l[c]); + } + } + else + { + if (c != log_blk.m_color_component_selector) + { + span_len += basisu::squaref((float)h[c] - (float)l[c]); + } + } + } + + span_len = sqrtf(span_len); + } + else + { + for (uint32_t i = 0; i < log_blk.m_num_partitions; i++) + { + color32 l, h; + decode_endpoints(log_blk.m_color_endpoint_modes[0], log_blk.m_endpoints + astc_helpers::get_num_cem_values(log_blk.m_color_endpoint_modes[0]) * i, log_blk.m_endpoint_ise_range, l, h); + + float part_span_len = sqrtf( + basisu::squaref((float)h.r - (float)l.r) + basisu::squaref((float)h.g - (float)l.g) + basisu::squaref((float)h.b - (float)l.b) + basisu::squaref((float)h.a - (float)l.a) + ); + + span_len = basisu::maximum(part_span_len, span_len); + } + } + + return span_len; + } + +#include "basisu_astc_cfgs.inl" + + void create_encoder_trial_modes_table(uint32_t block_width, uint32_t block_height, + basisu::vector& encoder_trial_modes, grouped_trial_modes& grouped_encoder_trial_modes, + bool print_debug_info, bool print_modes) + { + //interval_timer itm; + //itm.start(); + + uint32_t mode_index = 0; + uint32_t max_grid_width = 0, max_grid_height = 0, max_grid_samples = 0; + + //encoder_trial_modes.reserve(BU_TOTAL_ASTC_CFGS); + encoder_trial_modes.reserve(3072); + encoder_trial_modes.resize(0); + + grouped_encoder_trial_modes.clear(); + + for (uint32_t cfg_index = 0; cfg_index < BU_TOTAL_ASTC_CFGS; cfg_index++) + { + assert((cfg_index * 3 + 2) < std::size(s_astc_cfg_table)); + uint32_t packed_mode = s_astc_cfg_table[cfg_index * 3] | (s_astc_cfg_table[cfg_index * 3 + 1] << 8) | (s_astc_cfg_table[cfg_index * 3 + 2] << 16); + + uint32_t endpoint_ise_range, weight_ise_range, ccs_index, num_subsets, unique_cem_index, grid_wh; + +#define BU_UNPACK_FIELD(val, bits) do { val = packed_mode & ((1u << (bits)) - 1u); packed_mode >>= (bits); } while(0) + BU_UNPACK_FIELD(endpoint_ise_range, CFG_PACK_EISE_BITS); + BU_UNPACK_FIELD(weight_ise_range, CFG_PACK_WISE_BITS); + BU_UNPACK_FIELD(ccs_index, CFG_PACK_CCS_BITS); + BU_UNPACK_FIELD(num_subsets, CFG_PACK_SUBSETS_BITS); + BU_UNPACK_FIELD(unique_cem_index, CFG_PACK_CEM_BITS); + BU_UNPACK_FIELD(grid_wh, CFG_PACK_GRID_BITS); +#undef BU_UNPACK_FIELD + + assert(!packed_mode); + + const uint32_t grid_width = (grid_wh / 11) + 2; + + // modes are sorted by grid widths, which is at/near the MSB of the packed values, rest must be >= + if (grid_width > block_width) + break; + + const uint32_t grid_height = (grid_wh % 11) + 2; + if (grid_height > block_height) + continue; + + const uint32_t cem_index = s_unique_ldr_index_to_astc_cem[unique_cem_index]; + +#if defined(_DEBUG) || defined(DEBUG) + { + // Ensure configuration is actually valid. + astc_helpers::log_astc_block log_block; + log_block.clear(); + log_block.m_grid_width = (uint8_t)grid_width; + log_block.m_grid_height = (uint8_t)grid_height; + log_block.m_num_partitions = (uint8_t)(num_subsets + 1); + log_block.m_dual_plane = (ccs_index != 0); + log_block.m_color_component_selector = (uint8_t)(ccs_index ? (ccs_index - 1) : 0); + log_block.m_num_partitions = (uint8_t)(num_subsets + 1); + log_block.m_endpoint_ise_range = (uint8_t)(endpoint_ise_range + astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE); + log_block.m_weight_ise_range = (uint8_t)(weight_ise_range + astc_helpers::FIRST_VALID_WEIGHT_ISE_RANGE); + + for (uint32_t i = 0; i < log_block.m_num_partitions; i++) + log_block.m_color_endpoint_modes[i] = (uint8_t)cem_index; + + astc_helpers::astc_block phys_block; + bool pack_success = astc_helpers::pack_astc_block(phys_block, log_block, nullptr, nullptr, astc_helpers::cValidateSkipFinalEndpointWeightPacking); + assert(pack_success); + } +#endif + + const uint32_t tm_index = encoder_trial_modes.size_u32(); + + trial_mode& tm = *encoder_trial_modes.enlarge(1); + + tm.m_ccs_index = (int)ccs_index - 1; + tm.m_cem = cem_index; + tm.m_endpoint_ise_range = endpoint_ise_range + astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE; + tm.m_weight_ise_range = weight_ise_range; + tm.m_grid_width = grid_width; + tm.m_grid_height = grid_height; + tm.m_num_parts = num_subsets + 1; + + grouped_encoder_trial_modes.add(block_width, block_height, tm, tm_index); + + if (print_modes) + { + max_grid_width = basisu::maximum(max_grid_width, grid_width); + max_grid_height = basisu::maximum(max_grid_height, grid_height); + max_grid_samples = basisu::maximum(max_grid_samples, grid_width * grid_height); + + basisu::debug_printf("%u: CEM: %u DP: %u, CCS: %i, SUBSETS: %u, GRID: %ux%u, ENDPOINTS: %u, WEIGHTS: %u\n", + mode_index, + tm.m_cem, tm.m_ccs_index >= 0, tm.m_ccs_index, tm.m_num_parts, + tm.m_grid_width, tm.m_grid_height, + astc_helpers::get_ise_levels(tm.m_endpoint_ise_range), + astc_helpers::get_ise_levels(tm.m_weight_ise_range)); + } + + mode_index++; + } // cfg_index + + if (print_debug_info) + { + //fmt_debug_printf("create_encoder_trial_modes_table() time: {} secs\n", itm.get_elapsed_secs()); + basisu::debug_printf("create_encoder_trial_modes_table() - ASTC %ux%u modes\n", block_width, block_height); + basisu::debug_printf("Total used trial mode groups: %u\n", grouped_encoder_trial_modes.count_used_groups()); + basisu::debug_printf("Total ASTC configurations iterated: %u\n", mode_index); + if (print_modes) + basisu::fmt_debug_printf("Max grid dimensions: {}x{}, max grid samples: {}\n", max_grid_width, max_grid_height, max_grid_samples); + } + } + + // Cached encoder trial modes for each block size, to avoid having to compute this for every texture/mipmap level. + basisu::vector g_encoder_trial_modes[astc_helpers::cTOTAL_BLOCK_SIZES]; + grouped_trial_modes g_grouped_encoder_trial_modes[astc_helpers::cTOTAL_BLOCK_SIZES]; + + grid_weight_dct g_grid_weight_dcts[astc_helpers::cTOTAL_BLOCK_SIZES]; + + // These tables could be initialized per transcoded texture, but that would result in per-texture overhead. + void init_transcoding_tables() + { + if (g_encoder_trial_modes[0].size()) + return; + + // We don't know what ASTC block sizes they're going to transcode, to prepare for all of them. + for (uint32_t i = 0; i < astc_helpers::cTOTAL_BLOCK_SIZES; i++) + { + const uint32_t block_width = astc_helpers::g_astc_block_sizes[i][0]; + const uint32_t block_height = astc_helpers::g_astc_block_sizes[i][1]; + + auto& encoder_trial_modes = g_encoder_trial_modes[i]; + auto& grouped_encoder_trial_modes = g_grouped_encoder_trial_modes[i]; + + encoder_trial_modes.reserve(3072); + create_encoder_trial_modes_table(block_width, block_height, encoder_trial_modes, grouped_encoder_trial_modes, false, false); + + g_grid_weight_dcts[i].init(block_width, block_height); + } // i + } + + const uint16_t g_total_unique_patterns[astc_helpers::NUM_ASTC_BLOCK_SIZES][2] = + { + { 437, 329 }, { 559, 405 }, { 659, 486 }, { 720, 534 }, + { 521, 333 }, { 584, 377 }, { 640, 410 }, { 672, 436 }, + { 710, 468 }, { 701, 476 }, { 759, 528 }, { 799, 568 }, + { 818, 597 }, { 838, 626 } + }; + + inline uint32_t get_total_unique_patterns(uint32_t astc_block_size_index, uint32_t num_parts) + { + assert(astc_block_size_index < astc_helpers::NUM_ASTC_BLOCK_SIZES); + assert((num_parts >= 2) && (num_parts <= 3)); + + return g_total_unique_patterns[astc_block_size_index][num_parts - 2]; + } + + const uint16_t g_unique_to_seed_4x4_p2[] = { 1,2,3,4,5,7,8,9,10,11,13,14,15,16,17,18,20,21,23,24,25,26,27,28,29,30,33,36,37,39,42,43,44,45,47,48,49,50,51,53,54,55,56,58,59,61,62,63,65,66,68,69,70,71,72,73,74,75,76,78,83,87,89,90,91,94,95,98,99,100,101,107,108,109,110,111,113,114,115,116,119,121,122,124,125,128,129,130,131,134,135,137,138,139,142,143,144,146,147,149,150,151,156,158,159,161,165,167,168,169,170,171,172,174,175,177,181,183,184,191,194,195,196,198,199,203,204,206,207,208,210,211,213,214,215,216,218,220,222,226,227,230,231,232,235,236,239,245,246,247,248,249,250,252,253,254,255,257,258,260,262,264,270,271,273,277,278,279,280,281,284,291,293,299,302,304,305,306,307,309,314,319,324,325,326,327,329,330,335,337,339,341,343,344,347,348,351,352,354,355,359,362,368,370,373,374,375,376,380,386,387,388,389,394,395,399,404,409,411,412,418,419,422,423,426,430,432,438,441,443,445,447,453,455,463,471,474,475,476,478,479,484,487,488,489,490,491,495,496,498,500,504,510,511,513,517,518,523,524,526,527,529,530,531,534,539,542,546,547,549,553,558,567,578,581,583,586,587,591,593,594,595,598,600,601,602,605,607,611,612,614,615,619,622,625,627,631,633,634,638,639,643,647,649,655,658,661,662,663,664,666,672,673,674,681,683,684,686,690,693,694,695,696,700,703,705,707,713,716,719,720,724,726,727,730,731,732,736,742,751,754,756,762,764,766,769,770,773,774,778,780,789,791,796,798,799,801,802,804,807,810,811,812,818,819,821,826,828,831,833,834,836,839,840,842,847,849,852,868,872,873,877,881,886,887,888,890,895,897,898,899,902,903,906,911,914,915,919,923,924,930,934,937,938,943,945,947,948,950,951,954,958,959,963,964,966,967,971,976,983,987,988,993,994,995,998,999,1006,1007,1009,1013,1014,1015,1016,1019,1022,1023 } ; + const uint16_t g_unique_to_seed_5x4_p2[] = { 1,2,3,4,5,7,8,9,10,11,13,14,15,16,17,18,20,21,23,24,25,26,27,28,29,30,31,33,36,37,39,42,43,44,45,47,48,49,50,51,53,54,55,56,58,59,61,62,63,65,66,67,68,69,70,71,72,73,74,75,76,78,83,87,88,89,90,91,94,95,97,98,99,100,101,107,108,109,110,111,113,114,115,116,119,121,122,124,125,128,129,130,131,132,134,135,137,138,139,142,143,144,145,146,147,149,150,151,153,154,156,157,158,159,161,165,167,168,169,170,171,172,174,175,177,181,183,184,185,188,190,191,194,195,196,198,199,200,203,204,206,207,208,210,211,213,214,215,216,217,218,220,222,225,226,227,229,230,231,232,235,236,239,245,246,247,248,249,250,252,253,254,255,257,258,260,261,262,264,265,267,270,271,273,275,277,278,279,280,281,282,284,287,291,293,295,296,299,300,302,304,305,306,307,309,314,317,319,323,324,325,326,327,329,330,332,335,337,339,341,342,343,344,347,348,349,350,351,352,354,355,359,361,362,365,368,370,373,374,375,376,380,381,386,387,388,389,391,394,395,399,404,405,407,409,410,411,412,418,419,420,422,423,426,430,432,438,439,441,443,445,447,449,453,454,455,462,463,465,471,473,474,475,476,478,479,482,484,486,487,488,489,490,491,495,496,498,500,501,503,504,505,508,510,511,513,516,517,518,519,521,523,524,526,527,529,530,531,533,534,538,539,542,546,547,549,550,551,553,554,558,563,567,569,572,575,578,579,581,583,586,587,591,593,594,595,598,600,601,602,605,606,607,608,611,612,614,615,616,619,622,623,625,627,631,633,634,636,638,639,643,645,647,649,652,655,658,661,662,663,664,665,666,668,672,673,674,675,681,683,684,686,687,690,692,693,694,695,696,697,700,702,703,705,707,709,711,713,716,719,720,724,725,726,727,730,731,732,736,739,742,748,751,754,756,758,762,763,764,766,768,769,770,772,773,774,776,778,780,782,786,789,791,792,796,798,799,801,802,804,807,810,811,812,814,818,819,821,823,826,828,830,831,833,834,835,836,839,840,842,845,847,849,852,858,861,866,868,870,871,872,873,876,877,878,881,886,887,888,890,891,895,897,898,899,901,902,903,906,909,911,914,915,919,923,924,927,929,930,933,934,935,936,937,938,941,942,943,945,947,948,950,951,954,955,958,959,963,964,966,967,970,971,975,976,980,983,986,987,988,993,994,995,997,998,999,1001,1006,1007,1009,1010,1013,1014,1015,1016,1019,1020,1022,1023 }; + const uint16_t g_unique_to_seed_5x5_p2[] = { 1,2,3,4,5,7,8,9,10,11,13,14,15,16,17,18,20,21,23,24,25,26,27,28,29,30,31,33,34,36,37,39,42,43,44,45,47,48,49,50,51,53,54,55,56,58,59,60,61,62,63,65,66,67,68,69,70,71,72,73,74,75,76,77,78,81,82,83,87,88,89,90,91,92,94,95,97,98,99,100,101,105,107,108,109,110,111,113,114,115,116,119,120,121,122,124,125,128,129,130,131,132,134,135,137,138,139,142,143,144,145,146,147,148,149,150,151,153,154,155,156,157,158,159,161,165,167,168,169,170,171,172,174,175,176,177,181,182,183,184,185,188,190,191,193,194,195,196,198,199,200,203,204,206,207,208,209,210,211,213,214,215,216,217,218,220,222,224,225,226,227,229,230,231,232,235,236,239,240,242,245,246,247,248,249,250,252,253,254,255,256,257,258,260,261,262,264,265,267,270,271,273,275,276,277,278,279,280,281,282,283,284,286,287,289,291,293,294,295,296,298,299,300,302,303,304,305,306,307,309,313,314,317,318,319,323,324,325,326,327,329,330,331,332,335,337,338,339,341,342,343,344,347,348,349,350,351,352,354,355,356,359,361,362,365,368,370,373,374,375,376,377,378,380,381,386,387,388,389,390,391,392,393,394,395,399,403,404,405,407,409,410,411,412,415,418,419,420,421,422,423,424,426,430,432,433,437,438,439,440,441,443,444,445,446,447,449,453,454,455,458,462,463,465,466,469,470,471,473,474,475,476,478,479,481,482,484,486,487,488,489,490,491,492,495,496,498,500,501,502,503,504,505,506,508,509,510,511,513,516,517,518,519,521,523,524,526,527,529,530,531,533,534,538,539,542,546,547,548,549,550,551,553,554,558,559,561,562,563,567,569,572,575,578,579,580,581,583,585,586,587,590,591,593,594,595,596,598,599,600,601,602,605,606,607,608,609,610,611,612,614,615,616,617,619,620,621,622,623,625,627,631,633,634,636,638,639,643,645,646,647,648,649,652,654,655,658,661,662,663,664,665,666,668,672,673,674,675,681,683,684,686,687,688,690,692,693,694,695,696,697,699,700,701,702,703,705,707,709,711,713,714,716,719,720,722,724,725,726,727,730,731,732,736,738,739,742,748,751,753,754,756,758,760,762,763,764,766,768,769,770,772,773,774,776,778,780,782,783,786,789,791,792,796,798,799,801,802,804,806,807,808,810,811,812,813,814,818,819,821,823,826,828,830,831,833,834,835,836,839,840,841,842,845,847,849,851,852,858,861,866,868,869,870,871,872,873,874,876,877,878,879,881,886,887,888,890,891,893,894,895,897,898,899,901,902,903,906,909,910,911,912,914,915,917,919,920,921,923,924,927,929,930,933,934,935,936,937,938,941,942,943,945,947,948,950,951,954,955,958,959,961,962,963,964,966,967,968,970,971,975,976,977,980,983,986,987,988,993,994,995,996,997,998,999,1001,1006,1007,1009,1010,1013,1014,1015,1016,1018,1019,1020,1022,1023 }; + const uint16_t g_unique_to_seed_6x5_p2[] = { 1,2,3,4,5,7,8,9,10,11,13,14,15,16,17,18,20,21,23,24,25,26,27,28,29,30,31,33,34,36,37,39,42,43,44,45,47,48,49,50,51,52,53,54,55,56,58,59,60,61,62,63,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,86,87,88,89,90,91,92,94,95,97,98,99,100,101,105,107,108,109,110,111,112,113,114,115,116,117,119,120,121,122,123,124,125,128,129,130,131,132,134,135,136,137,138,139,142,143,144,145,146,147,148,149,150,151,153,154,155,156,157,158,159,161,165,167,168,169,170,171,172,174,175,176,177,178,181,182,183,184,185,188,190,191,193,194,195,196,198,199,200,201,203,204,206,207,208,209,210,211,212,213,214,215,216,217,218,220,222,223,224,225,226,227,229,230,231,232,235,236,239,240,242,243,245,246,247,248,249,250,252,253,254,255,256,257,258,260,261,262,264,265,266,267,270,271,273,275,276,277,278,279,280,281,282,283,284,286,287,289,291,293,294,295,296,298,299,300,302,303,304,305,306,307,309,310,313,314,317,318,319,323,324,325,326,327,329,330,331,332,335,337,338,339,341,342,343,344,347,348,349,350,351,352,354,355,356,357,359,360,361,362,363,365,367,368,370,371,373,374,375,376,377,378,380,381,383,386,387,388,389,390,391,392,393,394,395,399,402,403,404,405,407,409,410,411,412,415,418,419,420,421,422,423,424,426,430,432,433,436,437,438,439,440,441,443,444,445,446,447,449,453,454,455,458,461,462,463,465,466,469,470,471,473,474,475,476,478,479,481,482,484,486,487,488,489,490,491,492,493,495,496,498,499,500,501,502,503,504,505,506,508,509,510,511,513,516,517,518,519,521,523,524,526,527,529,530,531,532,533,534,536,538,539,542,543,545,546,547,548,549,550,551,553,554,558,559,561,562,563,564,567,569,570,572,575,578,579,580,581,583,584,585,586,587,590,591,593,594,595,596,597,598,599,600,601,602,605,606,607,608,609,610,611,612,614,615,616,617,619,620,621,622,623,624,625,627,628,629,631,633,634,636,638,639,643,645,646,647,648,649,651,652,654,655,657,658,661,662,663,664,665,666,668,672,673,674,675,681,683,684,686,687,688,690,692,693,694,695,696,697,699,700,701,702,703,705,706,707,709,711,713,714,716,719,720,722,723,724,725,726,727,730,731,732,736,738,739,742,745,747,748,751,753,754,756,758,760,762,763,764,766,768,769,770,772,773,774,776,778,780,782,783,784,786,788,789,791,792,795,796,798,799,801,802,804,806,807,808,810,811,812,813,814,818,819,820,821,823,826,828,830,831,833,834,835,836,839,840,841,842,845,847,849,851,852,856,858,861,866,868,869,870,871,872,873,874,875,876,877,878,879,881,883,886,887,888,890,891,893,894,895,896,897,898,899,901,902,903,906,908,909,910,911,912,914,915,917,919,920,921,923,924,927,929,930,932,933,934,935,936,937,938,941,942,943,945,947,948,950,951,954,955,956,958,959,960,961,962,963,964,966,967,968,970,971,972,975,976,977,979,980,982,983,986,987,988,993,994,995,996,997,998,999,1001,1004,1006,1007,1009,1010,1013,1014,1015,1016,1017,1018,1019,1020,1022,1023 }; + const uint16_t g_unique_to_seed_6x6_p2[] = { 1,2,3,4,5,7,8,9,10,11,14,15,16,17,18,19,21,23,24,25,26,27,28,29,31,33,36,37,39,42,43,44,45,46,47,48,49,50,51,53,54,55,56,58,59,60,61,62,63,65,66,67,68,69,70,71,73,74,75,76,77,78,79,83,86,87,88,89,90,91,94,95,98,99,100,101,103,107,108,109,110,112,114,115,116,119,121,122,125,128,129,130,131,132,134,135,136,137,138,139,142,144,146,147,148,149,151,153,154,156,158,159,167,168,169,170,171,172,174,175,177,178,181,183,190,191,193,194,195,196,198,199,203,206,207,210,211,213,214,215,216,217,218,220,222,223,225,226,227,229,230,231,232,235,236,237,240,247,249,250,252,254,255,257,258,260,262,264,266,267,270,271,272,273,277,278,279,281,283,284,286,289,291,292,293,295,298,299,302,303,305,306,307,309,314,316,318,319,323,324,326,327,329,335,339,341,343,347,348,351,352,353,354,355,361,362,363,366,367,368,370,373,374,376,380,386,387,388,389,390,392,393,394,395,402,403,404,407,409,411,414,415,418,419,422,423,426,430,432,433,436,437,438,439,440,441,445,447,450,452,453,455,458,461,463,470,471,474,475,476,478,480,482,486,487,488,490,495,496,498,499,500,504,506,508,510,513,517,518,519,521,523,524,526,527,529,530,531,533,534,538,539,543,545,546,547,549,550,551,553,554,558,562,567,569,572,578,581,583,585,586,587,591,593,594,595,596,598,600,601,602,606,607,608,610,611,612,614,615,616,619,621,622,623,625,627,631,633,634,638,639,643,645,646,647,648,649,652,655,658,661,662,663,664,665,670,672,673,674,675,681,683,684,685,686,687,688,690,691,693,694,695,696,697,700,702,703,705,713,714,716,719,720,722,724,726,727,730,731,732,736,740,742,751,753,754,756,758,760,762,763,764,766,769,770,774,778,780,786,789,791,794,797,798,799,801,802,804,805,807,808,810,811,812,817,819,821,822,826,828,831,834,835,836,839,842,847,852,854,858,862,868,869,870,872,874,876,877,878,886,887,890,891,893,895,897,898,899,902,903,906,909,910,911,912,914,919,929,930,934,936,937,938,942,943,945,947,948,950,951,954,955,958,959,963,964,966,967,968,976,980,983,987,988,993,994,995,998,999,1001,1006,1007,1008,1009,1013,1014,1015,1016,1019,1022,1023 }; + const uint16_t g_unique_to_seed_8x5_p2[] = { 1,2,3,4,5,7,8,9,10,11,14,15,16,17,18,19,20,21,23,24,25,26,27,28,29,30,31,32,33,36,37,39,42,43,45,46,47,48,49,50,51,53,54,55,56,58,59,61,62,63,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,83,86,87,88,89,90,91,94,95,98,99,100,101,103,107,108,109,110,112,114,115,116,119,121,122,123,125,128,129,130,131,132,134,135,136,137,138,139,142,143,144,146,147,148,149,150,151,154,156,158,159,165,167,168,169,170,171,172,174,175,177,178,181,183,184,191,193,194,195,196,198,199,203,204,206,207,209,210,211,213,214,215,216,217,218,220,222,225,226,227,229,230,231,232,235,236,237,240,247,248,249,250,252,253,254,255,257,258,260,261,262,264,266,267,270,271,272,273,277,278,279,281,284,286,287,289,291,292,293,295,296,298,299,300,302,303,304,305,306,307,309,314,316,317,318,319,323,324,325,326,327,329,330,335,337,339,341,342,343,344,347,348,351,352,353,354,355,359,361,362,363,366,367,368,370,371,374,375,376,380,381,386,387,388,389,390,391,392,393,394,395,399,402,403,404,405,407,409,410,411,412,415,418,419,422,423,424,426,430,432,433,436,437,438,439,440,441,444,445,446,447,450,451,452,453,454,455,458,461,462,463,465,470,471,473,474,475,476,478,479,482,484,486,487,488,490,491,495,496,498,499,500,501,502,504,505,506,508,510,511,513,517,518,519,521,523,524,526,527,530,531,533,534,535,538,539,543,545,546,547,550,551,554,558,559,562,567,569,572,578,579,581,583,585,586,587,591,593,594,595,598,600,601,602,606,607,608,610,611,612,614,615,618,619,621,622,623,625,627,631,633,636,638,639,643,645,646,647,648,649,650,651,652,655,658,659,661,662,663,664,665,666,668,672,673,674,675,683,684,685,686,687,688,690,691,692,693,694,695,696,697,700,701,702,703,705,707,711,713,716,719,720,722,724,725,726,727,730,731,732,736,739,740,742,748,751,753,754,756,758,760,762,763,764,766,768,770,774,775,778,780,786,789,791,794,796,798,799,801,802,804,805,807,808,810,811,812,813,817,819,821,825,826,831,834,836,839,841,842,845,847,849,851,852,854,856,862,868,869,870,871,872,874,876,877,879,881,886,887,890,891,893,895,897,898,899,902,903,906,909,910,911,914,915,918,919,921,923,924,927,929,930,934,936,937,938,942,943,945,947,948,951,954,955,958,959,962,963,964,966,967,968,970,971,976,977,979,983,987,988,989,993,994,995,997,998,999,1001,1006,1007,1008,1009,1013,1015,1016,1018,1019,1022,1023 }; + const uint16_t g_unique_to_seed_8x6_p2[] = { 1,2,3,4,5,7,8,9,10,11,13,14,15,16,17,18,19,20,21,23,24,25,26,27,28,29,30,31,32,33,36,37,39,42,43,44,45,46,47,48,49,50,51,53,54,55,56,58,59,60,61,62,63,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,83,86,87,88,89,90,91,94,95,98,99,100,101,103,107,108,109,110,112,113,114,115,116,119,121,122,123,125,128,129,130,131,132,134,135,136,137,138,139,142,143,144,146,147,148,149,150,151,153,154,156,158,159,165,167,168,169,170,171,172,174,175,177,178,181,183,184,190,191,193,194,195,196,198,199,200,203,204,206,207,208,209,210,211,213,214,215,216,217,218,220,222,223,225,226,227,229,230,231,232,235,236,237,240,245,246,247,248,249,250,252,253,254,255,257,258,260,261,262,264,266,267,270,271,272,273,277,278,279,281,283,284,286,287,289,291,292,293,294,295,296,298,299,300,302,303,304,305,306,307,309,314,316,317,318,319,323,324,325,326,327,329,330,335,337,339,341,342,343,344,347,348,349,350,351,352,353,354,355,359,361,362,363,365,366,367,368,370,371,373,374,375,376,380,381,386,387,388,389,390,391,392,393,394,395,399,402,403,404,405,407,409,410,411,412,414,415,418,419,422,423,424,426,430,432,433,436,437,438,439,440,441,443,444,445,446,447,450,451,452,453,454,455,458,461,462,463,465,470,471,473,474,475,476,478,479,480,482,484,486,487,488,489,490,491,495,496,498,499,500,501,502,504,505,506,508,510,511,513,517,518,519,521,523,524,526,527,529,530,531,533,534,535,538,539,543,545,546,547,549,550,551,553,554,558,559,562,567,569,572,575,578,579,581,583,585,586,587,591,593,594,595,596,598,600,601,602,606,607,608,610,611,612,614,615,616,618,619,621,622,623,625,627,631,633,634,636,638,639,643,645,646,647,648,649,650,651,652,655,658,659,661,662,663,664,665,666,668,670,672,673,674,675,681,683,684,685,686,687,688,690,691,692,693,694,695,696,697,700,701,702,703,705,707,711,713,714,716,719,720,722,724,725,726,727,730,731,732,736,739,740,742,748,751,753,754,756,758,760,762,763,764,766,768,769,770,773,774,775,776,778,780,786,789,791,792,794,796,797,798,799,801,802,804,805,807,808,810,811,812,813,817,818,819,821,822,825,826,828,830,831,833,834,835,836,839,841,842,845,847,849,851,852,854,856,858,861,862,868,869,870,871,872,873,874,876,877,878,879,881,886,887,888,890,891,893,895,897,898,899,902,903,906,909,910,911,912,914,915,918,919,921,923,924,927,929,930,933,934,935,936,937,938,940,942,943,945,947,948,950,951,954,955,958,959,962,963,964,966,967,968,970,971,976,977,979,980,983,986,987,988,989,993,994,995,997,998,999,1001,1006,1007,1008,1009,1013,1014,1015,1016,1018,1019,1021,1022,1023 }; + const uint16_t g_unique_to_seed_10x5_p2[] = { 1,2,3,4,5,7,8,9,10,11,13,14,15,16,17,18,19,20,21,23,24,25,26,27,28,29,30,31,32,33,36,37,39,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,58,59,61,62,63,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,83,84,85,86,87,88,89,90,91,94,95,97,98,99,100,101,103,107,108,109,110,112,113,114,115,116,119,121,122,123,125,128,129,130,131,132,134,135,136,137,138,139,142,143,144,145,146,147,148,149,150,151,153,154,155,156,157,158,159,165,167,168,169,170,171,172,174,175,177,178,181,183,184,185,188,191,193,194,195,196,198,199,200,203,204,206,207,208,209,210,211,213,214,215,216,217,218,219,220,222,223,225,226,227,229,230,231,232,235,236,237,238,239,240,247,248,249,250,252,253,254,255,257,258,260,261,262,263,264,265,266,267,270,271,272,273,275,277,278,279,281,282,283,284,286,287,289,291,292,293,294,295,296,298,299,300,302,303,304,305,306,307,309,314,316,317,318,319,323,324,325,326,327,329,330,332,335,337,339,341,342,343,344,347,348,349,350,351,352,353,354,355,358,359,361,362,363,365,366,367,368,370,371,374,375,376,380,381,386,387,388,389,390,391,392,393,394,395,399,402,403,404,405,407,409,410,411,412,415,418,419,422,423,424,426,429,430,432,433,436,437,438,439,440,441,443,444,445,446,447,449,450,451,452,453,454,455,458,461,462,463,465,470,471,473,474,475,476,478,479,481,482,484,486,487,488,489,490,491,492,495,496,498,499,500,501,502,503,504,505,506,508,510,511,513,517,518,519,521,523,524,526,527,529,530,531,533,534,535,538,539,542,543,545,546,547,548,550,551,554,558,559,562,567,569,570,571,572,575,578,579,580,581,583,585,586,587,591,593,594,595,597,598,600,601,602,606,607,608,609,610,611,612,614,615,616,617,618,619,621,622,623,624,625,627,631,633,634,636,638,639,643,645,646,647,648,649,650,651,652,655,657,658,659,661,662,663,664,665,666,668,672,673,674,675,681,683,684,685,686,687,688,690,691,692,693,694,695,696,697,699,700,701,702,703,705,706,707,711,713,716,719,720,722,723,724,725,726,727,730,731,732,736,739,740,742,748,751,753,754,755,756,757,758,760,762,763,764,766,768,770,773,774,775,776,778,780,786,789,791,794,795,796,798,799,801,802,804,805,807,808,810,811,812,813,814,817,818,819,821,822,823,825,826,828,830,831,834,835,836,839,841,842,845,847,849,851,852,854,856,862,868,869,870,871,872,873,874,876,877,878,879,881,883,886,887,888,890,891,893,895,897,898,899,901,902,903,906,909,910,911,914,915,918,919,920,921,923,924,927,929,930,932,933,934,935,936,937,938,941,942,943,945,947,948,950,951,954,955,956,958,959,961,962,963,964,966,967,968,969,970,971,975,976,977,979,980,983,986,987,988,989,993,994,995,996,997,998,999,1001,1006,1007,1008,1009,1013,1014,1015,1016,1017,1018,1019,1020,1022,1023 }; + const uint16_t g_unique_to_seed_10x6_p2[] = { 1,2,3,4,5,7,8,9,10,11,13,14,15,16,17,18,19,20,21,23,24,25,26,27,28,29,30,31,32,33,36,37,39,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,58,59,60,61,62,63,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,83,84,85,86,87,88,89,90,91,94,95,97,98,99,100,101,103,107,108,109,110,112,113,114,115,116,119,121,122,123,124,125,128,129,130,131,132,134,135,136,137,138,139,142,143,144,145,146,147,148,149,150,151,153,154,155,156,157,158,159,165,167,168,169,170,171,172,174,175,177,178,181,183,184,185,188,190,191,193,194,195,196,198,199,200,203,204,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,225,226,227,229,230,231,232,235,236,237,238,239,240,245,246,247,248,249,250,252,253,254,255,257,258,260,261,262,263,264,265,266,267,270,271,272,273,274,275,277,278,279,281,282,283,284,286,287,289,291,292,293,294,295,296,298,299,300,302,303,304,305,306,307,309,310,314,316,317,318,319,323,324,325,326,327,329,330,332,335,337,338,339,341,342,343,344,347,348,349,350,351,352,353,354,355,358,359,361,362,363,365,366,367,368,370,371,373,374,375,376,380,381,386,387,388,389,390,391,392,393,394,395,399,402,403,404,405,407,409,410,411,412,414,415,418,419,422,423,424,426,429,430,432,433,436,437,438,439,440,441,443,444,445,446,447,449,450,451,452,453,454,455,458,461,462,463,465,470,471,473,474,475,476,478,479,480,481,482,484,486,487,488,489,490,491,492,495,496,498,499,500,501,502,503,504,505,506,508,509,510,511,513,516,517,518,519,521,523,524,526,527,529,530,531,533,534,535,538,539,542,543,545,546,547,548,549,550,551,553,554,558,559,562,567,569,570,571,572,575,578,579,580,581,583,585,586,587,590,591,593,594,595,596,597,598,600,601,602,606,607,608,609,610,611,612,614,615,616,617,618,619,621,622,623,624,625,627,631,633,634,636,638,639,643,645,646,647,648,649,650,651,652,655,657,658,659,661,662,663,664,665,666,668,670,672,673,674,675,681,683,684,685,686,687,688,690,691,692,693,694,695,696,697,699,700,701,702,703,705,706,707,709,711,713,714,716,719,720,722,723,724,725,726,727,730,731,732,736,739,740,742,748,751,753,754,755,756,757,758,760,762,763,764,766,768,769,770,772,773,774,775,776,778,780,782,783,786,788,789,791,792,794,795,796,797,798,799,801,802,804,805,807,808,810,811,812,813,814,817,818,819,821,822,823,825,826,828,830,831,833,834,835,836,839,841,842,845,847,849,851,852,854,856,858,861,862,868,869,870,871,872,873,874,876,877,878,879,881,883,886,887,888,890,891,893,895,897,898,899,901,902,903,906,908,909,910,911,912,914,915,918,919,920,921,923,924,927,929,930,932,933,934,935,936,937,938,940,941,942,943,945,947,948,949,950,951,954,955,956,958,959,961,962,963,964,966,967,968,969,970,971,975,976,977,979,980,983,986,987,988,989,993,994,995,996,997,998,999,1001,1004,1006,1007,1008,1009,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023 }; + const uint16_t g_unique_to_seed_8x8_p2[] = { 1,2,3,4,5,7,8,9,10,11,13,14,15,16,17,18,19,20,21,23,24,25,26,27,28,29,30,31,32,33,36,37,39,42,43,44,45,46,47,48,49,50,51,53,54,55,56,58,59,60,61,62,63,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,83,86,87,88,89,90,91,94,95,97,98,99,100,101,102,103,107,108,109,110,111,112,113,114,115,116,118,119,120,121,122,123,124,125,128,129,130,131,132,134,135,136,137,138,139,142,143,144,146,147,148,149,150,151,152,153,154,155,156,158,159,161,165,167,168,169,170,171,172,174,175,177,178,181,182,183,184,188,190,191,193,194,195,196,198,199,200,201,203,204,206,207,208,209,210,211,213,214,215,216,217,218,220,222,223,224,225,226,227,229,230,231,232,235,236,237,239,240,242,245,246,247,248,249,250,252,253,254,255,257,258,260,261,262,264,265,266,267,270,271,272,273,276,277,278,279,280,281,282,283,284,286,287,289,291,292,293,294,295,296,298,299,300,302,303,304,305,306,307,309,314,316,317,318,319,323,324,325,326,327,329,330,331,332,333,335,337,338,339,341,342,343,344,347,348,349,350,351,352,353,354,355,359,361,362,363,365,366,367,368,370,371,373,374,375,376,380,381,386,387,388,389,390,391,392,393,394,395,399,400,402,403,404,405,407,409,410,411,412,414,415,418,419,420,422,423,424,426,430,432,433,436,437,438,439,440,441,443,444,445,446,447,450,451,452,453,454,455,458,461,462,463,465,466,470,471,473,474,475,476,478,479,480,481,482,484,486,487,488,489,490,491,495,496,498,499,500,501,502,503,504,505,506,508,510,511,513,516,517,518,519,521,523,524,526,527,529,530,531,533,534,535,538,539,542,543,545,546,547,548,549,550,551,553,554,558,559,562,563,567,569,572,575,578,579,580,581,583,585,586,587,590,591,593,594,595,596,598,599,600,601,602,605,606,607,608,609,610,611,612,614,615,616,618,619,620,621,622,623,625,627,631,633,634,636,638,639,643,645,646,647,648,649,650,651,652,655,658,659,661,662,663,664,665,666,668,670,672,673,674,675,681,683,684,685,686,687,688,690,691,692,693,694,695,696,697,700,701,702,703,705,707,709,711,713,714,716,719,720,722,724,725,726,727,730,731,732,736,739,740,742,743,748,751,753,754,755,756,757,758,760,762,763,764,766,768,769,770,772,773,774,775,776,778,780,782,783,786,789,791,792,793,794,796,797,798,799,801,802,804,805,806,807,808,810,811,812,813,814,817,818,819,821,822,825,826,828,830,831,833,834,835,836,839,840,841,842,845,847,849,851,852,854,856,858,861,862,866,868,869,870,871,872,873,874,876,877,878,879,881,886,887,888,890,891,893,894,895,897,898,899,901,902,903,906,908,909,910,911,912,914,915,916,917,918,919,921,923,924,927,929,930,933,934,935,936,937,938,940,942,943,945,947,948,949,950,951,954,955,956,958,959,962,963,964,966,967,968,969,970,971,975,976,977,979,980,983,986,987,988,989,993,994,995,997,998,999,1001,1006,1007,1008,1009,1010,1013,1014,1015,1016,1018,1019,1020,1021,1022,1023 }; + const uint16_t g_unique_to_seed_10x8_p2[] = { 1,2,3,4,5,7,8,9,10,11,13,14,15,16,17,18,19,20,21,23,24,25,26,27,28,29,30,31,32,33,34,36,37,39,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,58,59,60,61,62,63,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,83,84,85,86,87,88,89,90,91,92,94,95,97,98,99,100,101,102,103,105,107,108,109,110,111,112,113,114,115,116,118,119,120,121,122,123,124,125,128,129,130,131,132,134,135,136,137,138,139,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,161,165,167,168,169,170,171,172,174,175,177,178,179,180,181,182,183,184,185,188,190,191,193,194,195,196,198,199,200,201,203,204,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,229,230,231,232,235,236,237,238,239,240,242,245,246,247,248,249,250,252,253,254,255,257,258,260,261,262,263,264,265,266,267,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,286,287,289,291,292,293,294,295,296,298,299,300,302,303,304,305,306,307,309,310,314,316,317,318,319,321,323,324,325,326,327,329,330,331,332,333,335,337,338,339,341,342,343,344,347,348,349,350,351,352,353,354,355,358,359,361,362,363,365,366,367,368,370,371,373,374,375,376,378,380,381,385,386,387,388,389,390,391,392,393,394,395,399,400,402,403,404,405,407,409,410,411,412,414,415,418,419,420,422,423,424,426,429,430,432,433,436,437,438,439,440,441,443,444,445,446,447,449,450,451,452,453,454,455,458,461,462,463,465,466,469,470,471,473,474,475,476,478,479,480,481,482,484,486,487,488,489,490,491,492,495,496,498,499,500,501,502,503,504,505,506,508,509,510,511,513,516,517,518,519,520,521,523,524,526,527,529,530,531,533,534,535,538,539,542,543,545,546,547,548,549,550,551,553,554,558,559,562,563,567,569,570,571,572,575,578,579,580,581,583,584,585,586,587,590,591,593,594,595,596,597,598,599,600,601,602,605,606,607,608,609,610,611,612,614,615,616,617,618,619,620,621,622,623,624,625,627,628,629,631,633,634,636,638,639,643,645,646,647,648,649,650,651,652,655,657,658,659,661,662,663,664,665,666,668,670,672,673,674,675,681,683,684,685,686,687,688,690,691,692,693,694,695,696,697,699,700,701,702,703,705,706,707,709,711,713,714,716,719,720,722,723,724,725,726,727,730,731,732,736,739,740,742,743,745,747,748,751,753,754,755,756,757,758,760,762,763,764,766,768,769,770,772,773,774,775,776,778,780,781,782,783,786,788,789,791,792,793,794,795,796,797,798,799,801,802,804,805,806,807,808,810,811,812,813,814,817,818,819,821,822,823,825,826,828,830,831,833,834,835,836,839,840,841,842,845,847,849,851,852,854,856,858,861,862,866,868,869,870,871,872,873,874,876,877,878,879,880,881,883,886,887,888,890,891,893,894,895,897,898,899,901,902,903,906,908,909,910,911,912,914,915,916,917,918,919,920,921,923,924,927,929,930,932,933,934,935,936,937,938,940,941,942,943,945,947,948,949,950,951,954,955,956,958,959,960,961,962,963,964,966,967,968,969,970,971,975,976,977,979,980,982,983,986,987,988,989,993,994,995,996,997,998,999,1001,1004,1006,1007,1008,1009,1010,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023 }; + const uint16_t g_unique_to_seed_10x10_p2[] = { 1,2,3,4,5,7,8,9,10,11,13,14,15,16,17,18,19,20,21,23,24,25,26,27,28,29,30,31,32,33,34,36,37,39,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,58,59,60,61,62,63,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,81,82,83,84,85,86,87,88,89,90,91,92,94,95,97,98,99,100,101,102,103,105,107,108,109,110,111,112,113,114,115,116,118,119,120,121,122,123,124,125,128,129,130,131,132,134,135,136,137,138,139,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,161,165,167,168,169,170,171,172,174,175,176,177,178,179,180,181,182,183,184,185,188,190,191,193,194,195,196,198,199,200,201,202,203,204,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,229,230,231,232,235,236,237,238,239,240,242,245,246,247,248,249,250,252,253,254,255,256,257,258,260,261,262,263,264,265,266,267,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,286,287,289,291,292,293,294,295,296,298,299,300,302,303,304,305,306,307,309,310,313,314,316,317,318,319,321,323,324,325,326,327,329,330,331,332,333,335,337,338,339,341,342,343,344,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,365,366,367,368,370,371,373,374,375,376,377,378,380,381,385,386,387,388,389,390,391,392,393,394,395,399,400,402,403,404,405,407,409,410,411,412,414,415,417,418,419,420,421,422,423,424,425,426,427,428,429,430,432,433,436,437,438,439,440,441,443,444,445,446,447,449,450,451,452,453,454,455,458,459,461,462,463,465,466,469,470,471,473,474,475,476,478,479,480,481,482,484,486,487,488,489,490,491,492,493,494,495,496,498,499,500,501,502,503,504,505,506,508,509,510,511,513,514,516,517,518,519,520,521,522,523,524,526,527,528,529,530,531,532,533,534,535,536,538,539,542,543,545,546,547,548,549,550,551,553,554,558,559,561,562,563,566,567,569,570,571,572,575,578,579,580,581,583,584,585,586,587,590,591,593,594,595,596,597,598,599,600,601,602,605,606,607,608,609,610,611,612,614,615,616,617,618,619,620,621,622,623,624,625,627,628,629,631,633,634,636,638,639,643,645,646,647,648,649,650,651,652,654,655,657,658,659,661,662,663,664,665,666,667,668,670,672,673,674,675,681,683,684,685,686,687,688,690,691,692,693,694,695,696,697,699,700,701,702,703,705,706,707,709,711,713,714,716,719,720,722,723,724,725,726,727,730,731,732,736,738,739,740,742,743,745,747,748,751,753,754,755,756,757,758,760,762,763,764,766,768,769,770,771,772,773,774,775,776,778,780,781,782,783,784,786,788,789,791,792,793,794,795,796,797,798,799,801,802,803,804,805,806,807,808,810,811,812,813,814,817,818,819,820,821,822,823,825,826,828,829,830,831,833,834,835,836,839,840,841,842,845,846,847,848,849,851,852,854,855,856,858,861,862,866,867,868,869,870,871,872,873,874,876,877,878,879,880,881,883,886,887,888,890,891,893,894,895,896,897,898,899,901,902,903,904,906,908,909,910,911,912,914,915,916,917,918,919,920,921,923,924,927,929,930,932,933,934,935,936,937,938,940,941,942,943,945,947,948,949,950,951,954,955,956,958,959,960,961,962,963,964,966,967,968,969,970,971,972,975,976,977,979,980,982,983,986,987,988,989,993,994,995,996,997,998,999,1001,1004,1006,1007,1008,1009,1010,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023 }; + const uint16_t g_unique_to_seed_12x10_p2[] = { 1,2,3,4,5,7,8,9,10,11,13,14,15,16,17,18,19,20,21,23,24,25,26,27,28,29,30,31,32,33,34,36,37,39,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,94,95,97,98,99,100,101,102,103,104,105,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,128,129,130,131,132,134,135,136,137,138,139,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,161,165,167,168,169,170,171,172,174,175,176,177,178,179,180,181,182,183,184,185,188,190,191,193,194,195,196,198,199,200,201,202,203,204,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,229,230,231,232,235,236,237,238,239,240,242,243,244,245,246,247,248,249,250,252,253,254,255,256,257,258,260,261,262,263,264,265,266,267,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,286,287,289,291,292,293,294,295,296,298,299,300,302,303,304,305,306,307,309,310,313,314,316,317,318,319,321,323,324,325,326,327,329,330,331,332,333,335,337,338,339,341,342,343,344,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,365,366,367,368,370,371,373,374,375,376,377,378,380,381,383,385,386,387,388,389,390,391,392,393,394,395,399,400,402,403,404,405,407,409,410,411,412,414,415,417,418,419,420,421,422,423,424,425,426,427,428,429,430,432,433,436,437,438,439,440,441,443,444,445,446,447,449,450,451,452,453,454,455,458,459,461,462,463,465,466,469,470,471,473,474,475,476,478,479,480,481,482,484,486,487,488,489,490,491,492,493,494,495,496,498,499,500,501,502,503,504,505,506,508,509,510,511,513,514,515,516,517,518,519,520,521,522,523,524,526,527,528,529,530,531,532,533,534,535,536,537,538,539,542,543,545,546,547,548,549,550,551,553,554,558,559,561,562,563,564,566,567,569,570,571,572,575,578,579,580,581,583,584,585,586,587,589,590,591,593,594,595,596,597,598,599,600,601,602,605,606,607,608,609,610,611,612,614,615,616,617,618,619,620,621,622,623,624,625,627,628,629,631,633,634,636,638,639,643,645,646,647,648,649,650,651,652,654,655,657,658,659,661,662,663,664,665,666,667,668,670,671,672,673,674,675,681,683,684,685,686,687,688,690,691,692,693,694,695,696,697,699,700,701,702,703,705,706,707,709,711,713,714,716,719,720,722,723,724,725,726,727,730,731,732,736,738,739,740,742,743,745,747,748,751,753,754,755,756,757,758,760,762,763,764,766,768,769,770,771,772,773,774,775,776,778,780,781,782,783,784,785,786,787,788,789,791,792,793,794,795,796,797,798,799,801,802,803,804,805,806,807,808,810,811,812,813,814,817,818,819,820,821,822,823,825,826,828,829,830,831,833,834,835,836,839,840,841,842,845,846,847,848,849,851,852,854,855,856,857,858,861,862,866,867,868,869,870,871,872,873,874,875,876,877,878,879,880,881,883,886,887,888,890,891,893,894,895,896,897,898,899,901,902,903,904,906,908,909,910,911,912,913,914,915,916,917,918,919,920,921,923,924,927,929,930,932,933,934,935,936,937,938,940,941,942,943,944,945,947,948,949,950,951,954,955,956,958,959,960,961,962,963,964,966,967,968,969,970,971,972,975,976,977,979,980,982,983,986,987,988,989,993,994,995,996,997,998,999,1001,1004,1006,1007,1008,1009,1010,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023 }; + const uint16_t g_unique_to_seed_12x12_p2[] = { 1,2,3,4,5,7,8,9,10,11,13,14,15,16,17,18,19,20,21,23,24,25,26,27,28,29,30,31,32,33,34,36,37,39,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,94,95,97,98,99,100,101,102,103,104,105,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,128,129,130,131,132,134,135,136,137,138,139,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,161,165,167,168,169,170,171,172,174,175,176,177,178,179,180,181,182,183,184,185,186,188,190,191,193,194,195,196,198,199,200,201,202,203,204,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,229,230,231,232,235,236,237,238,239,240,242,243,244,245,246,247,248,249,250,252,253,254,255,256,257,258,260,261,262,263,264,265,266,267,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,286,287,289,290,291,292,293,294,295,296,298,299,300,302,303,304,305,306,307,309,310,313,314,316,317,318,319,321,323,324,325,326,327,329,330,331,332,333,335,337,338,339,340,341,342,343,344,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,365,366,367,368,370,371,373,374,375,376,377,378,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,399,400,402,403,404,405,407,409,410,411,412,414,415,417,418,419,420,421,422,423,424,425,426,427,428,429,430,432,433,436,437,438,439,440,441,443,444,445,446,447,449,450,451,452,453,454,455,456,458,459,461,462,463,465,466,469,470,471,473,474,475,476,478,479,480,481,482,484,486,487,488,489,490,491,492,493,494,495,496,498,499,500,501,502,503,504,505,506,508,509,510,511,513,514,515,516,517,518,519,520,521,522,523,524,526,527,528,529,530,531,532,533,534,535,536,537,538,539,542,543,545,546,547,548,549,550,551,553,554,557,558,559,561,562,563,564,566,567,569,570,571,572,575,576,578,579,580,581,583,584,585,586,587,589,590,591,593,594,595,596,597,598,599,600,601,602,605,606,607,608,609,610,611,612,614,615,616,617,618,619,620,621,622,623,624,625,627,628,629,631,633,634,636,638,639,640,643,644,645,646,647,648,649,650,651,652,654,655,657,658,659,660,661,662,663,664,665,666,667,668,670,671,672,673,674,675,681,683,684,685,686,687,688,690,691,692,693,694,695,696,697,699,700,701,702,703,705,706,707,709,711,713,714,716,717,719,720,721,722,723,724,725,726,727,730,731,732,736,738,739,740,742,743,745,747,748,751,753,754,755,756,757,758,760,762,763,764,766,768,769,770,771,772,773,774,775,776,778,780,781,782,783,784,785,786,787,788,789,791,792,793,794,795,796,797,798,799,800,801,802,803,804,805,806,807,808,810,811,812,813,814,815,817,818,819,820,821,822,823,825,826,828,829,830,831,833,834,835,836,837,839,840,841,842,844,845,846,847,848,849,851,852,854,855,856,857,858,861,862,863,866,867,868,869,870,871,872,873,874,875,876,877,878,879,880,881,883,886,887,888,890,891,893,894,895,896,897,898,899,901,902,903,904,906,908,909,910,911,912,913,914,915,916,917,918,919,920,921,923,924,927,929,930,932,933,934,935,936,937,938,940,941,942,943,944,945,947,948,949,950,951,954,955,956,958,959,960,961,962,963,964,966,967,968,969,970,971,972,975,976,977,979,980,982,983,986,987,988,989,993,994,995,996,997,998,999,1001,1004,1006,1007,1008,1009,1010,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023 }; + + const uint16_t g_unique_to_seed_4x4_p3[] = { 0,3,8,11,14,15,17,26,29,30,31,32,33,36,38,43,44,47,49,51,55,56,57,59,67,70,74,76,79,81,82,88,89,90,100,104,108,110,111,117,122,126,127,132,133,134,135,139,147,150,151,152,156,157,163,166,167,168,171,175,176,179,181,182,183,186,189,192,199,203,205,207,210,214,216,230,236,247,249,250,252,254,260,262,263,266,272,273,276,291,292,294,297,302,309,310,313,314,318,319,324,327,328,330,331,335,337,346,355,356,357,358,363,365,368,378,381,384,386,388,390,391,392,397,398,401,410,411,417,419,427,431,437,439,440,446,451,455,457,458,459,460,462,464,467,468,471,472,474,475,477,479,483,487,488,493,495,496,497,502,503,504,511,512,516,518,519,523,525,530,532,538,543,544,546,547,549,550,551,553,554,562,567,568,570,571,578,579,581,582,588,589,590,593,594,600,601,606,611,613,623,624,625,630,637,638,645,646,648,650,651,658,659,662,666,669,670,678,683,686,688,691,694,696,699,700,701,703,704,707,713,715,717,719,722,724,725,727,730,731,735,738,739,745,750,751,758,759,760,766,775,776,779,783,784,785,786,787,788,798,799,802,804,805,807,808,809,812,821,822,823,825,827,831,835,837,838,842,844,845,846,848,853,854,858,859,860,866,884,888,892,894,898,902,906,907,915,918,922,923,925,927,931,932,937,938,940,943,945,953,955,958,959,963,971,974,977,979,989,990,998,1005,1006,1007,1011,1012,1015,1020,1023 }; + const uint16_t g_unique_to_seed_5x4_p3[] = { 0,3,7,8,11,12,14,15,17,18,26,29,30,31,32,33,34,36,38,39,43,44,47,49,51,55,56,57,59,62,63,67,70,74,76,79,81,82,88,89,90,91,100,103,104,108,110,111,117,122,123,126,127,132,133,134,135,136,139,144,147,150,151,152,156,157,158,163,166,167,168,171,173,175,176,179,181,182,183,186,189,192,199,203,205,207,210,214,216,222,230,236,246,247,249,250,252,254,259,260,262,263,266,269,272,273,274,275,276,291,292,293,294,297,302,306,309,310,311,313,314,315,318,319,324,327,328,330,331,335,337,346,355,356,357,358,359,363,365,368,377,378,381,384,386,388,390,391,392,394,397,398,401,407,410,411,417,419,427,430,431,437,439,440,446,451,455,457,458,459,460,462,464,467,468,470,471,472,474,475,477,478,479,483,485,487,488,493,495,496,497,501,502,503,504,506,508,510,511,512,515,516,518,519,521,523,524,525,530,532,538,541,543,544,546,547,549,550,551,552,553,554,562,567,568,570,571,577,578,579,581,582,588,589,590,593,594,595,600,601,603,606,609,611,613,623,624,625,630,632,637,638,639,645,646,648,650,651,654,658,659,662,666,669,670,678,679,683,685,686,688,691,694,696,699,700,701,703,704,707,713,715,717,719,722,724,725,727,730,731,732,735,738,739,742,745,746,749,750,751,758,759,760,766,769,773,775,776,779,783,784,785,786,787,788,791,793,798,799,802,804,805,806,807,808,809,812,813,821,822,823,825,827,831,835,837,838,839,842,844,845,846,848,853,854,858,859,860,866,873,876,877,884,887,888,892,894,898,902,906,907,914,915,918,919,922,923,925,927,931,932,937,938,940,943,944,945,951,953,955,958,959,963,971,972,974,977,979,982,983,989,990,991,998,999,1005,1006,1007,1010,1011,1012,1015,1020,1022,1023 }; + const uint16_t g_unique_to_seed_5x5_p3[] = { 0,3,7,8,10,11,12,14,15,17,18,26,27,29,30,31,32,33,34,36,38,39,43,44,47,48,49,50,51,55,56,57,59,60,61,62,63,67,70,72,74,76,79,81,82,88,89,90,91,94,100,103,104,106,108,110,111,115,117,122,123,126,127,128,130,132,133,134,135,136,139,144,147,150,151,152,156,157,158,162,163,166,167,168,169,171,173,175,176,179,181,182,183,186,189,192,199,203,205,207,209,210,214,216,220,222,227,230,235,236,246,247,249,250,252,254,257,259,260,262,263,266,269,272,273,274,275,276,279,282,291,292,293,294,295,297,302,306,309,310,311,313,314,315,318,319,324,326,327,328,330,331,335,337,342,345,346,353,355,356,357,358,359,363,364,365,368,371,374,377,378,381,384,386,387,388,390,391,392,394,397,398,399,401,407,410,411,417,419,427,430,431,437,438,439,440,443,446,451,455,456,457,458,459,460,462,463,464,466,467,468,470,471,472,474,475,477,478,479,480,482,483,485,487,488,493,495,496,497,501,502,503,504,506,508,510,511,512,515,516,518,519,521,522,523,524,525,530,532,538,539,541,543,544,546,547,549,550,551,552,553,554,555,562,567,568,570,571,577,578,579,581,582,586,588,589,590,593,594,595,600,601,602,603,606,609,610,611,613,618,623,624,625,626,630,632,637,638,639,645,646,648,650,651,654,658,659,662,666,667,668,669,670,671,678,679,683,685,686,687,688,691,694,696,698,699,700,701,703,704,707,708,713,715,717,719,722,724,725,727,730,731,732,734,735,738,739,742,745,746,747,748,749,750,751,753,758,759,760,764,766,767,769,771,773,775,776,779,780,781,783,784,785,786,787,788,791,793,794,798,799,800,802,804,805,806,807,808,809,811,812,813,821,822,823,825,827,831,835,837,838,839,840,842,843,844,845,846,847,848,850,852,853,854,858,859,860,866,869,873,874,876,877,881,884,886,887,888,892,894,897,898,902,905,906,907,914,915,918,919,920,922,923,925,927,931,932,937,938,940,943,944,945,951,953,954,955,958,959,963,971,972,973,974,977,978,979,982,983,989,990,991,992,998,999,1004,1005,1006,1007,1010,1011,1012,1015,1020,1022,1023 }; + const uint16_t g_unique_to_seed_6x5_p3[] = { 0,3,7,8,10,11,12,14,15,17,18,21,23,26,27,29,30,31,32,33,34,35,36,38,39,42,43,44,47,48,49,50,51,55,56,57,59,60,61,62,63,67,70,72,74,76,79,81,82,88,89,90,91,94,100,102,103,104,106,108,110,111,114,115,117,120,122,123,126,127,128,130,132,133,134,135,136,139,140,144,147,150,151,152,153,156,157,158,162,163,166,167,168,169,171,173,175,176,179,181,182,183,186,189,192,198,199,200,203,205,207,209,210,214,216,220,222,227,230,231,235,236,245,246,247,249,250,252,254,257,259,260,262,263,266,269,272,273,274,275,276,279,281,282,288,291,292,293,294,295,297,300,302,306,309,310,311,313,314,315,318,319,324,326,327,328,330,331,335,337,342,345,346,348,353,355,356,357,358,359,363,364,365,368,371,372,374,377,378,379,381,384,386,387,388,390,391,392,394,395,397,398,399,401,407,410,411,412,413,417,419,427,430,431,437,438,439,440,443,446,450,451,455,456,457,458,459,460,461,462,463,464,465,466,467,468,470,471,472,474,475,477,478,479,480,482,483,485,487,488,493,495,496,497,500,501,502,503,504,505,506,508,510,511,512,513,515,516,518,519,521,522,523,524,525,527,530,532,538,539,541,543,544,546,547,549,550,551,552,553,554,555,557,558,562,566,567,568,570,571,577,578,579,580,581,582,584,586,588,589,590,593,594,595,600,601,602,603,606,609,610,611,613,614,618,623,624,625,626,630,632,637,638,639,644,645,646,648,650,651,654,658,659,662,666,667,668,669,670,671,678,679,683,685,686,687,688,689,691,694,696,698,699,700,701,703,704,707,708,711,713,715,717,719,722,724,725,727,730,731,732,734,735,738,739,742,743,745,746,747,748,749,750,751,753,758,759,760,764,766,767,769,771,773,775,776,779,780,781,783,784,785,786,787,788,791,793,794,798,799,800,802,804,805,806,807,808,809,810,811,812,813,821,822,823,824,825,827,831,835,836,837,838,839,840,841,842,843,844,845,846,847,848,850,852,853,854,858,859,860,866,869,873,874,876,877,881,884,886,887,888,892,894,897,898,900,902,905,906,907,914,915,918,919,920,922,923,925,927,931,932,937,938,940,943,944,945,951,953,954,955,957,958,959,963,967,971,972,973,974,977,978,979,982,983,986,989,990,991,992,998,999,1003,1004,1005,1006,1007,1010,1011,1012,1015,1020,1022,1023 }; + + const uint16_t g_unique_to_seed_6x6_p3[] = { 0,8,11,14,15,17,18,19,26,31,34,35,36,38,44,47,48,49,51,56,59,61,70,74,76,82,88,90,96,100,103,104,108,110,111,117,122,123,126,127,132,133,135,139,147,150,151,152,156,157,163,166,168,171,175,176,179,181,182,183,186,189,192,199,203,205,207,210,214,216,222,247,249,250,252,254,260,261,262,263,266,272,273,275,276,288,291,292,293,294,297,302,309,310,313,314,318,327,328,331,335,337,346,356,357,358,363,365,368,378,381,384,386,390,391,392,396,397,398,399,401,410,411,419,427,430,431,437,439,440,451,455,457,458,459,460,462,468,470,471,472,474,475,477,479,482,483,488,493,495,496,502,503,504,507,510,511,512,515,516,518,519,522,523,525,526,527,538,543,544,546,547,549,550,552,553,554,562,570,578,579,581,582,588,589,590,593,595,600,606,611,613,618,623,625,632,637,638,645,646,650,651,658,659,662,666,667,669,670,678,679,685,686,687,688,691,694,696,698,699,700,701,703,704,707,713,714,715,717,719,722,724,727,730,731,734,738,739,743,747,748,750,751,753,758,760,764,766,769,775,776,783,784,785,787,791,793,798,799,802,804,805,806,807,808,809,810,813,822,823,825,831,835,837,838,839,840,842,845,846,848,853,854,858,859,860,866,874,882,884,887,888,892,894,898,902,907,914,915,918,919,922,923,925,927,931,932,937,938,940,943,944,945,953,955,958,959,963,966,971,974,979,990,991,998,999,1007,1010,1011,1012,1015,1020,1023 }; + const uint16_t g_unique_to_seed_8x5_p3[] = { 0,3,8,11,14,15,17,18,19,23,26,27,29,31,33,34,35,36,38,43,44,47,48,49,51,55,56,59,61,67,70,76,79,81,82,88,89,90,96,100,103,104,108,110,111,117,122,123,126,127,132,133,134,135,139,147,150,151,152,156,157,163,166,167,168,171,173,175,176,179,181,182,183,186,189,192,199,203,205,207,210,214,216,227,230,247,249,250,254,260,261,262,263,266,272,273,275,276,279,288,291,292,293,294,297,302,307,309,310,313,314,315,318,319,327,328,331,335,337,346,355,356,357,358,359,363,365,377,378,381,384,386,390,391,392,394,396,397,398,399,401,407,410,411,419,424,427,430,431,437,439,440,450,451,455,457,458,459,460,462,464,467,468,470,471,472,474,475,477,478,479,482,483,487,488,493,495,496,502,503,504,507,508,511,512,515,516,518,519,522,523,524,526,527,538,543,544,547,549,550,552,553,554,557,562,568,570,578,579,581,582,588,589,590,593,595,600,602,603,606,609,611,613,614,624,625,632,637,638,639,645,646,650,651,658,659,662,666,667,669,670,678,679,685,686,687,688,689,691,694,696,699,700,701,703,704,707,712,713,715,717,719,722,724,727,730,731,734,738,739,743,745,747,750,751,758,759,760,763,764,766,769,771,775,776,779,781,783,784,785,787,791,793,798,799,802,804,805,806,807,809,810,812,813,822,823,825,831,835,837,838,840,842,844,845,846,848,853,854,858,859,860,866,873,876,882,884,887,888,892,894,895,898,902,906,907,914,915,918,919,922,923,925,927,931,932,937,938,940,943,944,945,947,951,953,955,958,959,963,966,971,974,977,979,983,989,990,991,998,999,1005,1007,1010,1011,1012,1015,1023 }; + const uint16_t g_unique_to_seed_8x6_p3[] = { 0,3,8,11,14,15,17,18,19,23,26,27,29,31,33,34,35,36,38,43,44,47,48,49,51,55,56,59,61,67,70,74,76,79,81,82,88,89,90,96,100,103,104,108,110,111,117,122,123,126,127,131,132,133,134,135,139,147,150,151,152,156,157,163,166,167,168,171,173,175,176,179,181,182,183,186,189,192,199,203,205,207,210,214,216,222,227,230,236,247,249,250,252,254,260,261,262,263,266,272,273,275,276,279,288,291,292,293,294,297,302,307,309,310,313,314,315,318,319,324,327,328,331,335,337,338,346,355,356,357,358,359,363,365,368,377,378,381,384,386,390,391,392,394,396,397,398,399,401,407,410,411,419,424,427,430,431,437,439,440,450,451,455,457,458,459,460,462,464,467,468,470,471,472,474,475,477,478,479,482,483,485,487,488,493,495,496,502,503,504,507,508,510,511,512,515,516,518,519,522,523,524,525,526,527,538,541,543,544,546,547,549,550,552,553,554,557,562,566,567,568,570,578,579,581,582,588,589,590,593,595,600,601,602,603,606,609,611,613,614,618,623,624,625,632,637,638,639,645,646,650,651,658,659,662,666,667,669,670,678,679,685,686,687,688,689,691,694,696,698,699,700,701,703,704,707,708,712,713,714,715,717,719,722,724,725,727,730,731,732,734,738,739,743,745,747,748,750,751,753,758,759,760,763,764,766,769,771,775,776,779,781,783,784,785,786,787,791,793,798,799,802,804,805,806,807,808,809,810,812,813,822,823,825,831,835,837,838,839,840,842,844,845,846,848,850,853,854,858,859,860,866,873,874,876,882,884,887,888,892,894,895,898,900,902,906,907,914,915,918,919,922,923,925,927,931,932,937,938,940,943,944,945,947,951,953,955,958,959,963,966,971,974,977,979,983,989,990,991,998,999,1005,1007,1010,1011,1012,1015,1020,1022,1023 }; + const uint16_t g_unique_to_seed_10x5_p3[] = { 0,3,7,8,11,14,15,17,18,19,23,26,27,29,31,33,34,35,36,38,43,44,47,48,49,51,55,56,59,61,62,67,70,72,76,79,81,82,88,89,90,91,95,96,100,103,104,108,110,111,114,117,122,123,126,127,131,132,133,134,135,139,140,147,150,151,152,156,157,158,163,166,167,168,171,173,175,176,179,181,182,183,186,189,192,199,203,205,207,210,213,214,216,227,230,245,247,249,250,254,259,260,261,262,263,266,269,272,273,274,275,276,279,281,288,291,292,293,294,295,297,302,307,309,310,313,314,315,318,319,327,328,331,335,337,346,355,356,357,358,359,363,365,377,378,381,384,386,390,391,392,394,396,397,398,399,401,407,410,411,412,413,419,424,427,430,431,437,439,440,450,451,455,457,458,459,460,462,464,467,468,470,471,472,474,475,477,478,479,482,483,487,488,493,495,496,500,501,502,503,504,506,507,508,510,511,512,515,516,518,519,521,522,523,524,526,527,530,538,541,543,544,547,549,550,552,553,554,555,557,562,565,568,570,577,578,579,581,582,588,589,590,593,595,600,601,602,603,606,609,611,613,614,618,624,625,632,637,638,639,645,646,650,651,654,658,659,662,666,667,669,670,678,679,685,686,687,688,689,691,694,695,696,698,699,700,701,703,704,707,712,713,715,717,719,722,724,725,727,730,731,732,734,738,739,742,743,745,747,749,750,751,758,759,760,763,764,765,766,769,771,773,775,776,779,781,783,784,785,786,787,791,793,798,799,802,804,805,806,807,809,810,812,813,821,822,823,825,827,831,835,836,837,838,839,840,841,842,844,845,846,848,853,854,858,859,860,866,869,873,876,877,882,884,887,888,891,892,894,895,898,900,902,905,906,907,909,914,915,918,919,922,923,925,927,931,932,937,938,939,940,943,944,945,947,951,953,954,955,957,958,959,961,963,966,967,971,974,975,977,978,979,983,989,990,991,993,998,999,1005,1007,1010,1011,1012,1015,1023 }; + + const uint16_t g_unique_to_seed_10x6_p3[] = { 0,3,7,8,11,12,14,15,17,18,19,23,26,27,29,31,33,34,35,36,38,43,44,47,48,49,51,55,56,59,61,62,67,70,72,74,76,79,81,82,88,89,90,91,95,96,100,103,104,108,110,111,114,117,122,123,126,127,131,132,133,134,135,139,140,147,150,151,152,156,157,158,163,166,167,168,171,173,175,176,179,181,182,183,186,189,192,199,203,205,207,210,213,214,216,222,227,230,236,245,246,247,249,250,252,254,259,260,261,262,263,266,269,272,273,274,275,276,279,281,288,291,292,293,294,295,297,302,306,307,309,310,311,313,314,315,318,319,324,327,328,330,331,335,337,338,346,355,356,357,358,359,363,364,365,368,377,378,381,384,386,390,391,392,394,396,397,398,399,401,407,410,411,412,413,419,424,427,430,431,437,439,440,450,451,455,457,458,459,460,462,464,467,468,470,471,472,474,475,477,478,479,482,483,485,487,488,493,495,496,500,501,502,503,504,506,507,508,510,511,512,515,516,518,519,521,522,523,524,525,526,527,530,538,539,541,543,544,546,547,549,550,552,553,554,555,557,562,565,566,567,568,570,577,578,579,581,582,588,589,590,593,595,600,601,602,603,606,609,611,613,614,618,623,624,625,632,637,638,639,645,646,648,650,651,654,658,659,662,666,667,669,670,678,679,685,686,687,688,689,691,694,695,696,698,699,700,701,703,704,707,708,712,713,714,715,717,719,722,724,725,727,730,731,732,734,735,738,739,742,743,745,747,748,749,750,751,753,758,759,760,763,764,765,766,769,771,773,775,776,779,781,783,784,785,786,787,791,793,798,799,802,804,805,806,807,808,809,810,812,813,821,822,823,825,827,831,835,836,837,838,839,840,841,842,844,845,846,848,850,853,854,858,859,860,866,869,873,874,876,877,882,884,887,888,891,892,894,895,898,900,902,905,906,907,909,914,915,918,919,922,923,925,927,931,932,937,938,939,940,943,944,945,947,951,953,954,955,957,958,959,961,963,966,967,971,974,975,977,978,979,982,983,989,990,991,993,998,999,1005,1007,1010,1011,1012,1015,1020,1022,1023 }; + const uint16_t g_unique_to_seed_8x8_p3[] = { 0,3,7,8,11,12,14,15,17,18,19,23,26,27,29,30,31,32,33,34,35,36,38,39,43,44,47,48,49,50,51,55,56,57,59,60,61,63,67,70,72,74,76,79,81,82,88,89,90,96,100,103,104,106,108,110,111,117,122,123,126,127,131,132,133,134,135,136,139,144,147,150,151,152,156,157,158,163,166,167,168,171,173,175,176,178,179,181,182,183,186,189,192,199,203,205,207,210,214,216,222,227,230,235,236,246,247,249,250,252,254,260,261,262,263,266,269,272,273,275,276,279,288,291,292,293,294,295,297,302,306,307,309,310,311,313,314,315,318,319,324,327,328,330,331,335,337,338,342,345,346,355,356,357,358,359,363,365,368,371,377,378,381,384,386,388,390,391,392,394,396,397,398,399,401,407,410,411,417,419,424,427,430,431,437,439,440,446,450,451,455,457,458,459,460,462,464,466,467,468,470,471,472,474,475,477,478,479,480,482,483,485,487,488,493,495,496,497,502,503,504,507,508,510,511,512,515,516,518,519,521,522,523,524,525,526,527,530,532,538,539,541,543,544,546,547,549,550,551,552,553,554,557,562,566,567,568,570,571,577,578,579,581,582,586,588,589,590,592,593,594,595,600,601,602,603,606,609,610,611,613,614,618,623,624,625,630,632,637,638,639,645,646,648,650,651,658,659,662,666,667,669,670,671,678,679,683,685,686,687,688,689,691,694,696,698,699,700,701,703,704,707,708,712,713,714,715,717,719,722,724,725,727,730,731,732,734,735,738,739,743,745,746,747,748,750,751,753,758,759,760,763,764,766,767,769,771,773,775,776,779,780,781,783,784,785,786,787,788,791,793,794,798,799,802,804,805,806,807,808,809,810,811,812,813,821,822,823,825,827,831,835,837,838,839,840,842,844,845,846,847,848,850,852,853,854,858,859,860,866,873,874,876,877,882,884,886,887,888,892,894,895,897,898,900,902,906,907,914,915,918,919,920,922,923,925,927,931,932,937,938,940,943,944,945,947,951,953,954,955,958,959,963,966,971,972,974,977,979,982,983,989,990,991,998,999,1005,1006,1007,1010,1011,1012,1015,1020,1022,1023 }; + const uint16_t g_unique_to_seed_10x8_p3[] = { 0,3,7,8,11,12,14,15,17,18,19,23,26,27,29,30,31,32,33,34,35,36,38,39,43,44,47,48,49,50,51,55,56,57,59,60,61,62,63,67,70,72,74,76,79,81,82,88,89,90,91,94,95,96,100,103,104,106,108,110,111,114,115,117,122,123,126,127,131,132,133,134,135,136,139,140,144,147,150,151,152,153,156,157,158,163,166,167,168,171,173,175,176,178,179,181,182,183,186,189,192,198,199,203,205,207,210,213,214,216,220,222,227,230,235,236,245,246,247,249,250,252,254,259,260,261,262,263,266,269,272,273,274,275,276,279,281,288,291,292,293,294,295,297,302,306,307,309,310,311,313,314,315,318,319,324,327,328,330,331,335,337,338,342,345,346,355,356,357,358,359,363,364,365,368,371,374,377,378,379,381,384,386,387,388,390,391,392,394,395,396,397,398,399,401,407,410,411,412,413,417,419,424,427,430,431,437,438,439,440,443,446,450,451,455,457,458,459,460,462,464,466,467,468,470,471,472,474,475,477,478,479,480,482,483,485,487,488,493,495,496,497,500,501,502,503,504,505,506,507,508,510,511,512,515,516,518,519,521,522,523,524,525,526,527,530,532,538,539,541,543,544,546,547,549,550,551,552,553,554,555,557,562,565,566,567,568,570,571,577,578,579,581,582,586,588,589,590,592,593,594,595,600,601,602,603,606,609,610,611,613,614,618,623,624,625,630,632,637,638,639,644,645,646,648,650,651,654,658,659,662,666,667,669,670,671,678,679,683,685,686,687,688,689,691,694,695,696,698,699,700,701,703,704,707,708,712,713,714,715,717,719,722,724,725,727,730,731,732,734,735,738,739,742,743,745,746,747,748,749,750,751,753,758,759,760,763,764,765,766,767,769,771,773,775,776,779,780,781,783,784,785,786,787,788,791,793,794,798,799,800,802,804,805,806,807,808,809,810,811,812,813,821,822,823,825,827,831,835,836,837,838,839,840,841,842,844,845,846,847,848,850,852,853,854,858,859,860,866,869,873,874,876,877,882,884,886,887,888,891,892,894,895,897,898,900,902,905,906,907,909,914,915,918,919,920,922,923,925,927,931,932,937,938,939,940,943,944,945,947,951,953,954,955,957,958,959,961,963,966,967,971,972,973,974,975,977,978,979,982,983,986,989,990,991,993,998,999,1005,1006,1007,1010,1011,1012,1015,1020,1022,1023}; + const uint16_t g_unique_to_seed_10x10_p3[] = { 0,3,7,8,10,11,12,14,15,17,18,19,23,26,27,29,30,31,32,33,34,35,36,38,39,40,43,44,47,48,49,50,51,55,56,57,59,60,61,62,63,67,70,72,74,75,76,79,81,82,88,89,90,91,94,95,96,100,103,104,106,108,110,111,114,115,117,120,122,123,126,127,128,130,131,132,133,134,135,136,139,140,144,147,150,151,152,153,156,157,158,162,163,166,167,168,169,171,173,175,176,178,179,181,182,183,186,189,192,198,199,200,203,205,207,209,210,213,214,216,218,220,222,227,230,235,236,238,242,245,246,247,249,250,252,254,257,259,260,261,262,263,266,269,272,273,274,275,276,279,281,282,288,291,292,293,294,295,297,302,306,307,308,309,310,311,313,314,315,318,319,324,326,327,328,330,331,335,337,338,342,345,346,347,350,353,355,356,357,358,359,363,364,365,368,371,372,374,377,378,379,381,384,386,387,388,390,391,392,394,395,396,397,398,399,401,407,408,410,411,412,413,417,419,424,427,430,431,435,437,438,439,440,443,446,450,451,455,456,457,458,459,460,462,463,464,466,467,468,470,471,472,474,475,477,478,479,480,482,483,485,487,488,493,495,496,497,500,501,502,503,504,505,506,507,508,510,511,512,513,515,516,518,519,521,522,523,524,525,526,527,530,532,538,539,541,543,544,546,547,549,550,551,552,553,554,555,557,562,565,566,567,568,570,571,577,578,579,580,581,582,586,588,589,590,592,593,594,595,600,601,602,603,606,609,610,611,613,614,618,623,624,625,626,630,632,634,637,638,639,644,645,646,648,650,651,654,658,659,662,666,667,668,669,670,671,678,679,683,685,686,687,688,689,691,694,695,696,698,699,700,701,703,704,707,708,712,713,714,715,717,719,722,724,725,727,730,731,732,734,735,738,739,742,743,745,746,747,748,749,750,751,753,758,759,760,763,764,765,766,767,769,771,773,775,776,779,780,781,783,784,785,786,787,788,789,790,791,793,794,798,799,800,802,804,805,806,807,808,809,810,811,812,813,821,822,823,825,827,831,835,836,837,838,839,840,841,842,843,844,845,846,847,848,850,852,853,854,858,859,860,866,869,873,874,876,877,881,882,884,886,887,888,891,892,894,895,897,898,900,902,905,906,907,909,914,915,918,919,920,922,923,925,927,931,932,937,938,939,940,943,944,945,947,951,952,953,954,955,957,958,959,961,963,966,967,971,972,973,974,975,977,978,979,980,982,983,986,989,990,991,992,993,998,999,1003,1004,1005,1006,1007,1010,1011,1012,1014,1015,1020,1022,1023 }; + + const uint16_t g_unique_to_seed_12x10_p3[] = { 0,3,7,8,10,11,12,14,15,16,17,18,19,21,23,26,27,29,30,31,32,33,34,35,36,38,39,40,42,43,44,45,47,48,49,50,51,55,56,57,59,60,61,62,63,67,70,72,74,75,76,79,81,82,88,89,90,91,94,95,96,100,102,103,104,106,108,110,111,114,115,117,120,122,123,126,127,128,129,130,131,132,133,134,135,136,139,140,144,147,150,151,152,153,156,157,158,161,162,163,166,167,168,169,171,173,175,176,178,179,181,182,183,185,186,189,192,195,198,199,200,203,205,207,209,210,213,214,216,218,220,222,227,230,231,235,236,238,242,245,246,247,249,250,251,252,254,257,259,260,261,262,263,266,269,272,273,274,275,276,279,281,282,283,288,291,292,293,294,295,297,300,302,306,307,308,309,310,311,313,314,315,318,319,324,326,327,328,330,331,335,337,338,342,345,346,347,348,350,353,355,356,357,358,359,363,364,365,368,371,372,374,377,378,379,381,384,386,387,388,390,391,392,394,395,396,397,398,399,401,407,408,410,411,412,413,415,417,419,424,427,430,431,435,437,438,439,440,443,446,450,451,455,456,457,458,459,460,461,462,463,464,465,466,467,468,470,471,472,474,475,477,478,479,480,482,483,485,487,488,493,495,496,497,500,501,502,503,504,505,506,507,508,510,511,512,513,515,516,518,519,521,522,523,524,525,526,527,530,532,538,539,541,543,544,546,547,549,550,551,552,553,554,555,557,558,562,563,565,566,567,568,570,571,577,578,579,580,581,582,584,586,588,589,590,592,593,594,595,600,601,602,603,604,606,609,610,611,613,614,618,623,624,625,626,630,632,634,637,638,639,643,644,645,646,648,650,651,654,658,659,662,666,667,668,669,670,671,673,678,679,683,685,686,687,688,689,691,694,695,696,698,699,700,701,703,704,707,708,711,712,713,714,715,717,719,722,724,725,727,730,731,732,734,735,738,739,742,743,745,746,747,748,749,750,751,753,758,759,760,763,764,765,766,767,769,771,773,775,776,779,780,781,783,784,785,786,787,788,789,790,791,793,794,798,799,800,802,804,805,806,807,808,809,810,811,812,813,821,822,823,824,825,827,828,831,835,836,837,838,839,840,841,842,843,844,845,846,847,848,850,852,853,854,858,859,860,866,869,873,874,876,877,881,882,884,886,887,888,891,892,894,895,897,898,900,902,905,906,907,909,914,915,918,919,920,922,923,925,927,931,932,937,938,939,940,943,944,945,947,951,952,953,954,955,957,958,959,961,963,966,967,971,972,973,974,975,977,978,979,980,982,983,986,989,990,991,992,993,995,998,999,1002,1003,1004,1005,1006,1007,1010,1011,1012,1014,1015,1020,1021,1022,1023 }; + const uint16_t g_unique_to_seed_12x12_p3[] = { 0,3,4,7,8,10,11,12,14,15,16,17,18,19,21,23,26,27,29,30,31,32,33,34,35,36,38,39,40,42,43,44,45,47,48,49,50,51,53,55,56,57,58,59,60,61,62,63,67,70,72,74,75,76,79,81,82,83,88,89,90,91,94,95,96,100,102,103,104,106,108,110,111,114,115,117,120,122,123,126,127,128,129,130,131,132,133,134,135,136,138,139,140,144,147,150,151,152,153,156,157,158,159,160,161,162,163,166,167,168,169,171,173,175,176,177,178,179,181,182,183,185,186,189,192,195,196,198,199,200,203,205,207,208,209,210,213,214,216,218,220,222,227,230,231,235,236,238,242,245,246,247,249,250,251,252,254,257,259,260,261,262,263,266,269,272,273,274,275,276,279,281,282,283,288,291,292,293,294,295,297,300,302,306,307,308,309,310,311,313,314,315,318,319,324,326,327,328,330,331,335,337,338,342,345,346,347,348,350,353,355,356,357,358,359,363,364,365,368,371,372,374,377,378,379,381,384,386,387,388,390,391,392,394,395,396,397,398,399,401,407,408,410,411,412,413,415,417,419,424,426,427,430,431,432,435,437,438,439,440,443,444,446,450,451,455,456,457,458,459,460,461,462,463,464,465,466,467,468,470,471,472,474,475,477,478,479,480,482,483,485,487,488,493,495,496,497,500,501,502,503,504,505,506,507,508,510,511,512,513,515,516,518,519,521,522,523,524,525,526,527,530,532,535,538,539,540,541,543,544,546,547,549,550,551,552,553,554,555,557,558,562,563,565,566,567,568,569,570,571,577,578,579,580,581,582,584,586,588,589,590,592,593,594,595,600,601,602,603,604,606,609,610,611,613,614,618,623,624,625,626,628,630,631,632,634,636,637,638,639,640,643,644,645,646,648,650,651,654,658,659,662,666,667,668,669,670,671,673,678,679,683,685,686,687,688,689,691,694,695,696,698,699,700,701,703,704,707,708,711,712,713,714,715,717,719,722,724,725,727,730,731,732,734,735,738,739,742,743,745,746,747,748,749,750,751,753,758,759,760,763,764,765,766,767,768,769,771,773,774,775,776,778,779,780,781,783,784,785,786,787,788,789,790,791,793,794,798,799,800,802,804,805,806,807,808,809,810,811,812,813,821,822,823,824,825,827,828,831,835,836,837,838,839,840,841,842,843,844,845,846,847,848,850,852,853,854,858,859,860,863,866,869,873,874,876,877,881,882,884,886,887,888,891,892,894,895,897,898,900,902,905,906,907,909,911,912,914,915,918,919,920,922,923,925,927,929,930,931,932,937,938,939,940,943,944,945,947,951,952,953,954,955,957,958,959,961,963,966,967,971,972,973,974,975,977,978,979,980,982,983,986,989,990,991,992,993,995,998,999,1000,1002,1003,1004,1005,1006,1007,1010,1011,1012,1014,1015,1020,1021,1022,1023 }; + + static const uint16_t* g_unique_index_to_astc_part_seed[2][astc_helpers::NUM_ASTC_BLOCK_SIZES] = // [num_parts][astc_block_size_index] + { + { + g_unique_to_seed_4x4_p2, g_unique_to_seed_5x4_p2, g_unique_to_seed_5x5_p2, g_unique_to_seed_6x5_p2, + g_unique_to_seed_6x6_p2, g_unique_to_seed_8x5_p2, g_unique_to_seed_8x6_p2, g_unique_to_seed_10x5_p2, + g_unique_to_seed_10x6_p2, g_unique_to_seed_8x8_p2, g_unique_to_seed_10x8_p2, g_unique_to_seed_10x10_p2, + g_unique_to_seed_12x10_p2, g_unique_to_seed_12x12_p2 + }, + { + g_unique_to_seed_4x4_p3, g_unique_to_seed_5x4_p3, g_unique_to_seed_5x5_p3, g_unique_to_seed_6x5_p3, + g_unique_to_seed_6x6_p3, g_unique_to_seed_8x5_p3, g_unique_to_seed_8x6_p3, g_unique_to_seed_10x5_p3, + g_unique_to_seed_10x6_p3, g_unique_to_seed_8x8_p3, g_unique_to_seed_10x8_p3, g_unique_to_seed_10x10_p3, + g_unique_to_seed_12x10_p3, g_unique_to_seed_12x12_p3 + } + }; + + static inline uint16_t unique_pat_index_to_part_seed(uint32_t astc_block_size_index, uint32_t num_parts, uint32_t unique_pat_index) + { + assert(astc_block_size_index < astc_helpers::NUM_ASTC_BLOCK_SIZES); + assert((num_parts >= 2) && (num_parts <= 3)); + assert(unique_pat_index < get_total_unique_patterns(astc_block_size_index, num_parts)); + + return g_unique_index_to_astc_part_seed[num_parts - 2][astc_block_size_index][unique_pat_index]; + } + + static bool zstd_decompress(const void *pComp_data, size_t comp_size, basisu::uint8_vec &uncomp_data) + { + if (!comp_size) + { + uncomp_data.resize(0); + return true; + } + +#if BASISD_SUPPORT_KTX2_ZSTD + const uint64_t decomp_size = ZSTD_getFrameContentSize(pComp_data, comp_size); + + if ((decomp_size == ZSTD_CONTENTSIZE_UNKNOWN) || (decomp_size == ZSTD_CONTENTSIZE_ERROR)) + { + BASISU_DEVEL_ERROR("zstd_decompress: ZSTD_getFrameContentSize failed\n"); + return false; + } + + // sanity check, not UINT32_MAX purposely, even INT_MAX is too high + if (decomp_size > (uint64_t)INT32_MAX) + { + BASISU_DEVEL_ERROR("zstd_decompress: decompressed size too large\n"); + return false; + } + + if (!uncomp_data.try_resize((size_t)decomp_size)) + { + BASISU_DEVEL_ERROR("zstd_decompress: Out of memory\n"); + return false; + } + + if (!decomp_size) + return true; + + const size_t actual_uncomp_size = ZSTD_decompress(uncomp_data.data(), uncomp_data.size(), pComp_data, comp_size); + if (ZSTD_isError(actual_uncomp_size)) + { + BASISU_DEVEL_ERROR("zstd_decompress: Zstd decompression failed, file is invalid or corrupted\n"); + return false; + } + + assert(actual_uncomp_size == decomp_size); + uncomp_data.resize(actual_uncomp_size); + + return true; +#else + BASISU_NOTE_UNUSED(pComp_data); + BASISU_DEVEL_ERROR("zstd_decompress: file uses ZStd compression, but ZStd support disabled (see BASISD_SUPPORT_KTX2_ZSTD) \n"); + return false; +#endif + } + + static bool zstd_decompress_and_advance(const uint8_t* &pComp_data, size_t comp_size, basisu::uint8_vec& uncomp_data, simplified_bitwise_decoder& dec) + { + if (!zstd_decompress(pComp_data, comp_size, uncomp_data)) + return false; + pComp_data += comp_size; + dec.init(uncomp_data.data(), uncomp_data.size()); + return true; + } + + bool xuastc_ldr_decompress_image_full_zstd( + const uint8_t* pComp_data_all, size_t comp_data_size_all, + uint32_t& astc_block_width, uint32_t& astc_block_height, + uint32_t& actual_width, uint32_t& actual_height, bool& has_alpha, bool& uses_srgb_astc_decode_mode, + bool debug_output, + xuastc_decomp_image_init_callback_ptr pInit_callback, void* pInit_callback_data, + xuastc_decomp_image_block_callback_ptr pBlock_callback, void* pBlock_callback_data) + { + if (comp_data_size_all < sizeof(xuastc_ldr_full_zstd_header)) + { + BASISU_DEVEL_ERROR("Compressed file is too small\n"); + return false; + } + + const xuastc_ldr_full_zstd_header* pHdr = (const xuastc_ldr_full_zstd_header*)pComp_data_all; + + if ((!pHdr->m_raw_bits_len) || (!pHdr->m_mode_bytes_len)) + { + BASISU_DEVEL_ERROR("Compressed file is too small\n"); + return false; + } + + const uint64_t total_comp_size = (uint64_t)((uint32_t)pHdr->m_raw_bits_len) + + pHdr->m_mode_bytes_len + pHdr->m_solid_dpcm_bytes_len + pHdr->m_endpoint_dpcm_reuse_indices_len + pHdr->m_use_bc_bits_len + + pHdr->m_endpoint_dpcm_3bit_len + pHdr->m_endpoint_dpcm_4bit_len + pHdr->m_endpoint_dpcm_5bit_len + pHdr->m_endpoint_dpcm_6bit_len + pHdr->m_endpoint_dpcm_7bit_len + pHdr->m_endpoint_dpcm_8bit_len + + pHdr->m_mean0_bits_len + pHdr->m_mean1_bytes_len + + pHdr->m_run_bytes_len + pHdr->m_coeff_bytes_len + pHdr->m_sign_bits_len + + pHdr->m_weight2_bits_len + pHdr->m_weight3_bits_len + pHdr->m_weight4_bits_len + pHdr->m_weight8_bytes_len; + + if (comp_data_size_all < (sizeof(xuastc_ldr_full_zstd_header) + total_comp_size)) + { + BASISU_DEVEL_ERROR("Compressed file is too small\n"); + return false; + } + + bitwise_decoder raw_bits; + simplified_bitwise_decoder comp_mode_dec, solid_dpcm_dec, endpoint_dpcm_reuse_indices_dec, use_bc_bits_dec; + simplified_bitwise_decoder endpoint_dpcm_3bit_dec, endpoint_dpcm_4bit_dec, endpoint_dpcm_5bit_dec, endpoint_dpcm_6bit_dec, endpoint_dpcm_7bit_dec, endpoint_dpcm_8bit_dec; + + basisu::uint8_vec uncomp_mode_bytes, uncomp_solid_dpcm_bytes, uncomp_endpoint_dpcm_reuse_indices, uncomp_use_bc_bits; + basisu::uint8_vec uncomp_endpoint_dpcm_3bit, uncomp_endpoint_dpcm_4bit, uncomp_endpoint_dpcm_5bit, uncomp_endpoint_dpcm_6bit, uncomp_endpoint_dpcm_7bit, uncomp_endpoint_dpcm_8bit; + + basisu::uint8_vec uncomp_mean0_bits, uncomp_mean1_bytes, uncomp_run_bytes, uncomp_coeff_bytes, uncomp_weight2_bytes, uncomp_weight3_bytes, uncomp_weight4_bytes, uncomp_weight8_bytes; + simplified_bitwise_decoder mean0_bits, mean1_bytes, run_bytes, coeff_bytes, sign_bits, weight2_bits, weight3_bits, weight4_bits, weight8_bytes; + + const uint8_t* pCur_buf = pComp_data_all + sizeof(xuastc_ldr_full_zstd_header); + + // raw bits + { + raw_bits.init(pCur_buf, pHdr->m_raw_bits_len); + pCur_buf += pHdr->m_raw_bits_len; + } + + if (!zstd_decompress_and_advance(pCur_buf, pHdr->m_mode_bytes_len, uncomp_mode_bytes, comp_mode_dec)) + return false; + if (!zstd_decompress_and_advance(pCur_buf, pHdr->m_solid_dpcm_bytes_len, uncomp_solid_dpcm_bytes, solid_dpcm_dec)) + return false; + if (!zstd_decompress_and_advance(pCur_buf, pHdr->m_endpoint_dpcm_reuse_indices_len, uncomp_endpoint_dpcm_reuse_indices, endpoint_dpcm_reuse_indices_dec)) + return false; + if (!zstd_decompress_and_advance(pCur_buf, pHdr->m_use_bc_bits_len, uncomp_use_bc_bits, use_bc_bits_dec)) + return false; + if (!zstd_decompress_and_advance(pCur_buf, pHdr->m_endpoint_dpcm_3bit_len, uncomp_endpoint_dpcm_3bit, endpoint_dpcm_3bit_dec)) + return false; + if (!zstd_decompress_and_advance(pCur_buf, pHdr->m_endpoint_dpcm_4bit_len, uncomp_endpoint_dpcm_4bit, endpoint_dpcm_4bit_dec)) + return false; + if (!zstd_decompress_and_advance(pCur_buf, pHdr->m_endpoint_dpcm_5bit_len, uncomp_endpoint_dpcm_5bit, endpoint_dpcm_5bit_dec)) + return false; + if (!zstd_decompress_and_advance(pCur_buf, pHdr->m_endpoint_dpcm_6bit_len, uncomp_endpoint_dpcm_6bit, endpoint_dpcm_6bit_dec)) + return false; + if (!zstd_decompress_and_advance(pCur_buf, pHdr->m_endpoint_dpcm_7bit_len, uncomp_endpoint_dpcm_7bit, endpoint_dpcm_7bit_dec)) + return false; + if (!zstd_decompress_and_advance(pCur_buf, pHdr->m_endpoint_dpcm_8bit_len, uncomp_endpoint_dpcm_8bit, endpoint_dpcm_8bit_dec)) + return false; + + if (!zstd_decompress_and_advance(pCur_buf, pHdr->m_mean0_bits_len, uncomp_mean0_bits, mean0_bits)) + return false; + if (!zstd_decompress_and_advance(pCur_buf, pHdr->m_mean1_bytes_len, uncomp_mean1_bytes, mean1_bytes)) + return false; + if (!zstd_decompress_and_advance(pCur_buf, pHdr->m_run_bytes_len, uncomp_run_bytes, run_bytes)) + return false; + if (!zstd_decompress_and_advance(pCur_buf, pHdr->m_coeff_bytes_len, uncomp_coeff_bytes, coeff_bytes)) + return false; + + // sign + { + sign_bits.init(pCur_buf, pHdr->m_sign_bits_len); + pCur_buf += pHdr->m_sign_bits_len; + } + + if (!zstd_decompress_and_advance(pCur_buf, pHdr->m_weight2_bits_len, uncomp_weight2_bytes, weight2_bits)) + return false; + if (!zstd_decompress_and_advance(pCur_buf, pHdr->m_weight3_bits_len, uncomp_weight3_bytes, weight3_bits)) + return false; + if (!zstd_decompress_and_advance(pCur_buf, pHdr->m_weight4_bits_len, uncomp_weight4_bytes, weight4_bits)) + return false; + if (!zstd_decompress_and_advance(pCur_buf, pHdr->m_weight8_bytes_len, uncomp_weight8_bytes, weight8_bytes)) + return false; + + // sanity check + const uint64_t total_read_size = pCur_buf - pComp_data_all; + if (total_read_size > comp_data_size_all) + { + BASISU_DEVEL_ERROR("Compressed file is too small\n"); + return false; + } + + const uint32_t header_val = raw_bits.get_bits(FULL_ZSTD_HEADER_MARKER_BITS); + if (header_val != FULL_ZSTD_HEADER_MARKER) + { + BASISU_DEVEL_ERROR("Invalid marker\n"); + return false; + } + + const uint32_t astc_block_size_index = raw_bits.get_bits(4); + if (astc_block_size_index >= astc_helpers::NUM_ASTC_BLOCK_SIZES) + { + BASISU_DEVEL_ERROR("Invalid block dimension index\n"); + return false; + } + + astc_block_width = astc_helpers::g_astc_block_sizes[astc_block_size_index][0]; + astc_block_height = astc_helpers::g_astc_block_sizes[astc_block_size_index][1]; + + uses_srgb_astc_decode_mode = raw_bits.get_bits(1); + + actual_width = raw_bits.get_bits(16); + actual_height = raw_bits.get_bits(16); + has_alpha = raw_bits.get_bits(1); + + const bool use_dct = (raw_bits.get_bits(1) != 0); + + int int_q = 0; + if (use_dct) + int_q = raw_bits.get_bits(8); + + const float dct_q = (float)int_q / 2.0f; + if ((use_dct) && ((dct_q <= 0.0f) || (dct_q > 100.0f))) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd::Invalid DCT global quality factor\n"); + return false; + } + + if (debug_output) + { + basisu::fmt_debug_printf("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd: block dim: {}x{}, image dim: {}x{}, sRGB decode profile: {}, has_alpha: {}, dct: {} dct_q: {}\n", + astc_block_width, astc_block_height, + actual_width, actual_height, + uses_srgb_astc_decode_mode, has_alpha, + use_dct, dct_q); + } + + const uint32_t num_blocks_x = (actual_width + astc_block_width - 1) / astc_block_width; + const uint32_t num_blocks_y = (actual_height + astc_block_height - 1) / astc_block_height; + + if (pInit_callback) + { + if (!(*pInit_callback)(num_blocks_x, num_blocks_y, astc_block_width, astc_block_height, uses_srgb_astc_decode_mode, dct_q, has_alpha, pInit_callback_data)) + return false; + } + + fvec dct_work; + + assert((size_t)astc_block_size_index < std::size(g_encoder_trial_modes)); + const auto& encoder_trial_modes = g_encoder_trial_modes[astc_block_size_index]; + + const grid_weight_dct& grid_dct = g_grid_weight_dcts[astc_block_size_index]; + + basisu::vector2D log_blocks; + if (!log_blocks.try_resize(num_blocks_x, 8)) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd: out of memory\n"); + return false; + } + + memset(log_blocks.get_ptr(), 0, log_blocks.size_in_bytes()); + + basisu::vector2D prev_block_states; + if (!prev_block_states.try_resize(num_blocks_x, 2)) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd: out of memory\n"); + return false; + } + + uint32_t cur_run_len = 0; + + int part2_hash[PART_HASH_SIZE]; + std::fill(part2_hash, part2_hash + PART_HASH_SIZE, -1); + + int part3_hash[PART_HASH_SIZE]; + std::fill(part3_hash, part3_hash + PART_HASH_SIZE, -1); + + int tm_hash[TM_HASH_SIZE]; + std::fill(tm_hash, tm_hash + TM_HASH_SIZE, -1); + + dct_syms syms; + + for (uint32_t by = 0; by < num_blocks_y; by++) + { + for (uint32_t bx = 0; bx < num_blocks_x; bx++) + { + prev_block_state_full_zstd& new_prev_state = prev_block_states(bx, by & 1); + + const prev_block_state_full_zstd* pLeft_state = bx ? &prev_block_states(bx - 1, by & 1) : nullptr; + const prev_block_state_full_zstd* pUpper_state = by ? &prev_block_states(bx, (by - 1) & 1) : nullptr; + + astc_helpers::log_astc_block& log_blk = log_blocks(bx, by & 7); + + if (cur_run_len) + { + const prev_block_state_full_zstd* pPrev_block_state = pLeft_state ? pLeft_state : pUpper_state; + const astc_helpers::log_astc_block& prev_log_blk = bx ? log_blocks(bx - 1, by & 7) : log_blocks(bx, (by - 1) & 7); + + memcpy((void*)&log_blk, (const void*)&prev_log_blk, sizeof(log_blk)); + + if (pBlock_callback) + { + if (!(*pBlock_callback)(bx, by, prev_log_blk, pBlock_callback_data)) + return false; + } + + new_prev_state.m_tm_index = pPrev_block_state->m_tm_index; + //new_prev_state.m_base_cem_index = pPrev_block_state->m_base_cem_index; + + cur_run_len--; + continue; + } + + const prev_block_state_full_zstd* pDiag_state = (bx && by) ? &prev_block_states(bx - 1, (by - 1) & 1) : nullptr; + + // TODO: End check + const uint32_t mode_byte = comp_mode_dec.get_bits8(); + + if ((mode_byte & 3) == (uint32_t)xuastc_zstd_mode::cMODE_RUN) + { + // run + cur_run_len = 1 + (mode_byte >> 2); + + if (!bx && !by) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd::Invalid run command\n"); + return false; + } + + const uint32_t max_possible_run_len = num_blocks_x - bx; + if (cur_run_len > max_possible_run_len) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd::Invalid run len\n"); + return false; + } + + const prev_block_state_full_zstd* pPrev_block_state = pLeft_state ? pLeft_state : pUpper_state; + const astc_helpers::log_astc_block& prev_log_blk = bx ? log_blocks(bx - 1, by & 7) : log_blocks(bx, (by - 1) & 7); + + memcpy((void*)&log_blk, (const void*)&prev_log_blk, sizeof(log_blk)); + + if (pBlock_callback) + { + if (!(*pBlock_callback)(bx, by, prev_log_blk, pBlock_callback_data)) + return false; + } + + new_prev_state.m_tm_index = pPrev_block_state->m_tm_index; + + cur_run_len--; + + continue; + } + else if ((mode_byte & 15) == (uint32_t)xuastc_zstd_mode::cMODE_SOLID) + { + // solid + const astc_helpers::log_astc_block* pPrev_log_blk = bx ? &log_blocks(bx - 1, by & 7) : (by ? &log_blocks(bx, (by - 1) & 7) : nullptr); + + uint32_t prev_solid_color[4] = { 0 }; + + if (pPrev_log_blk) + { + if (pPrev_log_blk->m_solid_color_flag_ldr) + { + prev_solid_color[0] = pPrev_log_blk->m_solid_color[0] >> 8; + prev_solid_color[1] = pPrev_log_blk->m_solid_color[1] >> 8; + prev_solid_color[2] = pPrev_log_blk->m_solid_color[2] >> 8; + prev_solid_color[3] = pPrev_log_blk->m_solid_color[3] >> 8; + } + else + { + // Decode previous block's first CEM, use the halfway point as the predictor. + color_rgba prev_l, prev_h; + decode_endpoints(pPrev_log_blk->m_color_endpoint_modes[0], pPrev_log_blk->m_endpoints, pPrev_log_blk->m_endpoint_ise_range, prev_l, prev_h); + + prev_solid_color[0] = (prev_l[0] + prev_h[0] + 1) >> 1; + prev_solid_color[1] = (prev_l[1] + prev_h[1] + 1) >> 1; + prev_solid_color[2] = (prev_l[2] + prev_h[2] + 1) >> 1; + prev_solid_color[3] = (prev_l[3] + prev_h[3] + 1) >> 1; + } + } + + uint32_t delta_r = solid_dpcm_dec.get_bits8(); + uint32_t delta_g = solid_dpcm_dec.get_bits8(); + uint32_t delta_b = solid_dpcm_dec.get_bits8(); + uint32_t delta_a = has_alpha ? solid_dpcm_dec.get_bits8() : 0; + + uint32_t r = (prev_solid_color[0] + delta_r) & 0xFF; + uint32_t g = (prev_solid_color[1] + delta_g) & 0xFF; + uint32_t b = (prev_solid_color[2] + delta_b) & 0xFF; + uint32_t a = 255; + if (has_alpha) + a = (prev_solid_color[3] + delta_a) & 0xFF; + + log_blk.clear(); + log_blk.m_solid_color_flag_ldr = true; + log_blk.m_solid_color[0] = (uint16_t)(r | (r << 8)); + log_blk.m_solid_color[1] = (uint16_t)(g | (g << 8)); + log_blk.m_solid_color[2] = (uint16_t)(b | (b << 8)); + log_blk.m_solid_color[3] = (uint16_t)(a | (a << 8)); + + if (pBlock_callback) + { + if (!(*pBlock_callback)(bx, by, log_blk, pBlock_callback_data)) + return false; + } + + new_prev_state.m_tm_index = -1; + + continue; + } + + new_prev_state.clear(); + + //log_blk.clear(); + memset((void*)&log_blk, 0, offsetof(astc_helpers::log_astc_block, m_weights)); + + uint32_t tm_index = 0; + uint32_t actual_cem = 0; + + if ((mode_byte & 1) == 0) + { + // raw + uint32_t config_reuse_index = (mode_byte >> 1) & 3; + + if (config_reuse_index < 3) + { + // 0 = left, 1 = upper, 2 = left-upper + int cfg_dx = 0, cfg_dy = 0; + const prev_block_state_full_zstd* pCfg_state = nullptr; + + switch (config_reuse_index) + { + case 0: cfg_dx = -1; pCfg_state = pLeft_state; break; + case 1: cfg_dx = 0; cfg_dy = -1; pCfg_state = pUpper_state; break; + case 2: cfg_dx = -1; cfg_dy = -1; pCfg_state = pDiag_state; break; + default: assert(0); break; + } + + if ((((cfg_dx + (int)bx) < 0) || + ((cfg_dy + (int)by) < 0)) || + (!pCfg_state)) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd::Invalid config reuse\n"); + return false; + } + + astc_helpers::log_astc_block& cfg_log_blk = log_blocks((int)bx + cfg_dx, ((int)by + cfg_dy) & 7); + + tm_index = pCfg_state->m_tm_index; + + if (pCfg_state->m_tm_index < 0) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd::Invalid config reuse\n"); + return false; + } + + log_blk.m_partition_id = cfg_log_blk.m_partition_id; + actual_cem = cfg_log_blk.m_color_endpoint_modes[0]; + + new_prev_state.m_tm_index = tm_index; + //new_prev_state.m_base_cem_index = pCfg_state->m_base_cem_index; // base cem not including base+ofs, not actual + } + else + { + if (mode_byte & XUASTC_LDR_MODE_BYTE_TM_HASH_HIT_FLAG) + { + uint32_t tm_hash_index = raw_bits.get_bits(TM_HASH_BITS); + tm_index = tm_hash[tm_hash_index]; + } + else + { + tm_index = raw_bits.decode_truncated_binary(encoder_trial_modes.size_u32()); + + tm_hash[tm_hash_index(tm_index)] = tm_index; + } + + if (tm_index >= encoder_trial_modes.size()) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd::Invalid tm_index\n"); + return false; + } + + new_prev_state.m_tm_index = tm_index; + + const trial_mode& tm = encoder_trial_modes[tm_index]; + + actual_cem = tm.m_cem; + + if ((tm.m_cem == astc_helpers::CEM_LDR_RGB_DIRECT) || (tm.m_cem == astc_helpers::CEM_LDR_RGBA_DIRECT)) + { + // Decode is_base_ofs bit + bool is_base_ofs = (mode_byte & XUASTC_LDR_MODE_BYTE_IS_BASE_OFS_FLAG) != 0; + + if (is_base_ofs) + { + if (actual_cem == astc_helpers::CEM_LDR_RGB_DIRECT) + actual_cem = astc_helpers::CEM_LDR_RGB_BASE_PLUS_OFFSET; + else if (actual_cem == astc_helpers::CEM_LDR_RGBA_DIRECT) + actual_cem = astc_helpers::CEM_LDR_RGBA_BASE_PLUS_OFFSET; + } + } + + if (tm.m_num_parts > 1) + { + const uint32_t total_unique_indices = get_total_unique_patterns(astc_block_size_index, tm.m_num_parts); + int* pPart_hash = (tm.m_num_parts == 2) ? part2_hash : part3_hash; + + const bool hash_hit_flag = (mode_byte & XUASTC_LDR_MODE_BYTE_PART_HASH_HIT) != 0; + + uint32_t unique_pat_index; + if (hash_hit_flag) + { + uint32_t h = raw_bits.get_bits(basist::astc_ldr_t::PART_HASH_BITS); + + unique_pat_index = pPart_hash[h]; + } + else + { + unique_pat_index = raw_bits.decode_truncated_binary(total_unique_indices); + + pPart_hash[basist::astc_ldr_t::part_hash_index(unique_pat_index)] = unique_pat_index; + } + + if (unique_pat_index >= get_total_unique_patterns(astc_block_size_index, tm.m_num_parts)) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd: invalid unique_pat_index, decompression failed (file corrupt)\n"); + return false; + } + + log_blk.m_partition_id = unique_pat_index_to_part_seed(astc_block_size_index, tm.m_num_parts, unique_pat_index); + } + + + } // if (config_reuse_index < 3) + + if (tm_index >= encoder_trial_modes.size()) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd: invalid tm_index, decompression failed (file corrupt)\n"); + return false; + } + + const trial_mode& tm = encoder_trial_modes[tm_index]; + + const bool actual_cem_supports_bc = astc_helpers::cem_supports_bc(actual_cem); + + const uint32_t total_endpoint_vals = astc_helpers::get_num_cem_values(actual_cem); + + for (uint32_t part_iter = 0; part_iter < tm.m_num_parts; part_iter++) + log_blk.m_color_endpoint_modes[part_iter] = (uint8_t)actual_cem; + + log_blk.m_num_partitions = (uint8_t)tm.m_num_parts; + log_blk.m_dual_plane = (tm.m_ccs_index >= 0); + if (log_blk.m_dual_plane) + log_blk.m_color_component_selector = (uint8_t)tm.m_ccs_index; + + log_blk.m_weight_ise_range = (uint8_t)tm.m_weight_ise_range; + log_blk.m_endpoint_ise_range = (uint8_t)tm.m_endpoint_ise_range; + log_blk.m_grid_width = (uint8_t)tm.m_grid_width; + log_blk.m_grid_height = (uint8_t)tm.m_grid_height; + + const bool used_dpcm_endpoints_flag = (mode_byte & XUASTC_LDR_MODE_BYTE_DPCM_ENDPOINTS_FLAG) != 0; + + if (used_dpcm_endpoints_flag) + { + const int num_endpoint_levels = astc_helpers::get_ise_levels(log_blk.m_endpoint_ise_range); + const auto& endpoint_rank_to_ise = astc_helpers::g_dequant_tables.get_endpoint_tab(log_blk.m_endpoint_ise_range).m_rank_to_ISE; + const auto& endpoint_ise_to_rank = astc_helpers::g_dequant_tables.get_endpoint_tab(log_blk.m_endpoint_ise_range).m_ISE_to_rank; + + uint32_t reuse_delta_index = endpoint_dpcm_reuse_indices_dec.get_bits8(); + if (reuse_delta_index >= NUM_REUSE_XY_DELTAS) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd::Invalid reuse delta\n"); + return false; + } + + const int reuse_bx = (int)bx + basist::astc_6x6_hdr::g_reuse_xy_deltas[reuse_delta_index].m_x; + const int reuse_by = (int)by + basist::astc_6x6_hdr::g_reuse_xy_deltas[reuse_delta_index].m_y; + + if ((reuse_bx < 0) || (reuse_by < 0) || (reuse_bx >= (int)num_blocks_x) || (reuse_by >= (int)num_blocks_y)) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd::Invalid reuse delta\n"); + return false; + } + + const astc_helpers::log_astc_block* pEndpoint_pred_log_blk = &log_blocks(reuse_bx, reuse_by & 7); + if (pEndpoint_pred_log_blk->m_solid_color_flag_ldr) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd::Invalid reuse delta\n"); + return false; + } + + bool endpoints_use_bc[astc_helpers::MAX_PARTITIONS] = { }; + + if (actual_cem_supports_bc) + { + for (uint32_t part_iter = 0; part_iter < log_blk.m_num_partitions; part_iter++) + endpoints_use_bc[part_iter] = (use_bc_bits_dec.get_bits1() != 0); + } + + uint8_t predicted_endpoints[astc_helpers::MAX_PARTITIONS][astc_helpers::MAX_CEM_ENDPOINT_VALS] = { }; + + for (uint32_t part_iter = 0; part_iter < log_blk.m_num_partitions; part_iter++) + { + const bool always_repack_flag = false; + bool blue_contraction_clamped_flag = false, base_ofs_clamped_flag = false; + + // Mini-CEM encoder, to cross CEM domains. + bool conv_status = convert_endpoints_across_cems( + pEndpoint_pred_log_blk->m_color_endpoint_modes[0], pEndpoint_pred_log_blk->m_endpoint_ise_range, pEndpoint_pred_log_blk->m_endpoints, + log_blk.m_color_endpoint_modes[0], log_blk.m_endpoint_ise_range, predicted_endpoints[part_iter], + always_repack_flag, + endpoints_use_bc[part_iter], false, + blue_contraction_clamped_flag, base_ofs_clamped_flag); + + if (!conv_status) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd::Failed predicting endpoints\n"); + return false; + } + } + + if (num_endpoint_levels <= 8) + { + for (uint32_t part_iter = 0; part_iter < tm.m_num_parts; part_iter++) + { + for (uint32_t val_iter = 0; val_iter < total_endpoint_vals; val_iter++) + { + const uint32_t endpoint_idx = part_iter * total_endpoint_vals + val_iter; + + int delta = endpoint_dpcm_3bit_dec.get_bits4(); + + int e_val = (delta + endpoint_ise_to_rank[predicted_endpoints[part_iter][val_iter]]) % num_endpoint_levels; + log_blk.m_endpoints[endpoint_idx] = endpoint_rank_to_ise[e_val]; + } + } + } + else if (num_endpoint_levels <= 16) + { + for (uint32_t part_iter = 0; part_iter < tm.m_num_parts; part_iter++) + { + for (uint32_t val_iter = 0; val_iter < total_endpoint_vals; val_iter++) + { + const uint32_t endpoint_idx = part_iter * total_endpoint_vals + val_iter; + + int delta = endpoint_dpcm_4bit_dec.get_bits4(); + + int e_val = (delta + endpoint_ise_to_rank[predicted_endpoints[part_iter][val_iter]]) % num_endpoint_levels; + log_blk.m_endpoints[endpoint_idx] = endpoint_rank_to_ise[e_val]; + } + } + } + else if (num_endpoint_levels <= 32) + { + for (uint32_t part_iter = 0; part_iter < tm.m_num_parts; part_iter++) + { + for (uint32_t val_iter = 0; val_iter < total_endpoint_vals; val_iter++) + { + const uint32_t endpoint_idx = part_iter * total_endpoint_vals + val_iter; + + int delta = endpoint_dpcm_5bit_dec.get_bits8(); + + int e_val = (delta + endpoint_ise_to_rank[predicted_endpoints[part_iter][val_iter]]) % num_endpoint_levels; + log_blk.m_endpoints[endpoint_idx] = endpoint_rank_to_ise[e_val]; + } + } + } + else if (num_endpoint_levels <= 64) + { + for (uint32_t part_iter = 0; part_iter < tm.m_num_parts; part_iter++) + { + for (uint32_t val_iter = 0; val_iter < total_endpoint_vals; val_iter++) + { + const uint32_t endpoint_idx = part_iter * total_endpoint_vals + val_iter; + + int delta = endpoint_dpcm_6bit_dec.get_bits8(); + + int e_val = (delta + endpoint_ise_to_rank[predicted_endpoints[part_iter][val_iter]]) % num_endpoint_levels; + log_blk.m_endpoints[endpoint_idx] = endpoint_rank_to_ise[e_val]; + } + } + } + else if (num_endpoint_levels <= 128) + { + for (uint32_t part_iter = 0; part_iter < tm.m_num_parts; part_iter++) + { + for (uint32_t val_iter = 0; val_iter < total_endpoint_vals; val_iter++) + { + const uint32_t endpoint_idx = part_iter * total_endpoint_vals + val_iter; + + int delta = endpoint_dpcm_7bit_dec.get_bits8(); + + int e_val = (delta + endpoint_ise_to_rank[predicted_endpoints[part_iter][val_iter]]) % num_endpoint_levels; + log_blk.m_endpoints[endpoint_idx] = endpoint_rank_to_ise[e_val]; + } + } + } + else + { + for (uint32_t part_iter = 0; part_iter < tm.m_num_parts; part_iter++) + { + for (uint32_t val_iter = 0; val_iter < total_endpoint_vals; val_iter++) + { + const uint32_t endpoint_idx = part_iter * total_endpoint_vals + val_iter; + + int delta = endpoint_dpcm_8bit_dec.get_bits8(); + + int e_val = (delta + endpoint_ise_to_rank[predicted_endpoints[part_iter][val_iter]]) % num_endpoint_levels; + log_blk.m_endpoints[endpoint_idx] = endpoint_rank_to_ise[e_val]; + } + } + } + } + else + { + if (!decode_values(raw_bits, tm.m_num_parts * total_endpoint_vals, log_blk.m_endpoint_ise_range, log_blk.m_endpoints)) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd: decode_values() failed\n"); + return false; + } + } + } + else if ((mode_byte & 15) >= (uint32_t)xuastc_zstd_mode::cMODE_REUSE_CFG_ENDPOINTS_LEFT) + { + // reuse full cfg+endpoints+part id + const uint32_t reuse_index = ((mode_byte >> 2) & 3) - 1; + + int cfg_dx = 0, cfg_dy = 0; + const prev_block_state_full_zstd* pCfg_state = nullptr; + + switch (reuse_index) + { + case 0: cfg_dx = -1; pCfg_state = pLeft_state; break; + case 1: cfg_dx = 0; cfg_dy = -1; pCfg_state = pUpper_state; break; + case 2: cfg_dx = -1; cfg_dy = -1; pCfg_state = pDiag_state; break; + default: assert(0); break; + } + + if ((((cfg_dx + (int)bx) < 0) || + ((cfg_dy + (int)by) < 0)) || + (!pCfg_state)) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd::Invalid config reuse\n"); + return false; + } + + const astc_helpers::log_astc_block& cfg_log_blk = log_blocks((int)bx + cfg_dx, ((int)by + cfg_dy) & 7); + + if (pCfg_state->m_tm_index < 0) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd::Invalid config reuse\n"); + return false; + } + + tm_index = pCfg_state->m_tm_index; + actual_cem = cfg_log_blk.m_color_endpoint_modes[0]; + + for (uint32_t i = 0; i < cfg_log_blk.m_num_partitions; i++) + log_blk.m_color_endpoint_modes[i] = (uint8_t)actual_cem; + + log_blk.m_dual_plane = cfg_log_blk.m_dual_plane; + log_blk.m_color_component_selector = cfg_log_blk.m_color_component_selector; + log_blk.m_num_partitions = cfg_log_blk.m_num_partitions; + log_blk.m_partition_id = cfg_log_blk.m_partition_id; + log_blk.m_endpoint_ise_range = cfg_log_blk.m_endpoint_ise_range; + log_blk.m_weight_ise_range = cfg_log_blk.m_weight_ise_range; + log_blk.m_grid_width = cfg_log_blk.m_grid_width; + log_blk.m_grid_height = cfg_log_blk.m_grid_height; + + const uint32_t total_endpoint_vals = astc_helpers::get_num_cem_values(actual_cem) * log_blk.m_num_partitions; + memcpy(log_blk.m_endpoints, cfg_log_blk.m_endpoints, total_endpoint_vals); + + new_prev_state.m_tm_index = tm_index; + } + else + { + // shouldn't actually get here + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd: decompression failed\n"); + return false; + } + + // Decode weights + + if (tm_index >= encoder_trial_modes.size()) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd: invalid tm_index, decompression failed (file corrupt)\n"); + return false; + } + + const trial_mode& tm = encoder_trial_modes[tm_index]; + + const uint32_t total_planes = (tm.m_ccs_index >= 0) ? 2 : 1; + const uint32_t total_weights = tm.m_grid_width * tm.m_grid_height; + + bool block_used_dct = false; + if (use_dct) + block_used_dct = ((mode_byte & XUASTC_LDR_MODE_BYTE_USE_DCT) != 0); + + if (block_used_dct) + { + const astc_block_grid_data* pGrid_data = find_astc_block_grid_data(astc_block_width, astc_block_height, log_blk.m_grid_width, log_blk.m_grid_height); + + const uint32_t num_dc_levels = grid_weight_dct::get_num_weight_dc_levels(log_blk.m_weight_ise_range); + syms.m_num_dc_levels = num_dc_levels; + + for (uint32_t plane_iter = 0; plane_iter < total_planes; plane_iter++) + { + syms.m_coeffs.resize(0); + + if (num_dc_levels == DCT_MEAN_LEVELS1) + syms.m_dc_sym = mean1_bytes.get_bits8(); + else + syms.m_dc_sym = mean0_bits.get_bits4(); + + uint32_t cur_zig_ofs = 1; + + while (cur_zig_ofs < total_weights) + { + uint32_t run_len = run_bytes.get_bits8(); + if (run_len == DCT_RUN_LEN_EOB_SYM_INDEX) + break; + + cur_zig_ofs += run_len; + + if (cur_zig_ofs >= total_weights) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd::DCT decode error\n"); + return false; + } + + int sign = sign_bits.get_bits1(); + int coeff = coeff_bytes.get_bits8() + 1; + + if (sign) + coeff = -coeff; + + syms.m_coeffs.push_back(dct_syms::coeff(basisu::safe_cast_uint16(run_len), basisu::safe_cast_int16(coeff))); + cur_zig_ofs++; + } + + // weight grid IDCT + if (!grid_dct.decode_block_weights(dct_q, plane_iter, log_blk, nullptr, pGrid_data, nullptr, dct_work, &syms)) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd::DCT decode failed\n"); + return false; + } + + } // plane_iter + } + else + { + // Weight grid DPCM (no dependency on other blocks, or between planes, for determinism even when IDCT is used) + const uint32_t num_weight_levels = astc_helpers::get_ise_levels(log_blk.m_weight_ise_range); + const auto& weight_rank_to_ise = astc_helpers::g_dequant_tables.get_weight_tab(log_blk.m_weight_ise_range).m_rank_to_ISE; + + for (uint32_t plane_iter = 0; plane_iter < total_planes; plane_iter++) + { + int prev_w = num_weight_levels / 2; + + if (num_weight_levels < 4) + { + for (uint32_t weight_iter = 0; weight_iter < total_weights; weight_iter++) + { + uint32_t r = weight2_bits.get_bits2(); + + uint32_t w = (prev_w + r) % num_weight_levels; + + prev_w = w; + + log_blk.m_weights[plane_iter + weight_iter * total_planes] = (uint8_t)weight_rank_to_ise[w]; + + } // weight_iter + } + else if (num_weight_levels == 4) + { + for (uint32_t weight_iter = 0; weight_iter < total_weights; weight_iter++) + { + uint32_t r = weight2_bits.get_bits2(); + + uint32_t w = (prev_w + r) & 3; + + prev_w = w; + + log_blk.m_weights[plane_iter + weight_iter * total_planes] = (uint8_t)weight_rank_to_ise[w]; + + } // weight_iter + } + else if (num_weight_levels < 8) + { + for (uint32_t weight_iter = 0; weight_iter < total_weights; weight_iter++) + { + uint32_t r = weight3_bits.get_bits4(); + + uint32_t w = (prev_w + r) % num_weight_levels; + + prev_w = w; + + log_blk.m_weights[plane_iter + weight_iter * total_planes] = (uint8_t)weight_rank_to_ise[w]; + + } // weight_iter + } + else if (num_weight_levels == 8) + { + for (uint32_t weight_iter = 0; weight_iter < total_weights; weight_iter++) + { + uint32_t r = weight3_bits.get_bits4(); + + uint32_t w = (prev_w + r) & 7; + + prev_w = w; + + log_blk.m_weights[plane_iter + weight_iter * total_planes] = (uint8_t)weight_rank_to_ise[w]; + + } // weight_iter + } + else if (num_weight_levels < 16) + { + for (uint32_t weight_iter = 0; weight_iter < total_weights; weight_iter++) + { + uint32_t r = weight4_bits.get_bits4(); + + uint32_t w = (prev_w + r) % num_weight_levels; + + prev_w = w; + + log_blk.m_weights[plane_iter + weight_iter * total_planes] = (uint8_t)weight_rank_to_ise[w]; + + } // weight_iter + } + else if (num_weight_levels == 16) + { + for (uint32_t weight_iter = 0; weight_iter < total_weights; weight_iter++) + { + uint32_t r = weight4_bits.get_bits4(); + + uint32_t w = (prev_w + r) & 15; + + prev_w = w; + + log_blk.m_weights[plane_iter + weight_iter * total_planes] = (uint8_t)weight_rank_to_ise[w]; + + } // weight_iter + } + else + { + for (uint32_t weight_iter = 0; weight_iter < total_weights; weight_iter++) + { + uint32_t r = weight8_bytes.get_bits8(); + + uint32_t w = (prev_w + r) % num_weight_levels; + + prev_w = w; + + log_blk.m_weights[plane_iter + weight_iter * total_planes] = (uint8_t)weight_rank_to_ise[w]; + + } // weight_iter + } + + } // plane_iter + + } // if (block_used_dct) + + if (pBlock_callback) + { + if (!(*pBlock_callback)(bx, by, log_blk, pBlock_callback_data)) + return false; + } + + } // bx + + } // by + + assert(!cur_run_len); + + const uint32_t final_sync_marker = raw_bits.get_bits(FINAL_SYNC_MARKER_BITS); + if (final_sync_marker != FINAL_SYNC_MARKER) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd::Final sync check failed (1)\n"); + return false; + } + + if (comp_mode_dec.m_pBuf != comp_mode_dec.m_pBuf_end) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd::Final sync check failed (2)\n"); + return false; + } + + return true; + } + + bool xuastc_ldr_decompress_image( + const uint8_t* pComp_data_all, size_t comp_data_size_all, + uint32_t& astc_block_width, uint32_t& astc_block_height, + uint32_t& actual_width, uint32_t& actual_height, bool& has_alpha, bool& uses_srgb_astc_decode_mode, + bool debug_output, + xuastc_decomp_image_init_callback_ptr pInit_callback, void* pInit_callback_data, + xuastc_decomp_image_block_callback_ptr pBlock_callback, void* pBlock_callback_data) + { + if (debug_output) + basisu::debug_printf("\n------------------- astc_ldr_t::decompress_image\n"); + + assert(g_initialized); + + astc_block_width = 0; + astc_block_height = 0; + actual_width = 0; + actual_height = 0; + has_alpha = false; + uses_srgb_astc_decode_mode = false; + + if (!g_initialized) + { + BASISU_DEVEL_ERROR("Not initialized"); + //dec_blocks.clear(); + return false; + } + + if (comp_data_size_all < 1) + { + BASISU_DEVEL_ERROR("Compressed file is too small\n"); + return false; + } + + const uint8_t first_comp_byte = pComp_data_all[0]; + + if (first_comp_byte == (uint8_t)xuastc_ldr_syntax::cFullZStd) + { + return xuastc_ldr_decompress_image_full_zstd( + pComp_data_all, comp_data_size_all, + astc_block_width, astc_block_height, + actual_width, actual_height, has_alpha, uses_srgb_astc_decode_mode, + debug_output, + pInit_callback, pInit_callback_data, + pBlock_callback, pBlock_callback_data); + } + + // Either full arith or hybrid arith+zstd now + + const xuastc_ldr_arith_header* pHdr = nullptr; + + const uint8_t* pComp_data = pComp_data_all + 1; + size_t comp_data_size = comp_data_size_all - 1; + + basisu::uint8_vec uncomp_mean0_bits, uncomp_mean1_bytes, uncomp_run_bytes, uncomp_coeff_bytes, uncomp_weight2_bytes, uncomp_weight3_bytes, uncomp_weight4_bytes, uncomp_weight8_bytes; + simplified_bitwise_decoder mean0_bits, mean1_bytes, run_bytes, coeff_bytes, sign_bits, weight2_bits, weight3_bits, weight4_bits, weight8_bytes; + bool use_fast_decoding = false; + + if (first_comp_byte == (uint8_t)xuastc_ldr_syntax::cHybridArithZStd) + { + if (comp_data_size_all < sizeof(xuastc_ldr_arith_header)) + { + BASISU_DEVEL_ERROR("Compressed file is too small\n"); + return false; + } + + pHdr = (const xuastc_ldr_arith_header*)pComp_data_all; + + if (pHdr->m_arith_bytes_len < arith::ArithMinExpectedDataBufSize) + { + BASISU_DEVEL_ERROR("Invalid header\n"); + return false; + } + + const uint64_t total_comp_size = (uint64_t)((uint32_t)pHdr->m_arith_bytes_len) + + pHdr->m_mean0_bits_len + pHdr->m_mean1_bytes_len + + pHdr->m_run_bytes_len + pHdr->m_coeff_bytes_len + pHdr->m_sign_bits_len + + pHdr->m_weight2_bits_len + pHdr->m_weight3_bits_len + pHdr->m_weight4_bits_len + pHdr->m_weight8_bytes_len; + + if ((sizeof(xuastc_ldr_arith_header) + total_comp_size) > comp_data_size_all) + { + BASISU_DEVEL_ERROR("Compressed file is too small\n"); + return false; + } + + pComp_data = pComp_data_all + sizeof(xuastc_ldr_arith_header); + comp_data_size = pHdr->m_arith_bytes_len; + + const uint8_t* pCur_buf = (const uint8_t*)pComp_data + comp_data_size; + + // mean0 + { + bool status = zstd_decompress(pCur_buf, pHdr->m_mean0_bits_len, uncomp_mean0_bits); + if (!status) + return false; + pCur_buf += pHdr->m_mean0_bits_len; + mean0_bits.init(uncomp_mean0_bits); + } + + // mean1 + { + bool status = zstd_decompress(pCur_buf, pHdr->m_mean1_bytes_len, uncomp_mean1_bytes); + if (!status) + return false; + pCur_buf += pHdr->m_mean1_bytes_len; + mean1_bytes.init(uncomp_mean1_bytes); + } + + // run + { + bool status = zstd_decompress(pCur_buf, pHdr->m_run_bytes_len, uncomp_run_bytes); + if (!status) + return false; + pCur_buf += pHdr->m_run_bytes_len; + run_bytes.init(uncomp_run_bytes); + } + + // coeff + { + bool status = zstd_decompress(pCur_buf, pHdr->m_coeff_bytes_len, uncomp_coeff_bytes); + if (!status) + return false; + pCur_buf += pHdr->m_coeff_bytes_len; + coeff_bytes.init(uncomp_coeff_bytes); + } + + // sign + { + sign_bits.init(pCur_buf, pHdr->m_sign_bits_len); + pCur_buf += pHdr->m_sign_bits_len; + } + + // weight2 + { + bool status = zstd_decompress(pCur_buf, pHdr->m_weight2_bits_len, uncomp_weight2_bytes); + if (!status) + return false; + pCur_buf += pHdr->m_weight2_bits_len; + weight2_bits.init(uncomp_weight2_bytes); + } + + // weight3 + { + bool status = zstd_decompress(pCur_buf, pHdr->m_weight3_bits_len, uncomp_weight3_bytes); + if (!status) + return false; + pCur_buf += pHdr->m_weight3_bits_len; + weight3_bits.init(uncomp_weight3_bytes); + } + + // weight4 + { + bool status = zstd_decompress(pCur_buf, pHdr->m_weight4_bits_len, uncomp_weight4_bytes); + if (!status) + return false; + pCur_buf += pHdr->m_weight4_bits_len; + weight4_bits.init(uncomp_weight4_bytes); + } + + // weight8 + { + bool status = zstd_decompress(pCur_buf, pHdr->m_weight8_bytes_len, uncomp_weight8_bytes); + if (!status) + return false; + pCur_buf += pHdr->m_weight8_bytes_len; + weight8_bytes.init(uncomp_weight8_bytes); + } + + // sanity check + const uint64_t total_read_size = pCur_buf - pComp_data_all; + if (total_read_size > comp_data_size_all) + { + BASISU_DEVEL_ERROR("Compressed file is too small\n"); + return false; + } + + use_fast_decoding = true; + } + + if (comp_data_size < arith::ArithMinExpectedDataBufSize) + { + BASISU_DEVEL_ERROR("Compressed file is too small\n"); + return false; + } + + //interval_timer itm; + //itm.start(); + + arith::arith_dec dec; + if (!dec.init(pComp_data, comp_data_size)) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::Invalid compressed data\n"); + return false; + } + + const uint32_t header_val = dec.get_bits(ARITH_HEADER_MARKER_BITS); + if (header_val != ARITH_HEADER_MARKER) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::Unexpected header marker\n"); + return false; + } + + const uint32_t astc_block_size_index = dec.get_bits(4); + if (astc_block_size_index >= astc_helpers::NUM_ASTC_BLOCK_SIZES) + { + BASISU_DEVEL_ERROR("Invalid block dimension index\n"); + return false; + } + + const uint32_t block_width = astc_helpers::g_astc_block_sizes[astc_block_size_index][0]; + const uint32_t block_height = astc_helpers::g_astc_block_sizes[astc_block_size_index][1]; + + // sanity checks + assert((int)astc_block_size_index == astc_helpers::find_astc_block_size_index(block_width, block_height)); + assert(astc_helpers::is_valid_block_size(block_width, block_height)); + + astc_block_width = block_width; + astc_block_height = block_height; + + //const uint32_t total_block_pixels = block_width * block_height; + + uses_srgb_astc_decode_mode = dec.get_bit(); + + //const astc_helpers::decode_mode dec_mode = uses_srgb_astc_decode_mode ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8; + + const uint32_t width = dec.get_bits(16); + const uint32_t height = dec.get_bits(16); + + if ((width < 1) || (height < 1)) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::Invalid image dimension\n"); + return false; + } + + actual_width = width; + actual_height = height; + + has_alpha = dec.get_bit(); + + const bool use_dct = (dec.get_bits(1) != 0); + + int int_q = 0; + if (use_dct) + int_q = dec.get_bits(8); + + const float dct_q = (float)int_q / 2.0f; + if ((use_dct) && ((dct_q <= 0.0f) || (dct_q > 100.0f))) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::Invalid DCT global quality factor\n"); + return false; + } + + if (debug_output) + { + basisu::fmt_debug_printf("astc_ldr_t::decompress_image: block dim: {}x{}, image dim: {}x{}, sRGB decode profile: {}, has_alpha: {}, dct: {} dct_q: {}\n", + block_width, block_height, + width, height, + uses_srgb_astc_decode_mode, has_alpha, + use_dct, dct_q); + } + + const uint32_t num_blocks_x = (width + block_width - 1) / block_width; + const uint32_t num_blocks_y = (height + block_height - 1) / block_height; + + if (pInit_callback) + { + if (!(*pInit_callback)(num_blocks_x, num_blocks_y, block_width, block_height, uses_srgb_astc_decode_mode, dct_q, has_alpha, pInit_callback_data)) + return false; + } + + assert((size_t)astc_block_size_index < std::size(g_encoder_trial_modes)); + const auto& encoder_trial_modes = g_encoder_trial_modes[astc_block_size_index]; + + assert((size_t)astc_block_size_index < std::size(g_grouped_encoder_trial_modes)); + const auto& grouped_encoder_trial_modes = g_grouped_encoder_trial_modes[astc_block_size_index]; + + arith::arith_data_model mode_model((uint32_t)xuastc_mode::cMODE_TOTAL); + + arith::arith_data_model solid_color_dpcm_model[4]; + for (uint32_t i = 0; i < 4; i++) + solid_color_dpcm_model[i].init(256, true); + + arith::arith_data_model raw_endpoint_models[astc_helpers::TOTAL_ENDPOINT_ISE_RANGES]; + for (uint32_t i = 0; i < astc_helpers::TOTAL_ENDPOINT_ISE_RANGES; i++) + raw_endpoint_models[i].init(astc_helpers::get_ise_levels(astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE + i)); + + arith::arith_data_model dpcm_endpoint_models[astc_helpers::TOTAL_ENDPOINT_ISE_RANGES]; + for (uint32_t i = 0; i < astc_helpers::TOTAL_ENDPOINT_ISE_RANGES; i++) + dpcm_endpoint_models[i].init(astc_helpers::get_ise_levels(astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE + i)); + + arith::arith_bit_model is_base_ofs_model; + arith::arith_bit_model use_dct_model[4]; + arith::arith_bit_model use_dpcm_endpoints_model; + + arith::arith_data_model cem_index_model[8]; + for (uint32_t i = 0; i < 8; i++) + cem_index_model[i].init(OTM_NUM_CEMS); + + arith::arith_data_model subset_index_model[OTM_NUM_SUBSETS]; + for (uint32_t i = 0; i < OTM_NUM_SUBSETS; i++) + subset_index_model[i].init(OTM_NUM_SUBSETS); + + arith::arith_data_model ccs_index_model[OTM_NUM_CCS]; + for (uint32_t i = 0; i < OTM_NUM_CCS; i++) + ccs_index_model[i].init(OTM_NUM_CCS); + + arith::arith_data_model grid_size_model[OTM_NUM_GRID_SIZES]; + for (uint32_t i = 0; i < OTM_NUM_GRID_SIZES; i++) + grid_size_model[i].init(OTM_NUM_GRID_SIZES); + + arith::arith_data_model grid_aniso_model[OTM_NUM_GRID_ANISOS]; + for (uint32_t i = 0; i < OTM_NUM_GRID_ANISOS; i++) + grid_aniso_model[i].init(OTM_NUM_GRID_ANISOS); + + arith::arith_data_model dct_run_len_model; // [0,63] or 64=EOB + arith::arith_data_model dct_coeff_mag; // [1,255] (blocks with larger mags go DPCM) + arith::arith_data_model weight_mean_models[2]; + arith::arith_data_model raw_weight_models[astc_helpers::TOTAL_WEIGHT_ISE_RANGES]; + + if (!use_fast_decoding) + { + // Models used for weight decompression in pure arithmetic mode. + dct_run_len_model.init(65); + dct_coeff_mag.init(255); + + weight_mean_models[0].init(DCT_MEAN_LEVELS0); + weight_mean_models[1].init(DCT_MEAN_LEVELS1); + + for (uint32_t i = 0; i < astc_helpers::TOTAL_WEIGHT_ISE_RANGES; i++) + raw_weight_models[i].init(astc_helpers::get_ise_levels(astc_helpers::FIRST_VALID_WEIGHT_ISE_RANGE + i)); + } + + const grid_weight_dct& grid_dct = g_grid_weight_dcts[astc_block_size_index]; + + basisu::vector2D log_blocks; + if (!log_blocks.try_resize(num_blocks_x, 8)) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image: out of memory\n"); + return false; + } + + memset(log_blocks.get_ptr(), 0, log_blocks.size_in_bytes()); + + basisu::vector2D prev_block_states; + if (!prev_block_states.try_resize(num_blocks_x, 2)) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image: out of memory\n"); + return false; + } + + arith::arith_data_model submode_models[OTM_NUM_CEMS][OTM_NUM_SUBSETS][OTM_NUM_CCS][OTM_NUM_GRID_SIZES][OTM_NUM_GRID_ANISOS]; + + arith::arith_bit_model endpoints_use_bc_models[4]; + + arith::arith_data_model endpoint_reuse_delta_model(basist::astc_6x6_hdr::NUM_REUSE_XY_DELTAS); + + arith::arith_data_model config_reuse_model[4]; + for (uint32_t i = 0; i < 4; i++) + config_reuse_model[i].init(4); + + arith::arith_gamma_contexts m_run_len_contexts; + uint32_t cur_run_len = 0; + + int part2_hash[PART_HASH_SIZE]; + std::fill(part2_hash, part2_hash + PART_HASH_SIZE, -1); + + int part3_hash[PART_HASH_SIZE]; + std::fill(part3_hash, part3_hash + PART_HASH_SIZE, -1); + + arith::arith_bit_model use_part_hash_model[4]; + arith::arith_data_model part2_hash_index_model(PART_HASH_SIZE, true); + arith::arith_data_model part3_hash_index_model(PART_HASH_SIZE, true); + + //if (debug_output) + // debug_printf("Decompressor init time finish: {} secs\n", itm.get_elapsed_secs()); + + //itm.start(); + + dct_syms syms; + + fvec dct_work; + + for (uint32_t by = 0; by < num_blocks_y; by++) + { + for (uint32_t bx = 0; bx < num_blocks_x; bx++) + { + prev_block_state& new_prev_state = prev_block_states(bx, by & 1); + new_prev_state.clear(); + + const prev_block_state* pLeft_state = bx ? &prev_block_states(bx - 1, by & 1) : nullptr; + const prev_block_state* pUpper_state = by ? &prev_block_states(bx, (by - 1) & 1) : nullptr; + const prev_block_state* pDiag_state = (bx && by) ? &prev_block_states(bx - 1, (by - 1) & 1) : nullptr; + const prev_block_state* pPred_state = pLeft_state ? pLeft_state : pUpper_state; // left or upper, or nullptr on first block + + astc_helpers::log_astc_block& log_blk = log_blocks(bx, by & 7); + + if (cur_run_len) + { + const prev_block_state* pPrev_block_state = pLeft_state ? pLeft_state : pUpper_state; + const astc_helpers::log_astc_block& prev_log_blk = bx ? log_blocks(bx - 1, by & 7) : log_blocks(bx, (by - 1) & 7); + + log_blk = prev_log_blk; + + if (pBlock_callback) + { + if (!(*pBlock_callback)(bx, by, prev_log_blk, pBlock_callback_data)) + return false; + } + + new_prev_state.m_was_solid_color = pPrev_block_state->m_was_solid_color; + new_prev_state.m_used_weight_dct = pPrev_block_state->m_used_weight_dct; + new_prev_state.m_first_endpoint_uses_bc = pPrev_block_state->m_first_endpoint_uses_bc; + new_prev_state.m_reused_full_cfg = true; + new_prev_state.m_tm_index = pPrev_block_state->m_tm_index; + new_prev_state.m_base_cem_index = pPrev_block_state->m_base_cem_index; + new_prev_state.m_subset_index = pPrev_block_state->m_subset_index; + new_prev_state.m_ccs_index = pPrev_block_state->m_ccs_index; + new_prev_state.m_grid_size = pPrev_block_state->m_grid_size; + new_prev_state.m_grid_aniso = pPrev_block_state->m_grid_aniso; + new_prev_state.m_used_part_hash = pPrev_block_state->m_used_part_hash; + + cur_run_len--; + continue; + } + + log_blk.clear(); + + uint32_t mode_index = dec.decode_sym(mode_model); + + switch (mode_index) + { + case (uint32_t)xuastc_mode::cMODE_SOLID: + { + const astc_helpers::log_astc_block* pPrev_log_blk = bx ? &log_blocks(bx - 1, by & 7) : (by ? &log_blocks(bx, (by - 1) & 7) : nullptr); + + uint32_t prev_solid_color[4] = { 0 }; + + if (pPrev_log_blk) + { + if (pPrev_log_blk->m_solid_color_flag_ldr) + { + prev_solid_color[0] = pPrev_log_blk->m_solid_color[0] >> 8; + prev_solid_color[1] = pPrev_log_blk->m_solid_color[1] >> 8; + prev_solid_color[2] = pPrev_log_blk->m_solid_color[2] >> 8; + prev_solid_color[3] = pPrev_log_blk->m_solid_color[3] >> 8; + } + else + { + // Decode previous block's first CEM, use the halfway point as the predictor. + color_rgba prev_l, prev_h; + decode_endpoints(pPrev_log_blk->m_color_endpoint_modes[0], pPrev_log_blk->m_endpoints, pPrev_log_blk->m_endpoint_ise_range, prev_l, prev_h); + + prev_solid_color[0] = (prev_l[0] + prev_h[0] + 1) >> 1; + prev_solid_color[1] = (prev_l[1] + prev_h[1] + 1) >> 1; + prev_solid_color[2] = (prev_l[2] + prev_h[2] + 1) >> 1; + prev_solid_color[3] = (prev_l[3] + prev_h[3] + 1) >> 1; + } + } + + uint32_t r = (prev_solid_color[0] + dec.decode_sym(solid_color_dpcm_model[0])) & 0xFF; + uint32_t g = (prev_solid_color[1] + dec.decode_sym(solid_color_dpcm_model[1])) & 0xFF; + uint32_t b = (prev_solid_color[2] + dec.decode_sym(solid_color_dpcm_model[2])) & 0xFF; + + uint32_t a = 255; + if (has_alpha) + a = (prev_solid_color[3] + dec.decode_sym(solid_color_dpcm_model[3])) & 0xFF; + + log_blk.m_solid_color_flag_ldr = true; + log_blk.m_solid_color[0] = (uint16_t)(r | (r << 8)); + log_blk.m_solid_color[1] = (uint16_t)(g | (g << 8)); + log_blk.m_solid_color[2] = (uint16_t)(b | (b << 8)); + log_blk.m_solid_color[3] = (uint16_t)(a | (a << 8)); + + if (pBlock_callback) + { + if (!(*pBlock_callback)(bx, by, log_blk, pBlock_callback_data)) + return false; + } + + // Bias the statistics towards using DCT (most common case). + if (use_dct) + new_prev_state.m_used_weight_dct = true; + + new_prev_state.m_first_endpoint_uses_bc = true; + new_prev_state.m_was_solid_color = true; + new_prev_state.m_tm_index = -1; + new_prev_state.m_base_cem_index = astc_helpers::CEM_LDR_RGB_DIRECT; + new_prev_state.m_subset_index = 0; + new_prev_state.m_ccs_index = 0; + new_prev_state.m_grid_size = 0; + new_prev_state.m_grid_aniso = 0; + new_prev_state.m_reused_full_cfg = false; + new_prev_state.m_used_part_hash = true; // bias to true + + break; + } + case (uint32_t)xuastc_mode::cMODE_RUN: + { + if (!bx && !by) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::Invalid run command\n"); + return false; + } + + cur_run_len = dec.decode_gamma(m_run_len_contexts); + if (!cur_run_len) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::Invalid run len\n"); + return false; + } + + const uint32_t max_possible_run_len = num_blocks_x - bx; + if (cur_run_len > max_possible_run_len) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::Invalid run len\n"); + return false; + } + + const prev_block_state* pPrev_block_state = pLeft_state ? pLeft_state : pUpper_state; + const astc_helpers::log_astc_block& prev_log_blk = bx ? log_blocks(bx - 1, by & 7) : log_blocks(bx, (by - 1) & 7); + + log_blk = prev_log_blk; + + if (pBlock_callback) + { + if (!(*pBlock_callback)(bx, by, prev_log_blk, pBlock_callback_data)) + return false; + } + + new_prev_state.m_was_solid_color = pPrev_block_state->m_was_solid_color; + new_prev_state.m_used_weight_dct = pPrev_block_state->m_used_weight_dct; + new_prev_state.m_first_endpoint_uses_bc = pPrev_block_state->m_first_endpoint_uses_bc; + new_prev_state.m_reused_full_cfg = true; + new_prev_state.m_tm_index = pPrev_block_state->m_tm_index; + new_prev_state.m_base_cem_index = pPrev_block_state->m_base_cem_index; + new_prev_state.m_subset_index = pPrev_block_state->m_subset_index; + new_prev_state.m_ccs_index = pPrev_block_state->m_ccs_index; + new_prev_state.m_grid_size = pPrev_block_state->m_grid_size; + new_prev_state.m_grid_aniso = pPrev_block_state->m_grid_aniso; + new_prev_state.m_used_part_hash = pPrev_block_state->m_used_part_hash; + + cur_run_len--; + + break; + } + case (uint32_t)xuastc_mode::cMODE_RAW: + case (uint32_t)xuastc_mode::cMODE_REUSE_CFG_ENDPOINTS_LEFT: + case (uint32_t)xuastc_mode::cMODE_REUSE_CFG_ENDPOINTS_UP: + case (uint32_t)xuastc_mode::cMODE_REUSE_CFG_ENDPOINTS_DIAG: + { + uint32_t tm_index = 0; + uint32_t actual_cem = 0; + + if (mode_index != (uint32_t)xuastc_mode::cMODE_RAW) + { + // Full config+part ID+endpoint reuse from an immediate neighbor + // + // 0 = left, 1 = upper, 2 = left-upper + int cfg_dx = 0, cfg_dy = 0; + const prev_block_state* pCfg_state = nullptr; + + switch (mode_index) + { + case (uint32_t)xuastc_mode::cMODE_REUSE_CFG_ENDPOINTS_LEFT: cfg_dx = -1; pCfg_state = pLeft_state; break; + case (uint32_t)xuastc_mode::cMODE_REUSE_CFG_ENDPOINTS_UP: cfg_dx = 0; cfg_dy = -1; pCfg_state = pUpper_state; break; + case (uint32_t)xuastc_mode::cMODE_REUSE_CFG_ENDPOINTS_DIAG: cfg_dx = -1; cfg_dy = -1; pCfg_state = pDiag_state; break; + default: assert(0); break; + } + + if ((((cfg_dx + (int)bx) < 0) || + ((cfg_dy + (int)by) < 0)) || + (!pCfg_state)) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::Invalid config reuse\n"); + return false; + } + + if (pCfg_state->m_tm_index < 0) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image::Invalid config reuse\n"); + return false; + } + + const astc_helpers::log_astc_block& cfg_log_blk = log_blocks((int)bx + cfg_dx, ((int)by + cfg_dy) & 7); + + tm_index = pCfg_state->m_tm_index; + actual_cem = cfg_log_blk.m_color_endpoint_modes[0]; + + for (uint32_t i = 0; i < cfg_log_blk.m_num_partitions; i++) + log_blk.m_color_endpoint_modes[i] = (uint8_t)actual_cem; + log_blk.m_dual_plane = cfg_log_blk.m_dual_plane; + log_blk.m_color_component_selector = cfg_log_blk.m_color_component_selector; + log_blk.m_num_partitions = cfg_log_blk.m_num_partitions; + log_blk.m_partition_id = cfg_log_blk.m_partition_id; + log_blk.m_endpoint_ise_range = cfg_log_blk.m_endpoint_ise_range; + log_blk.m_weight_ise_range = cfg_log_blk.m_weight_ise_range; + log_blk.m_grid_width = cfg_log_blk.m_grid_width; + log_blk.m_grid_height = cfg_log_blk.m_grid_height; + + const uint32_t total_endpoint_vals = astc_helpers::get_num_cem_values(actual_cem) * log_blk.m_num_partitions; + memcpy(log_blk.m_endpoints, cfg_log_blk.m_endpoints, total_endpoint_vals); + + new_prev_state.m_tm_index = pCfg_state->m_tm_index; + new_prev_state.m_base_cem_index = pCfg_state->m_base_cem_index; // base cem not including base+ofs, not actual + new_prev_state.m_subset_index = pCfg_state->m_subset_index; + new_prev_state.m_ccs_index = pCfg_state->m_ccs_index; + new_prev_state.m_grid_size = pCfg_state->m_grid_size; + new_prev_state.m_grid_aniso = pCfg_state->m_grid_aniso; + new_prev_state.m_used_part_hash = pCfg_state->m_used_part_hash; + new_prev_state.m_reused_full_cfg = true; + + const bool actual_cem_supports_bc = astc_helpers::cem_supports_bc(actual_cem); + if (actual_cem_supports_bc) + { + new_prev_state.m_first_endpoint_uses_bc = astc_helpers::used_blue_contraction(actual_cem, log_blk.m_endpoints, log_blk.m_endpoint_ise_range); + assert(new_prev_state.m_first_endpoint_uses_bc == pCfg_state->m_first_endpoint_uses_bc); + } + } + else + { + uint32_t reused_full_cfg_model_index = 0; + if (pLeft_state) + reused_full_cfg_model_index = pLeft_state->m_reused_full_cfg; + else + reused_full_cfg_model_index = 1; + + if (pUpper_state) + reused_full_cfg_model_index |= pUpper_state->m_reused_full_cfg ? 2 : 0; + else + reused_full_cfg_model_index |= 2; + + const uint32_t config_reuse_index = dec.decode_sym(config_reuse_model[reused_full_cfg_model_index]); + + // TODO: Shared with encoder, make global constant + + //if (config_reuse_index < ldr_astc_block_encode_image_output::cMaxConfigReuseNeighbors) + if (config_reuse_index < cMaxConfigReuseNeighbors) + { + // 0 = left, 1 = upper, 2 = left-upper + int cfg_dx = 0, cfg_dy = 0; + const prev_block_state* pCfg_state = nullptr; + + switch (config_reuse_index) + { + case 0: cfg_dx = -1; pCfg_state = pLeft_state; break; + case 1: cfg_dx = 0; cfg_dy = -1; pCfg_state = pUpper_state; break; + case 2: cfg_dx = -1; cfg_dy = -1; pCfg_state = pDiag_state; break; + default: assert(0); break; + } + + if ((((cfg_dx + (int)bx) < 0) || + ((cfg_dy + (int)by) < 0)) || + (!pCfg_state)) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::Invalid config reuse\n"); + return false; + } + + if (pCfg_state->m_tm_index < 0) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::Invalid config reuse\n"); + return false; + } + + astc_helpers::log_astc_block& cfg_log_blk = log_blocks((int)bx + cfg_dx, ((int)by + cfg_dy) & 7); + + tm_index = pCfg_state->m_tm_index; + log_blk.m_partition_id = cfg_log_blk.m_partition_id; + actual_cem = cfg_log_blk.m_color_endpoint_modes[0]; + + new_prev_state.m_tm_index = pCfg_state->m_tm_index; + new_prev_state.m_base_cem_index = pCfg_state->m_base_cem_index; // base cem not including base+ofs, not actual + new_prev_state.m_subset_index = pCfg_state->m_subset_index; + new_prev_state.m_ccs_index = pCfg_state->m_ccs_index; + new_prev_state.m_grid_size = pCfg_state->m_grid_size; + new_prev_state.m_grid_aniso = pCfg_state->m_grid_aniso; + new_prev_state.m_used_part_hash = pCfg_state->m_used_part_hash; + new_prev_state.m_reused_full_cfg = true; + } + else + { + // -------------------- Decode full ASTC config + { + uint32_t prev_cem_index = astc_helpers::CEM_LDR_RGB_DIRECT; + uint32_t prev_subset_index = 0, prev_ccs_index = 0, prev_grid_size = 0, prev_grid_aniso = 0; + + if (pPred_state) + { + prev_cem_index = pPred_state->m_base_cem_index; + prev_subset_index = pPred_state->m_subset_index; + prev_ccs_index = pPred_state->m_ccs_index; + prev_grid_size = pPred_state->m_grid_size; + prev_grid_aniso = pPred_state->m_grid_aniso; + } + + const uint32_t ldrcem_index = cem_to_ldrcem_index(prev_cem_index); + + uint32_t cem_index = dec.decode_sym(cem_index_model[ldrcem_index]); + uint32_t subset_index = dec.decode_sym(subset_index_model[prev_subset_index]); + uint32_t ccs_index = dec.decode_sym(ccs_index_model[prev_ccs_index]); + uint32_t grid_size_index = dec.decode_sym(grid_size_model[prev_grid_size]); + uint32_t grid_aniso_index = dec.decode_sym(grid_aniso_model[prev_grid_aniso]); + + const basisu::uint_vec& modes = get_tm_candidates(grouped_encoder_trial_modes, cem_index, subset_index, ccs_index, grid_size_index, grid_aniso_index); + uint32_t submode_index = 0; + + if (modes.size() > 1) + { + arith::arith_data_model& submode_model = submode_models[cem_index][subset_index][ccs_index][grid_size_index][grid_aniso_index]; + if (!submode_model.get_num_data_syms()) + submode_model.init(modes.size_u32(), true); + + submode_index = dec.decode_sym(submode_model); + } + + if (submode_index >= modes.size()) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::Invalid mode index\n"); + return false; + } + + tm_index = modes[submode_index]; + + new_prev_state.m_tm_index = tm_index; + new_prev_state.m_base_cem_index = cem_index; + new_prev_state.m_subset_index = subset_index; + new_prev_state.m_ccs_index = ccs_index; + new_prev_state.m_grid_size = grid_size_index; + new_prev_state.m_grid_aniso = grid_aniso_index; + new_prev_state.m_reused_full_cfg = false; + } + + if (tm_index >= encoder_trial_modes.size()) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image: invalid tm_index, decompression failed (file corrupt)\n"); + return false; + } + + const trial_mode& tm = encoder_trial_modes[tm_index]; + + actual_cem = tm.m_cem; + if ((tm.m_cem == astc_helpers::CEM_LDR_RGB_DIRECT) || (tm.m_cem == astc_helpers::CEM_LDR_RGBA_DIRECT)) + { + // Decode is_base_ofs bit + bool is_base_ofs = dec.decode_bit(is_base_ofs_model); + if (is_base_ofs) + { + if (actual_cem == astc_helpers::CEM_LDR_RGB_DIRECT) + actual_cem = astc_helpers::CEM_LDR_RGB_BASE_PLUS_OFFSET; + else if (actual_cem == astc_helpers::CEM_LDR_RGBA_DIRECT) + actual_cem = astc_helpers::CEM_LDR_RGBA_BASE_PLUS_OFFSET; + } + } + + if (tm.m_num_parts > 1) + { + const uint32_t total_unique_indices = get_total_unique_patterns(astc_block_size_index, tm.m_num_parts); + + uint32_t use_part_model_index = 0; + if (pLeft_state) + use_part_model_index = pLeft_state->m_used_part_hash; + else + use_part_model_index = 1; + if (pUpper_state) + use_part_model_index |= pUpper_state->m_used_part_hash ? 2 : 0; + else + use_part_model_index |= 2; + + int* pPart_hash = (tm.m_num_parts == 2) ? part2_hash : part3_hash; + + bool use_part_hash_flag = dec.decode_bit(use_part_hash_model[use_part_model_index]); + + uint32_t unique_pat_index; + if (!use_part_hash_flag) + { + unique_pat_index = dec.decode_truncated_binary(total_unique_indices); + pPart_hash[part_hash_index(unique_pat_index)] = unique_pat_index; + + new_prev_state.m_used_part_hash = false; + } + else + { + uint32_t hash_index = dec.decode_sym((tm.m_num_parts == 2) ? part2_hash_index_model : part3_hash_index_model); + unique_pat_index = pPart_hash[hash_index]; + + if ((int)unique_pat_index < 0) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image: invalid hash_index, decompression failed (file corrupt)\n"); + return false; + } + + new_prev_state.m_used_part_hash = true; + } + + if (unique_pat_index >= get_total_unique_patterns(astc_block_size_index, tm.m_num_parts)) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image: invalid unique_pat_index, decompression failed (file corrupt)\n"); + return false; + } + + log_blk.m_partition_id = unique_pat_index_to_part_seed(astc_block_size_index, tm.m_num_parts, unique_pat_index); + } + else + { + new_prev_state.m_used_part_hash = true; // bias to true + } + + } // if (config_reuse_index < ldr_astc_block_encode_image_output::cMaxConfigReuseNeighbors) + + if (tm_index >= encoder_trial_modes.size()) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image: invalid tm_index, decompression failed (file corrupt)\n"); + return false; + } + + const trial_mode& tm = encoder_trial_modes[tm_index]; + + const bool actual_cem_supports_bc = astc_helpers::cem_supports_bc(actual_cem); + + const uint32_t total_endpoint_vals = astc_helpers::get_num_cem_values(actual_cem); + + for (uint32_t part_iter = 0; part_iter < tm.m_num_parts; part_iter++) + log_blk.m_color_endpoint_modes[part_iter] = (uint8_t)actual_cem; + + log_blk.m_num_partitions = (uint8_t)tm.m_num_parts; + log_blk.m_dual_plane = (tm.m_ccs_index >= 0); + if (log_blk.m_dual_plane) + log_blk.m_color_component_selector = (uint8_t)tm.m_ccs_index; + + log_blk.m_weight_ise_range = (uint8_t)tm.m_weight_ise_range; + log_blk.m_endpoint_ise_range = (uint8_t)tm.m_endpoint_ise_range; + log_blk.m_grid_width = (uint8_t)tm.m_grid_width; + log_blk.m_grid_height = (uint8_t)tm.m_grid_height; + + // --------------------------------- Decode endpoints + const bool used_dpcm_endpoints_flag = dec.decode_bit(use_dpcm_endpoints_model); + + if (!used_dpcm_endpoints_flag) + { + auto& raw_model = raw_endpoint_models[log_blk.m_endpoint_ise_range - astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE]; + + for (uint32_t part_iter = 0; part_iter < tm.m_num_parts; part_iter++) + { + for (uint32_t val_iter = 0; val_iter < total_endpoint_vals; val_iter++) + { + log_blk.m_endpoints[part_iter * total_endpoint_vals + val_iter] = (uint8_t)dec.decode_sym(raw_model); + } // val_iter + + } // part_iter + } + else + { + // Endpoint DPCM + const int num_endpoint_levels = astc_helpers::get_ise_levels(log_blk.m_endpoint_ise_range); + const auto& endpoint_rank_to_ise = astc_helpers::g_dequant_tables.get_endpoint_tab(log_blk.m_endpoint_ise_range).m_rank_to_ISE; + const auto& endpoint_ise_to_rank = astc_helpers::g_dequant_tables.get_endpoint_tab(log_blk.m_endpoint_ise_range).m_ISE_to_rank; + + const uint32_t reuse_delta_index = dec.decode_sym(endpoint_reuse_delta_model); + const int reuse_bx = (int)bx + basist::astc_6x6_hdr::g_reuse_xy_deltas[reuse_delta_index].m_x; + const int reuse_by = (int)by + basist::astc_6x6_hdr::g_reuse_xy_deltas[reuse_delta_index].m_y; + + if ((reuse_bx < 0) || (reuse_by < 0) || (reuse_bx >= (int)num_blocks_x) || (reuse_by >= (int)num_blocks_y)) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::Invalid reuse delta\n"); + return false; + } + + const astc_helpers::log_astc_block* pEndpoint_pred_log_blk = &log_blocks(reuse_bx, reuse_by & 7); + if (pEndpoint_pred_log_blk->m_solid_color_flag_ldr) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::Invalid reuse delta\n"); + return false; + } + + uint32_t bc_model_index = 0; + if (pLeft_state) + bc_model_index = pLeft_state->m_first_endpoint_uses_bc; + else + bc_model_index = 1; + + if (pUpper_state) + bc_model_index |= pUpper_state->m_first_endpoint_uses_bc ? 2 : 0; + else + bc_model_index |= 2; + + if (!pEndpoint_pred_log_blk) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::Can't use endpoint DPCM here\n"); + return false; + } + + bool endpoints_use_bc[astc_helpers::MAX_PARTITIONS] = { false }; + + if (actual_cem_supports_bc) + { + for (uint32_t part_iter = 0; part_iter < log_blk.m_num_partitions; part_iter++) + { + endpoints_use_bc[part_iter] = dec.decode_bit(endpoints_use_bc_models[bc_model_index]); + } + } + + uint8_t predicted_endpoints[astc_helpers::MAX_PARTITIONS][astc_helpers::MAX_CEM_ENDPOINT_VALS] = { }; + + for (uint32_t part_iter = 0; part_iter < log_blk.m_num_partitions; part_iter++) + { + const bool always_repack_flag = false; + bool blue_contraction_clamped_flag = false, base_ofs_clamped_flag = false; + + // Mini-CEM encoder, to cross CEM domains. + bool conv_status = convert_endpoints_across_cems( + pEndpoint_pred_log_blk->m_color_endpoint_modes[0], pEndpoint_pred_log_blk->m_endpoint_ise_range, pEndpoint_pred_log_blk->m_endpoints, + log_blk.m_color_endpoint_modes[0], log_blk.m_endpoint_ise_range, predicted_endpoints[part_iter], + always_repack_flag, + endpoints_use_bc[part_iter], false, + blue_contraction_clamped_flag, base_ofs_clamped_flag); + + if (!conv_status) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::Failed predicting endpoints\n"); + return false; + } + } + + auto& dpcm_model = dpcm_endpoint_models[log_blk.m_endpoint_ise_range - astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE]; + + for (uint32_t part_iter = 0; part_iter < tm.m_num_parts; part_iter++) + { + for (uint32_t val_iter = 0; val_iter < total_endpoint_vals; val_iter++) + { + const uint32_t endpoint_idx = part_iter * total_endpoint_vals + val_iter; + + int delta = (uint8_t)dec.decode_sym(dpcm_model); + int e_val = basisu::imod(delta + endpoint_ise_to_rank[predicted_endpoints[part_iter][val_iter]], num_endpoint_levels); + + log_blk.m_endpoints[endpoint_idx] = endpoint_rank_to_ise[e_val]; + + } // val_iter + + } // part_iter + + } // if (!used_dpcm_endpoints_flag) + + if (actual_cem_supports_bc) + { + new_prev_state.m_first_endpoint_uses_bc = astc_helpers::used_blue_contraction(actual_cem, log_blk.m_endpoints, log_blk.m_endpoint_ise_range); + } + + } // if (mode_index != cMODE_RAW) + + // ----------------------------------- Decode weights + if (tm_index >= encoder_trial_modes.size()) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image: invalid tm_index, decompression failed (file corrupt)\n"); + return false; + } + + const trial_mode& tm = encoder_trial_modes[tm_index]; + + const uint32_t total_planes = (tm.m_ccs_index >= 0) ? 2 : 1; + const uint32_t total_weights = tm.m_grid_width * tm.m_grid_height; + + uint32_t use_dct_model_index = 0; + if (use_dct) + { + if (pLeft_state) + use_dct_model_index = pLeft_state->m_used_weight_dct; + else + use_dct_model_index = 1; + + if (pUpper_state) + use_dct_model_index |= pUpper_state->m_used_weight_dct ? 2 : 0; + else + use_dct_model_index |= 2; + } + + bool block_used_dct = false; + if (use_dct) + block_used_dct = dec.decode_bit(use_dct_model[use_dct_model_index]); + + if (use_fast_decoding) + { + if (block_used_dct) + { + new_prev_state.m_used_weight_dct = true; + + const astc_block_grid_data* pGrid_data = find_astc_block_grid_data(block_width, block_height, log_blk.m_grid_width, log_blk.m_grid_height); + + const uint32_t num_dc_levels = grid_weight_dct::get_num_weight_dc_levels(log_blk.m_weight_ise_range); + syms.m_num_dc_levels = num_dc_levels; + + for (uint32_t plane_iter = 0; plane_iter < total_planes; plane_iter++) + { + syms.m_coeffs.resize(0); + + if (num_dc_levels == DCT_MEAN_LEVELS1) + syms.m_dc_sym = mean1_bytes.get_bits8(); + else + syms.m_dc_sym = mean0_bits.get_bits4(); + + uint32_t cur_zig_ofs = 1; + + while (cur_zig_ofs < total_weights) + { + uint32_t run_len = run_bytes.get_bits8(); + if (run_len == DCT_RUN_LEN_EOB_SYM_INDEX) + break; + + cur_zig_ofs += run_len; + + if (cur_zig_ofs >= total_weights) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::DCT decode error\n"); + return false; + } + + int sign = sign_bits.get_bits1(); + int coeff = coeff_bytes.get_bits8() + 1; + + if (sign) + coeff = -coeff; + + syms.m_coeffs.push_back(dct_syms::coeff(basisu::safe_cast_uint16(run_len), basisu::safe_cast_int16(coeff))); + cur_zig_ofs++; + } + + // weight grid IDCT + if (!grid_dct.decode_block_weights(dct_q, plane_iter, log_blk, nullptr, pGrid_data, nullptr, dct_work, &syms)) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::DCT decode failed\n"); + return false; + } + + } // plane_iter + } + else + { + // Weight grid DPCM (no dependency on other blocks, or between planes, for determinism even when IDCT is used) + const uint32_t num_weight_levels = astc_helpers::get_ise_levels(log_blk.m_weight_ise_range); + const auto& weight_rank_to_ise = astc_helpers::g_dequant_tables.get_weight_tab(log_blk.m_weight_ise_range).m_rank_to_ISE; + + for (uint32_t plane_iter = 0; plane_iter < total_planes; plane_iter++) + { + int prev_w = num_weight_levels / 2; + + if (num_weight_levels <= 4) + { + for (uint32_t weight_iter = 0; weight_iter < total_weights; weight_iter++) + { + uint32_t r = weight2_bits.get_bits2(); + + uint32_t w = r; + w = basisu::imod(prev_w + r, num_weight_levels); + + prev_w = w; + + log_blk.m_weights[plane_iter + weight_iter * total_planes] = (uint8_t)weight_rank_to_ise[w]; + + } // weight_iter + } + else if (num_weight_levels <= 8) + { + for (uint32_t weight_iter = 0; weight_iter < total_weights; weight_iter++) + { + uint32_t r = weight3_bits.get_bits4(); + + uint32_t w = r; + w = basisu::imod(prev_w + r, num_weight_levels); + + prev_w = w; + + log_blk.m_weights[plane_iter + weight_iter * total_planes] = (uint8_t)weight_rank_to_ise[w]; + + } // weight_iter + } + else if (num_weight_levels <= 16) + { + for (uint32_t weight_iter = 0; weight_iter < total_weights; weight_iter++) + { + uint32_t r = weight4_bits.get_bits4(); + + uint32_t w = r; + w = basisu::imod(prev_w + r, num_weight_levels); + + prev_w = w; + + log_blk.m_weights[plane_iter + weight_iter * total_planes] = (uint8_t)weight_rank_to_ise[w]; + + } // weight_iter + } + else + { + for (uint32_t weight_iter = 0; weight_iter < total_weights; weight_iter++) + { + uint32_t r = weight8_bytes.get_bits8(); + + uint32_t w = r; + w = basisu::imod(prev_w + r, num_weight_levels); + + prev_w = w; + + log_blk.m_weights[plane_iter + weight_iter * total_planes] = (uint8_t)weight_rank_to_ise[w]; + + } // weight_iter + } + + } // plane_iter + + } // if (block_used_dct) + } + else + { + if (block_used_dct) + { + new_prev_state.m_used_weight_dct = true; + + const astc_block_grid_data* pGrid_data = find_astc_block_grid_data(block_width, block_height, log_blk.m_grid_width, log_blk.m_grid_height); + + const uint32_t num_dc_levels = grid_weight_dct::get_num_weight_dc_levels(log_blk.m_weight_ise_range); + syms.m_num_dc_levels = num_dc_levels; + + for (uint32_t plane_iter = 0; plane_iter < total_planes; plane_iter++) + { + syms.m_coeffs.resize(0); + + syms.m_dc_sym = dec.decode_sym(weight_mean_models[(num_dc_levels == DCT_MEAN_LEVELS1) ? 1 : 0]); + + uint32_t cur_zig_ofs = 1; + + while (cur_zig_ofs < total_weights) + { + uint32_t run_len = dec.decode_sym(dct_run_len_model); + if (run_len == DCT_RUN_LEN_EOB_SYM_INDEX) + break; + + cur_zig_ofs += run_len; + + if (cur_zig_ofs >= total_weights) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::DCT decode error\n"); + return false; + } + + int sign = dec.get_bit(); + int coeff = dec.decode_sym(dct_coeff_mag) + 1; + + if (sign) + coeff = -coeff; + + syms.m_coeffs.push_back(dct_syms::coeff(basisu::safe_cast_uint16(run_len), basisu::safe_cast_int16(coeff))); + cur_zig_ofs++; + } + + // weight grid IDCT + if (!grid_dct.decode_block_weights(dct_q, plane_iter, log_blk, nullptr, pGrid_data, nullptr, dct_work, &syms)) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::DCT decode failed\n"); + return false; + } + + } // plane_iter + } + else + { + // Weight grid DPCM (no dependency on other blocks, or between planes, for determinism even when IDCT is used) + const uint32_t num_weight_levels = astc_helpers::get_ise_levels(log_blk.m_weight_ise_range); + const auto& weight_rank_to_ise = astc_helpers::g_dequant_tables.get_weight_tab(log_blk.m_weight_ise_range).m_rank_to_ISE; + + for (uint32_t plane_iter = 0; plane_iter < total_planes; plane_iter++) + { + int prev_w = num_weight_levels / 2; + + for (uint32_t weight_iter = 0; weight_iter < total_weights; weight_iter++) + { + uint32_t r = dec.decode_sym(raw_weight_models[log_blk.m_weight_ise_range - astc_helpers::FIRST_VALID_WEIGHT_ISE_RANGE]); + + uint32_t w = r; + w = basisu::imod(prev_w + r, num_weight_levels); + + prev_w = w; + + log_blk.m_weights[plane_iter + weight_iter * total_planes] = (uint8_t)weight_rank_to_ise[w]; + + } // weight_iter + + } // plane_iter + } + + } // use_fast_decoding + + if (pBlock_callback) + { + if (!(*pBlock_callback)(bx, by, log_blk, pBlock_callback_data)) + return false; + } + + break; + } + default: + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::Invalid mode\n"); + return false; + } + } + + } // bx + + assert(!cur_run_len); + + } // by + + //if (debug_output) + // debug_printf("Decomp time: {3.3}ms\n", itm.get_elapsed_ms()); + + const uint32_t final_sync_marker = dec.get_bits(FINAL_SYNC_MARKER_BITS); + if (final_sync_marker != FINAL_SYNC_MARKER) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::Final sync failed\n"); + return false; + } + + if (debug_output) + basisu::debug_printf("astc_ldr_t::decompress_image: Decode sync OK\n"); + + return true; + } + +} // namespace astc_ldr_t + +#endif // #if BASISD_SUPPORT_XUASTC + +#if BASISD_SUPPORT_XUASTC + +namespace bc7u +{ + //------------------------------------------------------------------------------------------------ + // BC7 mode 0-7 decompression. + // Instead of one monster routine to unpack all the BC7 modes, we're lumping the 3 subset, 2 subset, 1 subset, and dual plane modes together into simple shared routines. + + static inline uint32_t bc7_dequant(uint32_t val, uint32_t pbit, uint32_t val_bits) { assert(val < (1U << val_bits)); assert(pbit < 2); assert(val_bits >= 4 && val_bits <= 8); const uint32_t total_bits = val_bits + 1; val = (val << 1) | pbit; val <<= (8 - total_bits); val |= (val >> total_bits); assert(val <= 255); return val; } + static inline uint32_t bc7_dequant(uint32_t val, uint32_t val_bits) { assert(val < (1U << val_bits)); assert(val_bits >= 4 && val_bits <= 8); val <<= (8 - val_bits); val |= (val >> val_bits); assert(val <= 255); return val; } + + static inline uint32_t bc7_interp2(uint32_t l, uint32_t h, uint32_t w) { assert(w < 4); return (l * (64 - basist::g_bc7_weights2[w]) + h * basist::g_bc7_weights2[w] + 32) >> 6; } + static inline uint32_t bc7_interp3(uint32_t l, uint32_t h, uint32_t w) { assert(w < 8); return (l * (64 - basist::g_bc7_weights3[w]) + h * basist::g_bc7_weights3[w] + 32) >> 6; } + static inline uint32_t bc7_interp4(uint32_t l, uint32_t h, uint32_t w) { assert(w < 16); return (l * (64 - basist::g_bc7_weights4[w]) + h * basist::g_bc7_weights4[w] + 32) >> 6; } + static inline uint32_t bc7_interp(uint32_t l, uint32_t h, uint32_t w, uint32_t bits) + { + assert(l <= 255 && h <= 255); + switch (bits) + { + case 2: return bc7_interp2(l, h, w); + case 3: return bc7_interp3(l, h, w); + case 4: return bc7_interp4(l, h, w); + default: + break; + } + return 0; + } + + inline uint32_t read_bits32(const uint8_t* pBuf, uint32_t& bit_offset, uint32_t codesize) + { + assert(codesize <= 32); + uint32_t bits = 0; + uint32_t total_bits = 0; + + while (total_bits < codesize) + { + uint32_t byte_bit_offset = bit_offset & 7; + uint32_t bits_to_read = basisu::minimum(codesize - total_bits, 8 - byte_bit_offset); + + uint32_t byte_bits = pBuf[bit_offset >> 3] >> byte_bit_offset; + byte_bits &= ((1 << bits_to_read) - 1); + + bits |= (byte_bits << total_bits); + + total_bits += bits_to_read; + bit_offset += bits_to_read; + } + + return bits; + } + + bool unpack_bc7_mode0_2(uint32_t mode, const void* pBlock_bits, color_rgba* pPixels) + { + //const uint32_t SUBSETS = 3; + const uint32_t ENDPOINTS = 6; + const uint32_t COMPS = 3; + const uint32_t WEIGHT_BITS = (mode == 0) ? 3 : 2; + const uint32_t ENDPOINT_BITS = (mode == 0) ? 4 : 5; + const uint32_t PBITS = (mode == 0) ? 6 : 0; + const uint32_t WEIGHT_VALS = 1 << WEIGHT_BITS; + + uint32_t bit_offset = 0; + const uint8_t* pBuf = static_cast(pBlock_bits); + + if (read_bits32(pBuf, bit_offset, mode + 1) != (1U << mode)) return false; + + const uint32_t part = read_bits32(pBuf, bit_offset, (mode == 0) ? 4 : 6); + + color_rgba endpoints[ENDPOINTS]; + for (uint32_t c = 0; c < COMPS; c++) + for (uint32_t e = 0; e < ENDPOINTS; e++) + endpoints[e][c] = (uint8_t)read_bits32(pBuf, bit_offset, ENDPOINT_BITS); + + uint32_t pbits[6]; + for (uint32_t p = 0; p < PBITS; p++) + pbits[p] = read_bits32(pBuf, bit_offset, 1); + + uint32_t weights[16]; + for (uint32_t i = 0; i < 16; i++) + weights[i] = read_bits32(pBuf, bit_offset, ((!i) || (i == basist::g_bc7_table_anchor_index_third_subset_1[part]) || (i == basist::g_bc7_table_anchor_index_third_subset_2[part])) ? (WEIGHT_BITS - 1) : WEIGHT_BITS); + + assert(bit_offset == 128); + + for (uint32_t e = 0; e < ENDPOINTS; e++) + for (uint32_t c = 0; c < 4; c++) + endpoints[e][c] = (uint8_t)((c == 3) ? 255 : (PBITS ? bc7_dequant(endpoints[e][c], pbits[e], ENDPOINT_BITS) : bc7_dequant(endpoints[e][c], ENDPOINT_BITS))); + + color_rgba block_colors[3][8]; + for (uint32_t s = 0; s < 3; s++) + for (uint32_t i = 0; i < WEIGHT_VALS; i++) + { + for (uint32_t c = 0; c < 3; c++) + block_colors[s][i][c] = (uint8_t)bc7_interp(endpoints[s * 2 + 0][c], endpoints[s * 2 + 1][c], i, WEIGHT_BITS); + block_colors[s][i][3] = 255; + } + + for (uint32_t i = 0; i < 16; i++) + pPixels[i] = block_colors[basist::g_bc7_partition3[part * 16 + i]][weights[i]]; + + return true; + } + + bool unpack_bc7_mode1_3_7(uint32_t mode, const void* pBlock_bits, color_rgba* pPixels) + { + //const uint32_t SUBSETS = 2; + const uint32_t ENDPOINTS = 4; + const uint32_t COMPS = (mode == 7) ? 4 : 3; + const uint32_t WEIGHT_BITS = (mode == 1) ? 3 : 2; + const uint32_t ENDPOINT_BITS = (mode == 7) ? 5 : ((mode == 1) ? 6 : 7); + const uint32_t PBITS = (mode == 1) ? 2 : 4; + const uint32_t SHARED_PBITS = (mode == 1) ? true : false; + const uint32_t WEIGHT_VALS = 1 << WEIGHT_BITS; + + uint32_t bit_offset = 0; + const uint8_t* pBuf = static_cast(pBlock_bits); + + if (read_bits32(pBuf, bit_offset, mode + 1) != (1U << mode)) return false; + + const uint32_t part = read_bits32(pBuf, bit_offset, 6); + + color_rgba endpoints[ENDPOINTS]; + for (uint32_t c = 0; c < COMPS; c++) + for (uint32_t e = 0; e < ENDPOINTS; e++) + endpoints[e][c] = (uint8_t)read_bits32(pBuf, bit_offset, ENDPOINT_BITS); + + uint32_t pbits[4]; + for (uint32_t p = 0; p < PBITS; p++) + pbits[p] = read_bits32(pBuf, bit_offset, 1); + + uint32_t weights[16]; + for (uint32_t i = 0; i < 16; i++) + weights[i] = read_bits32(pBuf, bit_offset, ((!i) || (i == basist::g_bc7_table_anchor_index_second_subset[part])) ? (WEIGHT_BITS - 1) : WEIGHT_BITS); + + assert(bit_offset == 128); + + for (uint32_t e = 0; e < ENDPOINTS; e++) + for (uint32_t c = 0; c < 4; c++) + endpoints[e][c] = (uint8_t)((c == ((mode == 7U) ? 4U : 3U)) ? 255 : bc7_dequant(endpoints[e][c], pbits[SHARED_PBITS ? (e >> 1) : e], ENDPOINT_BITS)); + + color_rgba block_colors[2][8]; + for (uint32_t s = 0; s < 2; s++) + for (uint32_t i = 0; i < WEIGHT_VALS; i++) + { + for (uint32_t c = 0; c < COMPS; c++) + block_colors[s][i][c] = (uint8_t)bc7_interp(endpoints[s * 2 + 0][c], endpoints[s * 2 + 1][c], i, WEIGHT_BITS); + block_colors[s][i][3] = (COMPS == 3) ? 255 : block_colors[s][i][3]; + } + + for (uint32_t i = 0; i < 16; i++) + pPixels[i] = block_colors[basist::g_bc7_partition2[part * 16 + i]][weights[i]]; + + return true; + } + + bool unpack_bc7_mode4_5(uint32_t mode, const void* pBlock_bits, color_rgba* pPixels) + { + const uint32_t ENDPOINTS = 2; + const uint32_t COMPS = 4; + const uint32_t WEIGHT_BITS = 2; + const uint32_t A_WEIGHT_BITS = (mode == 4) ? 3 : 2; + const uint32_t ENDPOINT_BITS = (mode == 4) ? 5 : 7; + const uint32_t A_ENDPOINT_BITS = (mode == 4) ? 6 : 8; + //const uint32_t WEIGHT_VALS = 1 << WEIGHT_BITS; + //const uint32_t A_WEIGHT_VALS = 1 << A_WEIGHT_BITS; + + uint32_t bit_offset = 0; + const uint8_t* pBuf = static_cast(pBlock_bits); + + if (read_bits32(pBuf, bit_offset, mode + 1) != (1U << mode)) return false; + + const uint32_t comp_rot = read_bits32(pBuf, bit_offset, 2); + const uint32_t index_mode = (mode == 4) ? read_bits32(pBuf, bit_offset, 1) : 0; + + color_rgba endpoints[ENDPOINTS]; + for (uint32_t c = 0; c < COMPS; c++) + for (uint32_t e = 0; e < ENDPOINTS; e++) + endpoints[e][c] = (uint8_t)read_bits32(pBuf, bit_offset, (c == 3) ? A_ENDPOINT_BITS : ENDPOINT_BITS); + + const uint32_t weight_bits[2] = { index_mode ? A_WEIGHT_BITS : WEIGHT_BITS, index_mode ? WEIGHT_BITS : A_WEIGHT_BITS }; + + uint32_t weights[16], a_weights[16]; + + for (uint32_t i = 0; i < 16; i++) + (index_mode ? a_weights : weights)[i] = read_bits32(pBuf, bit_offset, weight_bits[index_mode] - ((!i) ? 1 : 0)); + + for (uint32_t i = 0; i < 16; i++) + (index_mode ? weights : a_weights)[i] = read_bits32(pBuf, bit_offset, weight_bits[1 - index_mode] - ((!i) ? 1 : 0)); + + assert(bit_offset == 128); + + for (uint32_t e = 0; e < ENDPOINTS; e++) + for (uint32_t c = 0; c < 4; c++) + endpoints[e][c] = (uint8_t)bc7_dequant(endpoints[e][c], (c == 3) ? A_ENDPOINT_BITS : ENDPOINT_BITS); + + color_rgba block_colors[8]; + for (uint32_t i = 0; i < (1U << weight_bits[0]); i++) + for (uint32_t c = 0; c < 3; c++) + block_colors[i][c] = (uint8_t)bc7_interp(endpoints[0][c], endpoints[1][c], i, weight_bits[0]); + + for (uint32_t i = 0; i < (1U << weight_bits[1]); i++) + block_colors[i][3] = (uint8_t)bc7_interp(endpoints[0][3], endpoints[1][3], i, weight_bits[1]); + + for (uint32_t i = 0; i < 16; i++) + { + pPixels[i] = block_colors[weights[i]]; + pPixels[i].a = block_colors[a_weights[i]].a; + if (comp_rot >= 1) + std::swap(pPixels[i].a, pPixels[i].m_comps[comp_rot - 1]); + } + + return true; + } + + struct bc7_mode_6 + { + struct + { + uint64_t m_mode : 7; + uint64_t m_r0 : 7; + uint64_t m_r1 : 7; + uint64_t m_g0 : 7; + uint64_t m_g1 : 7; + uint64_t m_b0 : 7; + uint64_t m_b1 : 7; + uint64_t m_a0 : 7; + uint64_t m_a1 : 7; + uint64_t m_p0 : 1; + } m_lo; + + union + { + struct + { + uint64_t m_p1 : 1; + uint64_t m_s00 : 3; + uint64_t m_s10 : 4; + uint64_t m_s20 : 4; + uint64_t m_s30 : 4; + + uint64_t m_s01 : 4; + uint64_t m_s11 : 4; + uint64_t m_s21 : 4; + uint64_t m_s31 : 4; + + uint64_t m_s02 : 4; + uint64_t m_s12 : 4; + uint64_t m_s22 : 4; + uint64_t m_s32 : 4; + + uint64_t m_s03 : 4; + uint64_t m_s13 : 4; + uint64_t m_s23 : 4; + uint64_t m_s33 : 4; + + } m_hi; + + uint64_t m_hi_bits; + }; + }; + + bool unpack_bc7_mode6(const void* pBlock_bits, color_rgba* pPixels) + { + static_assert(sizeof(bc7_mode_6) == 16, "sizeof(bc7_mode_6) == 16"); + + const bc7_mode_6& block = *static_cast(pBlock_bits); + + if (block.m_lo.m_mode != (1 << 6)) + return false; + + const uint32_t r0 = (uint32_t)((block.m_lo.m_r0 << 1) | block.m_lo.m_p0); + const uint32_t g0 = (uint32_t)((block.m_lo.m_g0 << 1) | block.m_lo.m_p0); + const uint32_t b0 = (uint32_t)((block.m_lo.m_b0 << 1) | block.m_lo.m_p0); + const uint32_t a0 = (uint32_t)((block.m_lo.m_a0 << 1) | block.m_lo.m_p0); + const uint32_t r1 = (uint32_t)((block.m_lo.m_r1 << 1) | block.m_hi.m_p1); + const uint32_t g1 = (uint32_t)((block.m_lo.m_g1 << 1) | block.m_hi.m_p1); + const uint32_t b1 = (uint32_t)((block.m_lo.m_b1 << 1) | block.m_hi.m_p1); + const uint32_t a1 = (uint32_t)((block.m_lo.m_a1 << 1) | block.m_hi.m_p1); + + color_rgba vals[16]; + for (uint32_t i = 0; i < 16; i++) + { + const uint32_t w = basist::g_bc7_weights4[i]; + const uint32_t iw = 64 - w; + vals[i].set_noclamp_rgba( + (r0 * iw + r1 * w + 32) >> 6, + (g0 * iw + g1 * w + 32) >> 6, + (b0 * iw + b1 * w + 32) >> 6, + (a0 * iw + a1 * w + 32) >> 6); + } + + pPixels[0] = vals[block.m_hi.m_s00]; + pPixels[1] = vals[block.m_hi.m_s10]; + pPixels[2] = vals[block.m_hi.m_s20]; + pPixels[3] = vals[block.m_hi.m_s30]; + + pPixels[4] = vals[block.m_hi.m_s01]; + pPixels[5] = vals[block.m_hi.m_s11]; + pPixels[6] = vals[block.m_hi.m_s21]; + pPixels[7] = vals[block.m_hi.m_s31]; + + pPixels[8] = vals[block.m_hi.m_s02]; + pPixels[9] = vals[block.m_hi.m_s12]; + pPixels[10] = vals[block.m_hi.m_s22]; + pPixels[11] = vals[block.m_hi.m_s32]; + + pPixels[12] = vals[block.m_hi.m_s03]; + pPixels[13] = vals[block.m_hi.m_s13]; + pPixels[14] = vals[block.m_hi.m_s23]; + pPixels[15] = vals[block.m_hi.m_s33]; + + return true; + } + + int determine_bc7_mode(const void* pBlock) + { + const uint32_t first_byte = static_cast(pBlock)[0]; + + for (uint32_t mode = 0; mode <= 7; mode++) + { + if (first_byte & (1U << mode)) + return mode; + } + + return -1; + } + + int determine_bc7_mode_4_index_mode(const void* pBlock) + { + const uint32_t first_byte = static_cast(pBlock)[0]; + + // check for mode 4 + if ((first_byte & 31) != 0b10000) + return -1; + + return (first_byte >> 7); + } + + int determine_bc7_mode_4_or_5_rotation(const void* pBlock) + { + const uint32_t first_byte = static_cast(pBlock)[0]; + if ((first_byte & 31) == 0b10000) + { + // mode 4 + return (first_byte >> 5) & 3; + } + + if ((first_byte & 63) == 0b100000) + { + // mode 5 + return first_byte >> 6; + } + + return -1; + } + + bool unpack_bc7(const void* pBlock, color_rgba* pPixels) + { + const uint32_t first_byte = static_cast(pBlock)[0]; + + for (uint32_t mode = 0; mode <= 7; mode++) + { + if (first_byte & (1U << mode)) + { + switch (mode) + { + case 0: + case 2: + return unpack_bc7_mode0_2(mode, pBlock, pPixels); + case 1: + case 3: + case 7: + return unpack_bc7_mode1_3_7(mode, pBlock, pPixels); + case 4: + case 5: + return unpack_bc7_mode4_5(mode, pBlock, pPixels); + case 6: + return unpack_bc7_mode6(pBlock, pPixels); + default: + break; + } + } + } + + return false; + } +} + +// BASISU_BC7F_USE_SSE41 - only very minimally tested. Needs more testing and more variants. Only improves transcoding perf by ~10% in native so far. +#define BASISU_BC7F_USE_SSE41 (0) +#define BASISU_BC7F_PERF_STATS (0) + +namespace bc7f +{ + const uint32_t MAX_PATTERNS2_TO_CHECK = 64; + const uint32_t MAX_PATTERNS3_TO_CHECK = 64; + + const float UNIQUE_PBIT_DISCOUNT = .85f; + const float SHARED_PBIT_DISCOUNT = .95f; + + //static inline uint8_t mul_8(uint32_t v, uint32_t q) { v = v * q + 128; return (uint8_t)((v + (v >> 8)) >> 8); } + //static inline int mul_8bit(int a, int b) { int t = a * b + 128; return (t + (t >> 8)) >> 8; } + //static inline int lerp_8bit(int a, int b, int s) { assert(a >= 0 && a <= 255); assert(b >= 0 && b <= 255); assert(s >= 0 && s <= 255); return a + mul_8bit(b - a, s); } + + static int popcount32(uint32_t x) + { +#if defined(__EMSCRIPTEN__) || defined(__clang__) || defined(__GNUC__) + return __builtin_popcount(x); +#elif defined(_MSC_VER) + return __popcnt(x); +#else + int count = 0; + while (x) + { + x &= (x - 1); + ++count; + } + return count; +#endif + } + +#if BASISU_BC7F_PERF_STATS + // not thread safe (no need/for dev) + uint32_t g_total_rgb_calls; + uint32_t g_total_rgba_calls; + uint32_t g_total_solid_blocks; + + uint32_t g_total_trivial_mode6_blocks; + + uint32_t g_total_dp_valid_chans_rgb; + uint32_t g_total_dp_valid_chans_a; + uint32_t g_total_high_ortho_energy; + + uint32_t g_total_mode02_evals; + uint32_t g_total_mode02_bailouts; + + uint32_t g_total_mode13_evals; + uint32_t g_total_mode13_bailouts; + + uint32_t g_total_mode45_evals; + uint32_t g_total_mode45_bailouts; + + uint32_t g_total_mode7_evals; + uint32_t g_total_mode7_bailouts; +#endif + + inline int fast_roundf_pos_int(float x) + { + assert(x >= 0.0f); + return (int)(x + 0.5f); + } + + inline int fast_roundf_int(float x) + { + return (x >= 0.0f) ? (int)(x + 0.5f) : (int)(x - 0.5f); + } + + inline int fast_floorf_int(float x) + { + int xi = (int)x; // Truncate towards zero + return ((x < 0.0f) && (x != (float)xi)) ? (xi - 1) : xi; + } + + static inline uint32_t from_7(uint32_t v) + { + assert(v < 128); + return (v << 1) | (v >> 6); + } + + static inline uint32_t from_7(uint32_t v, uint32_t p) + { + assert((v < 128) && (p <= 1)); + return (v << 1) | p; + } + + static inline int to_7(int c8, int pbit) + { + assert((c8 >= 0) && (c8 <= 255) && (pbit >= 0) && (pbit <= 1)); + uint32_t e = (uint32_t(c8) + uint32_t(pbit ^ 1)) >> 1; + return basisu::minimum(127, e); + } + + static inline int to_7(int c8) + { + assert((c8 >= 0) && (c8 <= 255)); + return (c8 * 127 + 127) / 255; + } + + static inline int to_7(float c, int pbit) + { + assert((c >= 0) && (c <= 255.0f)); + return to_7(fast_roundf_pos_int(c), pbit); + } + + static inline int to_7_clamp(float c, int pbit) + { + return to_7(basisu::clamp(fast_roundf_int(c), 0, 255), pbit); + } + + static inline int to_5(int c8) + { + assert((c8 >= 0) && (c8 <= 255)); + return (c8 * 31 + 127) / 255; + } + + static inline int to_5_clamp(float c) + { + return basisu::clamp(fast_roundf_int(c * (31.0f / 255.0f)), 0, 31); + } + + static inline int to_6(int c8) + { + assert((c8 >= 0) && (c8 <= 255)); + return (c8 * 63 + 127) / 255; + } + + static inline int to_6(int c8, int pbit) + { + assert((c8 >= 0) && (c8 <= 255)); + assert((pbit == 0) || (pbit == 1)); + + int q7 = (c8 * 127 + 127) / 255; + + if ((q7 & 1) != pbit) + { + const int lhs = c8 * 127; + const int rhs = 255 * q7; + + if (lhs >= rhs) + { + q7 = (q7 < 127) ? (q7 + 1) : (q7 - 1); + } + else + { + q7 = (q7 > 0) ? (q7 - 1) : (q7 + 1); + } + } + + return q7 >> 1; + } + + static inline int to_6_clamp(float c, int pbit) + { + return to_6(basisu::clamp(fast_roundf_int(c), 0, 255), pbit); + } + + static inline uint32_t from_6(uint32_t v, uint32_t p) + { + assert((v < 64) && (p <= 1)); + v = (v << 1) | p; + v = (v << 1) | (v >> 6); + return v; + } + + static inline uint32_t from_4(uint32_t v, uint32_t p) + { + assert((v < 16) && (p <= 1)); + v = (v << 1) | p; + v = (v << 3) | (v >> 2); + return v; + } + + static inline uint32_t from_5(uint32_t v) + { + assert(v < 32); + v = (v << 3) | (v >> 2); + return v; + } + + static inline uint32_t from_5(uint32_t v, uint32_t p) + { + assert((v < 32) && (p <= 1)); + v = (v << 1) | p; + v = (v << 2) | (v >> 4); + return v; + } + + static inline uint32_t from_6(uint32_t v) + { + assert(v < 64); + v = (v << 2) | (v >> 4); + return v; + } + + static inline int to_5(int c8, int pbit) + { + assert((c8 >= 0) && (c8 <= 255)); + assert((pbit == 0) || (pbit == 1)); + + int q6 = (c8 * 63 + 127) / 255; + + if ((q6 & 1) != pbit) + { + const int lhs = c8 * 63; + const int rhs = 255 * q6; + + if (lhs >= rhs) + { + q6 = (q6 < 63) ? (q6 + 1) : (q6 - 1); + } + else + { + q6 = (q6 > 0) ? (q6 - 1) : (q6 + 1); + } + } + + return q6 >> 1; + } + +#if 0 + static inline int to_5(float c, int pbit) + { + assert((c >= 0.0f) && (c <= 255.0f)); + return to_5((int)fast_roundf_pos_int(c), pbit); + } +#endif + + static inline int to_5_clamp(float c, uint32_t pbit) + { + return to_5(basisu::clamp(fast_roundf_int(c), 0, 255), pbit); + } + + //static inline uint32_t bc7_interp(uint32_t l, uint32_t h, uint32_t w) { assert(w <= 64); return (l * (64 - w) + h * w + 32) >> 6; } + //static inline uint32_t bc7_interp2(uint32_t l, uint32_t h, uint32_t w) { assert(w <= 64); int d = h - l; return (int)l + ((d * (int)w + 32) >> 6); } + //static inline uint32_t bc7_interp3(int l, int d, uint32_t w) { assert(w <= 64); return l + ((d * (int)w + 32) >> 6); } + + static vec4F g_bc7_2bit_ls_tab[4]; + static vec4F g_bc7_3bit_ls_tab[8]; + static vec4F g_bc7_4bit_ls_tab[16]; + static uint16_t g_bc7_part2_bitmasks[64]; + static uint32_t g_part3_bitmasks[64]; + + void init() + { + for (uint32_t i = 0; i < 4; i++) + { + float w = (float)basist::g_bc7_weights2[i] * (1.0f / 64.0f); + g_bc7_2bit_ls_tab[i].set(w * w, (1.0f - w) * w, (1.0f - w) * (1.0f - w), w); + } + + for (uint32_t i = 0; i < 8; i++) + { + float w = (float)basist::g_bc7_weights3[i] * (1.0f / 64.0f); + g_bc7_3bit_ls_tab[i].set(w * w, (1.0f - w) * w, (1.0f - w) * (1.0f - w), w); + } + + for (uint32_t i = 0; i < 16; i++) + { + float w = (float)basist::g_bc7_weights4[i] * (1.0f / 64.0f); + g_bc7_4bit_ls_tab[i].set(w * w, (1.0f - w) * w, (1.0f - w) * (1.0f - w), w); + } + + for (uint32_t i = 0; i < 64; i++) + { + uint16_t y = 0; + + for (uint32_t x = 0; x < 16; x++) + y |= (g_bc7_partition2[i * 16 + x] << x); + + g_bc7_part2_bitmasks[i] = y; + } + + for (uint32_t i = 0; i < 64; i++) + { + const uint8_t* pPat = &g_bc7_partition3[i * 16]; + + for (uint32_t j = 0; j < 16; j++) + { + const uint32_t s = pPat[j]; + + if (s == 0) + g_part3_bitmasks[i] |= (1 << j); + else if (s == 1) + g_part3_bitmasks[i] |= (0x10000 << j); + } + } + } + + void encode_mode0_rgb_block(uint8_t* pBlock, uint32_t part_id, // 3 subsets, 4-bits part ID + uint32_t lr[3], uint32_t lg[3], uint32_t lb[3], // 4 bit endpoints + uint32_t hr[3], uint32_t hg[3], uint32_t hb[3], + uint32_t p[6], + const uint8_t* pWeights) // 3-bit weights + { + assert(part_id < 16); + assert((lr[0] | lr[1] | lr[2] | lg[0] | lg[1] | lg[2] | lb[0] | lb[1] | lb[2]) <= 15); + assert((hr[0] | hr[1] | hr[2] | hg[0] | hg[1] | hg[2] | hb[0] | hb[1] | hb[2]) <= 15); + assert((p[0] | p[1] | p[2] | p[3] | p[4] | p[5]) <= 1); + + const uint8_t* pPart_map = &g_bc7_partition3[part_id * 16]; + const uint32_t anchor_index0 = g_bc7_table_anchor_index_third_subset_1[part_id]; + const uint32_t anchor_index1 = g_bc7_table_anchor_index_third_subset_2[part_id]; + + uint32_t weight_inv[3] = { 0, 0, 0 }; + + if (pWeights[0] & 4) + { + std::swap(lr[0], hr[0]); + std::swap(lg[0], hg[0]); + std::swap(lb[0], hb[0]); + std::swap(p[0], p[1]); + weight_inv[0] = 7; + } + + if (pWeights[anchor_index0] & 4) + { + std::swap(lr[1], hr[1]); + std::swap(lg[1], hg[1]); + std::swap(lb[1], hb[1]); + std::swap(p[2], p[3]); + weight_inv[1] = 7; + } + + if (pWeights[anchor_index1] & 4) + { + std::swap(lr[2], hr[2]); + std::swap(lg[2], hg[2]); + std::swap(lb[2], hb[2]); + std::swap(p[4], p[5]); + weight_inv[2] = 7; + } + + uint64_t low = 1ULL | ((part_id) << 1) | + ((lr[0]) << 5) | ((hr[0]) << 9) | + ((lr[1]) << 13) | ((hr[1]) << 17) | + ((lr[2]) << 21) | ((hr[2]) << 25) | + (uint64_t(lg[0]) << 29) | (uint64_t(hg[0]) << 33) | + (uint64_t(lg[1]) << 37) | (uint64_t(hg[1]) << 41) | + (uint64_t(lg[2]) << 45) | (uint64_t(hg[2]) << 49) | + (uint64_t(lb[0]) << 53) | (uint64_t(hb[0]) << 57) | + (uint64_t(lb[1]) << 61); + + pBlock[0] = (uint8_t)low; + pBlock[1] = (uint8_t)(low >> 8); + pBlock[2] = (uint8_t)(low >> 16); + pBlock[3] = (uint8_t)(low >> 24); + pBlock[4] = (uint8_t)(low >> 32); + pBlock[5] = (uint8_t)(low >> 40); + pBlock[6] = (uint8_t)(low >> 48); + pBlock[7] = (uint8_t)(low >> 56); + + uint64_t high = (lb[1] >> 3) | ((hb[1]) << 1) | ((lb[2]) << 5) | ((hb[2]) << 9) | + ((p[0]) << 13) | ((p[1]) << 14) | ((p[2]) << 15) | ((p[3]) << 16) | ((p[4]) << 17) | ((p[5]) << 18); + + uint32_t ofs = 19; + for (uint32_t i = 0; i < 16; i++) + { + const uint32_t subset_index = pPart_map[i]; + uint64_t w = pWeights[i] ^ weight_inv[subset_index]; + +#ifdef _DEBUG + assert(w <= 7); + if ((i == 0) || (i == anchor_index0) || (i == anchor_index1)) + { + assert((w & 4) == 0); + } +#endif + + high |= (w << ofs); + ofs += (3 - ((i == 0) || (i == anchor_index0) || (i == anchor_index1))); + } + assert(64 == ofs); + + pBlock[8] = (uint8_t)high; + pBlock[9] = (uint8_t)(high >> 8); + pBlock[10] = (uint8_t)(high >> 16); + pBlock[11] = (uint8_t)(high >> 24); + pBlock[12] = (uint8_t)(high >> 32); + pBlock[13] = (uint8_t)(high >> 40); + pBlock[14] = (uint8_t)(high >> 48); + pBlock[15] = (uint8_t)(high >> 56); + } + + void encode_mode1_rgb_block(uint8_t* pBlock, uint32_t part_id, // 2 subsets, 6-bits part ID + uint32_t lr[2], uint32_t lg[2], uint32_t lb[2], // 6-bit endpoints, 2 shared pbits + uint32_t hr[2], uint32_t hg[2], uint32_t hb[2], + uint32_t p0, uint32_t p1, + const uint8_t* pWeights) // 3-bit weights + { + assert(part_id < 64); + assert((lr[0] | lr[1] | lg[0] | lg[1] | lb[0] | lb[1]) <= 63); + assert((hr[0] | hr[1] | hg[0] | hg[1] | hb[0] | hb[1]) <= 63); + assert((p0 | p1) <= 1); + + const uint8_t* pPart_map = &g_bc7_partition2[part_id * 16]; + const uint32_t anchor_index = g_bc7_table_anchor_index_second_subset[part_id]; + + uint32_t weight_inv[2] = { 0, 0 }; + if (pWeights[0] & 4) + { + std::swap(lr[0], hr[0]); + std::swap(lg[0], hg[0]); + std::swap(lb[0], hb[0]); + weight_inv[0] = 7; + } + + if (pWeights[anchor_index] & 4) + { + std::swap(lr[1], hr[1]); + std::swap(lg[1], hg[1]); + std::swap(lb[1], hb[1]); + weight_inv[1] = 7; + } + + pBlock[0] = (uint8_t)(0b10 | (part_id << 2)); + + uint64_t x = lr[0] | (hr[0] << (6 * 1)); + x |= (lr[1] << (6 * 2)) | (hr[1] << (6 * 3)); + + x |= (lg[0] << (6 * 4)) | (uint64_t(hg[0]) << (6 * 5)); + x |= (uint64_t(lg[1]) << (6 * 6)) | (uint64_t(hg[1]) << (6 * 7)); + + x |= (uint64_t(lb[0]) << (6 * 8)) | (uint64_t(hb[0]) << (6 * 9)); + x |= (uint64_t(lb[1]) << (6 * 10)); + + // 11*6=66 bits total, write first 64 + + pBlock[1] = (uint8_t)x; + pBlock[2] = (uint8_t)(x >> 8); + pBlock[3] = (uint8_t)(x >> 16); + pBlock[4] = (uint8_t)(x >> 24); + + pBlock[5] = (uint8_t)(x >> 32); + pBlock[6] = (uint8_t)(x >> 40); + pBlock[7] = (uint8_t)(x >> 48); + pBlock[8] = (uint8_t)(x >> 56); + + pBlock[9] = (uint8_t)((lb[1] >> 4) | (hb[1] << 2)); + + uint64_t y = p0 | (p1 << 1); + uint32_t ofs = 2; + + for (uint32_t i = 0; i < 16; i++) + { + const uint32_t subset_index = pPart_map[i]; + + uint64_t w = pWeights[i] ^ weight_inv[subset_index]; + +#ifdef _DEBUG + assert(w <= 7); + if ((i == 0) || (i == anchor_index)) + { + assert((w & 4) == 0); + } +#endif + y |= (w << ofs); + + ofs += (3 - ((i == 0) || (i == anchor_index))); + } + assert(48 == ofs); + + pBlock[10] = (uint8_t)y; + pBlock[11] = (uint8_t)(y >> 8); + pBlock[12] = (uint8_t)(y >> 16); + pBlock[13] = (uint8_t)(y >> 24); + pBlock[14] = (uint8_t)(y >> 32); + pBlock[15] = (uint8_t)(y >> 40); + } + + void encode_mode2_rgb_block(uint8_t* pBlock, uint32_t part_id, // 3 subsets, 6-bits part ID + uint32_t lr[3], uint32_t lg[3], uint32_t lb[3], // 5 bit endpoints, no pbits + uint32_t hr[3], uint32_t hg[3], uint32_t hb[3], + const uint8_t* pWeights) // 2-bit weights + { + assert(part_id < 64); + assert((lr[0] | lr[1] | lr[2] | lg[0] | lg[1] | lg[2] | lb[0] | lb[1] | lb[2]) <= 31); + assert((hr[0] | hr[1] | hr[2] | hg[0] | hg[1] | hg[2] | hb[0] | hb[1] | hb[2]) <= 31); + + const uint8_t* pPart_map = &g_bc7_partition3[part_id * 16]; + + uint32_t weight_inv[3] = { 0 }; + if (pWeights[0] & 2) + { + std::swap(lr[0], hr[0]); + std::swap(lg[0], hg[0]); + std::swap(lb[0], hb[0]); + weight_inv[0] = 3; + } + + const uint32_t anchor_index0 = g_bc7_table_anchor_index_third_subset_1[part_id]; + if (pWeights[anchor_index0] & 2) + { + std::swap(lr[1], hr[1]); + std::swap(lg[1], hg[1]); + std::swap(lb[1], hb[1]); + weight_inv[1] = 3; + } + + const uint32_t anchor_index1 = g_bc7_table_anchor_index_third_subset_2[part_id]; + if (pWeights[anchor_index1] & 2) + { + std::swap(lr[2], hr[2]); + std::swap(lg[2], hg[2]); + std::swap(lb[2], hb[2]); + weight_inv[2] = 3; + } + + uint64_t v = 0b100 | (part_id << 3); + v |= (lr[0] << 9) | (hr[0] << (9 + 5 * 1)); + v |= (lr[1] << (9 + 5 * 2)) | (hr[1] << (9 + 5 * 3)); + v |= (uint64_t(lr[2]) << (9 + 5 * 4)) | (uint64_t(hr[2]) << (9 + 5 * 5)); + + v |= (uint64_t(lg[0]) << (9 + 5 * 6)) | (uint64_t(hg[0]) << (9 + 5 * 7)); + v |= (uint64_t(lg[1]) << (9 + 5 * 8)) | (uint64_t(hg[1]) << (9 + 5 * 9)); + v |= (uint64_t(lg[2]) << (9 + 5 * 10)); + + pBlock[0] = (uint8_t)v; + pBlock[1] = (uint8_t)(v >> 8); + pBlock[2] = (uint8_t)(v >> 16); + pBlock[3] = (uint8_t)(v >> 24); + pBlock[4] = (uint8_t)(v >> 32); + pBlock[5] = (uint8_t)(v >> 40); + pBlock[6] = (uint8_t)(v >> 48); + pBlock[7] = (uint8_t)(v >> 56); + + uint64_t v1 = hg[2]; + v1 |= (lb[0] << (5 * 1)) | (hb[0] << (5 * 2)); + v1 |= (lb[1] << (5 * 3)) | (hb[1] << (5 * 4)); + v1 |= (lb[2] << (5 * 5)) | (uint64_t(hb[2]) << (5 * 6)); + + pBlock[8] = (uint8_t)(v1); + pBlock[9] = (uint8_t)(v1 >> 8); + pBlock[10] = (uint8_t)(v1 >> 16); + pBlock[11] = (uint8_t)(v1 >> 24); + + v1 >>= 32; + + // 3 bits left over + uint32_t ofs = 3; + + for (uint32_t i = 0; i < 16; i++) + { + const uint32_t subset_index = pPart_map[i]; + + uint64_t w = pWeights[i] ^ weight_inv[subset_index]; + +#ifdef _DEBUG + assert(w <= 3); + if ((i == 0) || (i == anchor_index0) || (i == anchor_index1)) + { + assert((w & 2) == 0); + } +#endif + v1 |= (w << ofs); + + ofs += (2 - ((i == 0) || (i == anchor_index0) || (i == anchor_index1))); + } + assert(32 == ofs); + + pBlock[12] = (uint8_t)v1; + pBlock[13] = (uint8_t)(v1 >> 8); + pBlock[14] = (uint8_t)(v1 >> 16); + pBlock[15] = (uint8_t)(v1 >> 24); + } + + void encode_mode3_rgb_block(uint8_t* pBlock, uint32_t part_id, // 2 subsets, 6-bits part ID + uint32_t lr[2], uint32_t lg[2], uint32_t lb[2], // 7-bit endpoints, 4 unique pbits + uint32_t hr[2], uint32_t hg[2], uint32_t hb[2], + uint32_t p[4], + const uint8_t* pWeights) // 2-bit weights + { + assert(part_id < 64); + assert((lr[0] | lr[1] | lg[0] | lg[1] | lb[0] | lb[1]) <= 127); + assert((hr[0] | hr[1] | hg[0] | hg[1] | hb[0] | hb[1]) <= 127); + assert((p[0] | p[1] | p[2] | p[3]) <= 1); + + const uint8_t* pPart_map = &g_bc7_partition2[part_id * 16]; + const uint32_t anchor_index = g_bc7_table_anchor_index_second_subset[part_id]; + + uint32_t weight_inv[2] = { 0, 0 }; + if (pWeights[0] & 2) + { + std::swap(lr[0], hr[0]); + std::swap(lg[0], hg[0]); + std::swap(lb[0], hb[0]); + std::swap(p[0], p[1]); + weight_inv[0] = 3; + } + + if (pWeights[anchor_index] & 2) + { + std::swap(lr[1], hr[1]); + std::swap(lg[1], hg[1]); + std::swap(lb[1], hb[1]); + std::swap(p[2], p[3]); + weight_inv[1] = 3; + } + + uint64_t x = 0b1000 | (part_id << 4) | + (lr[0] << 10) | (hr[0] << 17) | + (lr[1] << 24) | (uint64_t(hr[1]) << 31) | + (uint64_t(lg[0]) << 38) | (uint64_t(hg[0]) << 45) | + (uint64_t(lg[1]) << 52) | (uint64_t(hg[1]) << 59); + + pBlock[0] = (uint8_t)x; + pBlock[1] = (uint8_t)(x >> 8); + pBlock[2] = (uint8_t)(x >> 16); + pBlock[3] = (uint8_t)(x >> 24); + pBlock[4] = (uint8_t)(x >> 32); + pBlock[5] = (uint8_t)(x >> 40); + pBlock[6] = (uint8_t)(x >> 48); + pBlock[7] = (uint8_t)(x >> 56); + + // 2 bits of hg[1] remaining to pack + + uint64_t y = (hg[1] >> 5) | (lb[0] << 2) | (hb[0] << 9) | + (lb[1] << (9 + 7 * 1)) | (hb[1] << (9 + 7 * 2)) | + (uint64_t(p[0]) << (9 + 7 * 3)) | (uint64_t(p[1]) << (9 + 7 * 3 + 1)) | + (uint64_t(p[2]) << (9 + 7 * 3 + 2)) | (uint64_t(p[3]) << (9 + 7 * 3 + 3)); + + // now 34 total bits + + uint32_t ofs = 34; + for (uint32_t i = 0; i < 16; i++) + { + const uint32_t subset_index = pPart_map[i]; + uint64_t w = pWeights[i] ^ weight_inv[subset_index]; + +#ifdef _DEBUG + assert(w <= 3); + if ((i == 0) || (i == anchor_index)) + { + assert((w & 2) == 0); + } +#endif + + y |= (w << ofs); + ofs += (2 - ((i == 0) || (i == anchor_index))); + } + assert(64 == ofs); + + pBlock[8] = (uint8_t)y; + pBlock[9] = (uint8_t)(y >> 8); + pBlock[10] = (uint8_t)(y >> 16); + pBlock[11] = (uint8_t)(y >> 24); + pBlock[12] = (uint8_t)(y >> 32); + pBlock[13] = (uint8_t)(y >> 40); + pBlock[14] = (uint8_t)(y >> 48); + pBlock[15] = (uint8_t)(y >> 56); + } + + void encode_mode4_rgba_block(uint8_t* pBlock, + uint32_t lr, uint32_t lg, uint32_t lb, uint32_t la, // 5-bit RGB endpoints, 6-bit A endpoints, no p-bits + uint32_t hr, uint32_t hg, uint32_t hb, uint32_t ha, + const uint8_t* pWeights0, const uint8_t* pWeights1, // weights0 are 3-bits (RGB), weights1 are 2-bits (alpha) + uint32_t rot_index, uint32_t index_flag) // rot_index=0 no rotation, if index_flag is 1, the 3-bit indices are for RGB + { + assert((lr | lg | lb | hr | hg | hb) <= 31); + assert((la | ha) <= 63); + assert(rot_index <= 3); + assert(index_flag <= 1); + + // defaults: 2nd plane=always alpha, RGB=3-bit indices, A=2-bits (favoring RGB) + //const uint32_t rot_index = 0, index_flag = 1; + + uint32_t weights_inv[2] = { }; + + const uint8_t* p2BitWeights = index_flag ? pWeights1 : pWeights0; + const uint8_t* p3BitWeights = index_flag ? pWeights0 : pWeights1; + + // 3-bits + if (p3BitWeights[0] & 4) + { + weights_inv[0] = 7; + if (index_flag) + { + std::swap(lr, hr); + std::swap(lg, hg); + std::swap(lb, hb); + } + else + { + std::swap(la, ha); + } + } + + // 2-bits + if (p2BitWeights[0] & 2) + { + weights_inv[1] = 3; + if (index_flag) + { + std::swap(la, ha); + } + else + { + std::swap(lr, hr); + std::swap(lg, hg); + std::swap(lb, hb); + } + } + + pBlock[0] = (uint8_t)(0b10000 | (rot_index << 5) | (index_flag << 7)); + + // 6*5+6*2=42 bits + uint64_t x = lr | (hr << (5 * 1)); + x |= (lg << (5 * 2)) | (hg << (5 * 3)); + x |= (lb << (5 * 4)) | (hb << (5 * 5)); + x |= (uint64_t(la) << (5 * 6)) | (uint64_t(ha) << (5 * 6 + 6)); + + pBlock[1] = (uint8_t)x; + pBlock[2] = (uint8_t)(x >> 8); + pBlock[3] = (uint8_t)(x >> 16); + pBlock[4] = (uint8_t)(x >> 24); + + pBlock[5] = (uint8_t)(x >> 32); + + // 2 leftover bits + x >>= 40; + uint32_t ofs0 = 2; + + // alpha indices (2-bits) + for (uint32_t i = 0; i < 16; i++) + { + assert(p2BitWeights[i] <= 3); + uint64_t w = p2BitWeights[i] ^ weights_inv[1]; + + assert(i || ((w & 2) == 0)); + + x |= (w << ofs0); + + ofs0 += 2 - (i == 0); + } + + // x = 31+2=33 bits + + pBlock[6] = (uint8_t)x; + pBlock[7] = (uint8_t)(x >> 8); + pBlock[8] = (uint8_t)(x >> 16); + pBlock[9] = (uint8_t)(x >> 24); + + x >>= 32; + + // x = 1 bits + uint32_t ofs1 = 1; + + // rgb indices (3-bits) + for (uint32_t i = 0; i < 16; i++) + { + assert(p3BitWeights[i] <= 7); + uint64_t w = p3BitWeights[i] ^ weights_inv[0]; + + assert(i || ((w & 4) == 0)); + + x |= (w << ofs1); + + ofs1 += 3 - (i == 0); + } + + assert(ofs1 == 48); + + // x=48 bits + pBlock[10] = (uint8_t)x; + pBlock[11] = (uint8_t)(x >> 8); + pBlock[12] = (uint8_t)(x >> 16); + pBlock[13] = (uint8_t)(x >> 24); + pBlock[14] = (uint8_t)(x >> 32); + pBlock[15] = (uint8_t)(x >> 40); + } + + // lossless in RGBA + void pack_mode5_solid(uint8_t* pBlock, const color_rgba& c) + { + pBlock[0] = 0b00100000; + + uint32_t lr = basist::g_bc7_mode_5_optimal_endpoints[c[0]].m_lo; + uint32_t hr = basist::g_bc7_mode_5_optimal_endpoints[c[0]].m_hi; + + uint32_t lg = basist::g_bc7_mode_5_optimal_endpoints[c[1]].m_lo; + uint32_t hg = basist::g_bc7_mode_5_optimal_endpoints[c[1]].m_hi; + + uint32_t lb = basist::g_bc7_mode_5_optimal_endpoints[c[2]].m_lo; + uint32_t hb = basist::g_bc7_mode_5_optimal_endpoints[c[2]].m_hi; + + // 8 endpoints are 8-bits, nothing fancy needed + uint32_t a = c[3]; + + // 58 total bits + uint64_t x = lr | (hr << (7 * 1)); + x |= (lg << (7 * 2)) | (hg << (7 * 3)); + x |= (((uint64_t)lb) << (7 * 4)) | (((uint64_t)hb) << (7 * 5)); + x |= (((uint64_t)a) << (7 * 6)) | (((uint64_t)a) << (7 * 6 + 8)); + + // write 56 bits, leaving 2 left over + pBlock[1] = (uint8_t)(x); + pBlock[2] = (uint8_t)(x >> 8); + pBlock[3] = (uint8_t)(x >> 16); + pBlock[4] = (uint8_t)(x >> 24); + pBlock[5] = (uint8_t)(x >> 32); + pBlock[6] = (uint8_t)(x >> 40); + pBlock[7] = (uint8_t)(x >> 48); + + x >>= 56; + assert(x <= 3); + +#if 0 + x |= (0b0101010101010101010101010101011ull << 2); + + pBlock[8] = (uint8_t)(x); + pBlock[9] = (uint8_t)(x >> 8); + pBlock[10] = (uint8_t)(x >> 16); + pBlock[11] = (uint8_t)(x >> 24); + pBlock[12] = (uint8_t)(x >> 32); + pBlock[13] = 0; + pBlock[14] = 0; + pBlock[15] = 0; +#elif 0 + // 0xaaaaaaac | x + pBlock[8] = (uint8_t)(x) | 0xAC; + + static const uint8_t s_tail_bytes[7] = { 0xaa, 0xaa, 0xaa, 0, 0, 0, 0 }; + memcpy(pBlock + 9, s_tail_bytes, 7); +#elif 1 + static const uint8_t s_tail_bytes[8] = { 0xac, 0xaa, 0xaa, 0xaa, 0, 0, 0, 0 }; + memcpy(pBlock + 8, s_tail_bytes, 8); + pBlock[8] |= (uint8_t)x; +#endif + } + + void encode_mode5_rgba_block(uint8_t* pBlock, + uint32_t lr, uint32_t lg, uint32_t lb, uint32_t la, // 7-bit RGB endpoints, 8-bit alpha endpoints + uint32_t hr, uint32_t hg, uint32_t hb, uint32_t ha, + const uint8_t* pColorWeights, const uint8_t* pAlphaWeights, // both 2-bit weights + uint32_t rot_index = 0) // rot_index=0 no rotation + { + assert((lr | lg | lb | hr | hg | hb) <= 127); + assert((la | ha) <= 255); + assert(rot_index <= 3); + + uint32_t color_inv = 0, alpha_inv = 0; + + if (pColorWeights[0] & 2) + { + std::swap(lr, hr); + std::swap(lg, hg); + std::swap(lb, hb); + color_inv = 3; + } + + if (pAlphaWeights[0] & 2) + { + std::swap(la, ha); + alpha_inv = 3; + } + + uint64_t low = (1ULL << 5) | (rot_index << 6) | + (lr << 8) | (hr << 15) | + (lg << 22) | (uint64_t(hg) << 29) | + (uint64_t(lb) << 36) | (uint64_t(hb) << 43) | + (uint64_t(la) << 50) | (uint64_t(ha) << 58); + + pBlock[0] = (uint8_t)low; + pBlock[1] = (uint8_t)(low >> 8); + pBlock[2] = (uint8_t)(low >> 16); + pBlock[3] = (uint8_t)(low >> 24); + pBlock[4] = (uint8_t)(low >> 32); + pBlock[5] = (uint8_t)(low >> 40); + pBlock[6] = (uint8_t)(low >> 48); + pBlock[7] = (uint8_t)(low >> 56); + + uint64_t high = (ha >> 6) & 3; + + uint32_t ofs = 2; + + for (uint32_t i = 0; i < 16; i++) + { + uint64_t w = pColorWeights[i] ^ color_inv; +#ifdef _DEBUG + assert(w <= 3); + if (i == 0) + { + assert((w & 2) == 0); + } +#endif + high |= (w << ofs); + ofs += (2 - (i == 0)); + } + + assert(33 == ofs); + + for (uint32_t i = 0; i < 16; i++) + { + uint64_t w = pAlphaWeights[i] ^ alpha_inv; +#ifdef _DEBUG + assert(w <= 3); + if (i == 0) + { + assert((w & 2) == 0); + } +#endif + high |= (w << ofs); + ofs += (2 - (i == 0)); + } + + assert(64 == ofs); + + pBlock[8] = (uint8_t)high; + pBlock[9] = (uint8_t)(high >> 8); + pBlock[10] = (uint8_t)(high >> 16); + pBlock[11] = (uint8_t)(high >> 24); + pBlock[12] = (uint8_t)(high >> 32); + pBlock[13] = (uint8_t)(high >> 40); + pBlock[14] = (uint8_t)(high >> 48); + pBlock[15] = (uint8_t)(high >> 56); + } + + void encode_mode6_rgba_block(uint8_t* pBlock, + uint32_t lr, uint32_t lg, uint32_t lb, uint32_t la, uint32_t p0, // 7-bit endpoints, 2 shared p-bits + uint32_t hr, uint32_t hg, uint32_t hb, uint32_t ha, uint32_t p1, + const uint8_t* pWeights) // 4-bit weights + { + assert((lr | lg | lb | la | hr | hg | hb | ha) <= 127); + assert((p0 | p1) <= 1); + + uint32_t weight_inv = 0; + if (pWeights[0] & 8) + { + std::swap(lr, hr); + std::swap(lg, hg); + std::swap(lb, hb); + std::swap(la, ha); + std::swap(p0, p1); + weight_inv = 15; + } + + // 9*7=63 bits + uint64_t x = 0b1000000 | (lr << (7 * 1)) | (hr << (7 * 2)); + x |= (lg << (7 * 3)) | (uint64_t(hg) << (7 * 4)); + x |= (uint64_t(lb) << (7 * 5)) | (uint64_t(hb) << (7 * 6)); + x |= (uint64_t(la) << (7 * 7)) | (uint64_t(ha) << (7 * 8)); + + pBlock[0] = (uint8_t)x; + pBlock[1] = (uint8_t)(x >> 8); + pBlock[2] = (uint8_t)(x >> 16); + pBlock[3] = (uint8_t)(x >> 24); + + pBlock[4] = (uint8_t)(x >> 32); + pBlock[5] = (uint8_t)(x >> 40); + pBlock[6] = (uint8_t)(x >> 48); + x >>= 56; + + // x=7 bits + x |= (p0 << 7); + pBlock[7] = (uint8_t)x; + + uint64_t y = p1; + uint32_t ofs = 1; + // TODO: Unroll/optimize + for (uint32_t i = 0; i < 16; i++) + { + uint64_t w = pWeights[i] ^ weight_inv; + assert(w <= 15); + assert(i || ((w & 8) == 0)); + y |= (w << ofs); + ofs += 3 + (i > 0); + } + assert(64 == ofs); + + pBlock[8] = (uint8_t)y; + pBlock[9] = (uint8_t)(y >> 8); + pBlock[10] = (uint8_t)(y >> 16); + pBlock[11] = (uint8_t)(y >> 24); + pBlock[12] = (uint8_t)(y >> 32); + pBlock[13] = (uint8_t)(y >> 40); + pBlock[14] = (uint8_t)(y >> 48); + pBlock[15] = (uint8_t)(y >> 56); + } + + void encode_mode7_rgba_block(uint8_t* pBlock, uint32_t part_id, // 2 subsets, 6-bits part ID + uint32_t lr[2], uint32_t lg[2], uint32_t lb[2], uint32_t la[2], // 5-bit endpoints, unique pbits + uint32_t hr[2], uint32_t hg[2], uint32_t hb[2], uint32_t ha[2], + uint32_t p[4], + const uint8_t* pWeights) // 2-bit weights + { + assert(part_id < 64); + assert((lr[0] | lr[1] | lg[0] | lg[1] | lb[0] | lb[1] | la[0] | la[1]) <= 31); + assert((hr[0] | hr[1] | hg[0] | hg[1] | hb[0] | hb[1] | ha[0] | ha[1]) <= 31); + assert((p[0] | p[1] | p[2] | p[3]) <= 1); + + const uint8_t* pPart_map = &g_bc7_partition2[part_id * 16]; + const uint32_t anchor_index = g_bc7_table_anchor_index_second_subset[part_id]; + + uint32_t weight_inv[2] = { 0, 0 }; + if (pWeights[0] & 2) + { + std::swap(lr[0], hr[0]); std::swap(lg[0], hg[0]); std::swap(lb[0], hb[0]); std::swap(la[0], ha[0]); + std::swap(p[0], p[1]); + weight_inv[0] = 3; + } + + if (pWeights[anchor_index] & 2) + { + std::swap(lr[1], hr[1]); std::swap(lg[1], hg[1]); std::swap(lb[1], hb[1]); std::swap(la[1], ha[1]); + std::swap(p[2], p[3]); + weight_inv[1] = 3; + } + + uint64_t x = 0x80ULL | (part_id << 8) | + (lr[0] << 14) | (hr[0] << 19) | (lr[1] << 24) | (uint64_t(hr[1]) << 29) | + (uint64_t(lg[0]) << 34) | (uint64_t(hg[0]) << 39) | (uint64_t(lg[1]) << 44) | (uint64_t(hg[1]) << 49) | + (uint64_t(lb[0]) << 54) | (uint64_t(hb[0]) << 59); + + pBlock[0] = (uint8_t)x; + pBlock[1] = (uint8_t)(x >> 8); + pBlock[2] = (uint8_t)(x >> 16); + pBlock[3] = (uint8_t)(x >> 24); + + pBlock[4] = (uint8_t)(x >> 32); + pBlock[5] = (uint8_t)(x >> 40); + pBlock[6] = (uint8_t)(x >> 48); + pBlock[7] = (uint8_t)(x >> 56); + + uint64_t y = (lb[1] << 0) | (hb[1] << 5) | + (la[0] << 10) | (ha[0] << 15) | (la[1] << 20) | (ha[1] << 25) | + (uint64_t(p[0]) << 30) | (uint64_t(p[1]) << 31) | (uint64_t(p[2]) << 32) | (uint64_t(p[3]) << 33); + + uint32_t ofs = 34; + for (uint32_t i = 0; i < 16; i++) + { + const uint32_t subset_index = pPart_map[i]; + uint64_t w = pWeights[i] ^ weight_inv[subset_index]; + +#ifdef _DEBUG + assert(w <= 3); + if ((i == 0) || (i == anchor_index)) + { + assert((w & 2) == 0); + } +#endif + + y |= (w << ofs); + ofs += (2 - ((i == 0) || (i == anchor_index))); + } + assert(64 == ofs); + + pBlock[8] = (uint8_t)y; + pBlock[9] = (uint8_t)(y >> 8); + pBlock[10] = (uint8_t)(y >> 16); + pBlock[11] = (uint8_t)(y >> 24); + + pBlock[12] = (uint8_t)(y >> 32); + pBlock[13] = (uint8_t)(y >> 40); + pBlock[14] = (uint8_t)(y >> 48); + pBlock[15] = (uint8_t)(y >> 56); + } + + static bool compute_least_squares_endpoints_1D( + uint32_t N, const uint8_t* pWeights, uint32_t num_weights, + const vec4F* pSelector_weights, + float& xl, float& xh, + const color_rgba* pColors, uint32_t comp_index, + float t_r) + { + BASISU_NOTE_UNUSED(num_weights); + + float z00 = 0.0f, z10 = 0.0f, z11 = 0.0f; + float q00_r = 0.0f; + + for (uint32_t i = 0; i < N; i++) + { + const uint32_t sel = pWeights[i]; + assert(sel < num_weights); + + z00 += pSelector_weights[sel][0]; + z10 += pSelector_weights[sel][1]; + z11 += pSelector_weights[sel][2]; + + const float w = pSelector_weights[sel][3]; + + q00_r += w * (float)pColors[i][comp_index]; + } + + float q10_r = t_r - q00_r; + + float z01 = z10; + + float det = z00 * z11 - z01 * z10; + if (fabs(det) < 1e-8f) + return false; + + det = 1.0f / det; + + float iz00, iz01, iz10, iz11; + iz00 = z11 * det; + iz01 = -z01 * det; + iz10 = -z10 * det; + iz11 = z00 * det; + + xh = basisu::clamp(iz00 * q00_r + iz01 * q10_r, 0.0f, 255.0f); + xl = basisu::clamp(iz10 * q00_r + iz11 * q10_r, 0.0f, 255.0f); + + return true; + } + + static bool compute_least_squares_endpoints_3D( + uint32_t N, const uint8_t* pWeights, uint32_t num_weights, + const vec4F* pSelector_weights, + vec4F& xl, vec4F& xh, + const color_rgba* pColors, + float t_r, float t_g, float t_b) + { + BASISU_NOTE_UNUSED(num_weights); + + float z00 = 0.0f, z10 = 0.0f, z11 = 0.0f; + float q00_r = 0.0f, q00_g = 0.0f, q00_b = 0.0f; + + for (uint32_t i = 0; i < N; i++) + { + const uint32_t sel = pWeights[i]; + assert(sel < num_weights); + + z00 += pSelector_weights[sel][0]; + z10 += pSelector_weights[sel][1]; + z11 += pSelector_weights[sel][2]; + + const float w = pSelector_weights[sel][3]; + + q00_r += w * (float)pColors[i][0]; + q00_g += w * (float)pColors[i][1]; + q00_b += w * (float)pColors[i][2]; + } + + float q10_r = t_r - q00_r; + float q10_g = t_g - q00_g; + float q10_b = t_b - q00_b; + + float z01 = z10; + + float det = z00 * z11 - z01 * z10; + if (fabs(det) < 1e-8f) + return false; + + det = 1.0f / det; + + float iz00, iz01, iz10, iz11; + iz00 = z11 * det; + iz01 = -z01 * det; + iz10 = -z10 * det; + iz11 = z00 * det; + + xh[0] = basisu::clamp(iz00 * q00_r + iz01 * q10_r, 0.0f, 255.0f); + xl[0] = basisu::clamp(iz10 * q00_r + iz11 * q10_r, 0.0f, 255.0f); + + xh[1] = basisu::clamp(iz00 * q00_g + iz01 * q10_g, 0.0f, 255.0f); + xl[1] = basisu::clamp(iz10 * q00_g + iz11 * q10_g, 0.0f, 255.0f); + + xh[2] = basisu::clamp(iz00 * q00_b + iz01 * q10_b, 0.0f, 255.0f); + xl[2] = basisu::clamp(iz10 * q00_b + iz11 * q10_b, 0.0f, 255.0f); + + xh[3] = 0; + xl[3] = 0; + + return true; + } + + static bool compute_least_squares_endpoints_4D( + uint32_t N, const uint8_t* pWeights, uint32_t num_weights, + const vec4F* pSelector_weights, + vec4F& xl, vec4F& xh, + const color_rgba* pColors, + float t_r, float t_g, float t_b, float t_a) + { + BASISU_NOTE_UNUSED(num_weights); + + float z00 = 0.0f, z10 = 0.0f, z11 = 0.0f; + float q00_r = 0.0f, q00_g = 0.0f, q00_b = 0.0f, q00_a = 0.0f; + + for (uint32_t i = 0; i < N; i++) + { + const uint32_t sel = pWeights[i]; + assert(sel < num_weights); + + z00 += pSelector_weights[sel][0]; + z10 += pSelector_weights[sel][1]; + z11 += pSelector_weights[sel][2]; + + const float w = pSelector_weights[sel][3]; + + q00_r += w * (float)pColors[i][0]; + q00_g += w * (float)pColors[i][1]; + q00_b += w * (float)pColors[i][2]; + q00_a += w * (float)pColors[i][3]; + } + + float q10_r = t_r - q00_r; + float q10_g = t_g - q00_g; + float q10_b = t_b - q00_b; + float q10_a = t_a - q00_a; + + float z01 = z10; + + float det = z00 * z11 - z01 * z10; + if (fabs(det) < 1e-8f) + return false; + + det = 1.0f / det; + + float iz00, iz01, iz10, iz11; + iz00 = z11 * det; + iz01 = -z01 * det; + iz10 = -z10 * det; + iz11 = z00 * det; + + xh[0] = basisu::clamp(iz00 * q00_r + iz01 * q10_r, 0.0f, 255.0f); + xl[0] = basisu::clamp(iz10 * q00_r + iz11 * q10_r, 0.0f, 255.0f); + + xh[1] = basisu::clamp(iz00 * q00_g + iz01 * q10_g, 0.0f, 255.0f); + xl[1] = basisu::clamp(iz10 * q00_g + iz11 * q10_g, 0.0f, 255.0f); + + xh[2] = basisu::clamp(iz00 * q00_b + iz01 * q10_b, 0.0f, 255.0f); + xl[2] = basisu::clamp(iz10 * q00_b + iz11 * q10_b, 0.0f, 255.0f); + + xh[3] = basisu::clamp(iz00 * q00_a + iz01 * q10_a, 0.0f, 255.0f); + xl[3] = basisu::clamp(iz10 * q00_a + iz11 * q10_a, 0.0f, 255.0f); + + return true; + } + +#if BASISU_BC7F_USE_SSE41 + void bc7_proj_minmax_indices_sse41(const color_rgba* __restrict pPixels, int saxis_r, int saxis_g, int saxis_b, int* out_min_idx, int* out_max_idx) + { + __m128i coef32 = _mm_setr_epi32(saxis_r, saxis_g, saxis_b, 0); // 32-bit lanes + coef32 = _mm_srai_epi32(coef32, 4); // arithmetic >>4 in 32-bit + __m128i COEF = _mm_packs_epi32(coef32, coef32); + + const __m128i ZERO = _mm_setzero_si128(); + + __m128i vmin, vmax; + { + const __m128i px = _mm_loadu_si128((const __m128i*) & pPixels[0]); + const __m128i lo16 = _mm_unpacklo_epi8(px, ZERO); // [r0 g0 b0 a0 r1 g1 b1 a1] + const __m128i hi16 = _mm_unpackhi_epi8(px, ZERO); // [r2 g2 b2 a2 r3 g3 b3 a3] + + const __m128i lo32p = _mm_madd_epi16(lo16, COEF); + const __m128i hi32p = _mm_madd_epi16(hi16, COEF); + + const __m128i lo_sum = _mm_add_epi32(lo32p, _mm_shuffle_epi32(lo32p, _MM_SHUFFLE(2, 3, 0, 1))); + const __m128i hi_sum = _mm_add_epi32(hi32p, _mm_shuffle_epi32(hi32p, _MM_SHUFFLE(2, 3, 0, 1))); + + const __m128i pair01 = _mm_shuffle_epi32(lo_sum, _MM_SHUFFLE(2, 0, 2, 0)); + const __m128i pair23 = _mm_shuffle_epi32(hi_sum, _MM_SHUFFLE(2, 0, 2, 0)); + + const __m128i p32p = _mm_unpacklo_epi64(pair01, pair23); + + const __m128i p32 = _mm_slli_epi32(p32p, 4); + + const __m128i keyed = _mm_add_epi32(p32, _mm_set_epi32(3, 2, 1, 0)); + + vmin = keyed; + vmax = keyed; + } + + { + const __m128i px = _mm_loadu_si128((const __m128i*) & pPixels[4]); + const __m128i lo16 = _mm_unpacklo_epi8(px, ZERO); // [r0 g0 b0 a0 r1 g1 b1 a1] + const __m128i hi16 = _mm_unpackhi_epi8(px, ZERO); // [r2 g2 b2 a2 r3 g3 b3 a3] + + const __m128i lo32p = _mm_madd_epi16(lo16, COEF); + const __m128i hi32p = _mm_madd_epi16(hi16, COEF); + + const __m128i lo_sum = _mm_add_epi32(lo32p, _mm_shuffle_epi32(lo32p, _MM_SHUFFLE(2, 3, 0, 1))); + const __m128i hi_sum = _mm_add_epi32(hi32p, _mm_shuffle_epi32(hi32p, _MM_SHUFFLE(2, 3, 0, 1))); + + const __m128i pair01 = _mm_shuffle_epi32(lo_sum, _MM_SHUFFLE(2, 0, 2, 0)); + const __m128i pair23 = _mm_shuffle_epi32(hi_sum, _MM_SHUFFLE(2, 0, 2, 0)); + + const __m128i p32p = _mm_unpacklo_epi64(pair01, pair23); + + const __m128i p32 = _mm_slli_epi32(p32p, 4); + + const __m128i keyed = _mm_add_epi32(p32, _mm_set_epi32(7, 6, 5, 4)); + + vmin = _mm_min_epi32(vmin, keyed); + vmax = _mm_max_epi32(vmax, keyed); + } + + { + const __m128i px = _mm_loadu_si128((const __m128i*) & pPixels[8]); + const __m128i lo16 = _mm_unpacklo_epi8(px, ZERO); // [r0 g0 b0 a0 r1 g1 b1 a1] + const __m128i hi16 = _mm_unpackhi_epi8(px, ZERO); // [r2 g2 b2 a2 r3 g3 b3 a3] + + const __m128i lo32p = _mm_madd_epi16(lo16, COEF); + const __m128i hi32p = _mm_madd_epi16(hi16, COEF); + + const __m128i lo_sum = _mm_add_epi32(lo32p, _mm_shuffle_epi32(lo32p, _MM_SHUFFLE(2, 3, 0, 1))); + const __m128i hi_sum = _mm_add_epi32(hi32p, _mm_shuffle_epi32(hi32p, _MM_SHUFFLE(2, 3, 0, 1))); + + const __m128i pair01 = _mm_shuffle_epi32(lo_sum, _MM_SHUFFLE(2, 0, 2, 0)); + const __m128i pair23 = _mm_shuffle_epi32(hi_sum, _MM_SHUFFLE(2, 0, 2, 0)); + + const __m128i p32p = _mm_unpacklo_epi64(pair01, pair23); + + const __m128i p32 = _mm_slli_epi32(p32p, 4); + + const __m128i keyed = _mm_add_epi32(p32, _mm_set_epi32(11, 10, 9, 8)); + + vmin = _mm_min_epi32(vmin, keyed); + vmax = _mm_max_epi32(vmax, keyed); + } + + { + const __m128i px = _mm_loadu_si128((const __m128i*) & pPixels[12]); + const __m128i lo16 = _mm_unpacklo_epi8(px, ZERO); // [r0 g0 b0 a0 r1 g1 b1 a1] + const __m128i hi16 = _mm_unpackhi_epi8(px, ZERO); // [r2 g2 b2 a2 r3 g3 b3 a3] + + const __m128i lo32p = _mm_madd_epi16(lo16, COEF); + const __m128i hi32p = _mm_madd_epi16(hi16, COEF); + + const __m128i lo_sum = _mm_add_epi32(lo32p, _mm_shuffle_epi32(lo32p, _MM_SHUFFLE(2, 3, 0, 1))); + const __m128i hi_sum = _mm_add_epi32(hi32p, _mm_shuffle_epi32(hi32p, _MM_SHUFFLE(2, 3, 0, 1))); + + const __m128i pair01 = _mm_shuffle_epi32(lo_sum, _MM_SHUFFLE(2, 0, 2, 0)); + const __m128i pair23 = _mm_shuffle_epi32(hi_sum, _MM_SHUFFLE(2, 0, 2, 0)); + + const __m128i p32p = _mm_unpacklo_epi64(pair01, pair23); + + const __m128i p32 = _mm_slli_epi32(p32p, 4); + + const __m128i keyed = _mm_add_epi32(p32, _mm_set_epi32(15, 14, 13, 12)); + + vmin = _mm_min_epi32(vmin, keyed); + vmax = _mm_max_epi32(vmax, keyed); + } + + __m128i t = _mm_shuffle_epi32(vmin, _MM_SHUFFLE(2, 3, 0, 1)); + vmin = _mm_min_epi32(vmin, t); + t = _mm_shuffle_epi32(vmin, _MM_SHUFFLE(1, 0, 3, 2)); + vmin = _mm_min_epi32(vmin, t); + const int min_keyed = _mm_cvtsi128_si32(vmin); + + t = _mm_shuffle_epi32(vmax, _MM_SHUFFLE(2, 3, 0, 1)); + vmax = _mm_max_epi32(vmax, t); + t = _mm_shuffle_epi32(vmax, _MM_SHUFFLE(1, 0, 3, 2)); + vmax = _mm_max_epi32(vmax, t); + const int max_keyed = _mm_cvtsi128_si32(vmax); + + *out_min_idx = (min_keyed & 0xF); + *out_max_idx = (max_keyed & 0xF); + } + + void eval_weights_mode6_rgb_sse41( + const color_rgba* __restrict pPixels, uint8_t* __restrict pWeights, + int lr, int lg, int lb, + int hr, int hg, int hb, + uint32_t p0, uint32_t p1) + { + lr = from_7(lr, p0); lg = from_7(lg, p0); lb = from_7(lb, p0); + hr = from_7(hr, p1); hg = from_7(hg, p1); hb = from_7(hb, p1); + + const int dr = hr - lr; + const int dg = hg - lg; + const int db = hb - lb; + + const float denom = (float)(basisu::squarei(dr) + basisu::squarei(dg) + basisu::squarei(db)) + 0.00000125f; + const float f = 15.0f / denom; + + const __m128i ZEROi = _mm_setzero_si128(); + const __m128i FIFTEEN = _mm_set1_epi32(15); + const __m128 F = _mm_set1_ps(f); + const __m128 HALF = _mm_set1_ps(0.5f); + + const __m128i EP16 = _mm_setr_epi16((short)lr, (short)lg, (short)lb, 0, + (short)lr, (short)lg, (short)lb, 0); + + const __m128i COEF = _mm_setr_epi16((short)dr, (short)dg, (short)db, 0, + (short)dr, (short)dg, (short)db, 0); + + for (int i = 0; i < 16; i += 4) + { + const __m128i px = _mm_loadu_si128((const __m128i*) & pPixels[i]); + + const __m128i lo16 = _mm_unpacklo_epi8(px, ZEROi); + const __m128i hi16 = _mm_unpackhi_epi8(px, ZEROi); + + const __m128i lo_adj = _mm_sub_epi16(lo16, EP16); + const __m128i hi_adj = _mm_sub_epi16(hi16, EP16); + + const __m128i lo32p = _mm_madd_epi16(lo_adj, COEF); + const __m128i hi32p = _mm_madd_epi16(hi_adj, COEF); + + const __m128i lo_sum = _mm_add_epi32(lo32p, _mm_shuffle_epi32(lo32p, _MM_SHUFFLE(2, 3, 0, 1))); + const __m128i hi_sum = _mm_add_epi32(hi32p, _mm_shuffle_epi32(hi32p, _MM_SHUFFLE(2, 3, 0, 1))); + + const __m128i pair01 = _mm_shuffle_epi32(lo_sum, _MM_SHUFFLE(2, 0, 2, 0)); + const __m128i pair23 = _mm_shuffle_epi32(hi_sum, _MM_SHUFFLE(2, 0, 2, 0)); + const __m128i dot32 = _mm_unpacklo_epi64(pair01, pair23); + + __m128 y = _mm_add_ps(_mm_mul_ps(_mm_cvtepi32_ps(dot32), F), HALF); + __m128i sel32 = _mm_cvttps_epi32(y); + + sel32 = _mm_min_epi32(_mm_max_epi32(sel32, ZEROi), FIFTEEN); + + __m128i sel16 = _mm_packs_epi32(sel32, ZEROi); + __m128i sel8 = _mm_packus_epi16(sel16, ZEROi); + *(uint32_t*)&pWeights[i] = (uint32_t)_mm_cvtsi128_si32(sel8); + } + } +#endif + + BASISU_FORCE_INLINE uint32_t bc7_sse( + int pr, + int lr, + int dr, + int w) + { + assert((w >= 0) && (w <= 64)); + int re = pr - (lr + ((dr * (int)w + 32) >> 6)); + return (re * re); + } + + BASISU_FORCE_INLINE uint32_t bc7_sse( + int pr, int pg, int pb, + int lr, int lg, int lb, + int dr, int dg, int db, + int w) + { + assert((w >= 0) && (w <= 64)); + int re = pr - (lr + ((dr * (int)w + 32) >> 6)); + int ge = pg - (lg + ((dg * (int)w + 32) >> 6)); + int be = pb - (lb + ((db * (int)w + 32) >> 6)); + return (re * re) + (ge * ge) + (be * be); + } + + BASISU_FORCE_INLINE uint32_t bc7_sse( + int pr, int pg, int pb, int pa, + int lr, int lg, int lb, int la, + int dr, int dg, int db, int da, + int w) + { + assert((w >= 0) && (w <= 64)); + int re = pr - (lr + ((dr * (int)w + 32) >> 6)); + int ge = pg - (lg + ((dg * (int)w + 32) >> 6)); + int be = pb - (lb + ((db * (int)w + 32) >> 6)); + int ae = pa - (la + ((da * (int)w + 32) >> 6)); + return (re * re) + (ge * ge) + (be * be) + (ae * ae); + } + + void eval_weights_mode6_rgb(const color_rgba* pPixels, uint8_t* pWeights, // 4-bits + int lr, int lg, int lb, + int hr, int hg, int hb, + uint32_t p0, uint32_t p1) + { + lr = from_7(lr, p0); lg = from_7(lg, p0); lb = from_7(lb, p0); + hr = from_7(hr, p1); hg = from_7(hg, p1); hb = from_7(hb, p1); + + int dr = hr - lr; + int dg = hg - lg; + int db = hb - lb; + + const float f = 15.0f / (float)(basisu::squarei(dr) + basisu::squarei(dg) + basisu::squarei(db) + .00000125f); + + const int sofs = -(lr * dr + lg * dg + lb * db); + + for (uint32_t i = 0; i < 16; i += 4) + { + int sel0 = (int)(float(pPixels[i + 0][0] * dr + pPixels[i + 0][1] * dg + pPixels[i + 0][2] * db + sofs) * f + .5f); + int sel1 = (int)(float(pPixels[i + 1][0] * dr + pPixels[i + 1][1] * dg + pPixels[i + 1][2] * db + sofs) * f + .5f); + int sel2 = (int)(float(pPixels[i + 2][0] * dr + pPixels[i + 2][1] * dg + pPixels[i + 2][2] * db + sofs) * f + .5f); + int sel3 = (int)(float(pPixels[i + 3][0] * dr + pPixels[i + 3][1] * dg + pPixels[i + 3][2] * db + sofs) * f + .5f); + + if ((uint32_t)sel0 > 15) sel0 = (~sel0 >> 31) & 15; + if ((uint32_t)sel1 > 15) sel1 = (~sel1 >> 31) & 15; + if ((uint32_t)sel2 > 15) sel2 = (~sel2 >> 31) & 15; + if ((uint32_t)sel3 > 15) sel3 = (~sel3 >> 31) & 15; + + pWeights[i + 0] = (uint8_t)sel0; + pWeights[i + 1] = (uint8_t)sel1; + pWeights[i + 2] = (uint8_t)sel2; + pWeights[i + 3] = (uint8_t)sel3; + } + } + + uint32_t eval_weights_mode6_rgb_sse(const color_rgba* pPixels, uint8_t* pWeights, // 4-bits + int lr, int lg, int lb, + int hr, int hg, int hb, + uint32_t p0, uint32_t p1) + { + lr = from_7(lr, p0); lg = from_7(lg, p0); lb = from_7(lb, p0); + hr = from_7(hr, p1); hg = from_7(hg, p1); hb = from_7(hb, p1); + + // assumes packed a's are always 127 + const int la = from_7(127, p0); + const int ha = from_7(127, p1); + const int da = ha - la; + + int dr = hr - lr; + int dg = hg - lg; + int db = hb - lb; + + const float f = 15.0f / (float)(basisu::squarei(dr) + basisu::squarei(dg) + basisu::squarei(db) + .00000125f); + + const int sofs = -(lr * dr + lg * dg + lb * db); + + uint32_t sse = 0; + + for (uint32_t i = 0; i < 16; i += 4) + { + int sel0 = (int)(float(pPixels[i + 0][0] * dr + pPixels[i + 0][1] * dg + pPixels[i + 0][2] * db + sofs) * f + .5f); + int sel1 = (int)(float(pPixels[i + 1][0] * dr + pPixels[i + 1][1] * dg + pPixels[i + 1][2] * db + sofs) * f + .5f); + int sel2 = (int)(float(pPixels[i + 2][0] * dr + pPixels[i + 2][1] * dg + pPixels[i + 2][2] * db + sofs) * f + .5f); + int sel3 = (int)(float(pPixels[i + 3][0] * dr + pPixels[i + 3][1] * dg + pPixels[i + 3][2] * db + sofs) * f + .5f); + + if ((uint32_t)sel0 > 15) sel0 = (~sel0 >> 31) & 15; + if ((uint32_t)sel1 > 15) sel1 = (~sel1 >> 31) & 15; + if ((uint32_t)sel2 > 15) sel2 = (~sel2 >> 31) & 15; + if ((uint32_t)sel3 > 15) sel3 = (~sel3 >> 31) & 15; + + pWeights[i + 0] = (uint8_t)sel0; + pWeights[i + 1] = (uint8_t)sel1; + pWeights[i + 2] = (uint8_t)sel2; + pWeights[i + 3] = (uint8_t)sel3; + + sse += bc7_sse(pPixels[i + 0][0], pPixels[i + 0][1], pPixels[i + 0][2], pPixels[i + 0][3], lr, lg, lb, la, dr, dg, db, da, basist::g_bc7_weights4[sel0]); + sse += bc7_sse(pPixels[i + 1][0], pPixels[i + 1][1], pPixels[i + 1][2], pPixels[i + 1][3], lr, lg, lb, la, dr, dg, db, da, basist::g_bc7_weights4[sel1]); + sse += bc7_sse(pPixels[i + 2][0], pPixels[i + 2][1], pPixels[i + 2][2], pPixels[i + 2][3], lr, lg, lb, la, dr, dg, db, da, basist::g_bc7_weights4[sel2]); + sse += bc7_sse(pPixels[i + 3][0], pPixels[i + 3][1], pPixels[i + 3][2], pPixels[i + 3][3], lr, lg, lb, la, dr, dg, db, da, basist::g_bc7_weights4[sel3]); + } + + return sse; + } + + void eval_weights_mode6_rgba(const color_rgba* pPixels, uint8_t* pWeights, // 4-bits + int lr, int lg, int lb, int la, int p0, + int hr, int hg, int hb, int ha, int p1) + { + lr = from_7(lr, p0); lg = from_7(lg, p0); lb = from_7(lb, p0); la = from_7(la, p0); + hr = from_7(hr, p1); hg = from_7(hg, p1); hb = from_7(hb, p1); ha = from_7(ha, p1); + + int dr = hr - lr; + int dg = hg - lg; + int db = hb - lb; + int da = ha - la; + + const float f = 15.0f / (float)(basisu::squarei(dr) + basisu::squarei(dg) + basisu::squarei(db) + basisu::squarei(da) + .00000125f); + + const int sofs = -(lr * dr + lg * dg + lb * db + la * da); + + for (uint32_t i = 0; i < 16; i += 4) + { + int sel0 = (int)(float(pPixels[i + 0][0] * dr + pPixels[i + 0][1] * dg + pPixels[i + 0][2] * db + pPixels[i + 0][3] * da + sofs) * f + .5f); + int sel1 = (int)(float(pPixels[i + 1][0] * dr + pPixels[i + 1][1] * dg + pPixels[i + 1][2] * db + pPixels[i + 1][3] * da + sofs) * f + .5f); + int sel2 = (int)(float(pPixels[i + 2][0] * dr + pPixels[i + 2][1] * dg + pPixels[i + 2][2] * db + pPixels[i + 2][3] * da + sofs) * f + .5f); + int sel3 = (int)(float(pPixels[i + 3][0] * dr + pPixels[i + 3][1] * dg + pPixels[i + 3][2] * db + pPixels[i + 3][3] * da + sofs) * f + .5f); + + if ((uint32_t)sel0 > 15) sel0 = (~sel0 >> 31) & 15; + if ((uint32_t)sel1 > 15) sel1 = (~sel1 >> 31) & 15; + if ((uint32_t)sel2 > 15) sel2 = (~sel2 >> 31) & 15; + if ((uint32_t)sel3 > 15) sel3 = (~sel3 >> 31) & 15; + + pWeights[i + 0] = (uint8_t)sel0; + pWeights[i + 1] = (uint8_t)sel1; + pWeights[i + 2] = (uint8_t)sel2; + pWeights[i + 3] = (uint8_t)sel3; + } + } + + uint32_t eval_weights_mode6_rgba_sse(const color_rgba* pPixels, uint8_t* pWeights, // 4-bits + int lr, int lg, int lb, int la, int p0, + int hr, int hg, int hb, int ha, int p1) + { + lr = from_7(lr, p0); lg = from_7(lg, p0); lb = from_7(lb, p0); la = from_7(la, p0); + hr = from_7(hr, p1); hg = from_7(hg, p1); hb = from_7(hb, p1); ha = from_7(ha, p1); + + int dr = hr - lr; + int dg = hg - lg; + int db = hb - lb; + int da = ha - la; + + const float f = 15.0f / (float)(basisu::squarei(dr) + basisu::squarei(dg) + basisu::squarei(db) + basisu::squarei(da) + .00000125f); + + const int sofs = -(lr * dr + lg * dg + lb * db + la * da); + + uint32_t sse = 0; + + for (uint32_t i = 0; i < 16; i += 4) + { + int sel0 = (int)(float(pPixels[i + 0][0] * dr + pPixels[i + 0][1] * dg + pPixels[i + 0][2] * db + pPixels[i + 0][3] * da + sofs) * f + .5f); + int sel1 = (int)(float(pPixels[i + 1][0] * dr + pPixels[i + 1][1] * dg + pPixels[i + 1][2] * db + pPixels[i + 1][3] * da + sofs) * f + .5f); + int sel2 = (int)(float(pPixels[i + 2][0] * dr + pPixels[i + 2][1] * dg + pPixels[i + 2][2] * db + pPixels[i + 2][3] * da + sofs) * f + .5f); + int sel3 = (int)(float(pPixels[i + 3][0] * dr + pPixels[i + 3][1] * dg + pPixels[i + 3][2] * db + pPixels[i + 3][3] * da + sofs) * f + .5f); + + if ((uint32_t)sel0 > 15) sel0 = (~sel0 >> 31) & 15; + if ((uint32_t)sel1 > 15) sel1 = (~sel1 >> 31) & 15; + if ((uint32_t)sel2 > 15) sel2 = (~sel2 >> 31) & 15; + if ((uint32_t)sel3 > 15) sel3 = (~sel3 >> 31) & 15; + + pWeights[i + 0] = (uint8_t)sel0; + pWeights[i + 1] = (uint8_t)sel1; + pWeights[i + 2] = (uint8_t)sel2; + pWeights[i + 3] = (uint8_t)sel3; + + sse += bc7_sse(pPixels[i + 0][0], pPixels[i + 0][1], pPixels[i + 0][2], pPixels[i + 0][3], lr, lg, lb, la, dr, dg, db, da, basist::g_bc7_weights4[sel0]); + sse += bc7_sse(pPixels[i + 1][0], pPixels[i + 1][1], pPixels[i + 1][2], pPixels[i + 1][3], lr, lg, lb, la, dr, dg, db, da, basist::g_bc7_weights4[sel1]); + sse += bc7_sse(pPixels[i + 2][0], pPixels[i + 2][1], pPixels[i + 2][2], pPixels[i + 2][3], lr, lg, lb, la, dr, dg, db, da, basist::g_bc7_weights4[sel2]); + sse += bc7_sse(pPixels[i + 3][0], pPixels[i + 3][1], pPixels[i + 3][2], pPixels[i + 3][3], lr, lg, lb, la, dr, dg, db, da, basist::g_bc7_weights4[sel3]); + } + + return sse; + } + + void eval_weights_mode1_rgb(const color_rgba* pPixels, uint8_t* pWeights, // 3-bits + uint32_t blr[2], uint32_t blg[2], uint32_t blb[2], uint32_t bhr[2], uint32_t bhg[2], uint32_t bhb[2], + uint32_t pbits[2], uint32_t subset_bitmask) + { + int lr[2], lg[2], lb[2], hr[2], hg[2], hb[2], dr[2], dg[2], db[2]; + + for (uint32_t s = 0; s < 2; s++) + { + lr[s] = from_6(blr[s], pbits[s]); + lg[s] = from_6(blg[s], pbits[s]); + lb[s] = from_6(blb[s], pbits[s]); + + hr[s] = from_6(bhr[s], pbits[s]); + hg[s] = from_6(bhg[s], pbits[s]); + hb[s] = from_6(bhb[s], pbits[s]); + + dr[s] = hr[s] - lr[s]; + dg[s] = hg[s] - lg[s]; + db[s] = hb[s] - lb[s]; + } + + const float f[2] = + { + 7.0f / (float)(basisu::squarei(dr[0]) + basisu::squarei(dg[0]) + basisu::squarei(db[0]) + .00000125f), + 7.0f / (float)(basisu::squarei(dr[1]) + basisu::squarei(dg[1]) + basisu::squarei(db[1]) + .00000125f) + }; + + const int sofs[2] = { + lr[0] * dr[0] + lg[0] * dg[0] + lb[0] * db[0], + lr[1] * dr[1] + lg[1] * dg[1] + lb[1] * db[1] }; + + for (uint32_t i = 0; i < 16; i++) + { + const uint32_t subset_index = (subset_bitmask >> i) & 1; + + int sel = (int)((float)( + ((int)pPixels[i][0]) * dr[subset_index] + ((int)pPixels[i][1]) * dg[subset_index] + ((int)pPixels[i][2]) * db[subset_index] - sofs[subset_index]) * f[subset_index] + .5f); + + if ((uint32_t)sel > 7) + sel = (~sel >> 31) & 7; + + pWeights[i] = (uint8_t)sel; + } + } + + uint32_t eval_weights_mode1_rgb_sse(const color_rgba* pPixels, uint8_t* pWeights, // 3-bits + uint32_t blr[2], uint32_t blg[2], uint32_t blb[2], uint32_t bhr[2], uint32_t bhg[2], uint32_t bhb[2], + uint32_t pbits[2], uint32_t subset_bitmask) + { + int lr[2], lg[2], lb[2], hr[2], hg[2], hb[2], dr[2], dg[2], db[2]; + + for (uint32_t s = 0; s < 2; s++) + { + lr[s] = from_6(blr[s], pbits[s]); + lg[s] = from_6(blg[s], pbits[s]); + lb[s] = from_6(blb[s], pbits[s]); + + hr[s] = from_6(bhr[s], pbits[s]); + hg[s] = from_6(bhg[s], pbits[s]); + hb[s] = from_6(bhb[s], pbits[s]); + + dr[s] = hr[s] - lr[s]; + dg[s] = hg[s] - lg[s]; + db[s] = hb[s] - lb[s]; + } + + const float f[2] = + { + 7.0f / (float)(basisu::squarei(dr[0]) + basisu::squarei(dg[0]) + basisu::squarei(db[0]) + .00000125f), + 7.0f / (float)(basisu::squarei(dr[1]) + basisu::squarei(dg[1]) + basisu::squarei(db[1]) + .00000125f) + }; + + const int sofs[2] = { + lr[0] * dr[0] + lg[0] * dg[0] + lb[0] * db[0], + lr[1] * dr[1] + lg[1] * dg[1] + lb[1] * db[1] }; + + uint32_t sse = 0; + + for (uint32_t i = 0; i < 16; i++) + { + const uint32_t subset_index = (subset_bitmask >> i) & 1; + + int sel = (int)((float)( + ((int)pPixels[i][0]) * dr[subset_index] + ((int)pPixels[i][1]) * dg[subset_index] + ((int)pPixels[i][2]) * db[subset_index] - sofs[subset_index]) * f[subset_index] + .5f); + + if ((uint32_t)sel > 7) + sel = (~sel >> 31) & 7; + + pWeights[i] = (uint8_t)sel; + + sse += bc7_sse(pPixels[i][0], pPixels[i][1], pPixels[i][2], lr[subset_index], lg[subset_index], lb[subset_index], dr[subset_index], dg[subset_index], db[subset_index], basist::g_bc7_weights3[sel]); + } + + return sse; + } + + void eval_weights_mode7_rgba(const color_rgba* pPixels, uint8_t* pWeights, // 2-bits + uint32_t blr[2], uint32_t blg[2], uint32_t blb[2], uint32_t bla[2], + uint32_t bhr[2], uint32_t bhg[2], uint32_t bhb[2], uint32_t bha[2], + uint32_t pbits[4], uint32_t subset_bitmask) + { + int lr[2], lg[2], lb[2], la[2]; + int hr[2], hg[2], hb[2], ha[2]; + int dr[2], dg[2], db[2], da[2]; + + for (uint32_t s = 0; s < 2; s++) + { + const uint32_t l_pbit = pbits[s * 2 + 0], h_pbit = pbits[s * 2 + 1]; + + lr[s] = from_5(blr[s], l_pbit); + lg[s] = from_5(blg[s], l_pbit); + lb[s] = from_5(blb[s], l_pbit); + la[s] = from_5(bla[s], l_pbit); + + hr[s] = from_5(bhr[s], h_pbit); + hg[s] = from_5(bhg[s], h_pbit); + hb[s] = from_5(bhb[s], h_pbit); + ha[s] = from_5(bha[s], h_pbit); + + dr[s] = hr[s] - lr[s]; + dg[s] = hg[s] - lg[s]; + db[s] = hb[s] - lb[s]; + da[s] = ha[s] - la[s]; + } + + const float f[2] = + { + 3.0f / (float)(basisu::squarei(dr[0]) + basisu::squarei(dg[0]) + basisu::squarei(db[0]) + basisu::squarei(da[0]) + .00000125f), + 3.0f / (float)(basisu::squarei(dr[1]) + basisu::squarei(dg[1]) + basisu::squarei(db[1]) + basisu::squarei(da[1]) + .00000125f) + }; + + const int sofs[2] = { + lr[0] * dr[0] + lg[0] * dg[0] + lb[0] * db[0] + la[0] * da[0], + lr[1] * dr[1] + lg[1] * dg[1] + lb[1] * db[1] + la[1] * da[1] }; + + for (uint32_t i = 0; i < 16; i++) + { + const uint32_t subset_index = (subset_bitmask >> i) & 1; + + int sel = (int)((float)( + ((int)pPixels[i][0]) * dr[subset_index] + ((int)pPixels[i][1]) * dg[subset_index] + ((int)pPixels[i][2]) * db[subset_index] + ((int)pPixels[i][3]) * da[subset_index] - sofs[subset_index]) * f[subset_index] + .5f); + + if ((uint32_t)sel > 3) + sel = (~sel >> 31) & 3; + + pWeights[i] = (uint8_t)sel; + } + } + + uint32_t eval_weights_mode7_rgba_sse(const color_rgba* pPixels, uint8_t* pWeights, // 2-bits + uint32_t blr[2], uint32_t blg[2], uint32_t blb[2], uint32_t bla[2], + uint32_t bhr[2], uint32_t bhg[2], uint32_t bhb[2], uint32_t bha[2], + uint32_t pbits[4], uint32_t subset_bitmask) + { + int lr[2], lg[2], lb[2], la[2]; + int hr[2], hg[2], hb[2], ha[2]; + int dr[2], dg[2], db[2], da[2]; + + for (uint32_t s = 0; s < 2; s++) + { + const uint32_t l_pbit = pbits[s * 2 + 0], h_pbit = pbits[s * 2 + 1]; + + lr[s] = from_5(blr[s], l_pbit); + lg[s] = from_5(blg[s], l_pbit); + lb[s] = from_5(blb[s], l_pbit); + la[s] = from_5(bla[s], l_pbit); + + hr[s] = from_5(bhr[s], h_pbit); + hg[s] = from_5(bhg[s], h_pbit); + hb[s] = from_5(bhb[s], h_pbit); + ha[s] = from_5(bha[s], h_pbit); + + dr[s] = hr[s] - lr[s]; + dg[s] = hg[s] - lg[s]; + db[s] = hb[s] - lb[s]; + da[s] = ha[s] - la[s]; + } + + const float f[2] = + { + 3.0f / (float)(basisu::squarei(dr[0]) + basisu::squarei(dg[0]) + basisu::squarei(db[0]) + basisu::squarei(da[0]) + .00000125f), + 3.0f / (float)(basisu::squarei(dr[1]) + basisu::squarei(dg[1]) + basisu::squarei(db[1]) + basisu::squarei(da[1]) + .00000125f) + }; + + const int sofs[2] = { + lr[0] * dr[0] + lg[0] * dg[0] + lb[0] * db[0] + la[0] * da[0], + lr[1] * dr[1] + lg[1] * dg[1] + lb[1] * db[1] + la[1] * da[1] }; + + uint32_t sse = 0; + + for (uint32_t i = 0; i < 16; i++) + { + const uint32_t subset_index = (subset_bitmask >> i) & 1; + + int sel = (int)((float)( + ((int)pPixels[i][0]) * dr[subset_index] + ((int)pPixels[i][1]) * dg[subset_index] + ((int)pPixels[i][2]) * db[subset_index] + ((int)pPixels[i][3]) * da[subset_index] - sofs[subset_index]) * f[subset_index] + .5f); + + if ((uint32_t)sel > 3) + sel = (~sel >> 31) & 3; + + pWeights[i] = (uint8_t)sel; + + sse += bc7_sse(pPixels[i][0], pPixels[i][1], pPixels[i][2], pPixels[i][3], + lr[subset_index], lg[subset_index], lb[subset_index], la[subset_index], + dr[subset_index], dg[subset_index], db[subset_index], da[subset_index], basist::g_bc7_weights2[sel]); + } + + return sse; + } + + void eval_weights_mode3_rgb(const color_rgba* pPixels, uint8_t* pWeights, // 2-bits + uint32_t blr[2], uint32_t blg[2], uint32_t blb[2], uint32_t bhr[2], uint32_t bhg[2], uint32_t bhb[2], + uint32_t pbits[4], uint32_t subset_bitmask) + { + int lr[2], lg[2], lb[2], hr[2], hg[2], hb[2], dr[2], dg[2], db[2]; + + for (uint32_t s = 0; s < 2; s++) + { + lr[s] = from_7(blr[s], pbits[s * 2 + 0]); + lg[s] = from_7(blg[s], pbits[s * 2 + 0]); + lb[s] = from_7(blb[s], pbits[s * 2 + 0]); + + hr[s] = from_7(bhr[s], pbits[s * 2 + 1]); + hg[s] = from_7(bhg[s], pbits[s * 2 + 1]); + hb[s] = from_7(bhb[s], pbits[s * 2 + 1]); + + dr[s] = hr[s] - lr[s]; + dg[s] = hg[s] - lg[s]; + db[s] = hb[s] - lb[s]; + } + + const float f[2] = + { + 3.0f / (float)(basisu::squarei(dr[0]) + basisu::squarei(dg[0]) + basisu::squarei(db[0]) + .00000125f), + 3.0f / (float)(basisu::squarei(dr[1]) + basisu::squarei(dg[1]) + basisu::squarei(db[1]) + .00000125f) + }; + + const int sofs[2] = { + lr[0] * dr[0] + lg[0] * dg[0] + lb[0] * db[0], + lr[1] * dr[1] + lg[1] * dg[1] + lb[1] * db[1] }; + + for (uint32_t i = 0; i < 16; i++) + { + const uint32_t subset_index = (subset_bitmask >> i) & 1; + + int sel = (int)((float)( + ((int)pPixels[i][0]) * dr[subset_index] + ((int)pPixels[i][1]) * dg[subset_index] + ((int)pPixels[i][2]) * db[subset_index] - sofs[subset_index]) * f[subset_index] + .5f); + + if ((uint32_t)sel > 3) + sel = (~sel >> 31) & 3; + + pWeights[i] = (uint8_t)sel; + } + } + + uint32_t eval_weights_mode3_rgb_sse(const color_rgba* pPixels, uint8_t* pWeights, // 2-bits + uint32_t blr[2], uint32_t blg[2], uint32_t blb[2], uint32_t bhr[2], uint32_t bhg[2], uint32_t bhb[2], + uint32_t pbits[4], uint32_t subset_bitmask) + { + int lr[2], lg[2], lb[2], hr[2], hg[2], hb[2], dr[2], dg[2], db[2]; + + for (uint32_t s = 0; s < 2; s++) + { + lr[s] = from_7(blr[s], pbits[s * 2 + 0]); + lg[s] = from_7(blg[s], pbits[s * 2 + 0]); + lb[s] = from_7(blb[s], pbits[s * 2 + 0]); + + hr[s] = from_7(bhr[s], pbits[s * 2 + 1]); + hg[s] = from_7(bhg[s], pbits[s * 2 + 1]); + hb[s] = from_7(bhb[s], pbits[s * 2 + 1]); + + dr[s] = hr[s] - lr[s]; + dg[s] = hg[s] - lg[s]; + db[s] = hb[s] - lb[s]; + } + + const float f[2] = + { + 3.0f / (float)(basisu::squarei(dr[0]) + basisu::squarei(dg[0]) + basisu::squarei(db[0]) + .00000125f), + 3.0f / (float)(basisu::squarei(dr[1]) + basisu::squarei(dg[1]) + basisu::squarei(db[1]) + .00000125f) + }; + + const int sofs[2] = { + lr[0] * dr[0] + lg[0] * dg[0] + lb[0] * db[0], + lr[1] * dr[1] + lg[1] * dg[1] + lb[1] * db[1] }; + + uint32_t sse = 0; + + for (uint32_t i = 0; i < 16; i++) + { + const uint32_t subset_index = (subset_bitmask >> i) & 1; + + int sel = (int)((float)( + ((int)pPixels[i][0]) * dr[subset_index] + ((int)pPixels[i][1]) * dg[subset_index] + ((int)pPixels[i][2]) * db[subset_index] - sofs[subset_index]) * f[subset_index] + .5f); + + if ((uint32_t)sel > 3) + sel = (~sel >> 31) & 3; + + pWeights[i] = (uint8_t)sel; + + sse += bc7_sse(pPixels[i][0], pPixels[i][1], pPixels[i][2], lr[subset_index], lg[subset_index], lb[subset_index], dr[subset_index], dg[subset_index], db[subset_index], basist::g_bc7_weights2[sel]); + } + + return sse; + } + + void eval_weights_mode0_rgb(const color_rgba* pPixels, uint8_t* pWeights, // 3-bits + uint32_t blr[3], uint32_t blg[3], uint32_t blb[3], + uint32_t bhr[3], uint32_t bhg[3], uint32_t bhb[3], + uint32_t pbits[6], + uint32_t pat_index) + { + assert(pat_index <= 15); + int lr[3], lg[3], lb[3], hr[3], hg[3], hb[3], dr[3], dg[3], db[3]; + + for (uint32_t s = 0; s < 3; s++) + { + lr[s] = from_4(blr[s], pbits[s * 2 + 0]); + lg[s] = from_4(blg[s], pbits[s * 2 + 0]); + lb[s] = from_4(blb[s], pbits[s * 2 + 0]); + + hr[s] = from_4(bhr[s], pbits[s * 2 + 1]); + hg[s] = from_4(bhg[s], pbits[s * 2 + 1]); + hb[s] = from_4(bhb[s], pbits[s * 2 + 1]); + + dr[s] = hr[s] - lr[s]; + dg[s] = hg[s] - lg[s]; + db[s] = hb[s] - lb[s]; + } + + const float f[3] = + { + 7.0f / (float)(basisu::squarei(dr[0]) + basisu::squarei(dg[0]) + basisu::squarei(db[0]) + .00000125f), + 7.0f / (float)(basisu::squarei(dr[1]) + basisu::squarei(dg[1]) + basisu::squarei(db[1]) + .00000125f), + 7.0f / (float)(basisu::squarei(dr[2]) + basisu::squarei(dg[2]) + basisu::squarei(db[2]) + .00000125f) + }; + + const int sofs[3] = { + lr[0] * dr[0] + lg[0] * dg[0] + lb[0] * db[0], + lr[1] * dr[1] + lg[1] * dg[1] + lb[1] * db[1], + lr[2] * dr[2] + lg[2] * dg[2] + lb[2] * db[2] }; + + const uint8_t* pPart_map = &g_bc7_partition3[pat_index * 16]; + + for (uint32_t i = 0; i < 16; i++) + { + const uint32_t subset_index = pPart_map[i]; + + int sel = (int)((float)( + (int)pPixels[i][0] * dr[subset_index] + (int)pPixels[i][1] * dg[subset_index] + (int)pPixels[i][2] * db[subset_index] - sofs[subset_index]) * f[subset_index] + .5f); + + if ((uint32_t)sel > 7) + sel = (~sel >> 31) & 7; + + pWeights[i] = (uint8_t)sel; + } + } + + uint32_t eval_weights_mode0_rgb_sse(const color_rgba* pPixels, uint8_t* pWeights, // 3-bits + uint32_t blr[3], uint32_t blg[3], uint32_t blb[3], + uint32_t bhr[3], uint32_t bhg[3], uint32_t bhb[3], + uint32_t pbits[6], + uint32_t pat_index) + { + assert(pat_index <= 15); + int lr[3], lg[3], lb[3], hr[3], hg[3], hb[3], dr[3], dg[3], db[3]; + + for (uint32_t s = 0; s < 3; s++) + { + lr[s] = from_4(blr[s], pbits[s * 2 + 0]); + lg[s] = from_4(blg[s], pbits[s * 2 + 0]); + lb[s] = from_4(blb[s], pbits[s * 2 + 0]); + + hr[s] = from_4(bhr[s], pbits[s * 2 + 1]); + hg[s] = from_4(bhg[s], pbits[s * 2 + 1]); + hb[s] = from_4(bhb[s], pbits[s * 2 + 1]); + + dr[s] = hr[s] - lr[s]; + dg[s] = hg[s] - lg[s]; + db[s] = hb[s] - lb[s]; + } + + const float f[3] = + { + 7.0f / (float)(basisu::squarei(dr[0]) + basisu::squarei(dg[0]) + basisu::squarei(db[0]) + .00000125f), + 7.0f / (float)(basisu::squarei(dr[1]) + basisu::squarei(dg[1]) + basisu::squarei(db[1]) + .00000125f), + 7.0f / (float)(basisu::squarei(dr[2]) + basisu::squarei(dg[2]) + basisu::squarei(db[2]) + .00000125f) + }; + + const int sofs[3] = { + lr[0] * dr[0] + lg[0] * dg[0] + lb[0] * db[0], + lr[1] * dr[1] + lg[1] * dg[1] + lb[1] * db[1], + lr[2] * dr[2] + lg[2] * dg[2] + lb[2] * db[2] }; + + const uint8_t* pPart_map = &g_bc7_partition3[pat_index * 16]; + + uint32_t sse = 0; + + for (uint32_t i = 0; i < 16; i++) + { + const uint32_t subset_index = pPart_map[i]; + + int sel = (int)((float)( + (int)pPixels[i][0] * dr[subset_index] + (int)pPixels[i][1] * dg[subset_index] + (int)pPixels[i][2] * db[subset_index] - sofs[subset_index]) * f[subset_index] + .5f); + + if ((uint32_t)sel > 7) + sel = (~sel >> 31) & 7; + + pWeights[i] = (uint8_t)sel; + + sse += bc7_sse(pPixels[i][0], pPixels[i][1], pPixels[i][2], + lr[subset_index], lg[subset_index], lb[subset_index], + dr[subset_index], dg[subset_index], db[subset_index], basist::g_bc7_weights3[sel]); + } + + return sse; + } + + void eval_weights_mode2_rgb(const color_rgba* pPixels, uint8_t* pWeights, // 2-bits + uint32_t blr[3], uint32_t blg[3], uint32_t blb[3], uint32_t bhr[3], uint32_t bhg[3], uint32_t bhb[3], + uint32_t pat_index) + { + int lr[3], lg[3], lb[3], hr[3], hg[3], hb[3], dr[3], dg[3], db[3]; + + for (uint32_t s = 0; s < 3; s++) + { + lr[s] = from_5(blr[s]); + lg[s] = from_5(blg[s]); + lb[s] = from_5(blb[s]); + + hr[s] = from_5(bhr[s]); + hg[s] = from_5(bhg[s]); + hb[s] = from_5(bhb[s]); + + dr[s] = hr[s] - lr[s]; + dg[s] = hg[s] - lg[s]; + db[s] = hb[s] - lb[s]; + } + + const float f[3] = + { + 3.0f / (float)(basisu::squarei(dr[0]) + basisu::squarei(dg[0]) + basisu::squarei(db[0]) + .00000125f), + 3.0f / (float)(basisu::squarei(dr[1]) + basisu::squarei(dg[1]) + basisu::squarei(db[1]) + .00000125f), + 3.0f / (float)(basisu::squarei(dr[2]) + basisu::squarei(dg[2]) + basisu::squarei(db[2]) + .00000125f) + }; + + const int sofs[3] = { + lr[0] * dr[0] + lg[0] * dg[0] + lb[0] * db[0], + lr[1] * dr[1] + lg[1] * dg[1] + lb[1] * db[1], + lr[2] * dr[2] + lg[2] * dg[2] + lb[2] * db[2] }; + + const uint8_t* pPart_map = &g_bc7_partition3[pat_index * 16]; + + for (uint32_t i = 0; i < 16; i++) + { + const uint32_t subset_index = pPart_map[i]; + + int sel = (int)((float)( + (int)pPixels[i][0] * dr[subset_index] + (int)pPixels[i][1] * dg[subset_index] + (int)pPixels[i][2] * db[subset_index] - sofs[subset_index]) * f[subset_index] + .5f); + + if ((uint32_t)sel > 3) + sel = (~sel >> 31) & 3; + + pWeights[i] = (uint8_t)sel; + } + } + + uint32_t eval_weights_mode2_rgb_sse(const color_rgba* pPixels, uint8_t* pWeights, // 2-bits + uint32_t blr[3], uint32_t blg[3], uint32_t blb[3], uint32_t bhr[3], uint32_t bhg[3], uint32_t bhb[3], + uint32_t pat_index) + { + int lr[3], lg[3], lb[3], hr[3], hg[3], hb[3], dr[3], dg[3], db[3]; + + for (uint32_t s = 0; s < 3; s++) + { + lr[s] = from_5(blr[s]); + lg[s] = from_5(blg[s]); + lb[s] = from_5(blb[s]); + + hr[s] = from_5(bhr[s]); + hg[s] = from_5(bhg[s]); + hb[s] = from_5(bhb[s]); + + dr[s] = hr[s] - lr[s]; + dg[s] = hg[s] - lg[s]; + db[s] = hb[s] - lb[s]; + } + + const float f[3] = + { + 3.0f / (float)(basisu::squarei(dr[0]) + basisu::squarei(dg[0]) + basisu::squarei(db[0]) + .00000125f), + 3.0f / (float)(basisu::squarei(dr[1]) + basisu::squarei(dg[1]) + basisu::squarei(db[1]) + .00000125f), + 3.0f / (float)(basisu::squarei(dr[2]) + basisu::squarei(dg[2]) + basisu::squarei(db[2]) + .00000125f) + }; + + const int sofs[3] = { + lr[0] * dr[0] + lg[0] * dg[0] + lb[0] * db[0], + lr[1] * dr[1] + lg[1] * dg[1] + lb[1] * db[1], + lr[2] * dr[2] + lg[2] * dg[2] + lb[2] * db[2] }; + + const uint8_t* pPart_map = &g_bc7_partition3[pat_index * 16]; + + uint32_t sse = 0; + + for (uint32_t i = 0; i < 16; i++) + { + const uint32_t subset_index = pPart_map[i]; + + int sel = (int)((float)( + (int)pPixels[i][0] * dr[subset_index] + (int)pPixels[i][1] * dg[subset_index] + (int)pPixels[i][2] * db[subset_index] - sofs[subset_index]) * f[subset_index] + .5f); + + if ((uint32_t)sel > 3) + sel = (~sel >> 31) & 3; + + pWeights[i] = (uint8_t)sel; + + sse += bc7_sse(pPixels[i][0], pPixels[i][1], pPixels[i][2], + lr[subset_index], lg[subset_index], lb[subset_index], + dr[subset_index], dg[subset_index], db[subset_index], basist::g_bc7_weights2[sel]); + } + + return sse; + } + + void eval_weights_mode4_3bit_rgb(const color_rgba* pPixels, uint8_t* pWeights0, // 3-bits + int lr, int lg, int lb, + int hr, int hg, int hb) + { + lr = from_5(lr); lg = from_5(lg); lb = from_5(lb); + hr = from_5(hr); hg = from_5(hg); hb = from_5(hb); + + int dr = hr - lr; + int dg = hg - lg; + int db = hb - lb; + + const float f = 7.0f / (float)(basisu::squarei(dr) + basisu::squarei(dg) + basisu::squarei(db) + .00000125f); + + const int sofs = lr * dr + lg * dg + lb * db; + + for (uint32_t i = 0; i < 16; i += 4) + { + int sel0 = (int)((float)((int)pPixels[i + 0][0] * dr + (int)pPixels[i + 0][1] * dg + (int)pPixels[i + 0][2] * db - sofs) * f + .5f); + int sel1 = (int)((float)((int)pPixels[i + 1][0] * dr + (int)pPixels[i + 1][1] * dg + (int)pPixels[i + 1][2] * db - sofs) * f + .5f); + int sel2 = (int)((float)((int)pPixels[i + 2][0] * dr + (int)pPixels[i + 2][1] * dg + (int)pPixels[i + 2][2] * db - sofs) * f + .5f); + int sel3 = (int)((float)((int)pPixels[i + 3][0] * dr + (int)pPixels[i + 3][1] * dg + (int)pPixels[i + 3][2] * db - sofs) * f + .5f); + + if ((uint32_t)sel0 > 7) sel0 = (~sel0 >> 31) & 7; + if ((uint32_t)sel1 > 7) sel1 = (~sel1 >> 31) & 7; + if ((uint32_t)sel2 > 7) sel2 = (~sel2 >> 31) & 7; + if ((uint32_t)sel3 > 7) sel3 = (~sel3 >> 31) & 7; + + pWeights0[i + 0] = (uint8_t)sel0; + pWeights0[i + 1] = (uint8_t)sel1; + pWeights0[i + 2] = (uint8_t)sel2; + pWeights0[i + 3] = (uint8_t)sel3; + } + } + + uint32_t eval_weights_mode4_3bit_rgb_sse(const color_rgba* pPixels, uint8_t* pWeights0, // 3-bits + int lr, int lg, int lb, + int hr, int hg, int hb) + { + lr = from_5(lr); lg = from_5(lg); lb = from_5(lb); + hr = from_5(hr); hg = from_5(hg); hb = from_5(hb); + + int dr = hr - lr; + int dg = hg - lg; + int db = hb - lb; + + const float f = 7.0f / (float)(basisu::squarei(dr) + basisu::squarei(dg) + basisu::squarei(db) + .00000125f); + + const int sofs = lr * dr + lg * dg + lb * db; + + uint32_t sse = 0; + + for (uint32_t i = 0; i < 16; i += 4) + { + int sel0 = (int)((float)((int)pPixels[i + 0][0] * dr + (int)pPixels[i + 0][1] * dg + (int)pPixels[i + 0][2] * db - sofs) * f + .5f); + int sel1 = (int)((float)((int)pPixels[i + 1][0] * dr + (int)pPixels[i + 1][1] * dg + (int)pPixels[i + 1][2] * db - sofs) * f + .5f); + int sel2 = (int)((float)((int)pPixels[i + 2][0] * dr + (int)pPixels[i + 2][1] * dg + (int)pPixels[i + 2][2] * db - sofs) * f + .5f); + int sel3 = (int)((float)((int)pPixels[i + 3][0] * dr + (int)pPixels[i + 3][1] * dg + (int)pPixels[i + 3][2] * db - sofs) * f + .5f); + + if ((uint32_t)sel0 > 7) sel0 = (~sel0 >> 31) & 7; + if ((uint32_t)sel1 > 7) sel1 = (~sel1 >> 31) & 7; + if ((uint32_t)sel2 > 7) sel2 = (~sel2 >> 31) & 7; + if ((uint32_t)sel3 > 7) sel3 = (~sel3 >> 31) & 7; + + pWeights0[i + 0] = (uint8_t)sel0; + pWeights0[i + 1] = (uint8_t)sel1; + pWeights0[i + 2] = (uint8_t)sel2; + pWeights0[i + 3] = (uint8_t)sel3; + + sse += bc7_sse(pPixels[i + 0][0], pPixels[i + 0][1], pPixels[i + 0][2], lr, lg, lb, dr, dg, db, basist::g_bc7_weights3[sel0]); + sse += bc7_sse(pPixels[i + 1][0], pPixels[i + 1][1], pPixels[i + 1][2], lr, lg, lb, dr, dg, db, basist::g_bc7_weights3[sel1]); + sse += bc7_sse(pPixels[i + 2][0], pPixels[i + 2][1], pPixels[i + 2][2], lr, lg, lb, dr, dg, db, basist::g_bc7_weights3[sel2]); + sse += bc7_sse(pPixels[i + 3][0], pPixels[i + 3][1], pPixels[i + 3][2], lr, lg, lb, dr, dg, db, basist::g_bc7_weights3[sel3]); + } + + return sse; + } + + void eval_weights_mode4_2bit_rgb(const color_rgba* pPixels, uint8_t* pWeights0, // 2-bits + int lr, int lg, int lb, + int hr, int hg, int hb) + { + lr = from_5(lr); lg = from_5(lg); lb = from_5(lb); + hr = from_5(hr); hg = from_5(hg); hb = from_5(hb); + + int dr = hr - lr; + int dg = hg - lg; + int db = hb - lb; + + const float f = 3.0f / (float)(basisu::squarei(dr) + basisu::squarei(dg) + basisu::squarei(db) + .00000125f); + + const int sofs = lr * dr + lg * dg + lb * db; + + for (uint32_t i = 0; i < 16; i += 4) + { + int sel0 = (int)((float)((int)pPixels[i + 0][0] * dr + (int)pPixels[i + 0][1] * dg + (int)pPixels[i + 0][2] * db - sofs) * f + .5f); + int sel1 = (int)((float)((int)pPixels[i + 1][0] * dr + (int)pPixels[i + 1][1] * dg + (int)pPixels[i + 1][2] * db - sofs) * f + .5f); + int sel2 = (int)((float)((int)pPixels[i + 2][0] * dr + (int)pPixels[i + 2][1] * dg + (int)pPixels[i + 2][2] * db - sofs) * f + .5f); + int sel3 = (int)((float)((int)pPixels[i + 3][0] * dr + (int)pPixels[i + 3][1] * dg + (int)pPixels[i + 3][2] * db - sofs) * f + .5f); + + if ((uint32_t)sel0 > 3) sel0 = (~sel0 >> 31) & 3; + if ((uint32_t)sel1 > 3) sel1 = (~sel1 >> 31) & 3; + if ((uint32_t)sel2 > 3) sel2 = (~sel2 >> 31) & 3; + if ((uint32_t)sel3 > 3) sel3 = (~sel3 >> 31) & 3; + + pWeights0[i + 0] = (uint8_t)sel0; + pWeights0[i + 1] = (uint8_t)sel1; + pWeights0[i + 2] = (uint8_t)sel2; + pWeights0[i + 3] = (uint8_t)sel3; + } + } + + uint32_t eval_weights_mode4_2bit_rgb_sse(const color_rgba* pPixels, uint8_t* pWeights0, // 2-bits + int lr, int lg, int lb, + int hr, int hg, int hb) + { + lr = from_5(lr); lg = from_5(lg); lb = from_5(lb); + hr = from_5(hr); hg = from_5(hg); hb = from_5(hb); + + int dr = hr - lr; + int dg = hg - lg; + int db = hb - lb; + + const float f = 3.0f / (float)(basisu::squarei(dr) + basisu::squarei(dg) + basisu::squarei(db) + .00000125f); + + const int sofs = lr * dr + lg * dg + lb * db; + + uint32_t sse = 0; + + for (uint32_t i = 0; i < 16; i += 4) + { + int sel0 = (int)((float)((int)pPixels[i + 0][0] * dr + (int)pPixels[i + 0][1] * dg + (int)pPixels[i + 0][2] * db - sofs) * f + .5f); + int sel1 = (int)((float)((int)pPixels[i + 1][0] * dr + (int)pPixels[i + 1][1] * dg + (int)pPixels[i + 1][2] * db - sofs) * f + .5f); + int sel2 = (int)((float)((int)pPixels[i + 2][0] * dr + (int)pPixels[i + 2][1] * dg + (int)pPixels[i + 2][2] * db - sofs) * f + .5f); + int sel3 = (int)((float)((int)pPixels[i + 3][0] * dr + (int)pPixels[i + 3][1] * dg + (int)pPixels[i + 3][2] * db - sofs) * f + .5f); + + if ((uint32_t)sel0 > 3) sel0 = (~sel0 >> 31) & 3; + if ((uint32_t)sel1 > 3) sel1 = (~sel1 >> 31) & 3; + if ((uint32_t)sel2 > 3) sel2 = (~sel2 >> 31) & 3; + if ((uint32_t)sel3 > 3) sel3 = (~sel3 >> 31) & 3; + + pWeights0[i + 0] = (uint8_t)sel0; + pWeights0[i + 1] = (uint8_t)sel1; + pWeights0[i + 2] = (uint8_t)sel2; + pWeights0[i + 3] = (uint8_t)sel3; + + sse += bc7_sse(pPixels[i + 0][0], pPixels[i + 0][1], pPixels[i + 0][2], lr, lg, lb, dr, dg, db, basist::g_bc7_weights2[sel0]); + sse += bc7_sse(pPixels[i + 1][0], pPixels[i + 1][1], pPixels[i + 1][2], lr, lg, lb, dr, dg, db, basist::g_bc7_weights2[sel1]); + sse += bc7_sse(pPixels[i + 2][0], pPixels[i + 2][1], pPixels[i + 2][2], lr, lg, lb, dr, dg, db, basist::g_bc7_weights2[sel2]); + sse += bc7_sse(pPixels[i + 3][0], pPixels[i + 3][1], pPixels[i + 3][2], lr, lg, lb, dr, dg, db, basist::g_bc7_weights2[sel3]); + } + + return sse; + } + + void eval_weights_mode4_2bit_a(const color_rgba* pPixels, uint8_t* pWeights1, // 2-bits + int la, int ha) + { + la = from_6(la); + ha = from_6(ha); + + int da = ha - la; + + const float f = 3.0f / (float)(da + .00000125f); + + for (uint32_t i = 0; i < 16; i += 4) + { + int sel0 = (int)((float)(pPixels[i + 0][3] - la) * f + .5f); + int sel1 = (int)((float)(pPixels[i + 1][3] - la) * f + .5f); + int sel2 = (int)((float)(pPixels[i + 2][3] - la) * f + .5f); + int sel3 = (int)((float)(pPixels[i + 3][3] - la) * f + .5f); + + if ((uint32_t)sel0 > 3) sel0 = (~sel0 >> 31) & 3; + if ((uint32_t)sel1 > 3) sel1 = (~sel1 >> 31) & 3; + if ((uint32_t)sel2 > 3) sel2 = (~sel2 >> 31) & 3; + if ((uint32_t)sel3 > 3) sel3 = (~sel3 >> 31) & 3; + + pWeights1[i + 0] = (uint8_t)sel0; + pWeights1[i + 1] = (uint8_t)sel1; + pWeights1[i + 2] = (uint8_t)sel2; + pWeights1[i + 3] = (uint8_t)sel3; + } + } + + uint32_t eval_weights_mode4_2bit_a_sse(const color_rgba* pPixels, uint8_t* pWeights1, // 2-bits + int la, int ha) + { + la = from_6(la); + ha = from_6(ha); + + int da = ha - la; + + const float f = 3.0f / (float)(da + .00000125f); + + uint32_t sse = 0; + + for (uint32_t i = 0; i < 16; i += 4) + { + int sel0 = (int)((float)(pPixels[i + 0][3] - la) * f + .5f); + int sel1 = (int)((float)(pPixels[i + 1][3] - la) * f + .5f); + int sel2 = (int)((float)(pPixels[i + 2][3] - la) * f + .5f); + int sel3 = (int)((float)(pPixels[i + 3][3] - la) * f + .5f); + + if ((uint32_t)sel0 > 3) sel0 = (~sel0 >> 31) & 3; + if ((uint32_t)sel1 > 3) sel1 = (~sel1 >> 31) & 3; + if ((uint32_t)sel2 > 3) sel2 = (~sel2 >> 31) & 3; + if ((uint32_t)sel3 > 3) sel3 = (~sel3 >> 31) & 3; + + pWeights1[i + 0] = (uint8_t)sel0; + pWeights1[i + 1] = (uint8_t)sel1; + pWeights1[i + 2] = (uint8_t)sel2; + pWeights1[i + 3] = (uint8_t)sel3; + + sse += bc7_sse(pPixels[i + 0][3], la, da, basist::g_bc7_weights2[sel0]); + sse += bc7_sse(pPixels[i + 1][3], la, da, basist::g_bc7_weights2[sel1]); + sse += bc7_sse(pPixels[i + 2][3], la, da, basist::g_bc7_weights2[sel2]); + sse += bc7_sse(pPixels[i + 3][3], la, da, basist::g_bc7_weights2[sel3]); + } + + return sse; + } + + void eval_weights_mode4_3bit_a(const color_rgba* pPixels, uint8_t* pWeights1, // 3-bits + int la, int ha) + { + la = from_6(la); + ha = from_6(ha); + + int da = ha - la; + + const float f = 7.0f / (float)(da + .00000125f); + + for (uint32_t i = 0; i < 16; i += 4) + { + int sel0 = (int)((float)(pPixels[i + 0][3] - la) * f + .5f); + int sel1 = (int)((float)(pPixels[i + 1][3] - la) * f + .5f); + int sel2 = (int)((float)(pPixels[i + 2][3] - la) * f + .5f); + int sel3 = (int)((float)(pPixels[i + 3][3] - la) * f + .5f); + + if ((uint32_t)sel0 > 7) sel0 = (~sel0 >> 31) & 7; + if ((uint32_t)sel1 > 7) sel1 = (~sel1 >> 31) & 7; + if ((uint32_t)sel2 > 7) sel2 = (~sel2 >> 31) & 7; + if ((uint32_t)sel3 > 7) sel3 = (~sel3 >> 31) & 7; + + pWeights1[i + 0] = (uint8_t)sel0; + pWeights1[i + 1] = (uint8_t)sel1; + pWeights1[i + 2] = (uint8_t)sel2; + pWeights1[i + 3] = (uint8_t)sel3; + } + } + + uint32_t eval_weights_mode4_3bit_a_sse(const color_rgba* pPixels, uint8_t* pWeights1, // 3-bits + int la, int ha) + { + la = from_6(la); + ha = from_6(ha); + + int da = ha - la; + + const float f = 7.0f / (float)(da + .00000125f); + + uint32_t sse = 0; + + for (uint32_t i = 0; i < 16; i += 4) + { + int sel0 = (int)((float)(pPixels[i + 0][3] - la) * f + .5f); + int sel1 = (int)((float)(pPixels[i + 1][3] - la) * f + .5f); + int sel2 = (int)((float)(pPixels[i + 2][3] - la) * f + .5f); + int sel3 = (int)((float)(pPixels[i + 3][3] - la) * f + .5f); + + if ((uint32_t)sel0 > 7) sel0 = (~sel0 >> 31) & 7; + if ((uint32_t)sel1 > 7) sel1 = (~sel1 >> 31) & 7; + if ((uint32_t)sel2 > 7) sel2 = (~sel2 >> 31) & 7; + if ((uint32_t)sel3 > 7) sel3 = (~sel3 >> 31) & 7; + + pWeights1[i + 0] = (uint8_t)sel0; + pWeights1[i + 1] = (uint8_t)sel1; + pWeights1[i + 2] = (uint8_t)sel2; + pWeights1[i + 3] = (uint8_t)sel3; + + sse += bc7_sse(pPixels[i + 0][3], la, da, basist::g_bc7_weights3[sel0]); + sse += bc7_sse(pPixels[i + 1][3], la, da, basist::g_bc7_weights3[sel1]); + sse += bc7_sse(pPixels[i + 2][3], la, da, basist::g_bc7_weights3[sel2]); + sse += bc7_sse(pPixels[i + 3][3], la, da, basist::g_bc7_weights3[sel3]); + } + + return sse; + } + + void eval_weights_mode5_2bit_rgb(const color_rgba* pPixels, uint8_t* pWeights0, // 2-bits + int lr, int lg, int lb, + int hr, int hg, int hb) + { + lr = from_7(lr); lg = from_7(lg); lb = from_7(lb); + hr = from_7(hr); hg = from_7(hg); hb = from_7(hb); + + int dr = hr - lr; + int dg = hg - lg; + int db = hb - lb; + + const float f = 3.0f / (float)(basisu::squarei(dr) + basisu::squarei(dg) + basisu::squarei(db) + .00000125f); + + const int sofs = lr * dr + lg * dg + lb * db; + + for (uint32_t i = 0; i < 16; i += 4) + { + int sel0 = (int)((float)((int)pPixels[i + 0][0] * dr + (int)pPixels[i + 0][1] * dg + (int)pPixels[i + 0][2] * db - sofs) * f + .5f); + int sel1 = (int)((float)((int)pPixels[i + 1][0] * dr + (int)pPixels[i + 1][1] * dg + (int)pPixels[i + 1][2] * db - sofs) * f + .5f); + int sel2 = (int)((float)((int)pPixels[i + 2][0] * dr + (int)pPixels[i + 2][1] * dg + (int)pPixels[i + 2][2] * db - sofs) * f + .5f); + int sel3 = (int)((float)((int)pPixels[i + 3][0] * dr + (int)pPixels[i + 3][1] * dg + (int)pPixels[i + 3][2] * db - sofs) * f + .5f); + + if ((uint32_t)sel0 > 3) sel0 = (~sel0 >> 31) & 3; + if ((uint32_t)sel1 > 3) sel1 = (~sel1 >> 31) & 3; + if ((uint32_t)sel2 > 3) sel2 = (~sel2 >> 31) & 3; + if ((uint32_t)sel3 > 3) sel3 = (~sel3 >> 31) & 3; + + pWeights0[i + 0] = (uint8_t)sel0; + pWeights0[i + 1] = (uint8_t)sel1; + pWeights0[i + 2] = (uint8_t)sel2; + pWeights0[i + 3] = (uint8_t)sel3; + } + } + + uint32_t eval_weights_mode5_2bit_rgb_sse(const color_rgba* pPixels, uint8_t* pWeights0, // 2-bits + int lr, int lg, int lb, + int hr, int hg, int hb) + { + lr = from_7(lr); lg = from_7(lg); lb = from_7(lb); + hr = from_7(hr); hg = from_7(hg); hb = from_7(hb); + + int dr = hr - lr; + int dg = hg - lg; + int db = hb - lb; + + const float f = 3.0f / (float)(basisu::squarei(dr) + basisu::squarei(dg) + basisu::squarei(db) + .00000125f); + + const int sofs = lr * dr + lg * dg + lb * db; + + uint32_t sse = 0; + + for (uint32_t i = 0; i < 16; i += 4) + { + int sel0 = (int)((float)((int)pPixels[i + 0][0] * dr + (int)pPixels[i + 0][1] * dg + (int)pPixels[i + 0][2] * db - sofs) * f + .5f); + int sel1 = (int)((float)((int)pPixels[i + 1][0] * dr + (int)pPixels[i + 1][1] * dg + (int)pPixels[i + 1][2] * db - sofs) * f + .5f); + int sel2 = (int)((float)((int)pPixels[i + 2][0] * dr + (int)pPixels[i + 2][1] * dg + (int)pPixels[i + 2][2] * db - sofs) * f + .5f); + int sel3 = (int)((float)((int)pPixels[i + 3][0] * dr + (int)pPixels[i + 3][1] * dg + (int)pPixels[i + 3][2] * db - sofs) * f + .5f); + + if ((uint32_t)sel0 > 3) sel0 = (~sel0 >> 31) & 3; + if ((uint32_t)sel1 > 3) sel1 = (~sel1 >> 31) & 3; + if ((uint32_t)sel2 > 3) sel2 = (~sel2 >> 31) & 3; + if ((uint32_t)sel3 > 3) sel3 = (~sel3 >> 31) & 3; + + pWeights0[i + 0] = (uint8_t)sel0; + pWeights0[i + 1] = (uint8_t)sel1; + pWeights0[i + 2] = (uint8_t)sel2; + pWeights0[i + 3] = (uint8_t)sel3; + + sse += bc7_sse(pPixels[i + 0][0], pPixels[i + 0][1], pPixels[i + 0][2], lr, lg, lb, dr, dg, db, basist::g_bc7_weights2[sel0]); + sse += bc7_sse(pPixels[i + 1][0], pPixels[i + 1][1], pPixels[i + 1][2], lr, lg, lb, dr, dg, db, basist::g_bc7_weights2[sel1]); + sse += bc7_sse(pPixels[i + 2][0], pPixels[i + 2][1], pPixels[i + 2][2], lr, lg, lb, dr, dg, db, basist::g_bc7_weights2[sel2]); + sse += bc7_sse(pPixels[i + 3][0], pPixels[i + 3][1], pPixels[i + 3][2], lr, lg, lb, dr, dg, db, basist::g_bc7_weights2[sel3]); + } + + return sse; + } + + void eval_weights_mode5_2bit_a(const color_rgba* pPixels, uint8_t* pWeights1, // 2-bits + int la, int ha) + { + int da = ha - la; + + const float f = 3.0f / (float)(da + .00000125f); + + for (uint32_t i = 0; i < 16; i += 4) + { + int sel0 = (int)((float)(pPixels[i + 0][3] - la) * f + .5f); + int sel1 = (int)((float)(pPixels[i + 1][3] - la) * f + .5f); + int sel2 = (int)((float)(pPixels[i + 2][3] - la) * f + .5f); + int sel3 = (int)((float)(pPixels[i + 3][3] - la) * f + .5f); + + if ((uint32_t)sel0 > 3) sel0 = (~sel0 >> 31) & 3; + if ((uint32_t)sel1 > 3) sel1 = (~sel1 >> 31) & 3; + if ((uint32_t)sel2 > 3) sel2 = (~sel2 >> 31) & 3; + if ((uint32_t)sel3 > 3) sel3 = (~sel3 >> 31) & 3; + + pWeights1[i + 0] = (uint8_t)sel0; + pWeights1[i + 1] = (uint8_t)sel1; + pWeights1[i + 2] = (uint8_t)sel2; + pWeights1[i + 3] = (uint8_t)sel3; + } + } + + uint32_t eval_weights_mode5_2bit_a_sse(const color_rgba* pPixels, uint8_t* pWeights1, // 2-bits + int la, int ha) + { + int da = ha - la; + + const float f = 3.0f / (float)(da + .00000125f); + + uint32_t sse = 0; + + for (uint32_t i = 0; i < 16; i += 4) + { + int sel0 = (int)((float)(pPixels[i + 0][3] - la) * f + .5f); + int sel1 = (int)((float)(pPixels[i + 1][3] - la) * f + .5f); + int sel2 = (int)((float)(pPixels[i + 2][3] - la) * f + .5f); + int sel3 = (int)((float)(pPixels[i + 3][3] - la) * f + .5f); + + if ((uint32_t)sel0 > 3) sel0 = (~sel0 >> 31) & 3; + if ((uint32_t)sel1 > 3) sel1 = (~sel1 >> 31) & 3; + if ((uint32_t)sel2 > 3) sel2 = (~sel2 >> 31) & 3; + if ((uint32_t)sel3 > 3) sel3 = (~sel3 >> 31) & 3; + + pWeights1[i + 0] = (uint8_t)sel0; + pWeights1[i + 1] = (uint8_t)sel1; + pWeights1[i + 2] = (uint8_t)sel2; + pWeights1[i + 3] = (uint8_t)sel3; + + sse += bc7_sse(pPixels[i + 0][3], la, da, basist::g_bc7_weights2[sel0]); + sse += bc7_sse(pPixels[i + 1][3], la, da, basist::g_bc7_weights2[sel1]); + sse += bc7_sse(pPixels[i + 2][3], la, da, basist::g_bc7_weights2[sel2]); + sse += bc7_sse(pPixels[i + 3][3], la, da, basist::g_bc7_weights2[sel3]); + } + + return sse; + } + + // Determines the best unique pbits to use to encode xl/xh, which are [0,1] + static void determine_unique_pbits( + uint32_t total_comps, uint32_t comp_bits, float xl[4], float xh[4], + color_rgba& bestMinColor, color_rgba& bestMaxColor, uint32_t best_pbits[2]) + { +#ifdef _DEBUG + for (uint32_t c = 0; c < total_comps; c++) + { + assert((xl[c] >= 0.0f) && (xl[c] <= 1.0f)); + assert((xh[c] >= 0.0f) && (xh[c] <= 1.0f)); + } +#endif + + const uint32_t total_bits = comp_bits + 1; + const int iscalep = (1 << total_bits) - 1; + const float scalep = (float)iscalep; + + float best_err0 = 1e+9f; + float best_err1 = 1e+9f; + + for (int p = 0; p < 2; p++) + { + color_rgba xMinColor, xMaxColor; + + for (uint32_t c = 0; c < 4; c++) + { + xMinColor[c] = (uint8_t)(clampi(((int)((xl[c] * scalep - p) * (1.0f / 2.0f) + .5f)) * 2 + p, p, iscalep - 1 + p)); + xMaxColor[c] = (uint8_t)(clampi(((int)((xh[c] * scalep - p) * (1.0f / 2.0f) + .5f)) * 2 + p, p, iscalep - 1 + p)); + } + + color_rgba scaledLow, scaledHigh; + for (uint32_t i = 0; i < 4; i++) + { + scaledLow[i] = (xMinColor[i] << (8 - total_bits)); + scaledLow[i] |= (scaledLow[i] >> total_bits); + + scaledHigh[i] = (xMaxColor[i] << (8 - total_bits)); + scaledHigh[i] |= (scaledHigh[i] >> total_bits); + } + + float err0 = 0, err1 = 0; + for (uint32_t i = 0; i < total_comps; i++) + { + err0 += basisu::squaref(scaledLow[i] - xl[i] * 255.0f); + err1 += basisu::squaref(scaledHigh[i] - xh[i] * 255.0f); + } + + if (err0 < best_err0) + { + best_err0 = err0; + best_pbits[0] = p; + + bestMinColor[0] = xMinColor[0] >> 1; + bestMinColor[1] = xMinColor[1] >> 1; + bestMinColor[2] = xMinColor[2] >> 1; + bestMinColor[3] = xMinColor[3] >> 1; + } + + if (err1 < best_err1) + { + best_err1 = err1; + best_pbits[1] = p; + + bestMaxColor[0] = xMaxColor[0] >> 1; + bestMaxColor[1] = xMaxColor[1] >> 1; + bestMaxColor[2] = xMaxColor[2] >> 1; + bestMaxColor[3] = xMaxColor[3] >> 1; + } + } + } + + // Determines the best shared pbits to use to encode xl/xh, which are [0,1] + static void determine_shared_pbits( + uint32_t total_comps, uint32_t comp_bits, float xl[4], float xh[4], + color_rgba& bestMinColor, color_rgba& bestMaxColor, uint32_t best_pbits[2]) + { +#ifdef _DEBUG + for (uint32_t c = 0; c < total_comps; c++) + { + assert((xl[c] >= 0.0f) && (xl[c] <= 1.0f)); + assert((xh[c] >= 0.0f) && (xh[c] <= 1.0f)); + } +#endif + + const uint32_t total_bits = comp_bits + 1; + assert((total_bits >= 4) && (total_bits <= 8)); + + const int iscalep = (1 << total_bits) - 1; + const float scalep = (float)iscalep; + + float best_err = 1e+9f; + + for (int p = 0; p < 2; p++) + { + color_rgba xMinColor, xMaxColor; + for (uint32_t c = 0; c < 4; c++) + { + xMinColor[c] = (uint8_t)(clampi(((int)((xl[c] * scalep - p) * (1.0f / 2.0f) + .5f)) * 2 + p, p, iscalep - 1 + p)); + xMaxColor[c] = (uint8_t)(clampi(((int)((xh[c] * scalep - p) * (1.0f / 2.0f) + .5f)) * 2 + p, p, iscalep - 1 + p)); + } + + color_rgba scaledLow, scaledHigh; + + for (uint32_t i = 0; i < 4; i++) + { + scaledLow[i] = (xMinColor[i] << (8 - total_bits)); + scaledLow[i] |= (scaledLow[i] >> total_bits); + + scaledHigh[i] = (xMaxColor[i] << (8 - total_bits)); + scaledHigh[i] |= (scaledHigh[i] >> total_bits); + } + + float err = 0; + for (uint32_t i = 0; i < total_comps; i++) + err += basisu::squaref((scaledLow[i] * (1.0f / 255.0f)) - xl[i]) + basisu::squaref((scaledHigh[i] * (1.0f / 255.0f)) - xh[i]); + + if (err < best_err) + { + best_err = err; + best_pbits[0] = p; + best_pbits[1] = p; + for (uint32_t j = 0; j < 4; j++) + { + bestMinColor[j] = xMinColor[j] >> 1; + bestMaxColor[j] = xMaxColor[j] >> 1; + } + } + } + } + + // 4x4 ASTC blocks only, no dp, no subsets, outputs mode 6 + static void pack_from_astc_4x4_single_subset(uint8_t* pDst_block_u8, const astc_helpers::log_astc_block& log_blk) + { + assert(!log_blk.m_dual_plane && (log_blk.m_num_partitions == 1)); + assert((log_blk.m_grid_width <= 4) && (log_blk.m_grid_height <= 4)); + + color_rgba l, h; + astc_ldr_t::decode_endpoints(log_blk.m_color_endpoint_modes[0], log_blk.m_endpoints, log_blk.m_endpoint_ise_range, l, h); + + uint8_t dequantized_weights[16]; + uint8_t upsampled_weights[16]; + + const uint32_t total_weight_vals = log_blk.m_grid_width * log_blk.m_grid_height; + + const astc_helpers::dequant_table& weight_dequant_tab = astc_helpers::g_dequant_tables.get_weight_tab(log_blk.m_weight_ise_range); + const uint8_t* pWeight_dequant = weight_dequant_tab.m_ISE_to_val.data(); + + for (uint32_t i = 0; i < total_weight_vals; i++) + { + assert(log_blk.m_weights[i] < weight_dequant_tab.m_ISE_to_val.size_u32()); + + dequantized_weights[i] = pWeight_dequant[log_blk.m_weights[i]]; + } + + const uint8_t* pUpsampled_weights = dequantized_weights; + if ((log_blk.m_grid_width < 4) || (log_blk.m_grid_height < 4)) + { + astc_helpers::upsample_weight_grid_xuastc_ldr(4, 4, log_blk.m_grid_width, log_blk.m_grid_height, dequantized_weights, upsampled_weights, nullptr, nullptr); + pUpsampled_weights = upsampled_weights; + } + + const float q = 1.0f / 255.0f; + float sxl[4] = { (float)l.r * q, (float)l.g * q, (float)l.b * q, (float)l.a * q }; + float sxh[4] = { (float)h.r * q, (float)h.g * q, (float)h.b * q, (float)h.a * q }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_unique_pbits(4, 7, sxl, sxh, bestMinColor, bestMaxColor, best_pbits); + + uint8_t bc7_weights[16]; + // TODO: Potentially improve this mapping using a lookup table + for (uint32_t i = 0; i < 16; i++) + bc7_weights[i] = (uint8_t)((pUpsampled_weights[i] * 15 + 32) >> 6); + + encode_mode6_rgba_block(pDst_block_u8, + bestMinColor.r, bestMinColor.g, bestMinColor.b, bestMinColor.a, best_pbits[0], + bestMaxColor.r, bestMaxColor.g, bestMaxColor.b, bestMaxColor.a, best_pbits[1], + bc7_weights); + } + + // Outputs mode 6 + static void pack_from_astc_single_subset( + uint8_t* pDst_block_u8, + const astc_helpers::log_astc_block& log_blk, + const uint8_t *pUpsampled_weights, + uint32_t weight_ofs_x, uint32_t weight_ofs_y, + uint32_t block_width, uint32_t block_height) + { + BASISU_NOTE_UNUSED(block_width); + BASISU_NOTE_UNUSED(block_height); + + assert(!log_blk.m_dual_plane && (log_blk.m_num_partitions == 1)); + assert((log_blk.m_grid_width <= block_width) && (log_blk.m_grid_height <= block_height)); + assert((weight_ofs_x + 3) < block_width); + assert((weight_ofs_y + 3) < block_height); + + color_rgba l, h; + astc_ldr_t::decode_endpoints(log_blk.m_color_endpoint_modes[0], log_blk.m_endpoints, log_blk.m_endpoint_ise_range, l, h); + + const float q = 1.0f / 255.0f; + float sxl[4] = { (float)l.r * q, (float)l.g * q, (float)l.b * q, (float)l.a * q }; + float sxh[4] = { (float)h.r * q, (float)h.g * q, (float)h.b * q, (float)h.a * q }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_unique_pbits(4, 7, sxl, sxh, bestMinColor, bestMaxColor, best_pbits); + + uint8_t bc7_weights[16]; + + // TODO: Potentially improve this mapping using a lookup table + for (uint32_t y = 0; y < 4; y++) + { + for (uint32_t x = 0; x < 4; x++) + { + uint32_t w = pUpsampled_weights[(weight_ofs_x + x) + (weight_ofs_y + y) * block_width]; + assert(w <= 64); + + bc7_weights[x + y * 4] = (uint8_t)((w * 15 + 32) >> 6); + } // x + } // y + + encode_mode6_rgba_block(pDst_block_u8, + bestMinColor.r, bestMinColor.g, bestMinColor.b, bestMinColor.a, best_pbits[0], + bestMaxColor.r, bestMaxColor.g, bestMaxColor.b, bestMaxColor.a, best_pbits[1], + bc7_weights); + } + + // same or super close endpoints, 8x6 or 6x6 only + void pack_from_astc_to_single_subset_same_endpoints( + uint8_t* pDst_block_u8, + const astc_helpers::log_astc_block& b0, const uint8_t* pUpsampled_weights0, + const astc_helpers::log_astc_block& b1, const uint8_t* pUpsampled_weights1, + int dx, int dy, + uint32_t block_width, uint32_t block_height) + { + assert(!b0.m_dual_plane && (b0.m_num_partitions == 1)); + assert((b0.m_grid_width <= block_width) && (b0.m_grid_height <= block_height)); + assert(!b0.m_solid_color_flag_ldr); + + assert(!b1.m_dual_plane && (b1.m_num_partitions == 1)); + assert((b1.m_grid_width <= block_width) && (b1.m_grid_height <= block_height)); + assert(!b1.m_solid_color_flag_ldr); + + const bool is_6x6 = ((block_width == 6) && (block_height == 6)); + + // Only handles particular BC7 blocks in the 3x3 or 2x3 region. + if (is_6x6) + { + // 6x6 + assert( + (!dx && (dy == 1)) || ((dx == 2) && (dy == 1)) || + ((dx == 1) && !dy) || ((dx == 1) && (dy == 2)) + ); + } + else + { + // 8x6 + assert((block_width == 8) && (block_height == 6)); + assert((dx >= 0) && (dx <= 1)); + assert((dy >= 0) && (dy <= 2)); + } + + color_rgba l, h; + astc_ldr_t::decode_endpoints(b0.m_color_endpoint_modes[0], b0.m_endpoints, b0.m_endpoint_ise_range, l, h); + + color_rgba al, ah; + astc_ldr_t::decode_endpoints(b1.m_color_endpoint_modes[0], b1.m_endpoints, b1.m_endpoint_ise_range, al, ah); + + for (uint32_t c = 0; c < 4; c++) + { + l[c] = (l[c] + al[c] + 1) >> 1; + h[c] = (h[c] + ah[c] + 1) >> 1; + } + + const float q = 1.0f / 255.0f; + float sxl[4] = { (float)l.r * q, (float)l.g * q, (float)l.b * q, (float)l.a * q }; + float sxh[4] = { (float)h.r * q, (float)h.g * q, (float)h.b * q, (float)h.a * q }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_unique_pbits(4, 7, sxl, sxh, bestMinColor, bestMaxColor, best_pbits); + + bool top_or_bottom = false; + + if (is_6x6) + top_or_bottom = (dy == 0) || (dy == 2); + + uint8_t bc7_weights[16]; + + // TODO: Potentially improve this mapping using a lookup table + for (uint32_t y = 0; y < 4; y++) + { + for (uint32_t x = 0; x < 4; x++) + { + uint32_t w; + + if (is_6x6) + { + if (top_or_bottom) + { + if (x < 2) + w = pUpsampled_weights0[basisu::open_range_check((x + 4) + (y + ((dy == 2) ? 2 : 0)) * 6, 0, 36)]; + else + w = pUpsampled_weights1[basisu::open_range_check((x - 2) + (y + ((dy == 2) ? 2 : 0)) * 6, 0, 36)]; + } + else + { + if (y < 2) + w = pUpsampled_weights0[basisu::open_range_check((x + ((dx == 2) ? 2 : 0)) + (y + 4) * 6, 0, 36)]; + else + w = pUpsampled_weights1[basisu::open_range_check((x + ((dx == 2) ? 2 : 0)) + (y - 2) * 6, 0, 36)]; + } + } + else + { + // 8x6 + if (y < 2) + w = pUpsampled_weights0[basisu::open_range_check((dx * 4 + x) + (y + 4) * 8, 0, 48)]; + else + w = pUpsampled_weights1[basisu::open_range_check((dx * 4 + x) + (y - 2) * 8, 0, 48)]; + } + + assert(w <= 64); + + bc7_weights[x + y * 4] = (uint8_t)((w * 15 + 32) >> 6); + } // x + } // y + + encode_mode6_rgba_block(pDst_block_u8, + bestMinColor.r, bestMinColor.g, bestMinColor.b, bestMinColor.a, best_pbits[0], + bestMaxColor.r, bestMaxColor.g, bestMaxColor.b, bestMaxColor.a, best_pbits[1], + bc7_weights); + } + + bool pack_from_astc_6x6_to_two_subsets_different_endpoints( + uint8_t *pDst_block_u8, + const astc_helpers::log_astc_block& b0, const uint8_t* pUpsampled_weights0, + const astc_helpers::log_astc_block& b1, const uint8_t* pUpsampled_weights1, + int dx, int dy) + { + const bool b0_solid = b0.m_solid_color_flag_ldr; + const bool b1_solid = b1.m_solid_color_flag_ldr; + + assert(b0_solid || (!b0.m_dual_plane && (b0.m_num_partitions == 1))); + assert((b0.m_grid_width <= 6) && (b0.m_grid_height <= 6)); + + assert(b1_solid || (!b1.m_dual_plane && (b1.m_num_partitions == 1))); + assert((b1.m_grid_width <= 6) && (b1.m_grid_height <= 6)); + + // Only handles particular BC7 blocks in the 3x3 region. + assert( + (!dx && (dy == 1)) || ((dx == 2) && (dy == 1)) || + ((dx == 1) && !dy) || ((dx == 1) && (dy == 2)) + ); + + color_rgba l[2], h[2]; + if (b0_solid) + { + l[0][0] = h[0][0] = (uint8_t)(b0.m_solid_color[0] >> 8); + l[0][1] = h[0][1] = (uint8_t)(b0.m_solid_color[1] >> 8); + l[0][2] = h[0][2] = (uint8_t)(b0.m_solid_color[2] >> 8); + l[0][3] = h[0][3] = (uint8_t)(b0.m_solid_color[3] >> 8); + } + else + { + astc_ldr_t::decode_endpoints(b0.m_color_endpoint_modes[0], b0.m_endpoints, b0.m_endpoint_ise_range, l[0], h[0]); + } + + if (b1_solid) + { + l[1][0] = h[1][0] = (uint8_t)(b1.m_solid_color[0] >> 8); + l[1][1] = h[1][1] = (uint8_t)(b1.m_solid_color[1] >> 8); + l[1][2] = h[1][2] = (uint8_t)(b1.m_solid_color[2] >> 8); + l[1][3] = h[1][3] = (uint8_t)(b1.m_solid_color[3] >> 8); + } + else + { + astc_ldr_t::decode_endpoints(b1.m_color_endpoint_modes[0], b1.m_endpoints, b1.m_endpoint_ise_range, l[1], h[1]); + } + + float sxl[2][4], sxh[2][4]; + for (uint32_t i = 0; i < 2; i++) + { + for (uint32_t j = 0; j < 4; j++) + { + const float q = 1.0f / 255.0f; + + sxl[i][j] = l[i][j] * q; + sxh[i][j] = h[i][j] * q; + } // j + } // i + + color_rgba bestMinColor[2], bestMaxColor[2]; + + uint32_t best_p0[2]; + determine_shared_pbits(3, 6, sxl[0], &sxh[0][0], bestMinColor[0], bestMaxColor[0], best_p0); + + uint32_t best_p1[2]; + determine_shared_pbits(3, 6, sxl[1], &sxh[1][0], bestMinColor[1], bestMaxColor[1], best_p1); + + const bool top_or_bottom = (dy == 0) || (dy == 2); + + uint8_t bc7_weights[16]; + + uint32_t part_id = 0; + if ((dx == 0) || (dx == 2)) + part_id = 13; + +#if 0 + const uint8_t* pPart_map = &g_bc7_partition2[part_id * 16]; + uint32_t min_qw[2] = { 256, 256 }, max_qw[2] = { 0, 0 }; +#endif + + // TODO: Potentially improve this mapping using a lookup table + for (uint32_t y = 0; y < 4; y++) + { + for (uint32_t x = 0; x < 4; x++) + { + uint32_t w; + + if (top_or_bottom) + { + if (x < 2) + w = b0_solid ? 0 : pUpsampled_weights0[basisu::open_range_check((x + 4) + (y + ((dy == 2) ? 2 : 0)) * 6, 0, 36)]; + else + w = b1_solid ? 0 : pUpsampled_weights1[basisu::open_range_check((x - 2) + (y + ((dy == 2) ? 2 : 0)) * 6, 0, 36)]; + } + else + { + if (y < 2) + w = b0_solid ? 0 : pUpsampled_weights0[basisu::open_range_check((x + ((dx == 2) ? 2 : 0)) + (y + 4) * 6, 0, 36)]; + else + w = b1_solid ? 0 : pUpsampled_weights1[basisu::open_range_check((x + ((dx == 2) ? 2 : 0)) + (y - 2) * 6, 0, 36)]; + } + + assert(w <= 64); + + uint32_t qw = ((w * 7 + 32) >> 6); + +#if 0 + const uint8_t s = pPart_map[x + y * 4]; + min_qw[s] = basisu::minimum(min_qw[s], qw); + max_qw[s] = basisu::maximum(max_qw[s], qw); +#endif + + bc7_weights[x + y * 4] = (uint8_t)qw; + } // x + } // y + +#if 0 + const uint32_t w_range_0 = max_qw[0] - min_qw[0]; + const uint32_t w_range_1 = max_qw[1] - min_qw[1]; + const uint32_t W_RANGE_THRESH = 2; + if ((w_range_0 <= W_RANGE_THRESH) || (w_range_1 <= W_RANGE_THRESH)) + return false; +#endif + + uint32_t lr[2] = { bestMinColor[0][0], bestMinColor[1][0] }; + uint32_t lg[2] = { bestMinColor[0][1], bestMinColor[1][1] }; + uint32_t lb[2] = { bestMinColor[0][2], bestMinColor[1][2] }; + + uint32_t hr[2] = { bestMaxColor[0][0], bestMaxColor[1][0] }; + uint32_t hg[2] = { bestMaxColor[0][1], bestMaxColor[1][1] }; + uint32_t hb[2] = { bestMaxColor[0][2], bestMaxColor[1][2] }; + + encode_mode1_rgb_block(pDst_block_u8, part_id, + lr, lg, lb, + hr, hg, hb, + best_p0[0], best_p1[0], + bc7_weights); + + return true; + } + + bool pack_from_astc_8x6_to_two_subsets_different_endpoints( + uint8_t* pDst_block_u8, + const astc_helpers::log_astc_block& b0, const uint8_t* pUpsampled_weights0, + const astc_helpers::log_astc_block& b1, const uint8_t* pUpsampled_weights1, + int dx, int dy) + { + BASISU_NOTE_UNUSED(dy); + + const bool b0_solid = b0.m_solid_color_flag_ldr; + const bool b1_solid = b1.m_solid_color_flag_ldr; + + assert(b0_solid || (!b0.m_dual_plane && (b0.m_num_partitions == 1))); + assert((b0.m_grid_width <= 8) && (b0.m_grid_height <= 6)); + + assert(b1_solid || (!b1.m_dual_plane && (b1.m_num_partitions == 1))); + assert((b1.m_grid_width <= 8) && (b1.m_grid_height <= 6)); + + // Only handles particular BC7 blocks in the 2x3 region. + assert((dx >= 0) && (dx <= 1) && + (dy >= 0) && (dy <= 2)); + + color_rgba l[2], h[2]; + if (b0_solid) + { + l[0][0] = h[0][0] = (uint8_t)(b0.m_solid_color[0] >> 8); + l[0][1] = h[0][1] = (uint8_t)(b0.m_solid_color[1] >> 8); + l[0][2] = h[0][2] = (uint8_t)(b0.m_solid_color[2] >> 8); + l[0][3] = h[0][3] = (uint8_t)(b0.m_solid_color[3] >> 8); + } + else + { + astc_ldr_t::decode_endpoints(b0.m_color_endpoint_modes[0], b0.m_endpoints, b0.m_endpoint_ise_range, l[0], h[0]); + } + + if (b1_solid) + { + l[1][0] = h[1][0] = (uint8_t)(b1.m_solid_color[0] >> 8); + l[1][1] = h[1][1] = (uint8_t)(b1.m_solid_color[1] >> 8); + l[1][2] = h[1][2] = (uint8_t)(b1.m_solid_color[2] >> 8); + l[1][3] = h[1][3] = (uint8_t)(b1.m_solid_color[3] >> 8); + } + else + { + astc_ldr_t::decode_endpoints(b1.m_color_endpoint_modes[0], b1.m_endpoints, b1.m_endpoint_ise_range, l[1], h[1]); + } + + float sxl[2][4], sxh[2][4]; + for (uint32_t i = 0; i < 2; i++) + { + for (uint32_t j = 0; j < 4; j++) + { + const float q = 1.0f / 255.0f; + + sxl[i][j] = l[i][j] * q; + sxh[i][j] = h[i][j] * q; + } // j + } // i + + color_rgba bestMinColor[2], bestMaxColor[2]; + + uint32_t best_p0[2]; + determine_shared_pbits(3, 6, sxl[0], &sxh[0][0], bestMinColor[0], bestMaxColor[0], best_p0); + + uint32_t best_p1[2]; + determine_shared_pbits(3, 6, sxl[1], &sxh[1][0], bestMinColor[1], bestMaxColor[1], best_p1); + + uint8_t bc7_weights[16]; + + uint32_t part_id = 13; + + // TODO: Potentially improve this mapping using a lookup table + for (uint32_t y = 0; y < 4; y++) + { + for (uint32_t x = 0; x < 4; x++) + { + uint32_t w; + + if (y < 2) + w = b0_solid ? 0 : pUpsampled_weights0[basisu::open_range_check((dx * 4 + x) + (y + 4) * 8, 0, 48)]; + else + w = b1_solid ? 0 : pUpsampled_weights1[basisu::open_range_check((dx * 4 + x) + (y - 2) * 8, 0, 48)]; + + assert(w <= 64); + + uint32_t qw = ((w * 7 + 32) >> 6); + + bc7_weights[x + y * 4] = (uint8_t)qw; + } // x + } // y + + uint32_t lr[2] = { bestMinColor[0][0], bestMinColor[1][0] }; + uint32_t lg[2] = { bestMinColor[0][1], bestMinColor[1][1] }; + uint32_t lb[2] = { bestMinColor[0][2], bestMinColor[1][2] }; + + uint32_t hr[2] = { bestMaxColor[0][0], bestMaxColor[1][0] }; + uint32_t hg[2] = { bestMaxColor[0][1], bestMaxColor[1][1] }; + uint32_t hb[2] = { bestMaxColor[0][2], bestMaxColor[1][2] }; + + encode_mode1_rgb_block(pDst_block_u8, part_id, + lr, lg, lb, + hr, hg, hb, + best_p0[0], best_p1[0], + bc7_weights); + + return true; + } + + uint32_t fast_pack_bc7_rgb_partial_analytical(uint8_t* pBlock, const color_rgba* pPixels, uint32_t flags); + + bool pack_from_astc_8x6_to_two_subsets_different_endpoints_hq( + uint8_t* pDst_block_u8, + const astc_helpers::log_astc_block& b0, const uint8_t* pUpsampled_weights0, + const astc_helpers::log_astc_block& b1, const uint8_t* pUpsampled_weights1, + int dx, int dy, bool astc_srgb_decode, bool &fallback_encode_flag) + { + BASISU_NOTE_UNUSED(dy); + + const bool b_solid[2] = { b0.m_solid_color_flag_ldr, b1.m_solid_color_flag_ldr }; + + assert(b_solid[0] || (!b0.m_dual_plane && (b0.m_num_partitions == 1))); + assert((b0.m_grid_width <= 8) && (b0.m_grid_height <= 6)); + + assert(b_solid[1] || (!b1.m_dual_plane && (b1.m_num_partitions == 1))); + assert((b1.m_grid_width <= 8) && (b1.m_grid_height <= 6)); + + // Only handles particular BC7 blocks in the 2x3 region. + assert((dx >= 0) && (dx <= 1) && + (dy >= 0) && (dy <= 2)); + + color_rgba l[2], h[2]; + if (b_solid[0]) + { + l[0][0] = h[0][0] = (uint8_t)(b0.m_solid_color[0] >> 8); + l[0][1] = h[0][1] = (uint8_t)(b0.m_solid_color[1] >> 8); + l[0][2] = h[0][2] = (uint8_t)(b0.m_solid_color[2] >> 8); + l[0][3] = h[0][3] = (uint8_t)(b0.m_solid_color[3] >> 8); + } + else + { + astc_ldr_t::decode_endpoints(b0.m_color_endpoint_modes[0], b0.m_endpoints, b0.m_endpoint_ise_range, l[0], h[0]); + } + + if (b_solid[1]) + { + l[1][0] = h[1][0] = (uint8_t)(b1.m_solid_color[0] >> 8); + l[1][1] = h[1][1] = (uint8_t)(b1.m_solid_color[1] >> 8); + l[1][2] = h[1][2] = (uint8_t)(b1.m_solid_color[2] >> 8); + l[1][3] = h[1][3] = (uint8_t)(b1.m_solid_color[3] >> 8); + } + else + { + astc_ldr_t::decode_endpoints(b1.m_color_endpoint_modes[0], b1.m_endpoints, b1.m_endpoint_ise_range, l[1], h[1]); + } + + uint32_t low_w[2] = { UINT32_MAX, UINT32_MAX }; + uint32_t high_w[2] = { 0, 0 }; + + for (uint32_t y = 0; y < 4; y++) + { + for (uint32_t x = 0; x < 4; x++) + { + uint32_t s; + uint32_t w; + + if (y < 2) + { + w = b_solid[0] ? 0 : pUpsampled_weights0[basisu::open_range_check((dx * 4 + x) + (y + 4) * 8, 0, 48)]; + s = 0; + } + else + { + w = b_solid[1] ? 0 : pUpsampled_weights1[basisu::open_range_check((dx * 4 + x) + (y - 2) * 8, 0, 48)]; + s = 1; + } + + assert(w <= 64); + + low_w[s] = basisu::minimum(low_w[s], w); + high_w[s] = basisu::maximum(high_w[s], w); + } // x + } // y + + color_rgba orig_l[2], orig_h[2]; + memcpy(orig_l, l, sizeof(l)); + memcpy(orig_h, h, sizeof(h)); + +#if 1 + uint32_t num_low_stddev = 0; +#endif + + for (uint32_t s = 0; s < 2; s++) + { + if (b_solid[s]) + continue; + + if ((low_w[s] > 0) || (high_w[s] < 64)) + { + for (uint32_t c = 0; c < 3; c++) + { + l[s][c] = (uint8_t)astc_helpers::channel_interpolate(orig_l[s][c], orig_h[s][c], low_w[s], astc_srgb_decode); + h[s][c] = (uint8_t)astc_helpers::channel_interpolate(orig_l[s][c], orig_h[s][c], high_w[s], astc_srgb_decode); + } + } + +#if 1 + uint32_t e_delta = basisu::squarei((int)h[s][0] - (int)l[s][0]) + + basisu::squarei((int)h[s][1] - (int)l[s][1]) + + basisu::squarei((int)h[s][2] - (int)l[s][2]); + + const uint32_t E_DELTA_THRESH = 60; + num_low_stddev += (e_delta < E_DELTA_THRESH); +#endif + } + +#if 1 + if (num_low_stddev == 2) + { + //bc7f::pack_mode5_solid(pDst_block_u8, color_rgba(200, 0, 0, 255)); + //return true; + + assert(!b_solid[0] && !b_solid[1]); + + color_rgba dec_pixels[16]; + + int ep_l[2][3], ep_h[2][3]; + for (uint32_t s = 0; s < 2; s++) + { + for (uint32_t c = 0; c < 3; c++) + { + int le = l[s][c], he = h[s][c]; + + if (astc_srgb_decode) + { + le = (le << 8) | 0x80; + he = (he << 8) | 0x80; + } + else + { + le = (le << 8) | le; + he = (he << 8) | he; + } + + ep_l[s][c] = le; + ep_h[s][c] = he; + } + } + + color_rgba* pDst = dec_pixels; + for (uint32_t y = 0; y < 4; y++) + { + for (uint32_t x = 0; x < 4; x++) + { + uint32_t s = (y < 2) ? 0 : 1; + + int w; + if (y < 2) + w = pUpsampled_weights0[basisu::open_range_check((dx * 4 + x) + (y + 4) * 8, 0, 48)]; + else + w = pUpsampled_weights1[basisu::open_range_check((dx * 4 + x) + (y - 2) * 8, 0, 48)]; + + pDst->r = (uint8_t)(astc_helpers::weight_interpolate(ep_l[s][0], ep_h[s][0], w) >> 8); + pDst->g = (uint8_t)(astc_helpers::weight_interpolate(ep_l[s][1], ep_h[s][1], w) >> 8); + pDst->b = (uint8_t)(astc_helpers::weight_interpolate(ep_l[s][2], ep_h[s][2], w) >> 8); + pDst->a = 255; + + ++pDst; + + } // x + } // y + + const uint32_t flags = cPackBC7FlagUseTrivialMode6 | cPackBC7FlagPBitOptMode6; + //const uint32_t flags = cPackBC7FlagUse2SubsetsRGB | cPackBC7FlagPBitOpt | cPackBC7FlagPBitOptMode6 | cPackBC7FlagUseTrivialMode6; + bc7f::fast_pack_bc7_rgb_analytical(pDst_block_u8, dec_pixels, flags); + fallback_encode_flag = true; + return true; + } +#endif + + float sxl[2][4], sxh[2][4]; + for (uint32_t i = 0; i < 2; i++) + { + for (uint32_t j = 0; j < 4; j++) + { + const float q = 1.0f / 255.0f; + + sxl[i][j] = (float)l[i][j] * q; + sxh[i][j] = (float)h[i][j] * q; + } // j + } // i + + color_rgba bestMinColor[2], bestMaxColor[2]; + + uint32_t best_p0[2]; + determine_shared_pbits(3, 6, sxl[0], &sxh[0][0], bestMinColor[0], bestMaxColor[0], best_p0); + + uint32_t best_p1[2]; + determine_shared_pbits(3, 6, sxl[1], &sxh[1][0], bestMinColor[1], bestMaxColor[1], best_p1); + + uint8_t bc7_weights[16]; + + uint32_t part_id = 13; + + float one_over_w_range_scaled[2]; + for (uint32_t s = 0; s < 2; s++) + { + if (low_w[s] == high_w[s]) + one_over_w_range_scaled[s] = 0; + else + one_over_w_range_scaled[s] = 7.0f / (high_w[s] - low_w[s]); + } + + for (uint32_t y = 0; y < 4; y++) + { + for (uint32_t x = 0; x < 4; x++) + { + uint32_t s = (y < 2) ? 0 : 1; + + int qw = 0; + + if (!b_solid[s]) + { + int w; + + if (y < 2) + w = pUpsampled_weights0[basisu::open_range_check((dx * 4 + x) + (y + 4) * 8, 0, 48)]; + else + w = pUpsampled_weights1[basisu::open_range_check((dx * 4 + x) + (y - 2) * 8, 0, 48)]; + + assert(w <= 64); + + if (low_w[s] != high_w[s]) + { + float f = ((float)w - (float)low_w[s]) * one_over_w_range_scaled[s]; + + qw = (int)(f + .5f); + + if ((uint32_t)qw > 7) + { + qw = basisu::clamp(qw, 0, 7); + } + } + } + + bc7_weights[x + y * 4] = (uint8_t)qw; + } // x + } // y + + uint32_t lr[2] = { bestMinColor[0][0], bestMinColor[1][0] }; + uint32_t lg[2] = { bestMinColor[0][1], bestMinColor[1][1] }; + uint32_t lb[2] = { bestMinColor[0][2], bestMinColor[1][2] }; + + uint32_t hr[2] = { bestMaxColor[0][0], bestMaxColor[1][0] }; + uint32_t hg[2] = { bestMaxColor[0][1], bestMaxColor[1][1] }; + uint32_t hb[2] = { bestMaxColor[0][2], bestMaxColor[1][2] }; + + encode_mode1_rgb_block(pDst_block_u8, part_id, + lr, lg, lb, + hr, hg, hb, + best_p0[0], best_p1[0], + bc7_weights); + + return true; + } + + void pack_astc_6x6_to_two_subsets_middle_block( + uint8_t* pDst_block_u8, + const astc_helpers::log_astc_block* blocks[2][2], + const uint8_t(&weights)[2][2][36], + bool do_left_right) + { + const astc_helpers::log_astc_block* p[2]; + const astc_helpers::log_astc_block* q[2]; + + if (do_left_right) + { + // left and right into separate subsets + p[0] = blocks[0][0]; + q[0] = blocks[0][1]; + + p[1] = blocks[1][0]; + q[1] = blocks[1][1]; + } + else + { + // top and bottom into separate subsets + p[0] = blocks[0][0]; + q[0] = blocks[1][0]; + + p[1] = blocks[0][1]; + q[1] = blocks[1][1]; + } + + assert(p[0]->m_solid_color_flag_ldr || (!p[0]->m_dual_plane && (p[0]->m_num_partitions == 1))); + assert((p[0]->m_grid_width <= 6) && (p[0]->m_grid_height <= 6)); + + assert(p[1]->m_solid_color_flag_ldr || (!p[1]->m_dual_plane && (p[1]->m_num_partitions == 1))); + assert((p[1]->m_grid_width <= 6) && (p[1]->m_grid_height <= 6)); + + assert(q[0]->m_solid_color_flag_ldr || (!q[0]->m_dual_plane && (q[0]->m_num_partitions == 1))); + assert((q[0]->m_grid_width <= 6) && (q[0]->m_grid_height <= 6)); + + assert(q[1]->m_solid_color_flag_ldr || (!q[1]->m_dual_plane && (q[1]->m_num_partitions == 1))); + assert((q[1]->m_grid_width <= 6) && (q[1]->m_grid_height <= 6)); + + color_rgba el[2], eh[2]; + color_rgba el2[2], eh2[2]; + + for (uint32_t i = 0; i < 2; i++) + { + if (p[i]->m_solid_color_flag_ldr) + { + el[i][0] = eh[i][0] = (uint8_t)(p[i]->m_solid_color[0] >> 8); + el[i][1] = eh[i][1] = (uint8_t)(p[i]->m_solid_color[1] >> 8); + el[i][2] = eh[i][2] = (uint8_t)(p[i]->m_solid_color[2] >> 8); + el[i][3] = eh[i][3] = (uint8_t)(p[i]->m_solid_color[3] >> 8); + } + else + { + astc_ldr_t::decode_endpoints(p[i]->m_color_endpoint_modes[0], p[i]->m_endpoints, p[i]->m_endpoint_ise_range, el[i], eh[i]); + } + + if (q[i]->m_solid_color_flag_ldr) + { + el2[i][0] = eh2[i][0] = (uint8_t)(q[i]->m_solid_color[0] >> 8); + el2[i][1] = eh2[i][1] = (uint8_t)(q[i]->m_solid_color[1] >> 8); + el2[i][2] = eh2[i][2] = (uint8_t)(q[i]->m_solid_color[2] >> 8); + el2[i][3] = eh2[i][3] = (uint8_t)(q[i]->m_solid_color[3] >> 8); + } + else + { + astc_ldr_t::decode_endpoints(q[i]->m_color_endpoint_modes[0], q[i]->m_endpoints, q[i]->m_endpoint_ise_range, el2[i], eh2[i]); + } + + assert(el[i][3] == 255); + assert(el2[i][3] == 255); + + assert(eh[i][3] == 255); + assert(eh2[i][3] == 255); + } + + for (uint32_t i = 0; i < 2; i++) + { + for (uint32_t c = 0; c < 3; c++) + { + el[i][c] = (el[i][c] + el2[i][c] + 1) >> 1; + eh[i][c] = (eh[i][c] + eh2[i][c] + 1) >> 1; + } // c + } // i + + float sxl[2][4], sxh[2][4]; + for (uint32_t i = 0; i < 2; i++) + { + for (uint32_t j = 0; j < 4; j++) + { + const float S = 1.0f / 255.0f; + + sxl[i][j] = el[i][j] * S; + sxh[i][j] = eh[i][j] * S; + } // j + } // i + + color_rgba bestMinColor[2], bestMaxColor[2]; + + uint32_t best_p0[2]; + determine_shared_pbits(3, 6, sxl[0], &sxh[0][0], bestMinColor[0], bestMaxColor[0], best_p0); + + uint32_t best_p1[2]; + determine_shared_pbits(3, 6, sxl[1], &sxh[1][0], bestMinColor[1], bestMaxColor[1], best_p1); + + uint8_t bc7_weights[16]; + + // TODO: Potentially improve this mapping using a lookup table + for (uint32_t y = 0; y < 4; y++) + { + for (uint32_t x = 0; x < 4; x++) + { + uint32_t w = 0; + + if (y < 2) + { + if (x < 2) + { + if (!blocks[0][0]->m_solid_color_flag_ldr) + w = weights[0][0][(x + 4) + (y + 4) * 6]; + } + else + { + if (!blocks[1][0]->m_solid_color_flag_ldr) + w = weights[1][0][(x - 2) + (y + 4) * 6]; + } + } + else + { + if (x < 2) + { + if (!blocks[0][1]->m_solid_color_flag_ldr) + w = weights[0][1][(x + 4) + (y - 2) * 6]; + } + else + { + if (!blocks[1][1]->m_solid_color_flag_ldr) + w = weights[1][1][(x - 2) + (y - 2) * 6]; + } + } + + assert(w <= 64); + + bc7_weights[x + y * 4] = (uint8_t)((w * 7 + 32) >> 6); + } // x + } // y + + uint32_t part_id = 13; + if (do_left_right) + part_id = 0; + + uint32_t lr[2] = { bestMinColor[0][0], bestMinColor[1][0] }; + uint32_t lg[2] = { bestMinColor[0][1], bestMinColor[1][1] }; + uint32_t lb[2] = { bestMinColor[0][2], bestMinColor[1][2] }; + + uint32_t hr[2] = { bestMaxColor[0][0], bestMaxColor[1][0] }; + uint32_t hg[2] = { bestMaxColor[0][1], bestMaxColor[1][1] }; + uint32_t hb[2] = { bestMaxColor[0][2], bestMaxColor[1][2] }; + + encode_mode1_rgb_block(pDst_block_u8, part_id, + lr, lg, lb, + hr, hg, hb, + best_p0[0], best_p1[0], + bc7_weights); + } + +#if 0 + // var must be variance (divided by N, # pixels), not SSE + static inline int calc_span_est(int min_c, int max_c, int mean_c, float var) + { + // variance-implied span: span_var = ~sqrt(12 * var) + int span_var = (int)fast_roundf_pos_int(std::sqrtf((float)(12.0f * var))); + + // take into account available headroom on the low/high end + span_var = basisu::minimum(span_var, 2 * basisu::minimum(mean_c, 255 - mean_c)); + + return basisu::minimum(max_c - min_c, span_var); + } +#endif + + // Multi-channel estimates + // returns total SSE (pixel SSE * num_pixels), span_weights can be nullptr + float analytical_quant_est_sse(int e_levels, int w_levels, int num_chans, const int spans[4], const float span_weights[4], float endpoint_weight_scale, int num_pixels) + { + assert((e_levels >= 2) && (e_levels <= 256) && (w_levels >= 2) && (num_chans)); + assert(spans); + + const float Dep = 1.0f / (float)(e_levels - 1); // endpoint quant step + const float Dw = 1.0f / (float)(w_levels - 1); // weight quant step + + // TODO: precompute + const float N = float(w_levels); + const float ab_sum = (2.0f * N - 1.0f) / (3.0f * (N - 1.0f)); + + float pixel_sse = (e_levels == 256) ? 0.0f : ((Dep * Dep) * ((1.0f / 12.0f) * ab_sum * (255.0f * 255.0f)) * (float)num_chans * endpoint_weight_scale); + + const float k = (Dw * Dw) * (1.0f / 12.0f); + for (int i = 0; i < num_chans; i++) + { + pixel_sse += k * (float)(spans[i] * spans[i]) * (span_weights ? span_weights[i] : 1.0f); + } + + return pixel_sse * float(num_pixels); + } + + // Single channel estimates + float analytical_quant_est_sse(int e_levels, int w_levels, int span, float span_weight, float endpoint_weight_scale, int num_pixels) + { + assert((e_levels >= 2) && (e_levels <= 256) && (w_levels >= 2)); + + const float Dep = 1.0f / (float)(e_levels - 1); // endpoint quant step + const float Dw = 1.0f / (float)(w_levels - 1); // weight quant step + + // TODO: precompute + const float N = float(w_levels); + const float ab_sum = (2.0f * N - 1.0f) / (3.0f * (N - 1.0f)); + + float pixel_sse = (e_levels == 256) ? 0.0f : ((Dep * Dep) * ((1.0f / 12.0f) * ab_sum * (255.0f * 255.0f)) * endpoint_weight_scale); + + pixel_sse += (Dw * Dw) * (1.0f / 12.0f) * (float)(span * span) * span_weight; + + return pixel_sse * float(num_pixels); + } + + // if cov[] wasn't divided by the # of pixels, this is SSE + float estimate_slam_to_line_sse_3D(const float cov[6], float xr, float yr, float zr, float* pOrtho_ratio = nullptr) + { + // total var + const float total_var = cov[0] + cov[3] + cov[5]; + + float l = sqrtf(xr * xr + yr * yr + zr * zr); + if (l < basisu::SMALL_FLOAT_VAL) + { + xr = yr = zr = 0.577350269f; + } + else + { + l = 1.0f / l; + xr *= l; yr *= l; zr *= l; + } + + float xr2 = cov[0] * xr + cov[1] * yr + cov[2] * zr; + float xg2 = cov[1] * xr + cov[3] * yr + cov[4] * zr; + float xb2 = cov[2] * xr + cov[4] * yr + cov[5] * zr; + + // Rayleigh quotient/est var of principal axis + const float principal_axis_var = xr2 * xr + xg2 * yr + xb2 * zr; + + // Compute leftover var, this is the var unexplaind by the principal axis + const float ortho_var = basisu::maximum(0.0f, total_var - principal_axis_var); + + if (pOrtho_ratio) + *pOrtho_ratio = (total_var > basisu::SMALL_FLOAT_VAL) ? (ortho_var / total_var) : 0.0f; + + return ortho_var; + } + + float estimate_slam_to_line_sse_4D(const float cov[10], float xr, float yr, float zr, float wr, float* pOrtho_ratio = nullptr) + { + // total var + const float total_var = cov[0] + cov[4] + cov[7] + cov[9]; + + float l = sqrtf(xr * xr + yr * yr + zr * zr + wr * wr); + if (l < basisu::SMALL_FLOAT_VAL) + { + xr = yr = zr = wr = .5f; + } + else + { + l = 1.0f / l; + xr *= l; yr *= l; zr *= l; wr *= l; + } + + float xr2 = cov[0] * xr + cov[1] * yr + cov[2] * zr + cov[3] * wr; + float xg2 = cov[1] * xr + cov[4] * yr + cov[5] * zr + cov[6] * wr; + float xb2 = cov[2] * xr + cov[5] * yr + cov[7] * zr + cov[8] * wr; + float xa2 = cov[3] * xr + cov[6] * yr + cov[8] * zr + cov[9] * wr; + + // Rayleigh quotient/est var of principal axis + const float principal_axis_var = xr2 * xr + xg2 * yr + xb2 * zr + xa2 * wr; + + // Compute leftover var, this is the var unexplaind by the principal axis + const float ortho_var = basisu::maximum(0.0f, total_var - principal_axis_var); + + if (pOrtho_ratio) + *pOrtho_ratio = (total_var > basisu::SMALL_FLOAT_VAL) ? (ortho_var / total_var) : 0.0f; + + return ortho_var; + } + + uint32_t calc_sse(const uint8_t* pBlock, const color_rgba* pPixels) + { + color_rgba unpacked_pixels[16]; + bool status = bc7u::unpack_bc7(pBlock, unpacked_pixels); + if (!status) + { + assert(0); + return UINT32_MAX; + } + + uint32_t sse = 0; + for (uint32_t i = 0; i < 16; i++) + sse += basisu::squarei(pPixels[i][0] - unpacked_pixels[i][0]) + basisu::squarei(pPixels[i][1] - unpacked_pixels[i][1]) + basisu::squarei(pPixels[i][2] - unpacked_pixels[i][2]) + basisu::squarei(pPixels[i][3] - unpacked_pixels[i][3]); + + return sse; + } + + bool pack_mode1_or_3_rgb(uint8_t* pBlock, const color_rgba* pPixels, + float block_xr, float block_xg, float block_xb, + int block_mean_r, int block_mean_g, int block_mean_b, + float sse_est_to_beat, uint32_t flags, + float* pFinal_sse_est = nullptr, + uint32_t* pActual_sse = nullptr) + { +#if BASISU_BC7F_PERF_STATS + g_total_mode13_evals++; +#endif + + uint32_t desired_pat_bits = 0; + + for (uint32_t i = 0; i < 16; i++) + { + const float r = (float)(pPixels[i].r - block_mean_r); + const float g = (float)(pPixels[i].g - block_mean_g); + const float b = (float)(pPixels[i].b - block_mean_b); + + const uint32_t subset = (r * block_xr + g * block_xg + b * block_xb) > 0.0f; + + desired_pat_bits |= (subset << i); + } + + uint32_t best_diff = UINT32_MAX; + for (uint32_t p = 0; p < MAX_PATTERNS2_TO_CHECK; p++) + { + const uint32_t bc6h_pat_bits = g_bc7_part2_bitmasks[p]; + + int diff = popcount32(bc6h_pat_bits ^ desired_pat_bits); + int diff_inv = 16 - diff; + + uint32_t min_diff = (basisu::minimum(diff, diff_inv) << 8) | p; + if (min_diff < best_diff) + best_diff = min_diff; + } // p + + const uint32_t best_pat_index = best_diff & 0xFF; + const uint32_t best_pat_bits = g_bc7_part2_bitmasks[best_pat_index]; + + int total_r[2] = { }, total_g[2] = { }, total_b[2] = { }, total_c[2] = { }; + for (uint32_t i = 0; i < 16; i++) + { + const int r = pPixels[i].r, g = pPixels[i].g, b = pPixels[i].b; + const int subset = (best_pat_bits >> i) & 1; + + total_r[subset] += r; total_g[subset] += g; total_b[subset] += b; + total_c[subset]++; + } + + int mean_r[2], mean_g[2], mean_b[2]; + for (uint32_t s = 0; s < 2; s++) + { + const uint32_t t = total_c[s]; + const uint32_t h = (t >> 1); + + mean_r[s] = (total_r[s] + h) / t; + mean_g[s] = (total_g[s] + h) / t; + mean_b[s] = (total_b[s] + h) / t; + } + + int icov[2][6] = { { }, { } }; + + for (uint32_t i = 0; i < 16; i++) + { + const int subset = (best_pat_bits >> i) & 1; + + int r = (int)pPixels[i].r - mean_r[subset]; + int g = (int)pPixels[i].g - mean_g[subset]; + int b = (int)pPixels[i].b - mean_b[subset]; + icov[subset][0] += r * r; icov[subset][1] += r * g; icov[subset][2] += r * b; + icov[subset][3] += g * g; icov[subset][4] += g * b; + icov[subset][5] += b * b; + } + + int ar[2], ag[2], ab[2]; + + // Slam to line SSE estimate is the same for both mode 1 and 3. + float slam_to_line_sse_est = 0.0f; + + for (uint32_t s = 0; s < 2; s++) + { + int block_max_var = basisu::maximum(icov[s][0], icov[s][3], icov[s][5]); + + float cov[6]; + for (uint32_t i = 0; i < 6; i++) + cov[i] = (float)icov[s][i]; + + const float sc = 1.0f / ((float)block_max_var + .0000125f); + const float wx = sc * cov[0], wy = sc * cov[3], wz = sc * cov[5]; + + const float alt_xr = cov[0] * wx + cov[1] * wy + cov[2] * wz; + const float alt_xg = cov[1] * wx + cov[3] * wy + cov[4] * wz; + const float alt_xb = cov[2] * wx + cov[4] * wy + cov[5] * wz; + + slam_to_line_sse_est += estimate_slam_to_line_sse_3D(cov, alt_xr, alt_xg, alt_xb); + + int saxis_r = 306, saxis_g = 601, saxis_b = 117; + + float k = basisu::maximum(fabsf(alt_xr), fabsf(alt_xg), fabsf(alt_xb)); + if (fabs(k) >= basisu::SMALL_FLOAT_VAL) + { + float m = 2048.0f / k; + saxis_r = (int)(alt_xr * m); + saxis_g = (int)(alt_xg * m); + saxis_b = (int)(alt_xb * m); + } + + ar[s] = (int)((uint32_t)saxis_r << 4U); + ag[s] = (int)((uint32_t)saxis_g << 4U); + ab[s] = (int)((uint32_t)saxis_b << 4U); + } // s + + int low_dot[2] = { INT_MAX, INT_MAX }; + int high_dot[2] = { INT_MIN, INT_MIN }; + + for (uint32_t i = 0; i < 16; i++) + { + const int subset = (best_pat_bits >> i) & 1; + const int saxis_r = ar[subset], saxis_g = ag[subset], saxis_b = ab[subset]; + + int dot = (pPixels[i].r * saxis_r + pPixels[i].g * saxis_g + pPixels[i].b * saxis_b) + i; + + low_dot[subset] = basisu::minimum(low_dot[subset], dot); + high_dot[subset] = basisu::maximum(high_dot[subset], dot); + } + + int low_c[2] = { low_dot[0] & 15, low_dot[1] & 15 }; + int high_c[2] = { high_dot[0] & 15, high_dot[1] & 15 }; + + int spans[4]; + spans[3] = 0; + + // Endpoint/weight quant error estimates for modes 1 and 3 + float quant_err_sse_est[2] = { }; + + for (uint32_t subset = 0; subset < 2; subset++) + { + const uint32_t low_pixel = low_c[subset]; + const uint32_t high_pixel = high_c[subset]; + + for (uint32_t c = 0; c < 3; c++) + spans[c] = pPixels[high_pixel][c] - pPixels[low_pixel][c]; + + // mode 1: 6-bit endpoints, unique pbits, 3 bit weights + quant_err_sse_est[0] += analytical_quant_est_sse(64, 8, 3, spans, nullptr, (flags & cPackBC7FlagPBitOpt) ? UNIQUE_PBIT_DISCOUNT : 1.0f, total_c[subset]); + + // mode 3, 7-bit endpoints, shared pbits, 2-bit weights + quant_err_sse_est[1] += analytical_quant_est_sse(128, 4, 3, spans, nullptr, (flags & cPackBC7FlagPBitOpt) ? SHARED_PBIT_DISCOUNT : 1.0f, total_c[subset]); + + } // subset + + const float total_mode1_est_sse = slam_to_line_sse_est + quant_err_sse_est[0]; + const float total_mode3_est_sse = slam_to_line_sse_est + quant_err_sse_est[1]; + + if (total_mode1_est_sse < total_mode3_est_sse) + { + if (pFinal_sse_est) + *pFinal_sse_est = total_mode1_est_sse; + + // Mode 1: Large span + if (total_mode1_est_sse >= sse_est_to_beat) + { +#if BASISU_BC7F_PERF_STATS + g_total_mode13_bailouts++; +#endif + return false; + } + + uint32_t lr[2], lg[2], lb[2]; + uint32_t hr[2], hg[2], hb[2]; + uint32_t pbits[2] = { 0, 0 }; + + for (uint32_t s = 0; s < 2; s++) + { + const int lc = low_c[s], hc = high_c[s]; + + if (flags & cPackBC7FlagPBitOpt) + { + const float q = 1.0f / 255.0f; + float sxl[4] = { (float)pPixels[lc].r * q, (float)pPixels[lc].g * q, (float)pPixels[lc].b * q, 0 }; + float sxh[4] = { (float)pPixels[hc].r * q, (float)pPixels[hc].g * q, (float)pPixels[hc].b * q, 0 }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_shared_pbits(3, 6, sxl, sxh, bestMinColor, bestMaxColor, best_pbits); + + pbits[s] = best_pbits[0]; + lr[s] = bestMinColor.r, lg[s] = bestMinColor.g, lb[s] = bestMinColor.b; + hr[s] = bestMaxColor.r, hg[s] = bestMaxColor.g, hb[s] = bestMaxColor.b; + } + else + { + int l = pPixels[lc].r + pPixels[lc].g + pPixels[lc].b; + int h = pPixels[hc].r + pPixels[hc].g + pPixels[hc].b; + + if (basisu::maximum(l, h) >= 129 * 3) + pbits[s] = 1; + + lr[s] = to_6(pPixels[lc].r, pbits[s]); + lg[s] = to_6(pPixels[lc].g, pbits[s]); + lb[s] = to_6(pPixels[lc].b, pbits[s]); + + hr[s] = to_6(pPixels[hc].r, pbits[s]); + hg[s] = to_6(pPixels[hc].g, pbits[s]); + hb[s] = to_6(pPixels[hc].b, pbits[s]); + } + } // s + + uint8_t cur_weights[16]; + eval_weights_mode1_rgb(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, pbits, best_pat_bits); + + float z00[2] = { 0.0f }, z10[2] = { 0.0f }, z11[2] = { 0.0f }; + float q00_r[2] = { 0.0f }; + float q00_g[2] = { 0.0f }; + float q00_b[2] = { 0.0f }; + + for (uint32_t i = 0; i < 16; i++) + { + const int subset = (best_pat_bits >> i) & 1; + const uint32_t sel = cur_weights[i]; + assert(sel <= 7); + + z00[subset] += g_bc7_3bit_ls_tab[sel][0]; + z10[subset] += g_bc7_3bit_ls_tab[sel][1]; + z11[subset] += g_bc7_3bit_ls_tab[sel][2]; + + const float w = g_bc7_3bit_ls_tab[sel][3]; + + q00_r[subset] += w * (float)pPixels[i][0]; + q00_g[subset] += w * (float)pPixels[i][1]; + q00_b[subset] += w * (float)pPixels[i][2]; + } // i + + for (uint32_t s = 0; s < 2; s++) + { + float q10_r = (float)total_r[s] - q00_r[s]; + float q10_g = (float)total_g[s] - q00_g[s]; + float q10_b = (float)total_b[s] - q00_b[s]; + + float z01 = z10[s]; + + float det = z00[s] * z11[s] - z01 * z10[s]; + if (fabs(det) < 1e-8f) + continue; + + det = 1.0f / det; + + float iz00, iz01, iz10, iz11; + iz00 = z11[s] * det; + iz01 = -z01 * det; + iz10 = -z10[s] * det; + iz11 = z00[s] * det; + + const float shr = iz00 * q00_r[s] + iz01 * q10_r; + const float slr = iz10 * q00_r[s] + iz11 * q10_r; + + const float shg = iz00 * q00_g[s] + iz01 * q10_g; + const float slg = iz10 * q00_g[s] + iz11 * q10_g; + + const float shb = iz00 * q00_b[s] + iz01 * q10_b; + const float slb = iz10 * q00_b[s] + iz11 * q10_b; + + if (flags & cPackBC7FlagPBitOpt) + { + const float q = 1.0f / 255.0f; + float sxl[4] = { basisu::clamp(slr * q, 0.0f, 1.0f), basisu::clamp(slg * q, 0.0f, 1.0f), basisu::clamp(slb * q, 0.0f, 1.0f), 0 }; + float sxh[4] = { basisu::clamp(shr * q, 0.0f, 1.0f), basisu::clamp(shg * q, 0.0f, 1.0f), basisu::clamp(shb * q, 0.0f, 1.0f), 0 }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_shared_pbits(3, 6, sxl, sxh, bestMinColor, bestMaxColor, best_pbits); + + pbits[s] = best_pbits[0]; + lr[s] = bestMinColor.r, lg[s] = bestMinColor.g, lb[s] = bestMinColor.b; + hr[s] = bestMaxColor.r, hg[s] = bestMaxColor.g, hb[s] = bestMaxColor.b; + } + else + { + const float l = slr + slg + slb, h = shr + shg + shb; + + pbits[s] = (basisu::maximum(l, h) >= 129.0f * 3.0f); + + lr[s] = to_6_clamp(slr, pbits[s]); + hr[s] = to_6_clamp(shr, pbits[s]); + + lg[s] = to_6_clamp(slg, pbits[s]); + hg[s] = to_6_clamp(shg, pbits[s]); + + lb[s] = to_6_clamp(slb, pbits[s]); + hb[s] = to_6_clamp(shb, pbits[s]); + } + + } // s + + if (pActual_sse) + *pActual_sse = eval_weights_mode1_rgb_sse(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, pbits, best_pat_bits); + else + eval_weights_mode1_rgb(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, pbits, best_pat_bits); + + encode_mode1_rgb_block(pBlock, best_pat_index, + lr, lg, lb, hr, hg, hb, pbits[0], pbits[1], cur_weights); + } + else + { + // Mode 3: Small span + if (pFinal_sse_est) + *pFinal_sse_est = total_mode3_est_sse; + + if (total_mode3_est_sse >= sse_est_to_beat) + { +#if BASISU_BC7F_PERF_STATS + g_total_mode13_bailouts++; +#endif + return false; + } + + uint32_t lr[2], lg[2], lb[2]; + uint32_t hr[2], hg[2], hb[2]; + uint32_t pbits[4]; + + for (uint32_t s = 0; s < 2; s++) + { + const int lc = low_c[s]; + const int hc = high_c[s]; + + if (flags & cPackBC7FlagPBitOpt) + { + const float q = 1.0f / 255.0f; + float sxl[4] = { (float)pPixels[lc].r * q, (float)pPixels[lc].g * q, (float)pPixels[lc].b * q, 0 }; + float sxh[4] = { (float)pPixels[hc].r * q, (float)pPixels[hc].g * q, (float)pPixels[hc].b * q, 0 }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_unique_pbits(3, 7, sxl, sxh, bestMinColor, bestMaxColor, best_pbits); + + pbits[s * 2 + 0] = best_pbits[0]; + pbits[s * 2 + 1] = best_pbits[1]; + lr[s] = bestMinColor.r, lg[s] = bestMinColor.g, lb[s] = bestMinColor.b; + hr[s] = bestMaxColor.r, hg[s] = bestMaxColor.g, hb[s] = bestMaxColor.b; + } + else + { + const int l = pPixels[lc].r + pPixels[lc].g + pPixels[lc].b; + const int l_pbit = (l >= 129); + pbits[s * 2 + 0] = l_pbit; + + lr[s] = to_7(pPixels[lc].r, l_pbit); + lg[s] = to_7(pPixels[lc].g, l_pbit); + lb[s] = to_7(pPixels[lc].b, l_pbit); + + int h = pPixels[hc].r + pPixels[hc].g + pPixels[hc].b; + const int h_pbit = (h >= 129); + pbits[s * 2 + 1] = h_pbit; + + hr[s] = to_7(pPixels[hc].r, h_pbit); + hg[s] = to_7(pPixels[hc].g, h_pbit); + hb[s] = to_7(pPixels[hc].b, h_pbit); + } + } // s + + uint8_t cur_weights[16]; + eval_weights_mode3_rgb(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, pbits, best_pat_bits); + + float z00[2] = { 0.0f }, z10[2] = { 0.0f }, z11[2] = { 0.0f }; + float q00_r[2] = { 0.0f }; + float q00_g[2] = { 0.0f }; + float q00_b[2] = { 0.0f }; + + for (uint32_t i = 0; i < 16; i++) + { + const int subset = (best_pat_bits >> i) & 1; + const uint32_t sel = cur_weights[i]; + assert(sel <= 3); + + z00[subset] += g_bc7_2bit_ls_tab[sel][0]; + z10[subset] += g_bc7_2bit_ls_tab[sel][1]; + z11[subset] += g_bc7_2bit_ls_tab[sel][2]; + + const float w = g_bc7_2bit_ls_tab[sel][3]; + + q00_r[subset] += w * (float)pPixels[i][0]; + q00_g[subset] += w * (float)pPixels[i][1]; + q00_b[subset] += w * (float)pPixels[i][2]; + } // i + + for (uint32_t s = 0; s < 2; s++) + { + float q10_r = (float)total_r[s] - q00_r[s]; + float q10_g = (float)total_g[s] - q00_g[s]; + float q10_b = (float)total_b[s] - q00_b[s]; + + float z01 = z10[s]; + + float det = z00[s] * z11[s] - z01 * z10[s]; + if (fabs(det) < 1e-8f) + continue; + + det = 1.0f / det; + + float iz00, iz01, iz10, iz11; + iz00 = z11[s] * det; + iz01 = -z01 * det; + iz10 = -z10[s] * det; + iz11 = z00[s] * det; + + const float shr = iz00 * q00_r[s] + iz01 * q10_r; + const float slr = iz10 * q00_r[s] + iz11 * q10_r; + + const float shg = iz00 * q00_g[s] + iz01 * q10_g; + const float slg = iz10 * q00_g[s] + iz11 * q10_g; + + const float shb = iz00 * q00_b[s] + iz01 * q10_b; + const float slb = iz10 * q00_b[s] + iz11 * q10_b; + + if (flags & cPackBC7FlagPBitOpt) + { + const float q = 1.0f / 255.0f; + float sxl[4] = { basisu::clamp(slr * q, 0.0f, 1.0f), basisu::clamp(slg * q, 0.0f, 1.0f), basisu::clamp(slb * q, 0.0f, 1.0f), 0 }; + float sxh[4] = { basisu::clamp(shr * q, 0.0f, 1.0f), basisu::clamp(shg * q, 0.0f, 1.0f), basisu::clamp(shb * q, 0.0f, 1.0f), 0 }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_unique_pbits(3, 7, sxl, sxh, bestMinColor, bestMaxColor, best_pbits); + + pbits[s * 2 + 0] = best_pbits[0]; + pbits[s * 2 + 1] = best_pbits[1]; + lr[s] = bestMinColor.r, lg[s] = bestMinColor.g, lb[s] = bestMinColor.b; + hr[s] = bestMaxColor.r, hg[s] = bestMaxColor.g, hb[s] = bestMaxColor.b; + } + else + { + const float l = slr + slg + slb; + const int l_pbit = (l >= 129.0f * 3.0f); + pbits[s * 2 + 0] = l_pbit; + + lr[s] = to_7_clamp(slr, l_pbit); + lg[s] = to_7_clamp(slg, l_pbit); + lb[s] = to_7_clamp(slb, l_pbit); + + const float h = shr + shg + shb; + const int h_pbit = (h >= 129.0f * 3.0f); + pbits[s * 2 + 1] = h_pbit; + + hr[s] = to_7_clamp(shr, h_pbit); + hg[s] = to_7_clamp(shg, h_pbit); + hb[s] = to_7_clamp(shb, h_pbit); + } + + } // s + + if (pActual_sse) + *pActual_sse = eval_weights_mode3_rgb_sse(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, pbits, best_pat_bits); + else + eval_weights_mode3_rgb(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, pbits, best_pat_bits); + + encode_mode3_rgb_block(pBlock, best_pat_index, + lr, lg, lb, hr, hg, hb, pbits, cur_weights); + } + +#ifdef _DEBUG + if (pActual_sse) + { + const uint32_t expected_sse = calc_sse(pBlock, pPixels); + assert(expected_sse == *pActual_sse); + } +#endif + + return true; + } + + inline int dist3(int lr, int lg, int lb, int hr, int hg, int hb) + { + return basisu::squarei(hr - lr) + basisu::squarei(hg - lg) + basisu::squarei(hb - lb); + } + + bool determine_3subsets(uint8_t* pFinal_3subsets, + const color_rgba* pPixels, + float block_xr, float block_xg, float block_xb, + int block_mean_r, int block_mean_g, int block_mean_b) + { + uint32_t subset_indices[16]; + int subset_means[2][3] = { }; + int subset_total[2] = { }; + + for (uint32_t i = 0; i < 16; i++) + { + const int rd = pPixels[i].r - block_mean_r; + const int gd = pPixels[i].g - block_mean_g; + const int bd = pPixels[i].b - block_mean_b; + + const uint32_t subset_index = ((float)rd * block_xr + (float)gd * block_xg + (float)bd * block_xb) > 0.0f; + + subset_indices[i] = subset_index; + + subset_means[subset_index][0] += pPixels[i].r; + subset_means[subset_index][1] += pPixels[i].g; + subset_means[subset_index][2] += pPixels[i].b; + + subset_total[subset_index]++; + } + + for (uint32_t i = 0; i < 2; i++) + { + const uint32_t t = subset_total[i]; + if (!t) + return false; + + subset_means[i][0] = (subset_means[i][0] + (t >> 1)) / t; + subset_means[i][1] = (subset_means[i][1] + (t >> 1)) / t; + subset_means[i][2] = (subset_means[i][2] + (t >> 1)) / t; + } + + int subset_sses[2] = { }; + for (uint32_t i = 0; i < 16; i++) + { + const uint32_t subset_index = subset_indices[i]; + + subset_sses[subset_index] += dist3(pPixels[i].r, pPixels[i].g, pPixels[i].b, subset_means[subset_index][0], subset_means[subset_index][1], subset_means[subset_index][2]); + } + + const uint32_t subset_to_split = (subset_sses[1] > subset_sses[0]); + if (subset_total[subset_to_split] < 2) + return false; + + int lo_y = INT_MAX, hi_y = 0; + for (uint32_t i = 0; i < 16; i++) + { + if (subset_indices[i] != subset_to_split) + continue; + + int y = ((pPixels[i].r + pPixels[i].g + pPixels[i].b) << 4) + i; + + lo_y = basisu::minimum(lo_y, y); + hi_y = basisu::maximum(hi_y, y); + } + + const int lo_y_index = lo_y & 15, hi_y_index = hi_y & 15; + if (lo_y_index == hi_y_index) + return false; + + const int lr = pPixels[lo_y_index].r, lg = pPixels[lo_y_index].g, lb = pPixels[lo_y_index].b; + const int hr = pPixels[hi_y_index].r, hg = pPixels[hi_y_index].g, hb = pPixels[hi_y_index].b; + + memset(pFinal_3subsets, 2, 16); + + for (uint32_t i = 0; i < 16; i++) + { + if (subset_indices[i] == subset_to_split) + { + const int dist0 = dist3(lr, lg, lb, pPixels[i].r, pPixels[i].g, pPixels[i].b); + const int dist1 = dist3(hr, hg, hb, pPixels[i].r, pPixels[i].g, pPixels[i].b); + + pFinal_3subsets[i] = dist1 > dist0; + } + } + + return true; + } + + static inline int pop16(uint32_t x) + { +#if defined(_MSC_VER) + return __popcnt16((unsigned short)x); +#else + return __builtin_popcount(x & 0xFFFFu); +#endif + } + + int pick_3subset_pat_index(const uint8_t* pDesired_subsets, uint32_t& best_pat_index_first16) + { + best_pat_index_first16 = 0; + + uint16_t M[3]; + memset(M, 0, sizeof(M)); + + for (uint32_t i = 0; i < 16; i++) + { + uint32_t s = pDesired_subsets[i]; + M[s] |= (1 << i); + } + + const int n0 = pop16(M[0]), n1 = pop16(M[1]), n2 = 16 - n0 - n1; + + int best_score = -1; + int best_pat = 0; + + for (int p = 0; p < (int)MAX_PATTERNS3_TO_CHECK; ++p) + { + uint16_t S0 = (uint16_t)(g_part3_bitmasks[p] & 0xFFFFu); + uint16_t S1 = (uint16_t)(g_part3_bitmasks[p] >> 16); + + // Row sums for subsets 0 and 1 via 6 popcnts; derive subset 2 by subtraction + int C00 = pop16(M[0] & S0), C01 = pop16(M[0] & S1), C02 = n0 - C00 - C01; + int C10 = pop16(M[1] & S0), C11 = pop16(M[1] & S1), C12 = n1 - C10 - C11; + int C20 = pop16(M[2] & S0), C21 = pop16(M[2] & S1), C22 = n2 - C20 - C21; + + int s0 = C00 + C11 + C22; // (0,1,2) + int s1 = C00 + C12 + C21; // (0,2,1) + int s2 = C01 + C10 + C22; // (1,0,2) + int s3 = C01 + C12 + C20; // (1,2,0) + int s4 = C02 + C10 + C21; // (2,0,1) + int s5 = C02 + C11 + C20; // (2,1,0) + + // Argmax over 6 + int s = s0; + if (s1 > s) { s = s1; } + if (s2 > s) { s = s2; } + if (s3 > s) { s = s3; } + if (s4 > s) { s = s4; } + if (s5 > s) { s = s5; } + + if (s > best_score) + { + best_score = s; + best_pat = p; + + if (s == 16) + { + // perfect match so early out + if (p <= 15) + best_pat_index_first16 = best_pat; + break; + } + } + + if (p == 15) + { + // for mode 0 + best_pat_index_first16 = best_pat; + } + } + + return best_pat; + } + +#if 0 + static const uint8_t s_perms3[6][3] = { {0,1,2}, {0,2,1}, {1,0,2}, {1,2,0}, {2,0,1}, {2,1,0} }; + + int pick_3subset_pat_index_slow(const uint8_t* pDesired_subsets, uint32_t& best_pat_index_first16) + { + int best_pat = 0, best_dist = INT_MAX; + + for (uint32_t m = 0; m < 64; m++) + { + const uint8_t* pPat = &g_bc7_partition3[m * 16]; + + for (uint32_t p = 0; p < 6; p++) + { + int trial_dist = 0; + + for (uint32_t i = 0; i < 16; i++) + { + uint32_t s = s_perms3[p][pDesired_subsets[i]]; + + trial_dist += (s != pPat[i]); + + } // i + + if (trial_dist < best_dist) + { + best_dist = trial_dist; + best_pat = m; + } + + } // p + + if (m == 15) + best_pat_index_first16 = best_pat; + + } // m + + return best_pat; + } +#endif + + // false if packing failed (not enough unique colors) + bool pack_mode0_or_2_rgb(uint8_t* pBlock, const color_rgba* pPixels, + float block_xr, float block_xg, float block_xb, + int block_mean_r, int block_mean_g, int block_mean_b, float sse_est_to_beat, uint32_t flags, + float* pFinal_sse_est = nullptr, + uint32_t* pActual_sse = nullptr) + { + (void)flags; + +#if BASISU_BC7F_PERF_STATS + g_total_mode02_evals++; +#endif + + uint8_t desired_3subsets[16]; + if (!determine_3subsets(desired_3subsets, pPixels, block_xr, block_xg, block_xb, block_mean_r, block_mean_g, block_mean_b)) + { +#if BASISU_BC7F_PERF_STATS + g_total_mode02_bailouts++; +#endif + if (pFinal_sse_est) + *pFinal_sse_est = 1e+9f; + + return false; + } + + uint32_t best_pat_indices[2]; // mode 0 and 2 + best_pat_indices[1] = pick_3subset_pat_index(desired_3subsets, best_pat_indices[0]); + + assert((best_pat_indices[0] <= 15) && (best_pat_indices[1] <= 63)); + + float total_quant_sse_mode[2] = { }; + float total_slam_to_line_sse_mode[2] = { }; + + int mode_total_c[2][3] = { }; + int mode_low_c[2][3] = { }, mode_high_c[2][3] = { }; + int mode_total_r[2][3] = { }, mode_total_g[2][3] = { }, mode_total_b[2][3] = { }; + + int spans[4] = { }; + + for (uint32_t mode_iter = 0; mode_iter < 2; mode_iter++) // mode 0 vs. mode 2 + { + if ((mode_iter) && (best_pat_indices[0] == best_pat_indices[1])) + { + for (uint32_t s = 0; s < 3; s++) + { + mode_total_c[1][s] = mode_total_c[0][s]; + + mode_low_c[1][s] = mode_low_c[0][s]; + mode_high_c[1][s] = mode_high_c[0][s]; + + mode_total_r[1][s] = mode_total_r[0][s]; + mode_total_g[1][s] = mode_total_g[0][s]; + mode_total_b[1][s] = mode_total_b[0][s]; + + total_slam_to_line_sse_mode[1] = total_slam_to_line_sse_mode[0]; + + } // subset + } + else + { + const uint32_t best_pat_index = best_pat_indices[mode_iter]; + const uint8_t* pBest_pat = &g_bc7_partition3[best_pat_index * 16]; + + int* pTotal_r = &mode_total_r[mode_iter][0]; + int* pTotal_g = &mode_total_g[mode_iter][0]; + int* pTotal_b = &mode_total_b[mode_iter][0]; + + int* pTotal_c = mode_total_c[mode_iter]; + + for (uint32_t i = 0; i < 16; i++) + { + const int r = pPixels[i].r, g = pPixels[i].g, b = pPixels[i].b; + const int subset = pBest_pat[i]; + + pTotal_r[subset] += r; pTotal_g[subset] += g; pTotal_b[subset] += b; + pTotal_c[subset]++; + } + + int mean_r[3], mean_g[3], mean_b[3]; + for (uint32_t s = 0; s < 3; s++) + { + const uint32_t t = pTotal_c[s]; + const uint32_t h = (t >> 1); + + mean_r[s] = (pTotal_r[s] + h) / t; + mean_g[s] = (pTotal_g[s] + h) / t; + mean_b[s] = (pTotal_b[s] + h) / t; + } + + int icov[3][6] = { }; + + for (uint32_t i = 0; i < 16; i++) + { + const int subset = pBest_pat[i]; + + int r = (int)pPixels[i].r - mean_r[subset]; + int g = (int)pPixels[i].g - mean_g[subset]; + int b = (int)pPixels[i].b - mean_b[subset]; + icov[subset][0] += r * r; icov[subset][1] += r * g; icov[subset][2] += r * b; + icov[subset][3] += g * g; icov[subset][4] += g * b; + icov[subset][5] += b * b; + } + + int ar[3], ag[3], ab[3]; + + float total_slam_to_line_sse = 0.0f; + + for (uint32_t s = 0; s < 3; s++) + { + int block_max_var = basisu::maximum(icov[s][0], icov[s][3], icov[s][5]); + + float cov[6]; + for (uint32_t i = 0; i < 6; i++) + cov[i] = (float)icov[s][i]; + + const float sc = 1.0f / ((float)block_max_var + .0000125f); + const float wx = sc * cov[0], wy = sc * cov[3], wz = sc * cov[5]; + + const float alt_xr = cov[0] * wx + cov[1] * wy + cov[2] * wz; + const float alt_xg = cov[1] * wx + cov[3] * wy + cov[4] * wz; + const float alt_xb = cov[2] * wx + cov[4] * wy + cov[5] * wz; + + total_slam_to_line_sse += estimate_slam_to_line_sse_3D(cov, alt_xr, alt_xg, alt_xb); + + int saxis_r = 306, saxis_g = 601, saxis_b = 117; + + float k = basisu::maximum(fabsf(alt_xr), fabsf(alt_xg), fabsf(alt_xb)); + if (fabs(k) >= basisu::SMALL_FLOAT_VAL) + { + float m = 2048.0f / k; + saxis_r = (int)(alt_xr * m); + saxis_g = (int)(alt_xg * m); + saxis_b = (int)(alt_xb * m); + } + + ar[s] = (int)((uint32_t)saxis_r << 4U); + ag[s] = (int)((uint32_t)saxis_g << 4U); + ab[s] = (int)((uint32_t)saxis_b << 4U); + } // s + + total_slam_to_line_sse_mode[mode_iter] = total_slam_to_line_sse; + + int low_dot[3] = { INT_MAX, INT_MAX, INT_MAX }; + int high_dot[3] = { INT_MIN, INT_MIN, INT_MIN }; + + for (uint32_t i = 0; i < 16; i++) + { + const int subset = pBest_pat[i]; + const int saxis_r = ar[subset], saxis_g = ag[subset], saxis_b = ab[subset]; + + int dot = (pPixels[i].r * saxis_r + pPixels[i].g * saxis_g + pPixels[i].b * saxis_b) + i; + + low_dot[subset] = basisu::minimum(low_dot[subset], dot); + high_dot[subset] = basisu::maximum(high_dot[subset], dot); + } + + for (uint32_t subset = 0; subset < 3; subset++) + { + mode_low_c[mode_iter][subset] = low_dot[subset] & 15; + mode_high_c[mode_iter][subset] = high_dot[subset] & 15; + } // subset + + } // if ((mode_iter) && (best_pat_indices[0] == best_pat_indices[1])) + + for (uint32_t subset = 0; subset < 3; subset++) + { + const uint32_t low_pixel = mode_low_c[mode_iter][subset]; + const uint32_t high_pixel = mode_high_c[mode_iter][subset]; + + for (uint32_t c = 0; c < 3; c++) + spans[c] = pPixels[high_pixel][c] - pPixels[low_pixel][c]; + + float subset_sse; + if (mode_iter == 0) + { + // mode 0: 4-bit endpoints, unique p-bits, 3-bit weights, slight p-bit endpoint scale factor + subset_sse = analytical_quant_est_sse(16, 8, 3, spans, nullptr, UNIQUE_PBIT_DISCOUNT, mode_total_c[mode_iter][subset]); + } + else + { + // mode 2: 5-bit endpoints, no p-bits, 2-bit weights, no endpoint scale factor + subset_sse = analytical_quant_est_sse(32, 4, 3, spans, nullptr, 1.0f, mode_total_c[mode_iter][subset]); + } + + total_quant_sse_mode[mode_iter] += subset_sse; + } // subset + + } // mode_iter + + const float total_sse_est_mode0 = total_quant_sse_mode[0] + total_slam_to_line_sse_mode[0]; + const float total_sse_est_mode2 = total_quant_sse_mode[1] + total_slam_to_line_sse_mode[1]; + + if (total_sse_est_mode0 < total_sse_est_mode2) + { + if (pFinal_sse_est) + *pFinal_sse_est = total_sse_est_mode0; + + // Use mode 0 (high span) + if (total_sse_est_mode0 >= sse_est_to_beat) + { +#if BASISU_BC7F_PERF_STATS + g_total_mode02_bailouts++; +#endif + return false; + } + + const uint32_t best_pat_index = best_pat_indices[0]; + const uint8_t* pBest_pat = &g_bc7_partition3[best_pat_index * 16]; + + const int* pLow_c = &mode_low_c[0][0]; + const int* pHigh_c = &mode_high_c[0][0]; + + const int* pTotal_r = &mode_total_r[0][0]; + const int* pTotal_g = &mode_total_g[0][0]; + const int* pTotal_b = &mode_total_b[0][0]; + + float xl[3][4], xh[3][4]; + + for (uint32_t s = 0; s < 3; s++) + { + const int lc = pLow_c[s]; + const int hc = pHigh_c[s]; + + xl[s][0] = (float)pPixels[lc].r * (1.0f / 255.0f); + xl[s][1] = (float)pPixels[lc].g * (1.0f / 255.0f); + xl[s][2] = (float)pPixels[lc].b * (1.0f / 255.0f); + xl[s][3] = 0.0f; + + xh[s][0] = (float)pPixels[hc].r * (1.0f / 255.0f); + xh[s][1] = (float)pPixels[hc].g * (1.0f / 255.0f); + xh[s][2] = (float)pPixels[hc].b * (1.0f / 255.0f); + xh[s][3] = 0.0f; + } // s + + uint32_t lr[3], lg[3], lb[3], hr[3], hg[3], hb[3], pbits[6]; + + for (uint32_t s = 0; s < 3; s++) + { + color_rgba el, eh; + determine_unique_pbits(3, 4, xl[s], xh[s], el, eh, &pbits[s << 1]); + + lr[s] = el[0]; lg[s] = el[1]; lb[s] = el[2]; + hr[s] = eh[0]; hg[s] = eh[1]; hb[s] = eh[2]; + + } // s + + uint8_t cur_weights[16]; + eval_weights_mode0_rgb(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, pbits, best_pat_index); + + float z00[3] = { 0.0f }, z10[3] = { 0.0f }, z11[3] = { 0.0f }; + float q00_r[3] = { 0.0f }; + float q00_g[3] = { 0.0f }; + float q00_b[3] = { 0.0f }; + + for (uint32_t i = 0; i < 16; i++) + { + const int subset = pBest_pat[i]; + const uint32_t sel = cur_weights[i]; + assert(sel <= 7); + + z00[subset] += g_bc7_3bit_ls_tab[sel][0]; + z10[subset] += g_bc7_3bit_ls_tab[sel][1]; + z11[subset] += g_bc7_3bit_ls_tab[sel][2]; + + const float w = g_bc7_3bit_ls_tab[sel][3]; + + q00_r[subset] += w * (float)pPixels[i][0]; + q00_g[subset] += w * (float)pPixels[i][1]; + q00_b[subset] += w * (float)pPixels[i][2]; + } // i + + for (uint32_t s = 0; s < 3; s++) + { + float q10_r = (float)pTotal_r[s] - q00_r[s]; + float q10_g = (float)pTotal_g[s] - q00_g[s]; + float q10_b = (float)pTotal_b[s] - q00_b[s]; + + float z01 = z10[s]; + + float det = z00[s] * z11[s] - z01 * z10[s]; + if (fabs(det) < 1e-8f) + continue; + + det = 1.0f / det; + + float iz00, iz01, iz10, iz11; + iz00 = z11[s] * det; + iz01 = -z01 * det; + iz10 = -z10[s] * det; + iz11 = z00[s] * det; + + const float q = 1.0f / 255.0f; + + xl[s][0] = basisu::clamp(q * (iz10 * q00_r[s] + iz11 * q10_r), 0.0f, 1.0f); + xh[s][0] = basisu::clamp(q * (iz00 * q00_r[s] + iz01 * q10_r), 0.0f, 1.0f); + + xl[s][1] = basisu::clamp(q * (iz10 * q00_g[s] + iz11 * q10_g), 0.0f, 1.0f); + xh[s][1] = basisu::clamp(q * (iz00 * q00_g[s] + iz01 * q10_g), 0.0f, 1.0f); + + xl[s][2] = basisu::clamp(q * (iz10 * q00_b[s] + iz11 * q10_b), 0.0f, 1.0f); + xh[s][2] = basisu::clamp(q * (iz00 * q00_b[s] + iz01 * q10_b), 0.0f, 1.0f); + } // s + + for (uint32_t s = 0; s < 3; s++) + { + color_rgba el, eh; + determine_unique_pbits(3, 4, xl[s], xh[s], el, eh, &pbits[s << 1]); // fills in both pbit entries + + lr[s] = el[0]; lg[s] = el[1]; lb[s] = el[2]; + hr[s] = eh[0]; hg[s] = eh[1]; hb[s] = eh[2]; + + } // s + + if (pActual_sse) + *pActual_sse = eval_weights_mode0_rgb_sse(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, pbits, best_pat_index); + else + eval_weights_mode0_rgb(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, pbits, best_pat_index); + + encode_mode0_rgb_block(pBlock, best_pat_index, lr, lg, lb, hr, hg, hb, pbits, cur_weights); + } + else + { + if (pFinal_sse_est) + *pFinal_sse_est = total_sse_est_mode2; + + // Use mode 2 (low span) + if (total_sse_est_mode2 >= sse_est_to_beat) + { +#if BASISU_BC7F_PERF_STATS + g_total_mode02_bailouts++; +#endif + return false; + } + + const uint32_t best_pat_index = best_pat_indices[1]; + const uint8_t* pBest_pat = &g_bc7_partition3[best_pat_index * 16]; + + const int* pLow_c = &mode_low_c[1][0]; + const int* pHigh_c = &mode_high_c[1][0]; + + const int* pTotal_r = &mode_total_r[1][0]; + const int* pTotal_g = &mode_total_g[1][0]; + const int* pTotal_b = &mode_total_b[1][0]; + + uint32_t lr[3], lg[3], lb[3]; + uint32_t hr[3], hg[3], hb[3]; + + for (uint32_t s = 0; s < 3; s++) + { + const int lc = pLow_c[s]; + const int hc = pHigh_c[s]; + + lr[s] = to_5(pPixels[lc].r); + lg[s] = to_5(pPixels[lc].g); + lb[s] = to_5(pPixels[lc].b); + + hr[s] = to_5(pPixels[hc].r); + hg[s] = to_5(pPixels[hc].g); + hb[s] = to_5(pPixels[hc].b); + } + + uint8_t cur_weights[16]; + eval_weights_mode2_rgb(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, best_pat_index); + + float z00[3] = { 0.0f }, z10[3] = { 0.0f }, z11[3] = { 0.0f }; + float q00_r[3] = { 0.0f }; + float q00_g[3] = { 0.0f }; + float q00_b[3] = { 0.0f }; + + for (uint32_t i = 0; i < 16; i++) + { + const int subset = pBest_pat[i]; + const uint32_t sel = cur_weights[i]; + assert(sel <= 3); + + z00[subset] += g_bc7_2bit_ls_tab[sel][0]; + z10[subset] += g_bc7_2bit_ls_tab[sel][1]; + z11[subset] += g_bc7_2bit_ls_tab[sel][2]; + + const float w = g_bc7_2bit_ls_tab[sel][3]; + + q00_r[subset] += w * (float)pPixels[i][0]; + q00_g[subset] += w * (float)pPixels[i][1]; + q00_b[subset] += w * (float)pPixels[i][2]; + } // i + + for (uint32_t s = 0; s < 3; s++) + { + float q10_r = (float)pTotal_r[s] - q00_r[s]; + float q10_g = (float)pTotal_g[s] - q00_g[s]; + float q10_b = (float)pTotal_b[s] - q00_b[s]; + + float z01 = z10[s]; + + float det = z00[s] * z11[s] - z01 * z10[s]; + if (fabs(det) < 1e-8f) + continue; + + det = 1.0f / det; + + float iz00, iz01, iz10, iz11; + iz00 = z11[s] * det; + iz01 = -z01 * det; + iz10 = -z10[s] * det; + iz11 = z00[s] * det; + + hr[s] = to_5_clamp(iz00 * q00_r[s] + iz01 * q10_r); + lr[s] = to_5_clamp(iz10 * q00_r[s] + iz11 * q10_r); + + hg[s] = to_5_clamp(iz00 * q00_g[s] + iz01 * q10_g); + lg[s] = to_5_clamp(iz10 * q00_g[s] + iz11 * q10_g); + + hb[s] = to_5_clamp(iz00 * q00_b[s] + iz01 * q10_b); + lb[s] = to_5_clamp(iz10 * q00_b[s] + iz11 * q10_b); + } // s + + if (pActual_sse) + *pActual_sse = eval_weights_mode2_rgb_sse(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, best_pat_index); + else + eval_weights_mode2_rgb(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, best_pat_index); + + encode_mode2_rgb_block(pBlock, best_pat_index, + lr, lg, lb, hr, hg, hb, cur_weights); + } + +#ifdef _DEBUG + if (pActual_sse) + { + const uint32_t expected_sse = calc_sse(pBlock, pPixels); + assert(expected_sse == *pActual_sse); + } +#endif + + return true; + } + + bool pack_mode4_or_5(uint8_t* pBlock, const color_rgba* pOrig_pixels, uint32_t dp_chan_index, float sse_est_to_beat, uint32_t flags, + float* pFinal_sse_est = nullptr, + uint32_t* pActual_sse = nullptr) + { + (void)flags; + +#if BASISU_BC7F_PERF_STATS + g_total_mode45_evals++; +#endif + + color_rgba pixels[16]; + const color_rgba* pPixels = pOrig_pixels; + + if (dp_chan_index != 3) + { + memcpy(pixels, pOrig_pixels, sizeof(color_rgba) * 16); + pPixels = pixels; + + for (uint32_t i = 0; i < 16; i++) + { + const uint8_t c = pixels[i][dp_chan_index]; + pixels[i][dp_chan_index] = pixels[i][3]; + pixels[i][3] = c; + } + } + + int total_r = 0, total_g = 0, total_b = 0, total_a = 0; + + int min_r = 255, min_g = 255, min_b = 255, min_a = 255; + int max_r = 0, max_g = 0, max_b = 0, max_a = 0; + + for (uint32_t i = 0; i < 16; i++) + { + const int r = pPixels[i].r, g = pPixels[i].g, b = pPixels[i].b, a = pPixels[i].a; + + total_r += r; total_g += g; total_b += b; total_a += a; + + min_r = basisu::minimum(min_r, r); min_g = basisu::minimum(min_g, g); min_b = basisu::minimum(min_b, b); min_a = basisu::minimum(min_a, a); + max_r = basisu::maximum(max_r, r); max_g = basisu::maximum(max_g, g); max_b = basisu::maximum(max_b, b); max_a = basisu::maximum(max_a, a); + } + + int mean_r = (total_r + 8) >> 4, mean_g = (total_g + 8) >> 4, mean_b = (total_b + 8) >> 4; + + // covar rows are: + // 0, 1, 2 + // 1, 3, 4 + // 2, 4, 5 + int icov[6] = { 0, 0, 0, 0, 0, 0 }; + + for (uint32_t i = 0; i < 16; i++) + { + const int r = (int)pPixels[i].r - mean_r; + const int g = (int)pPixels[i].g - mean_g; + const int b = (int)pPixels[i].b - mean_b; + icov[0] += r * r; icov[1] += r * g; icov[2] += r * b; + icov[3] += g * g; icov[4] += g * b; + icov[5] += b * b; + } + + float cov3[6]; + for (uint32_t i = 0; i < 6; i++) + cov3[i] = (float)icov[i]; + + const int block_max_var3 = basisu::maximum(icov[0], icov[3], icov[5]); // not divided by 16, i.e. scaled by 16 + + const float sc3 = block_max_var3 ? (1.0f / (float)block_max_var3) : 0; + const float wx3 = sc3 * cov3[0], wy3 = sc3 * cov3[3], wz3 = sc3 * cov3[5]; + + const float alt_xr = cov3[0] * wx3 + cov3[1] * wy3 + cov3[2] * wz3; + const float alt_xg = cov3[1] * wx3 + cov3[3] * wy3 + cov3[4] * wz3; + const float alt_xb = cov3[2] * wx3 + cov3[4] * wy3 + cov3[5] * wz3; + + // Same for mode 4/5 + const float rgb_slam_to_line_sse_est = estimate_slam_to_line_sse_3D(cov3, alt_xr, alt_xg, alt_xb); + + int saxis_r = 306, saxis_g = 601, saxis_b = 117; + + float k = basisu::maximum(fabsf(alt_xr), fabsf(alt_xg), fabsf(alt_xb)); + if (fabs(k) >= basisu::SMALL_FLOAT_VAL) + { + float m = 2048.0f / k; + saxis_r = (int)(alt_xr * m); + saxis_g = (int)(alt_xg * m); + saxis_b = (int)(alt_xb * m); + } + + saxis_r = (int)((uint32_t)saxis_r << 4U); + saxis_g = (int)((uint32_t)saxis_g << 4U); + saxis_b = (int)((uint32_t)saxis_b << 4U); + + int low_dot = INT_MAX, high_dot = INT_MIN; + + for (uint32_t i = 0; i < 16; i += 4) + { + int dot0 = (pPixels[i].r * saxis_r + pPixels[i].g * saxis_g + pPixels[i].b * saxis_b) + i; + int dot1 = (pPixels[i + 1].r * saxis_r + pPixels[i + 1].g * saxis_g + pPixels[i + 1].b * saxis_b) + i + 1; + int dot2 = (pPixels[i + 2].r * saxis_r + pPixels[i + 2].g * saxis_g + pPixels[i + 2].b * saxis_b) + i + 2; + int dot3 = (pPixels[i + 3].r * saxis_r + pPixels[i + 3].g * saxis_g + pPixels[i + 3].b * saxis_b) + i + 3; + + int min_d01 = basisu::minimum(dot0, dot1); + int max_d01 = basisu::maximum(dot0, dot1); + + int min_d23 = basisu::minimum(dot2, dot3); + int max_d23 = basisu::maximum(dot2, dot3); + + int min_d = basisu::minimum(min_d01, min_d23); + int max_d = basisu::maximum(max_d01, max_d23); + + low_dot = basisu::minimum(low_dot, min_d); + high_dot = basisu::maximum(high_dot, max_d); + } + + const int low_c = low_dot & 15; + const int high_c = high_dot & 15; + + const int rgb_spans[4] = { pPixels[high_c][0] - pPixels[low_c][0], pPixels[high_c][1] - pPixels[low_c][1], pPixels[high_c][2] - pPixels[low_c][2], 0 }; + const int a_span = max_a - min_a; + + const float SECOND_PLANE_SPAN_WEIGHT = (dp_chan_index == 3) ? 1.0f : 1.0f; + + const float mode_4_rgb_3bit_quant_sse_est = analytical_quant_est_sse(32, 8, 3, rgb_spans, nullptr, 1.0f, 16); // mode 4 rgb: 5-bit endpoints, using 3-bit weights for RGB + const float mode_4_a_2bit_quant_sse_est = analytical_quant_est_sse(64, 4, a_span, SECOND_PLANE_SPAN_WEIGHT, 1.0f, 16); // mode 4 a: 6-bit endpoints, using 2-bit weights for RGB + + const float mode_4_rgb_2bit_quant_sse_est = analytical_quant_est_sse(32, 4, 3, rgb_spans, nullptr, 1.0f, 16); // mode 4 rgb: 5-bit endpoints, using 2-bit weights for RGB + const float mode_4_a_3bit_quant_sse_est = analytical_quant_est_sse(64, 8, a_span, SECOND_PLANE_SPAN_WEIGHT, 1.0f, 16); // mode 4 a: 6-bit endpoints, using 3-bit weights for RGB + + const float total_mode_4_rgb3_a2_sse_est = rgb_slam_to_line_sse_est + mode_4_rgb_3bit_quant_sse_est + mode_4_a_2bit_quant_sse_est; + const float total_mode_4_rgb2_a3_sse_est = rgb_slam_to_line_sse_est + mode_4_rgb_2bit_quant_sse_est + mode_4_a_3bit_quant_sse_est; + + const float mode_5_rgb_quant_sse_est = analytical_quant_est_sse(128, 4, 3, rgb_spans, nullptr, 1.0f, 16); // mode 5 rgb: 7-bit endpoints, using 2-bit weights for RGB + const float mode_5_a_quant_sse_est = analytical_quant_est_sse(256, 4, a_span, SECOND_PLANE_SPAN_WEIGHT, 1.0f, 16); // mode 5 a: 8-bit endpoints, using 2-bit weights for RGB + const float total_mode_5_rgba_sse_est = rgb_slam_to_line_sse_est + mode_5_rgb_quant_sse_est + mode_5_a_quant_sse_est; + + if (total_mode_5_rgba_sse_est < basisu::minimum(total_mode_4_rgb3_a2_sse_est, total_mode_4_rgb2_a3_sse_est)) + { + if (pFinal_sse_est) + *pFinal_sse_est = total_mode_5_rgba_sse_est; + + // Mode 5 - low RGB/A span + if (total_mode_5_rgba_sse_est >= sse_est_to_beat) + { +#if BASISU_BC7F_PERF_STATS + g_total_mode45_bailouts++; +#endif + return false; + } + + int lr = to_7(pPixels[low_c].r), lg = to_7(pPixels[low_c].g), lb = to_7(pPixels[low_c].b), la = min_a; + int hr = to_7(pPixels[high_c].r), hg = to_7(pPixels[high_c].g), hb = to_7(pPixels[high_c].b), ha = max_a; + + uint8_t cur_weights0[16]; // rgb 2-bits + if (pActual_sse) + *pActual_sse = eval_weights_mode5_2bit_rgb_sse(pPixels, cur_weights0, lr, lg, lb, hr, hg, hb); + else + eval_weights_mode5_2bit_rgb(pPixels, cur_weights0, lr, lg, lb, hr, hg, hb); + + vec4F xl, xh; + bool res = compute_least_squares_endpoints_3D( + 16, cur_weights0, 4, + g_bc7_2bit_ls_tab, + xl, xh, + pPixels, + (float)total_r, (float)total_g, (float)total_b); + + if (res) + { + lr = fast_roundf_int(xl[0] * (127.0f / 255.0f)); + lg = fast_roundf_int(xl[1] * (127.0f / 255.0f)); + lb = fast_roundf_int(xl[2] * (127.0f / 255.0f)); + + hr = fast_roundf_int(xh[0] * (127.0f / 255.0f)); + hg = fast_roundf_int(xh[1] * (127.0f / 255.0f)); + hb = fast_roundf_int(xh[2] * (127.0f / 255.0f)); + + if (pActual_sse) + *pActual_sse = eval_weights_mode5_2bit_rgb_sse(pPixels, cur_weights0, lr, lg, lb, hr, hg, hb); + else + eval_weights_mode5_2bit_rgb(pPixels, cur_weights0, lr, lg, lb, hr, hg, hb); + } + + uint8_t cur_weights1[16]; // alpha 2-bits + uint32_t a_sse = 0; + if (pActual_sse) + a_sse = eval_weights_mode5_2bit_a_sse(pPixels, cur_weights1, la, ha); + else + eval_weights_mode5_2bit_a(pPixels, cur_weights1, la, ha); + + float nal, nah; + if (compute_least_squares_endpoints_1D( + 16, cur_weights1, 4, + g_bc7_2bit_ls_tab, + nal, nah, + pPixels, 3, + (float)total_a)) + { + la = fast_roundf_int(nal); + ha = fast_roundf_int(nah); + + if (pActual_sse) + a_sse = eval_weights_mode5_2bit_a_sse(pPixels, cur_weights1, la, ha); + else + eval_weights_mode5_2bit_a(pPixels, cur_weights1, la, ha); + } + + if (pActual_sse) + *pActual_sse += a_sse; + + encode_mode5_rgba_block(pBlock, + lr, lg, lb, la, + hr, hg, hb, ha, + cur_weights0, cur_weights1, (dp_chan_index + 1) & 3); + } + else if (total_mode_4_rgb3_a2_sse_est < total_mode_4_rgb2_a3_sse_est) + { + if (pFinal_sse_est) + *pFinal_sse_est = total_mode_4_rgb3_a2_sse_est; + + // mode 4, rgb 3-bits, alpha 2-bits - high span RGB, low span in A, index bit=1 + if (total_mode_4_rgb3_a2_sse_est >= sse_est_to_beat) + { +#if BASISU_BC7F_PERF_STATS + g_total_mode45_bailouts++; +#endif + return false; + } + + int lr = to_5(pPixels[low_c].r), lg = to_5(pPixels[low_c].g), lb = to_5(pPixels[low_c].b), la = to_6(min_a); + int hr = to_5(pPixels[high_c].r), hg = to_5(pPixels[high_c].g), hb = to_5(pPixels[high_c].b), ha = to_6(max_a); + + uint8_t cur_weights0[16]; // rgb 3-bits + if (pActual_sse) + *pActual_sse = eval_weights_mode4_3bit_rgb_sse(pPixels, cur_weights0, lr, lg, lb, hr, hg, hb); + else + eval_weights_mode4_3bit_rgb(pPixels, cur_weights0, lr, lg, lb, hr, hg, hb); + + vec4F xl, xh; + bool res = compute_least_squares_endpoints_3D( + 16, cur_weights0, 8, + g_bc7_3bit_ls_tab, + xl, xh, + pPixels, + (float)total_r, (float)total_g, (float)total_b); + + if (res) + { + lr = fast_roundf_int(xl[0] * (31.0f / 255.0f)); + lg = fast_roundf_int(xl[1] * (31.0f / 255.0f)); + lb = fast_roundf_int(xl[2] * (31.0f / 255.0f)); + + hr = fast_roundf_int(xh[0] * (31.0f / 255.0f)); + hg = fast_roundf_int(xh[1] * (31.0f / 255.0f)); + hb = fast_roundf_int(xh[2] * (31.0f / 255.0f)); + + if (pActual_sse) + *pActual_sse = eval_weights_mode4_3bit_rgb_sse(pPixels, cur_weights0, lr, lg, lb, hr, hg, hb); + else + eval_weights_mode4_3bit_rgb(pPixels, cur_weights0, lr, lg, lb, hr, hg, hb); + } + + uint8_t cur_weights1[16]; // alpha 2-bits + + uint32_t a_sse = 0; + if (pActual_sse) + a_sse = eval_weights_mode4_2bit_a_sse(pPixels, cur_weights1, la, ha); + else + eval_weights_mode4_2bit_a(pPixels, cur_weights1, la, ha); + + float nal, nah; + if (compute_least_squares_endpoints_1D( + 16, cur_weights1, 4, + g_bc7_2bit_ls_tab, + nal, nah, + pPixels, 3, + (float)total_a)) + { + la = fast_roundf_int(nal * (63.0f / 255.0f)); + ha = fast_roundf_int(nah * (63.0f / 255.0f)); + + if (pActual_sse) + a_sse = eval_weights_mode4_2bit_a_sse(pPixels, cur_weights1, la, ha); + else + eval_weights_mode4_2bit_a(pPixels, cur_weights1, la, ha); + } + + if (pActual_sse) + *pActual_sse += a_sse; + + encode_mode4_rgba_block(pBlock, + lr, lg, lb, la, + hr, hg, hb, ha, + cur_weights0, cur_weights1, (dp_chan_index + 1) & 3, 1); + } + else + { + if (pFinal_sse_est) + *pFinal_sse_est = total_mode_4_rgb2_a3_sse_est; + + // mode 4, rgb 2-bits, alpha 3-bits - low span RGB, high span in A, index bit=0 + if (total_mode_4_rgb2_a3_sse_est >= sse_est_to_beat) + { +#if BASISU_BC7F_PERF_STATS + g_total_mode45_bailouts++; +#endif + return false; + } + + int lr = to_5(pPixels[low_c].r), lg = to_5(pPixels[low_c].g), lb = to_5(pPixels[low_c].b), la = to_6(min_a); + int hr = to_5(pPixels[high_c].r), hg = to_5(pPixels[high_c].g), hb = to_5(pPixels[high_c].b), ha = to_6(max_a); + + uint8_t cur_weights0[16]; // rgb 2-bits + if (pActual_sse) + *pActual_sse = eval_weights_mode4_2bit_rgb_sse(pPixels, cur_weights0, lr, lg, lb, hr, hg, hb); + else + eval_weights_mode4_2bit_rgb(pPixels, cur_weights0, lr, lg, lb, hr, hg, hb); + + vec4F xl, xh; + bool res = compute_least_squares_endpoints_3D( + 16, cur_weights0, 4, + g_bc7_2bit_ls_tab, + xl, xh, + pPixels, + (float)total_r, (float)total_g, (float)total_b); + + if (res) + { + lr = fast_roundf_int(xl[0] * (31.0f / 255.0f)); + lg = fast_roundf_int(xl[1] * (31.0f / 255.0f)); + lb = fast_roundf_int(xl[2] * (31.0f / 255.0f)); + + hr = fast_roundf_int(xh[0] * (31.0f / 255.0f)); + hg = fast_roundf_int(xh[1] * (31.0f / 255.0f)); + hb = fast_roundf_int(xh[2] * (31.0f / 255.0f)); + + if (pActual_sse) + *pActual_sse = eval_weights_mode4_2bit_rgb_sse(pPixels, cur_weights0, lr, lg, lb, hr, hg, hb); + else + eval_weights_mode4_2bit_rgb(pPixels, cur_weights0, lr, lg, lb, hr, hg, hb); + } + + uint8_t cur_weights1[16]; // alpha 2-bits + uint32_t a_sse = 0; + if (pActual_sse) + a_sse = eval_weights_mode4_3bit_a_sse(pPixels, cur_weights1, la, ha); + else + eval_weights_mode4_3bit_a(pPixels, cur_weights1, la, ha); + + float nal, nah; + if (compute_least_squares_endpoints_1D( + 16, cur_weights1, 8, + g_bc7_3bit_ls_tab, + nal, nah, + pPixels, 3, + (float)total_a)) + { + la = fast_roundf_int(nal * (63.0f / 255.0f)); + ha = fast_roundf_int(nah * (63.0f / 255.0f)); + + if (pActual_sse) + a_sse = eval_weights_mode4_3bit_a_sse(pPixels, cur_weights1, la, ha); + else + eval_weights_mode4_3bit_a(pPixels, cur_weights1, la, ha); + } + + if (pActual_sse) + *pActual_sse += a_sse; + + encode_mode4_rgba_block(pBlock, + lr, lg, lb, la, + hr, hg, hb, ha, + cur_weights0, cur_weights1, (dp_chan_index + 1) & 3, 0); + } + +#ifdef _DEBUG + if (pActual_sse) + { + const uint32_t expected_sse = calc_sse(pBlock, pOrig_pixels); + assert(expected_sse == *pActual_sse); + } +#endif + + return true; + } + + bool pack_mode7_rgba(uint8_t* pBlock, const color_rgba* pPixels, + float block_xr, float block_xg, float block_xb, float block_xa, + int block_mean_r, int block_mean_g, int block_mean_b, int block_mean_a, + float sse_est_to_beat, uint32_t flags, + float* pFinal_sse_est = nullptr, + uint32_t* pActual_sse = nullptr) + { +#if BASISU_BC7F_PERF_STATS + g_total_mode7_evals++; +#endif + + uint32_t desired_pat_bits = 0; + + for (uint32_t i = 0; i < 16; i++) + { + const float r = (float)(pPixels[i].r - block_mean_r); + const float g = (float)(pPixels[i].g - block_mean_g); + const float b = (float)(pPixels[i].b - block_mean_b); + const float a = (float)(pPixels[i].a - block_mean_a); + + const uint32_t subset = (r * block_xr + g * block_xg + b * block_xb + a * block_xa) > 0.0f; + + desired_pat_bits |= (subset << i); + } + + uint32_t best_diff = UINT32_MAX; + for (uint32_t p = 0; p < MAX_PATTERNS2_TO_CHECK; p++) + { + const uint32_t bc6h_pat_bits = g_bc7_part2_bitmasks[p]; + + int diff = popcount32(bc6h_pat_bits ^ desired_pat_bits); + int diff_inv = 16 - diff; + + uint32_t min_diff = (basisu::minimum(diff, diff_inv) << 8) | p; + if (min_diff < best_diff) + best_diff = min_diff; + } // p + + const uint32_t best_pat_index = best_diff & 0xFF; + const uint32_t best_pat_bits = g_bc7_part2_bitmasks[best_pat_index]; + + int total_r[2] = { }, total_g[2] = { }, total_b[2] = { }, total_a[2] = { }, total_c[2] = { }; + for (uint32_t i = 0; i < 16; i++) + { + const int r = pPixels[i].r, g = pPixels[i].g, b = pPixels[i].b, a = pPixels[i].a; + const int subset = (best_pat_bits >> i) & 1; + + total_r[subset] += r; total_g[subset] += g; total_b[subset] += b; total_a[subset] += a; + total_c[subset]++; + } + + int mean_r[2], mean_g[2], mean_b[2], mean_a[2]; + for (uint32_t s = 0; s < 2; s++) + { + const uint32_t t = total_c[s]; + const uint32_t h = (t >> 1); + + mean_r[s] = (total_r[s] + h) / t; + mean_g[s] = (total_g[s] + h) / t; + mean_b[s] = (total_b[s] + h) / t; + mean_a[s] = (total_a[s] + h) / t; + } + + int icov4[2][10] = { { }, { } }; + + // 0=rr + // 1=rg + // 2=rb + // 3=ra + // + // 4=gg + // 5=gb + // 6=ga + // + // 7=bb + // 8=ba + // + // 9=aa + + // 0 1 2 3 + // 4 5 6 + // 7 8 + // 9 + + // 0 1 2 3 + // 1 4 5 6 + // 2 5 7 8 + // 3 6 8 9 + + // trace at 0,4,7,9 + + for (uint32_t i = 0; i < 16; i++) + { + const int s = (best_pat_bits >> i) & 1; + + int r = (int)pPixels[i].r - mean_r[s]; + int g = (int)pPixels[i].g - mean_g[s]; + int b = (int)pPixels[i].b - mean_b[s]; + int a = (int)pPixels[i].a - mean_a[s]; + + icov4[s][0] += r * r; icov4[s][1] += r * g; icov4[s][2] += r * b; icov4[s][3] += r * a; + icov4[s][4] += g * g; icov4[s][5] += g * b; icov4[s][6] += g * a; + icov4[s][7] += b * b; icov4[s][8] += b * a; + icov4[s][9] += a * a; + } + + int ar[2], ag[2], ab[2], aa[2]; + + float slam_to_line_sse_est = 0.0f; + + for (uint32_t s = 0; s < 2; s++) + { + const int block_max_var4 = basisu::maximum(icov4[s][0], icov4[s][4], icov4[s][7], icov4[s][9]); + + float cov4[10]; + for (uint32_t i = 0; i < 10; i++) + cov4[i] = (float)icov4[s][i]; + + const float sc4 = block_max_var4 ? (1.0f / (float)block_max_var4) : 0; + const float wx = sc4 * cov4[0], wy = sc4 * cov4[4], wz = sc4 * cov4[7], wa = sc4 * cov4[9]; + + // 0 1 2 3 + // 1 4 5 6 + // 2 5 7 8 + // 3 6 8 9 + + const float x0 = cov4[0] * wx + cov4[1] * wy + cov4[2] * wz + cov4[3] * wa; + const float y0 = cov4[1] * wx + cov4[4] * wy + cov4[5] * wz + cov4[6] * wa; + const float z0 = cov4[2] * wx + cov4[5] * wy + cov4[7] * wz + cov4[8] * wa; + const float w0 = cov4[3] * wx + cov4[6] * wy + cov4[8] * wz + cov4[9] * wa; + + const float x1 = cov4[0] * x0 + cov4[1] * y0 + cov4[2] * z0 + cov4[3] * w0; + const float y1 = cov4[1] * x0 + cov4[4] * y0 + cov4[5] * z0 + cov4[6] * w0; + const float z1 = cov4[2] * x0 + cov4[5] * y0 + cov4[7] * z0 + cov4[8] * w0; + const float w1 = cov4[3] * x0 + cov4[6] * y0 + cov4[8] * z0 + cov4[9] * w0; + + slam_to_line_sse_est += estimate_slam_to_line_sse_4D(cov4, x1, y1, z1, w1); + + int saxis_r = 256, saxis_g = 256, saxis_b = 256, saxis_a = 256; + + float k = basisu::maximum(fabsf(x1), fabsf(y1), fabsf(z1), fabsf(w1)); + if (fabsf(k) >= basisu::SMALL_FLOAT_VAL) + { + float m = 2048.0f / k; + saxis_r = (int)(x1 * m); + saxis_g = (int)(y1 * m); + saxis_b = (int)(z1 * m); + saxis_a = (int)(w1 * m); + } + + ar[s] = (int)((uint32_t)saxis_r << 4U); + ag[s] = (int)((uint32_t)saxis_g << 4U); + ab[s] = (int)((uint32_t)saxis_b << 4U); + aa[s] = (int)((uint32_t)saxis_a << 4U); + } // s + + int low_dot[2] = { INT_MAX, INT_MAX }; + int high_dot[2] = { INT_MIN, INT_MIN }; + + for (uint32_t i = 0; i < 16; i++) + { + const int subset = (best_pat_bits >> i) & 1; + const int saxis_r = ar[subset], saxis_g = ag[subset], saxis_b = ab[subset], saxis_a = aa[subset]; + + assert(((pPixels[i].r * saxis_r + pPixels[i].g * saxis_g + pPixels[i].b * saxis_b + pPixels[i].a * saxis_a) & 0xF) == 0); // sanity + const int dot = (pPixels[i].r * saxis_r + pPixels[i].g * saxis_g + pPixels[i].b * saxis_b + pPixels[i].a * saxis_a) + i; + + low_dot[subset] = basisu::minimum(low_dot[subset], dot); + high_dot[subset] = basisu::maximum(high_dot[subset], dot); + } + + int low_c[2] = { low_dot[0] & 15, low_dot[1] & 15 }; + int high_c[2] = { high_dot[0] & 15, high_dot[1] & 15 }; + + float quant_err_sse_est = 0; + + for (uint32_t subset = 0; subset < 2; subset++) + { + const uint32_t low_pixel = low_c[subset]; + const uint32_t high_pixel = high_c[subset]; + + int spans[4]; + for (uint32_t c = 0; c < 4; c++) + spans[c] = pPixels[high_pixel][c] - pPixels[low_pixel][c]; + + // mode 7: 5-bit endpoints, unique pbits, 2 bit weights, 4 chans + quant_err_sse_est += analytical_quant_est_sse(32, 4, 4, spans, nullptr, (flags & cPackBC7FlagPBitOpt) ? UNIQUE_PBIT_DISCOUNT : 1.0f, total_c[subset]); + + } // subset + + const float total_mode7_est_sse = slam_to_line_sse_est + quant_err_sse_est; + + if (pFinal_sse_est) + *pFinal_sse_est = total_mode7_est_sse; + + if (total_mode7_est_sse >= sse_est_to_beat) + { +#if BASISU_BC7F_PERF_STATS + g_total_mode7_bailouts++; +#endif + return false; + } + + uint32_t lr[2], lg[2], lb[2], la[2]; + uint32_t hr[2], hg[2], hb[2], ha[2]; + uint32_t pbits[4]; + + for (uint32_t s = 0; s < 2; s++) + { + const int lc = low_c[s], hc = high_c[s]; + + if (flags & cPackBC7FlagPBitOpt) + { + const float q = 1.0f / 255.0f; + float sxl[4] = { (float)pPixels[lc].r * q, (float)pPixels[lc].g * q, (float)pPixels[lc].b * q, (float)pPixels[lc].a * q }; + float sxh[4] = { (float)pPixels[hc].r * q, (float)pPixels[hc].g * q, (float)pPixels[hc].b * q, (float)pPixels[hc].a * q }; + + color_rgba bestMinColor, bestMaxColor; + determine_unique_pbits(4, 5, sxl, sxh, bestMinColor, bestMaxColor, &pbits[s * 2]); + + lr[s] = bestMinColor.r, lg[s] = bestMinColor.g, lb[s] = bestMinColor.b; la[s] = bestMinColor.a; + hr[s] = bestMaxColor.r, hg[s] = bestMaxColor.g, hb[s] = bestMaxColor.b; ha[s] = bestMaxColor.a; + } + else + { + const uint32_t l_pbit = (pPixels[lc].a >= 129); + const uint32_t h_pbit = (pPixels[hc].a >= 129); + + pbits[s * 2 + 0] = l_pbit; + pbits[s * 2 + 1] = h_pbit; + + lr[s] = to_5(pPixels[lc].r, l_pbit); + lg[s] = to_5(pPixels[lc].g, l_pbit); + lb[s] = to_5(pPixels[lc].b, l_pbit); + la[s] = to_5(pPixels[lc].a, l_pbit); + + hr[s] = to_5(pPixels[hc].r, h_pbit); + hg[s] = to_5(pPixels[hc].g, h_pbit); + hb[s] = to_5(pPixels[hc].b, h_pbit); + ha[s] = to_5(pPixels[hc].a, h_pbit); + } + } // s + + uint8_t cur_weights[16]; + + eval_weights_mode7_rgba(pPixels, cur_weights, + lr, lg, lb, la, + hr, hg, hb, ha, + pbits, best_pat_bits); + + float z00[2] = { 0.0f }, z10[2] = { 0.0f }, z11[2] = { 0.0f }; + float q00_r[2] = { 0.0f }; + float q00_g[2] = { 0.0f }; + float q00_b[2] = { 0.0f }; + float q00_a[2] = { 0.0f }; + + for (uint32_t i = 0; i < 16; i++) + { + const int subset = (best_pat_bits >> i) & 1; + const uint32_t sel = cur_weights[i]; + assert(sel <= 3); + + z00[subset] += g_bc7_2bit_ls_tab[sel][0]; + z10[subset] += g_bc7_2bit_ls_tab[sel][1]; + z11[subset] += g_bc7_2bit_ls_tab[sel][2]; + + const float w = g_bc7_2bit_ls_tab[sel][3]; + + q00_r[subset] += w * (float)pPixels[i][0]; + q00_g[subset] += w * (float)pPixels[i][1]; + q00_b[subset] += w * (float)pPixels[i][2]; + q00_a[subset] += w * (float)pPixels[i][3]; + } // i + + for (uint32_t s = 0; s < 2; s++) + { + float q10_r = (float)total_r[s] - q00_r[s]; + float q10_g = (float)total_g[s] - q00_g[s]; + float q10_b = (float)total_b[s] - q00_b[s]; + float q10_a = (float)total_a[s] - q00_a[s]; + + float z01 = z10[s]; + + float det = z00[s] * z11[s] - z01 * z10[s]; + if (fabsf(det) < 1e-8f) + continue; + + det = 1.0f / det; + + float iz00, iz01, iz10, iz11; + iz00 = z11[s] * det; + iz01 = -z01 * det; + iz10 = -z10[s] * det; + iz11 = z00[s] * det; + + const float slr = iz10 * q00_r[s] + iz11 * q10_r; + const float shr = iz00 * q00_r[s] + iz01 * q10_r; + + const float slg = iz10 * q00_g[s] + iz11 * q10_g; + const float shg = iz00 * q00_g[s] + iz01 * q10_g; + + const float slb = iz10 * q00_b[s] + iz11 * q10_b; + const float shb = iz00 * q00_b[s] + iz01 * q10_b; + + const float sla = iz10 * q00_a[s] + iz11 * q10_a; + const float sha = iz00 * q00_a[s] + iz01 * q10_a; + + if (flags & cPackBC7FlagPBitOpt) + { + const float q = 1.0f / 255.0f; + float sxl[4] = { basisu::clamp(slr * q, 0.0f, 1.0f), basisu::clamp(slg * q, 0.0f, 1.0f), basisu::clamp(slb * q, 0.0f, 1.0f), basisu::clamp(sla * q, 0.0f, 1.0f) }; + float sxh[4] = { basisu::clamp(shr * q, 0.0f, 1.0f), basisu::clamp(shg * q, 0.0f, 1.0f), basisu::clamp(shb * q, 0.0f, 1.0f), basisu::clamp(sha * q, 0.0f, 1.0f) }; + + color_rgba bestMinColor, bestMaxColor; + determine_unique_pbits(4, 5, sxl, sxh, bestMinColor, bestMaxColor, &pbits[s * 2]); + + lr[s] = bestMinColor.r, lg[s] = bestMinColor.g, lb[s] = bestMinColor.b; la[s] = bestMinColor.a; + hr[s] = bestMaxColor.r, hg[s] = bestMaxColor.g, hb[s] = bestMaxColor.b; ha[s] = bestMaxColor.a; + } + else + { + const uint32_t l_pbit = (sla >= 129.0f); + const uint32_t h_pbit = (sha >= 129.0f); + + pbits[s * 2 + 0] = l_pbit; + pbits[s * 2 + 1] = h_pbit; + + lr[s] = to_5_clamp(slr, l_pbit); + lg[s] = to_5_clamp(slg, l_pbit); + lb[s] = to_5_clamp(slb, l_pbit); + la[s] = to_5_clamp(sla, l_pbit); + + hr[s] = to_5_clamp(shr, h_pbit); + hg[s] = to_5_clamp(shg, h_pbit); + hb[s] = to_5_clamp(shb, h_pbit); + ha[s] = to_5_clamp(sha, h_pbit); + } + + } // s + + if (pActual_sse) + { + *pActual_sse = eval_weights_mode7_rgba_sse(pPixels, cur_weights, + lr, lg, lb, la, + hr, hg, hb, ha, + pbits, best_pat_bits); + } + else + { + eval_weights_mode7_rgba(pPixels, cur_weights, + lr, lg, lb, la, + hr, hg, hb, ha, + pbits, best_pat_bits); + } + + encode_mode7_rgba_block(pBlock, best_pat_index, + lr, lg, lb, la, + hr, hg, hb, ha, + pbits, cur_weights); + +#ifdef _DEBUG + if (pActual_sse) + { + const uint32_t expected_sse = calc_sse(pBlock, pPixels); + assert(expected_sse == *pActual_sse); + } +#endif + + return true; + } + + const int TRIVIAL_BLOCK_THRESH_RGB = 20 * 16; // skip PCA/LS threshold (uses trivial mode 6 encoder) + const int TRIVIAL_BLOCK_THRESH_RGBA = 2 * 16; + + // dual plane +#if 0 + const int DP_BLOCK_VAR_THRESH = 1 * 16; // use dual plane threshold + const float STRONG_CORR_THRESH = .98f; +#else + const int DP_BLOCK_VAR_THRESH = 2 * 16; // use dual plane threshold + const float STRONG_CORR_THRESH = .85f; +#endif + + // 2-3 subsets +#if 0 + const float HIGH_ORTHO_ENERGY_THRESH = 1.0f * 16.0f; // use 2+ subsets threshold + const int MIN_BLOCK_MAX_VAR_23SUBSETS = 4 * 16; + const float ORTHO_RATIO_23SUBSET_RATIO_THRESH = .004f; +#else + const int MIN_BLOCK_MAX_VAR_23SUBSETS = 100 * 16; + const float HIGH_ORTHO_ENERGY_THRESH = 1.0f * 16.0f; // use 2+ subsets threshold + const float ORTHO_RATIO_23SUBSET_RATIO_THRESH = .004f; +#endif + +#if 0 + const int DP_BLOCK_VAR_THRESH_RGBA = 2 * 16; // use dual plane threshold + const float ALPHA_DECORR_THRESHOLD = .9f; + const float STRONG_DECORR_THRESH_RGBA = .85f; +#else + const int DP_BLOCK_VAR_THRESH_RGBA = 1 * 16; // use dual plane threshold + //const float ALPHA_DECORR_THRESHOLD = .98f; + const float ALPHA_DECORR_THRESHOLD = .995f; + const float STRONG_DECORR_THRESH_RGBA = .85f; +#endif + + // 3 subsets + const int MIN_BLOCK_MAX_VAR_3SUBSETS = 500 * 16; // use 3 subsets threshold + + //------------------------------------------------------------------------------------------------------- + + // Note: solid block check assumes A's all == 255. + void fast_pack_bc7_rgb_analytical(uint8_t* pBlock, const color_rgba* pPixels, uint32_t flags) + { + assert(g_bc7_4bit_ls_tab[1][0]); + +#if BASISU_BC7F_PERF_STATS + g_total_rgb_calls++; +#endif + + const uint32_t fc = *(const uint32_t*)&pPixels[0]; + if (fc == *(const uint32_t*)&pPixels[15]) + { + int k; + for (k = 1; k < 15; k++) + if (*(const uint32_t*)&pPixels[k] != fc) + break; + + if (k == 15) + { +#if BASISU_BC7F_PERF_STATS + g_total_solid_blocks++; +#endif + + pack_mode5_solid(pBlock, pPixels[0]); + return; + } + } + + int total_r = 0, total_g = 0, total_b = 0; + + int min_r = 255, min_g = 255, min_b = 255; + int max_r = 0, max_g = 0, max_b = 0; + + for (uint32_t i = 0; i < 16; i++) + { + int r = pPixels[i].r, g = pPixels[i].g, b = pPixels[i].b; + + total_r += r; total_g += g; total_b += b; + + min_r = basisu::minimum(min_r, r); min_g = basisu::minimum(min_g, g); min_b = basisu::minimum(min_b, b); + max_r = basisu::maximum(max_r, r); max_g = basisu::maximum(max_g, g); max_b = basisu::maximum(max_b, b); + } + + int mean_r = (total_r + 8) >> 4, mean_g = (total_g + 8) >> 4, mean_b = (total_b + 8) >> 4; + + // covar rows are: + // 0, 1, 2 + // 1, 3, 4 + // 2, 4, 5 + int icov[6] = { 0, 0, 0, 0, 0, 0 }; + + for (uint32_t i = 0; i < 16; i++) + { + int r = (int)pPixels[i].r - mean_r; + int g = (int)pPixels[i].g - mean_g; + int b = (int)pPixels[i].b - mean_b; + icov[0] += r * r; icov[1] += r * g; icov[2] += r * b; + icov[3] += g * g; icov[4] += g * b; + icov[5] += b * b; + } + + int block_max_var = basisu::maximum(icov[0], icov[3], icov[5]); // not divided by 16, i.e. scaled by 16 + + // not redundant due to uint32_t test above, which could be fooled by alpha accidentally passed in + if (!block_max_var) + { +#if BASISU_BC7F_PERF_STATS + g_total_solid_blocks++; +#endif + pack_mode5_solid(pBlock, pPixels[0]); + return; + } + + // check for dual plane, if a single component is very strongly decorrelated then switch to modes 4/5 + int desired_dp_chan = -1; + + if ((flags & cPackBC7FlagUseDualPlaneRGB) && (block_max_var >= DP_BLOCK_VAR_THRESH)) + { + // 0,1 + // 0,2 + // 1,2 + const bool has_r = icov[0] > 16, has_g = icov[3] > 16, has_b = icov[5] > 16; + + const uint32_t total_active_chans = has_r + has_g + has_b; + + if (total_active_chans >= 2) + { + const float r_var = (float)icov[0], g_var = (float)icov[3], b_var = (float)icov[5]; + + const float rg_corr = (has_r && has_g) ? fabs((float)icov[1] / sqrtf(r_var * g_var)) : 1.0f; + const float rb_corr = (has_r && has_b) ? fabs((float)icov[2] / sqrtf(r_var * b_var)) : 1.0f; + const float gb_corr = (has_g && has_b) ? fabs((float)icov[4] / sqrtf(g_var * b_var)) : 1.0f; + + float min_p = basisu::minimum(rg_corr, rb_corr, gb_corr); + if (min_p < STRONG_CORR_THRESH) + { + if (total_active_chans == 2) + { + if (!has_r) + desired_dp_chan = 1; + else if (!has_g) + desired_dp_chan = 0; + else + desired_dp_chan = 0; + } + else + { + // see if rg/rb is weakly correlated vs. gb + if ((rg_corr < gb_corr) && (rb_corr < gb_corr)) + desired_dp_chan = 0; + // see if gr/gb is weakly correlated vs. rb + else if ((rg_corr < rb_corr) && (gb_corr < rb_corr)) + desired_dp_chan = 1; + // assume b is weakest + else + desired_dp_chan = 2; + } +#if BASISU_BC7F_PERF_STATS + g_total_dp_valid_chans_rgb++; +#endif + } + } + } + + if ((flags & cPackBC7FlagUseTrivialMode6) && ((desired_dp_chan == -1) && (block_max_var < TRIVIAL_BLOCK_THRESH_RGB))) + { + //pack_mode5_solid(pBlock, color_rgba(0, 255, 0, 255)); + //return; + + int low_c = INT_MAX, high_c = 0; + + for (uint32_t i = 0; i < 16; i++) + { + int y = ((16 * 2) * pPixels[i].r + (16 * 4) * pPixels[i].g + 16 * pPixels[i].b) + i; + low_c = basisu::minimum(low_c, y); + high_c = basisu::maximum(high_c, y); + } + + low_c &= 0xF; + high_c &= 0xF; + + int p0, p1, lr, lg, lb, hr, hg, hb; + + if (flags & cPackBC7FlagPBitOptMode6) + { + // An alternative would be to set A's=1.0 here and bias the p-bit optimizer to lower A RMSE. + const float q = 1.0f / 255.0f; + float sxl[4] = { (float)pPixels[low_c].r * q, (float)pPixels[low_c].g * q, (float)pPixels[low_c].b * q, 0 }; + float sxh[4] = { (float)pPixels[high_c].r * q, (float)pPixels[high_c].g * q, (float)pPixels[high_c].b * q, 0 }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_unique_pbits(3, 7, sxl, sxh, bestMinColor, bestMaxColor, best_pbits); + + p0 = best_pbits[0], p1 = best_pbits[1]; + lr = bestMinColor.r, lg = bestMinColor.g, lb = bestMinColor.b; + hr = bestMaxColor.r, hg = bestMaxColor.g, hb = bestMaxColor.b; + } + else + { + p0 = 1; + p1 = 1; + + lr = to_7(pPixels[low_c].r, p0), lg = to_7(pPixels[low_c].g, p0), lb = to_7(pPixels[low_c].b, p0); + hr = to_7(pPixels[high_c].r, p1), hg = to_7(pPixels[high_c].g, p1), hb = to_7(pPixels[high_c].b, p1); + } + + uint8_t cur_weights[16]; + +#if BASISU_BC7F_USE_SSE41 + eval_weights_mode6_rgb_sse41(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, p0, p1); +#else + eval_weights_mode6_rgb(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, p0, p1); +#endif + + encode_mode6_rgba_block(pBlock, + lr, lg, lb, 127, p0, + hr, hg, hb, 127, p1, + cur_weights); + +#if BASISU_BC7F_PERF_STATS + g_total_trivial_mode6_blocks++; +#endif + return; + } + + float cov[6]; + for (uint32_t i = 0; i < 6; i++) + cov[i] = (float)icov[i]; + + const float sc = block_max_var ? (1.0f / (float)block_max_var) : 0; + const float wx = sc * cov[0], wy = sc * cov[3], wz = sc * cov[5]; + + const float alt_xr = cov[0] * wx + cov[1] * wy + cov[2] * wz; + const float alt_xg = cov[1] * wx + cov[3] * wy + cov[4] * wz; + const float alt_xb = cov[2] * wx + cov[4] * wy + cov[5] * wz; + + // quite rough mode 6 SSE estimate (explictly higher bound): if some other mode can't even beat this, don't use it and we fall back to a decently strong mode 6 + const int spans[4] = { max_r - min_r, max_g - min_g, max_b - min_b, 0 }; + + // need_sse_estimates MUST be set correctly or subtle mode selection issues will occur. + const bool need_sse_estimates = ((flags & cPackBC7FlagUse2SubsetsRGB) != 0) || (desired_dp_chan >= 0); + + float mode6_ortho_ratio = 0; + const float mode6_slam_to_line_sse_est = need_sse_estimates ? estimate_slam_to_line_sse_3D(cov, alt_xr, alt_xg, alt_xb, &mode6_ortho_ratio) : 0; + const float mode6_sse_est = need_sse_estimates ? (mode6_slam_to_line_sse_est + analytical_quant_est_sse(128, 16, 3, spans, nullptr, 1.0f, 16)) : 0; + + // Prefer 2/3-subsets over dual plane + // TODO: Use mode 6 sse est? + if ((flags & cPackBC7FlagUse2SubsetsRGB) && (block_max_var >= MIN_BLOCK_MAX_VAR_23SUBSETS) && (mode6_ortho_ratio > ORTHO_RATIO_23SUBSET_RATIO_THRESH)) + { + assert(need_sse_estimates); + + const bool high_ortho_energy_flag = (mode6_slam_to_line_sse_est >= HIGH_ORTHO_ENERGY_THRESH); + + if (high_ortho_energy_flag) + { +#if BASISU_BC7F_PERF_STATS + g_total_high_ortho_energy++; +#endif + //pack_mode5_solid(pBlock, color_rgba(255, 255, 0, 255)); + //return; + + if ((flags & cPackBC7FlagUse3SubsetsRGB) && (block_max_var >= MIN_BLOCK_MAX_VAR_3SUBSETS)) + { + //pack_mode5_solid(pBlock, color_rgba(255, 0, 255, 255)); + //return; + +#if 0 + if (pack_mode0_or_2_rgb(pBlock, pPixels, alt_xr, alt_xg, alt_xb, mean_r, mean_g, mean_b, mode6_sse_est, flags)) + { + return; + } +#else + float mode0_or_2_sse_est = 1e+9f; + if (pack_mode0_or_2_rgb(pBlock, pPixels, alt_xr, alt_xg, alt_xb, mean_r, mean_g, mean_b, mode6_sse_est, flags, &mode0_or_2_sse_est)) + { + float mode1_or_3_sse_est = 1e+9f; + + uint8_t temp_2subset_block[sizeof(basist::bc7_block)]; + if (pack_mode1_or_3_rgb(temp_2subset_block, pPixels, alt_xr, alt_xg, alt_xb, mean_r, mean_g, mean_b, mode0_or_2_sse_est, flags, &mode1_or_3_sse_est)) + { + assert(mode1_or_3_sse_est < mode0_or_2_sse_est); + memcpy(pBlock, temp_2subset_block, sizeof(basist::bc7_block)); + } + + return; + } +#endif + } + + if (pack_mode1_or_3_rgb(pBlock, pPixels, alt_xr, alt_xg, alt_xb, mean_r, mean_g, mean_b, mode6_sse_est, flags)) + return; + } + } + + // Use dual plane over mode 6 + if (desired_dp_chan >= 0) + { + assert(need_sse_estimates); + + if (pack_mode4_or_5(pBlock, pPixels, desired_dp_chan, mode6_sse_est, flags)) + return; + + } // if (desired_dp_chan >= 0) + + int saxis_r = 306, saxis_g = 601, saxis_b = 117; + + float k = basisu::maximum(fabsf(alt_xr), fabsf(alt_xg), fabsf(alt_xb)); + if (fabs(k) >= basisu::SMALL_FLOAT_VAL) + { + float m = 2048.0f / k; + saxis_r = (int)(alt_xr * m); + saxis_g = (int)(alt_xg * m); + saxis_b = (int)(alt_xb * m); + } + + saxis_r = (int)((uint32_t)saxis_r << 4U); + saxis_g = (int)((uint32_t)saxis_g << 4U); + saxis_b = (int)((uint32_t)saxis_b << 4U); + + int low_dot = INT_MAX, high_dot = INT_MIN; + +#if BASISU_BC7F_USE_SSE41 + int low_c, high_c; + bc7_proj_minmax_indices_sse41(pPixels, saxis_r, saxis_g, saxis_b, &low_c, &high_c); +#else + for (uint32_t i = 0; i < 16; i += 4) + { + assert(((pPixels[i].r * saxis_r + pPixels[i].g * saxis_g + pPixels[i].b * saxis_b) & 0xF) == 0); // sanity + assert(((pPixels[i + 1].r * saxis_r + pPixels[i + 1].g * saxis_g + pPixels[i + 1].b * saxis_b) & 0xF) == 0); + assert(((pPixels[i + 2].r * saxis_r + pPixels[i + 2].g * saxis_g + pPixels[i + 2].b * saxis_b) & 0xF) == 0); + assert(((pPixels[i + 3].r * saxis_r + pPixels[i + 3].g * saxis_g + pPixels[i + 3].b * saxis_b) & 0xF) == 0); + + const int dot0 = (pPixels[i].r * saxis_r + pPixels[i].g * saxis_g + pPixels[i].b * saxis_b) + i; + const int dot1 = (pPixels[i + 1].r * saxis_r + pPixels[i + 1].g * saxis_g + pPixels[i + 1].b * saxis_b) + i + 1; + const int dot2 = (pPixels[i + 2].r * saxis_r + pPixels[i + 2].g * saxis_g + pPixels[i + 2].b * saxis_b) + i + 2; + const int dot3 = (pPixels[i + 3].r * saxis_r + pPixels[i + 3].g * saxis_g + pPixels[i + 3].b * saxis_b) + i + 3; + + int min_d01 = basisu::minimum(dot0, dot1); + int max_d01 = basisu::maximum(dot0, dot1); + + int min_d23 = basisu::minimum(dot2, dot3); + int max_d23 = basisu::maximum(dot2, dot3); + + int min_d = basisu::minimum(min_d01, min_d23); + int max_d = basisu::maximum(max_d01, max_d23); + + low_dot = basisu::minimum(low_dot, min_d); + high_dot = basisu::maximum(high_dot, max_d); + } + + int low_c = low_dot & 15; + int high_c = high_dot & 15; +#endif + + int p0, p1, lr, lg, lb, hr, hg, hb; + + if (flags & cPackBC7FlagPBitOptMode6) + { + const float q = 1.0f / 255.0f; + float sxl[4] = { (float)pPixels[low_c].r * q, (float)pPixels[low_c].g * q, (float)pPixels[low_c].b * q, 0 }; + float sxh[4] = { (float)pPixels[high_c].r * q, (float)pPixels[high_c].g * q, (float)pPixels[high_c].b * q, 0 }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_unique_pbits(3, 7, sxl, sxh, bestMinColor, bestMaxColor, best_pbits); + + p0 = best_pbits[0], p1 = best_pbits[1]; + lr = bestMinColor.r, lg = bestMinColor.g, lb = bestMinColor.b; + hr = bestMaxColor.r, hg = bestMaxColor.g, hb = bestMaxColor.b; + } + else + { + // explictly force pbits to 1, that way alpha is always 255 and we don't slow down the entire encoder by 4-8% for a tiny ~.1 dB PSNR gain (not worth it) + p0 = 1, p1 = 1; + lr = to_7(pPixels[low_c].r, p0), lg = to_7(pPixels[low_c].g, p0), lb = to_7(pPixels[low_c].b, p0); + hr = to_7(pPixels[high_c].r, p1), hg = to_7(pPixels[high_c].g, p1), hb = to_7(pPixels[high_c].b, p1); + } + + uint8_t cur_weights[16]; + +#if BASISU_BC7F_USE_SSE41 + eval_weights_mode6_rgb_sse41(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, p0, p1); +#else + eval_weights_mode6_rgb(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, p0, p1); +#endif + + vec4F xl, xh; + bool res = compute_least_squares_endpoints_3D( + 16, cur_weights, 16, + g_bc7_4bit_ls_tab, + xl, xh, + pPixels, + (float)total_r, (float)total_g, (float)total_b); + + if (res) + { + if (flags & cPackBC7FlagPBitOptMode6) + { + const float q = 1.0f / 255.0f; + float sxl[4] = { xl[0] * q, xl[1] * q, xl[2] * q, 0.0f }; + float sxh[4] = { xh[0] * q, xh[1] * q, xh[2] * q, 0.0f }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_unique_pbits( + 3, 7, sxl, sxh, + bestMinColor, bestMaxColor, best_pbits); + + p0 = best_pbits[0], p1 = best_pbits[1]; + lr = bestMinColor.r, lg = bestMinColor.g, lb = bestMinColor.b; + hr = bestMaxColor.r, hg = bestMaxColor.g, hb = bestMaxColor.b; + } + else + { + p0 = 1; p1 = 1; + lr = to_7(xl[0], p0); + lg = to_7(xl[1], p0); + lb = to_7(xl[2], p0); + + hr = to_7(xh[0], p1); + hg = to_7(xh[1], p1); + hb = to_7(xh[2], p1); + } + +#if BASISU_BC7F_USE_SSE41 + eval_weights_mode6_rgb_sse41(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, p0, p1); +#else + eval_weights_mode6_rgb(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, p0, p1); +#endif + } + + //pack_mode5_solid(pBlock, color_rgba(0, 0, 255, 255)); + //return; + + // pbits set to 1 to ensure alpha is always decoded to fully opaque (255) + encode_mode6_rgba_block(pBlock, + lr, lg, lb, 127, p0, + hr, hg, hb, 127, p1, + cur_weights); + } + + //------------------------------------------------------------------------------------------------------- + const int MIN_BLOCK_MAX_VAR_23SUBSETS_RGBA = 100 * 16; + const float HIGH_ORTHO_ENERGY_THRESH_RGBA = 1.0f * 16.0f; // use 2+ subsets threshold + const float ORTHO_RATIO_23SUBSET_RATIO_THRESH_RGBA = .004f; + + uint32_t fast_pack_bc7_rgb_partial_analytical(uint8_t* pBlock, const color_rgba* pPixels, uint32_t flags) + { + assert(g_bc7_4bit_ls_tab[1][0]); + +#if BASISU_BC7F_PERF_STATS + g_total_rgb_calls++; +#endif + + const uint32_t fc = *(const uint32_t*)&pPixels[0]; + if (fc == *(const uint32_t*)&pPixels[15]) + { + int k; + for (k = 1; k < 15; k++) + if (*(const uint32_t*)&pPixels[k] != fc) + break; + + if (k == 15) + { +#if BASISU_BC7F_PERF_STATS + g_total_solid_blocks++; +#endif + + pack_mode5_solid(pBlock, pPixels[0]); + return 0; + } + } + + int total_r = 0, total_g = 0, total_b = 0; + + int min_r = 255, min_g = 255, min_b = 255; + int max_r = 0, max_g = 0, max_b = 0; + + for (uint32_t i = 0; i < 16; i++) + { + int r = pPixels[i].r, g = pPixels[i].g, b = pPixels[i].b; + + total_r += r; total_g += g; total_b += b; + + min_r = basisu::minimum(min_r, r); min_g = basisu::minimum(min_g, g); min_b = basisu::minimum(min_b, b); + max_r = basisu::maximum(max_r, r); max_g = basisu::maximum(max_g, g); max_b = basisu::maximum(max_b, b); + } + + int mean_r = (total_r + 8) >> 4, mean_g = (total_g + 8) >> 4, mean_b = (total_b + 8) >> 4; + + // covar rows are: + // 0, 1, 2 + // 1, 3, 4 + // 2, 4, 5 + int icov[6] = { 0, 0, 0, 0, 0, 0 }; + + for (uint32_t i = 0; i < 16; i++) + { + int r = (int)pPixels[i].r - mean_r; + int g = (int)pPixels[i].g - mean_g; + int b = (int)pPixels[i].b - mean_b; + icov[0] += r * r; icov[1] += r * g; icov[2] += r * b; + icov[3] += g * g; icov[4] += g * b; + icov[5] += b * b; + } + + int block_max_var = basisu::maximum(icov[0], icov[3], icov[5]); // not divided by 16, i.e. scaled by 16 + + // not redundant due to uint32_t test above, which could be fooled by alpha accidentally passed in + if (!block_max_var) + { +#if BASISU_BC7F_PERF_STATS + g_total_solid_blocks++; +#endif + pack_mode5_solid(pBlock, pPixels[0]); + return 0; + } + + // check for dual plane, if a single component is very strongly decorrelated then switch to modes 4/5 + int desired_dp_chan = -1; + + const bool non_analytical_flag = (flags & cPackBC7FlagNonAnalyticalRGB) != 0; + + if ((flags & cPackBC7FlagUseDualPlaneRGB) && + ((!non_analytical_flag && (block_max_var >= DP_BLOCK_VAR_THRESH)) || (non_analytical_flag && (block_max_var >= 16)))) + { + // 0,1 + // 0,2 + // 1,2 + const bool has_r = icov[0] > 16, has_g = icov[3] > 16, has_b = icov[5] > 16; + + const uint32_t total_active_chans = has_r + has_g + has_b; + + if (total_active_chans >= 2) + { + const float r_var = (float)icov[0], g_var = (float)icov[3], b_var = (float)icov[5]; + + const float rg_corr = (has_r && has_g) ? fabs((float)icov[1] / sqrtf(r_var * g_var)) : 1.0f; + const float rb_corr = (has_r && has_b) ? fabs((float)icov[2] / sqrtf(r_var * b_var)) : 1.0f; + const float gb_corr = (has_g && has_b) ? fabs((float)icov[4] / sqrtf(g_var * b_var)) : 1.0f; + + float min_p = basisu::minimum(rg_corr, rb_corr, gb_corr); + + const float corr_thresh = non_analytical_flag ? .999f : STRONG_CORR_THRESH; + + if (min_p < corr_thresh) + { + if (total_active_chans == 2) + { + if (!has_r) + desired_dp_chan = 1; + else if (!has_g) + desired_dp_chan = 0; + else + desired_dp_chan = 0; + } + else + { + // see if rg/rb is weakly correlated vs. gb + if ((rg_corr < gb_corr) && (rb_corr < gb_corr)) + desired_dp_chan = 0; + // see if gr/gb is weakly correlated vs. rb + else if ((rg_corr < rb_corr) && (gb_corr < rb_corr)) + desired_dp_chan = 1; + // assume b is weakest + else + desired_dp_chan = 2; + } +#if BASISU_BC7F_PERF_STATS + g_total_dp_valid_chans_rgb++; +#endif + } + } + } + + if ((flags & cPackBC7FlagUseTrivialMode6) && ((desired_dp_chan == -1) && (block_max_var < TRIVIAL_BLOCK_THRESH_RGB))) + { + int low_c = INT_MAX, high_c = 0; + + for (uint32_t i = 0; i < 16; i++) + { + int y = ((16 * 2) * pPixels[i].r + (16 * 4) * pPixels[i].g + 16 * pPixels[i].b) + i; + low_c = basisu::minimum(low_c, y); + high_c = basisu::maximum(high_c, y); + } + + low_c &= 0xF; + high_c &= 0xF; + + int p0, p1, lr, lg, lb, hr, hg, hb; + + if (flags & cPackBC7FlagPBitOptMode6) + { + // An alternative would be to set A's=1.0 here and bias the p-bit optimizer to lower A RMSE. + const float q = 1.0f / 255.0f; + float sxl[4] = { (float)pPixels[low_c].r * q, (float)pPixels[low_c].g * q, (float)pPixels[low_c].b * q, 0 }; + float sxh[4] = { (float)pPixels[high_c].r * q, (float)pPixels[high_c].g * q, (float)pPixels[high_c].b * q, 0 }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_unique_pbits(3, 7, sxl, sxh, bestMinColor, bestMaxColor, best_pbits); + + p0 = best_pbits[0], p1 = best_pbits[1]; + lr = bestMinColor.r, lg = bestMinColor.g, lb = bestMinColor.b; + hr = bestMaxColor.r, hg = bestMaxColor.g, hb = bestMaxColor.b; + } + else + { + p0 = 1; + p1 = 1; + + lr = to_7(pPixels[low_c].r, p0), lg = to_7(pPixels[low_c].g, p0), lb = to_7(pPixels[low_c].b, p0); + hr = to_7(pPixels[high_c].r, p1), hg = to_7(pPixels[high_c].g, p1), hb = to_7(pPixels[high_c].b, p1); + } + + uint8_t cur_weights[16]; + uint32_t mode6_actual_sse = eval_weights_mode6_rgb_sse(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, p0, p1); + + encode_mode6_rgba_block(pBlock, + lr, lg, lb, 127, p0, + hr, hg, hb, 127, p1, + cur_weights); + +#if BASISU_BC7F_PERF_STATS + g_total_trivial_mode6_blocks++; +#endif + +#ifdef _DEBUG + { + // Final sanity checking. + uint32_t expected_actual_sse = calc_sse(pBlock, pPixels); + assert(expected_actual_sse == mode6_actual_sse); + } +#endif + + return mode6_actual_sse; + } + + float cov[6]; + for (uint32_t i = 0; i < 6; i++) + cov[i] = (float)icov[i]; + + const float sc = block_max_var ? (1.0f / (float)block_max_var) : 0; + const float wx = sc * cov[0], wy = sc * cov[3], wz = sc * cov[5]; + + const float alt_xr = cov[0] * wx + cov[1] * wy + cov[2] * wz; + const float alt_xg = cov[1] * wx + cov[3] * wy + cov[4] * wz; + const float alt_xb = cov[2] * wx + cov[4] * wy + cov[5] * wz; + + int saxis_r = 306, saxis_g = 601, saxis_b = 117; + + float k = basisu::maximum(fabsf(alt_xr), fabsf(alt_xg), fabsf(alt_xb)); + if (fabs(k) >= basisu::SMALL_FLOAT_VAL) + { + float m = 2048.0f / k; + saxis_r = (int)(alt_xr * m); + saxis_g = (int)(alt_xg * m); + saxis_b = (int)(alt_xb * m); + } + + saxis_r = (int)((uint32_t)saxis_r << 4U); + saxis_g = (int)((uint32_t)saxis_g << 4U); + saxis_b = (int)((uint32_t)saxis_b << 4U); + + int low_dot = INT_MAX, high_dot = INT_MIN; + +#if BASISU_BC7F_USE_SSE41 + int low_c, high_c; + bc7_proj_minmax_indices_sse41(pPixels, saxis_r, saxis_g, saxis_b, &low_c, &high_c); +#else + for (uint32_t i = 0; i < 16; i += 4) + { + assert(((pPixels[i].r * saxis_r + pPixels[i].g * saxis_g + pPixels[i].b * saxis_b) & 0xF) == 0); // sanity + assert(((pPixels[i + 1].r * saxis_r + pPixels[i + 1].g * saxis_g + pPixels[i + 1].b * saxis_b) & 0xF) == 0); + assert(((pPixels[i + 2].r * saxis_r + pPixels[i + 2].g * saxis_g + pPixels[i + 2].b * saxis_b) & 0xF) == 0); + assert(((pPixels[i + 3].r * saxis_r + pPixels[i + 3].g * saxis_g + pPixels[i + 3].b * saxis_b) & 0xF) == 0); + + const int dot0 = (pPixels[i].r * saxis_r + pPixels[i].g * saxis_g + pPixels[i].b * saxis_b) + i; + const int dot1 = (pPixels[i + 1].r * saxis_r + pPixels[i + 1].g * saxis_g + pPixels[i + 1].b * saxis_b) + i + 1; + const int dot2 = (pPixels[i + 2].r * saxis_r + pPixels[i + 2].g * saxis_g + pPixels[i + 2].b * saxis_b) + i + 2; + const int dot3 = (pPixels[i + 3].r * saxis_r + pPixels[i + 3].g * saxis_g + pPixels[i + 3].b * saxis_b) + i + 3; + + int min_d01 = basisu::minimum(dot0, dot1); + int max_d01 = basisu::maximum(dot0, dot1); + + int min_d23 = basisu::minimum(dot2, dot3); + int max_d23 = basisu::maximum(dot2, dot3); + + int min_d = basisu::minimum(min_d01, min_d23); + int max_d = basisu::maximum(max_d01, max_d23); + + low_dot = basisu::minimum(low_dot, min_d); + high_dot = basisu::maximum(high_dot, max_d); + } + + int low_c = low_dot & 15; + int high_c = high_dot & 15; +#endif + + int p0, p1, lr, lg, lb, hr, hg, hb; + + if (flags & cPackBC7FlagPBitOptMode6) + { + const float q = 1.0f / 255.0f; + float sxl[4] = { (float)pPixels[low_c].r * q, (float)pPixels[low_c].g * q, (float)pPixels[low_c].b * q, 0 }; + float sxh[4] = { (float)pPixels[high_c].r * q, (float)pPixels[high_c].g * q, (float)pPixels[high_c].b * q, 0 }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_unique_pbits(3, 7, sxl, sxh, bestMinColor, bestMaxColor, best_pbits); + + p0 = best_pbits[0], p1 = best_pbits[1]; + lr = bestMinColor.r, lg = bestMinColor.g, lb = bestMinColor.b; + hr = bestMaxColor.r, hg = bestMaxColor.g, hb = bestMaxColor.b; + } + else + { + // explictly force pbits to 1, that way alpha is always 255 and we don't slow down the entire encoder by 4-8% for a tiny ~.1 dB PSNR gain (not worth it) + p0 = 1, p1 = 1; + lr = to_7(pPixels[low_c].r, p0), lg = to_7(pPixels[low_c].g, p0), lb = to_7(pPixels[low_c].b, p0); + hr = to_7(pPixels[high_c].r, p1), hg = to_7(pPixels[high_c].g, p1), hb = to_7(pPixels[high_c].b, p1); + } + + uint8_t cur_weights[16]; + + uint32_t mode6_actual_sse = eval_weights_mode6_rgb_sse(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, p0, p1); + + if (mode6_actual_sse) + { + vec4F xl, xh; + bool res = compute_least_squares_endpoints_3D( + 16, cur_weights, 16, + g_bc7_4bit_ls_tab, + xl, xh, + pPixels, + (float)total_r, (float)total_g, (float)total_b); + + if (res) + { + int trial_p0, trial_p1, trial_lr, trial_lg, trial_lb, trial_hr, trial_hg, trial_hb; + + if (flags & cPackBC7FlagPBitOptMode6) + { + const float q = 1.0f / 255.0f; + float sxl[4] = { xl[0] * q, xl[1] * q, xl[2] * q, 0.0f }; + float sxh[4] = { xh[0] * q, xh[1] * q, xh[2] * q, 0.0f }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_unique_pbits( + 3, 7, sxl, sxh, + bestMinColor, bestMaxColor, best_pbits); + + trial_p0 = best_pbits[0], trial_p1 = best_pbits[1]; + trial_lr = bestMinColor.r, trial_lg = bestMinColor.g, trial_lb = bestMinColor.b; + trial_hr = bestMaxColor.r, trial_hg = bestMaxColor.g, trial_hb = bestMaxColor.b; + } + else + { + trial_p0 = 1; trial_p1 = 1; + trial_lr = to_7(xl[0], trial_p0); + trial_lg = to_7(xl[1], trial_p0); + trial_lb = to_7(xl[2], trial_p0); + + trial_hr = to_7(xh[0], trial_p1); + trial_hg = to_7(xh[1], trial_p1); + trial_hb = to_7(xh[2], trial_p1); + } + + uint8_t trial_weights[16]; + uint32_t mode6_ls_actual_sse = eval_weights_mode6_rgb_sse(pPixels, trial_weights, trial_lr, trial_lg, trial_lb, trial_hr, trial_hg, trial_hb, trial_p0, trial_p1); + if (mode6_ls_actual_sse < mode6_actual_sse) + { + mode6_actual_sse = mode6_ls_actual_sse; + memcpy(cur_weights, trial_weights, 16); + p0 = trial_p0; p1 = trial_p1; + lr = trial_lr; lg = trial_lg; lb = trial_lb; + hr = trial_hr; hg = trial_hg; hb = trial_hb; + } + } + } + + uint32_t mode02_actual_sse = UINT32_MAX; + uint8_t mode02_candidate_block[sizeof(basist::bc7_block)]; + + uint32_t mode13_actual_sse = UINT32_MAX; + uint8_t mode13_candidate_block[sizeof(basist::bc7_block)]; + + uint32_t mode45_actual_sse = UINT32_MAX; + uint8_t mode45_candidate_block[sizeof(basist::bc7_block)]; + + if (mode6_actual_sse) + { + if (non_analytical_flag) + { + // No gates: very expensive. + if (flags & cPackBC7FlagUse2SubsetsRGB) + { + if (flags & cPackBC7FlagUse3SubsetsRGB) + { + pack_mode0_or_2_rgb(mode02_candidate_block, pPixels, alt_xr, alt_xg, alt_xb, mean_r, mean_g, mean_b, 1e+9f, flags, nullptr, &mode02_actual_sse); + } + + pack_mode1_or_3_rgb(mode13_candidate_block, pPixels, alt_xr, alt_xg, alt_xb, mean_r, mean_g, mean_b, 1e+9f, flags, nullptr, &mode13_actual_sse); + } + + if (flags & cPackBC7FlagUseDualPlaneRGB) + pack_mode4_or_5(mode45_candidate_block, pPixels, (desired_dp_chan >= 0) ? desired_dp_chan : 1, 1e+9f, flags, nullptr, &mode45_actual_sse); // todo: determine best def channel here + } + else + { + float mode6_ortho_ratio; + const float mode6_slam_to_line_sse_est = estimate_slam_to_line_sse_3D(cov, alt_xr, alt_xg, alt_xb, &mode6_ortho_ratio); + + if ((flags & cPackBC7FlagUse2SubsetsRGB) && (block_max_var >= MIN_BLOCK_MAX_VAR_23SUBSETS) && (mode6_ortho_ratio > ORTHO_RATIO_23SUBSET_RATIO_THRESH)) + { + const bool high_ortho_energy_flag = (mode6_slam_to_line_sse_est >= HIGH_ORTHO_ENERGY_THRESH); + + if (high_ortho_energy_flag) + { +#if BASISU_BC7F_PERF_STATS + g_total_high_ortho_energy++; +#endif + + if ((flags & cPackBC7FlagUse3SubsetsRGB) && (block_max_var >= MIN_BLOCK_MAX_VAR_3SUBSETS)) + { + pack_mode0_or_2_rgb(mode02_candidate_block, pPixels, alt_xr, alt_xg, alt_xb, mean_r, mean_g, mean_b, 1e+9f, flags, nullptr, &mode02_actual_sse); + pack_mode1_or_3_rgb(mode13_candidate_block, pPixels, alt_xr, alt_xg, alt_xb, mean_r, mean_g, mean_b, 1e+9f, flags, nullptr, &mode13_actual_sse); + } + else + { + pack_mode1_or_3_rgb(mode13_candidate_block, pPixels, alt_xr, alt_xg, alt_xb, mean_r, mean_g, mean_b, 1e+9f, flags, nullptr, &mode13_actual_sse); + } + } + } + + if (desired_dp_chan >= 0) + { + assert(flags & cPackBC7FlagUseDualPlaneRGB); + + pack_mode4_or_5(mode45_candidate_block, pPixels, desired_dp_chan, 1e+9f, flags, nullptr, &mode45_actual_sse); + + } // if (desired_dp_chan >= 0) + } + } + + const uint32_t best_actual_sse = basisu::minimum(mode6_actual_sse, mode02_actual_sse, mode13_actual_sse, mode45_actual_sse); + + if ((mode45_actual_sse != UINT32_MAX) && (best_actual_sse == mode45_actual_sse)) + { + memcpy(pBlock, mode45_candidate_block, sizeof(basist::bc7_block)); + } + else if ((mode02_actual_sse != UINT32_MAX) && (best_actual_sse == mode02_actual_sse)) + { + memcpy(pBlock, mode02_candidate_block, sizeof(basist::bc7_block)); + } + else if ((mode13_actual_sse != UINT32_MAX) && (best_actual_sse == mode13_actual_sse)) + { + memcpy(pBlock, mode13_candidate_block, sizeof(basist::bc7_block)); + } + else + { + assert(mode6_actual_sse == best_actual_sse); + + // pbits set to 1 to ensure alpha is always decoded to fully opaque (255) + encode_mode6_rgba_block(pBlock, + lr, lg, lb, 127, p0, + hr, hg, hb, 127, p1, + cur_weights); + } + +#ifdef _DEBUG + { + // Final sanity checking. + uint32_t expected_actual_sse = calc_sse(pBlock, pPixels); + assert(expected_actual_sse == best_actual_sse); + } +#endif + + return best_actual_sse; + } + + //------------------------------------------------------------------------------------------------------- + + void fast_pack_bc7_rgba_analytical(uint8_t* pBlock, const color_rgba* pPixels, uint32_t flags) + { + assert(g_bc7_4bit_ls_tab[1][0]); + +#if BASISU_BC7F_PERF_STATS + g_total_rgba_calls++; +#endif + + const uint32_t fc = *(const uint32_t*)&pPixels[0]; + if (fc == *(const uint32_t*)&pPixels[15]) + { + int k; + for (k = 1; k < 15; k++) + if (*(const uint32_t*)&pPixels[k] != fc) + break; + + if (k == 15) + { + pack_mode5_solid(pBlock, pPixels[0]); + return; + } + } + + int total_r = 0, total_g = 0, total_b = 0, total_a = 0; + int min_r = 255, min_g = 255, min_b = 255, min_a = 255; + int max_r = 0, max_g = 0, max_b = 0, max_a = 0; + + for (uint32_t i = 0; i < 16; i++) + { + int r = pPixels[i].r, g = pPixels[i].g, b = pPixels[i].b, a = pPixels[i].a; + + total_r += r; total_g += g; total_b += b; total_a += a; + + min_r = basisu::minimum(min_r, r); min_g = basisu::minimum(min_g, g); min_b = basisu::minimum(min_b, b); min_a = basisu::minimum(min_a, a); + max_r = basisu::maximum(max_r, r); max_g = basisu::maximum(max_g, g); max_b = basisu::maximum(max_b, b); max_a = basisu::maximum(max_a, a); + } + + assert((min_r != max_r) || (min_g != max_g) || (min_b != max_b) || (min_a != max_a)); + + const int mean_r = (total_r + 8) >> 4, mean_g = (total_g + 8) >> 4, mean_b = (total_b + 8) >> 4, mean_a = (total_a + 8) >> 4; + + // covar rows are: + int icov4[10] = { }; + + // 0=rr + // 1=rg + // 2=rb + // 3=ra + // + // 4=gg + // 5=gb + // 6=ga + // + // 7=bb + // 8=ba + // + // 9=aa + + // 0 1 2 3 + // 4 5 6 + // 7 8 + // 9 + + // 0 1 2 3 + // 1 4 5 6 + // 2 5 7 8 + // 3 6 8 9 + + // trace at 0,4,7,9 + + for (uint32_t i = 0; i < 16; i++) + { + const int r = (int)pPixels[i].r - mean_r, g = (int)pPixels[i].g - mean_g, b = (int)pPixels[i].b - mean_b, a = (int)pPixels[i].a - mean_a; + + icov4[0] += r * r; icov4[1] += r * g; icov4[2] += r * b; icov4[3] += r * a; + icov4[4] += g * g; icov4[5] += g * b; icov4[6] += g * a; + icov4[7] += b * b; icov4[8] += b * a; + icov4[9] += a * a; + } + + const int block_max_var4 = basisu::maximum(icov4[0], icov4[4], icov4[7], icov4[9]); // not divided by 16, i.e. scaled by 16 + assert(block_max_var4); // solid blocks already filtered out + + // check for dual plane, if a single component is very strongly decorrelated then switch to modes 4/5 + int desired_dp_chan = -1; + + if ((flags & cPackBC7FlagUseDualPlaneRGBA) && (block_max_var4 >= DP_BLOCK_VAR_THRESH_RGBA)) + { + // Prefer A, if not strongly decorrelated then check RGB. + const float r_var = (float)icov4[0], g_var = (float)icov4[4], b_var = (float)icov4[7], a_var = (float)icov4[9]; + + const bool has_a = icov4[9] > 0; + + if (has_a) + { + const float p_03 = icov4[0] ? fabs((float)icov4[3] / sqrtf(r_var * a_var)) : 1.0f; + const float p_13 = icov4[4] ? fabs((float)icov4[6] / sqrtf(g_var * a_var)) : 1.0f; + const float p_23 = icov4[7] ? fabs((float)icov4[8] / sqrtf(b_var * a_var)) : 1.0f; + + const float min_p = basisu::minimum(p_03, p_13, p_23); + if (min_p < ALPHA_DECORR_THRESHOLD) + { + desired_dp_chan = 3; +#if BASISU_BC7F_PERF_STATS + g_total_dp_valid_chans_a++; +#endif + } + } + + if (desired_dp_chan < 0) + { + const bool has_r = icov4[0] > 16, has_g = icov4[4] > 16, has_b = icov4[7] > 16; + const uint32_t total_active_chans_rgb = has_r + has_g + has_b; + + if (total_active_chans_rgb >= 2) + { + const float rg_corr = (has_r && has_g) ? fabs((float)icov4[1] / sqrtf(r_var * g_var)) : 1.0f; + const float rb_corr = (has_r && has_b) ? fabs((float)icov4[2] / sqrtf(r_var * b_var)) : 1.0f; + const float gb_corr = (has_g && has_b) ? fabs((float)icov4[5] / sqrtf(g_var * b_var)) : 1.0f; + + float min_p = basisu::minimum(rg_corr, rb_corr, gb_corr); + if (min_p < STRONG_DECORR_THRESH_RGBA) + { + if (total_active_chans_rgb == 2) + { + if (!has_r) + desired_dp_chan = 1; + else if (!has_g) + desired_dp_chan = 0; + else + desired_dp_chan = 0; + } + else + { + // see if rg/rb is weakly correlated vs. gb + if ((rg_corr < gb_corr) && (rb_corr < gb_corr)) + desired_dp_chan = 0; + // see if gr/gb is weakly correlated vs. rb + else if ((rg_corr < rb_corr) && (gb_corr < rb_corr)) + desired_dp_chan = 1; + // assume b is weakest + else + desired_dp_chan = 2; + } +#if BASISU_BC7F_PERF_STATS + g_total_dp_valid_chans_rgb++; +#endif + } + } + } + } + + if ((flags & cPackBC7FlagUseTrivialMode6) && ((desired_dp_chan == -1) && (block_max_var4 < TRIVIAL_BLOCK_THRESH_RGBA))) + { + //pack_mode5_solid(pBlock, color_rgba(0, 255, 0, 255)); + //return; + + int low_c = INT_MAX, high_c = 0; + + for (uint32_t i = 0; i < 16; i++) + { + int y = ((16 * 2) * pPixels[i].r + (16 * 4) * pPixels[i].g + 16 * pPixels[i].b + (16 * 4) * pPixels[i].a); + assert((y & 0xF) == 0); + y += i; + low_c = basisu::minimum(low_c, y); + high_c = basisu::maximum(high_c, y); + } + + low_c &= 0xF; + high_c &= 0xF; + + int p0, p1, lr, lg, lb, la, hr, hg, hb, ha; + + if (flags & cPackBC7FlagPBitOptMode6) + { + const float q = 1.0f / 255.0f; + float sxl[4] = { (float)pPixels[low_c].r * q, (float)pPixels[low_c].g * q, (float)pPixels[low_c].b * q, (float)pPixels[low_c].a * q }; + float sxh[4] = { (float)pPixels[high_c].r * q, (float)pPixels[high_c].g * q, (float)pPixels[high_c].b * q, (float)pPixels[high_c].a * q }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_unique_pbits(4, 7, sxl, sxh, bestMinColor, bestMaxColor, best_pbits); + + p0 = best_pbits[0], p1 = best_pbits[1]; + lr = bestMinColor.r, lg = bestMinColor.g, lb = bestMinColor.b, la = bestMinColor.a; + hr = bestMaxColor.r, hg = bestMaxColor.g, hb = bestMaxColor.b, ha = bestMaxColor.a; + } + else + { + p0 = pPixels[low_c].a > 128; + p1 = pPixels[high_c].a > 128; + + lr = to_7(pPixels[low_c].r, p0), lg = to_7(pPixels[low_c].g, p0), lb = to_7(pPixels[low_c].b, p0), la = to_7(pPixels[low_c].a, p0); + hr = to_7(pPixels[high_c].r, p1), hg = to_7(pPixels[high_c].g, p1), hb = to_7(pPixels[high_c].b, p1), ha = to_7(pPixels[high_c].a, p1); + } + + uint8_t cur_weights[16]; + eval_weights_mode6_rgba(pPixels, cur_weights, + lr, lg, lb, la, p0, + hr, hg, hb, ha, p1); + + encode_mode6_rgba_block(pBlock, + lr, lg, lb, la, p0, + hr, hg, hb, ha, p1, + cur_weights); + +#if BASISU_BC7F_PERF_STATS + g_total_trivial_mode6_blocks++; +#endif + return; + } + + float cov4[10]; + for (uint32_t i = 0; i < 10; i++) + cov4[i] = (float)icov4[i]; + + // all channel pairs: + // 0,1=1 + // 0,2=2 + // 0,3=3 + // 1,2=5 + // 1,3=6 + // 2,3=8 + //const float r_var = cov4[0], g_var = cov4[4], b_var = cov4[7], a_var = cov4[9]; + + const float sc4 = block_max_var4 ? (1.0f / (float)block_max_var4) : 0; + float wx = sc4 * cov4[0], wy = sc4 * cov4[4], wz = sc4 * cov4[7], wa = sc4 * cov4[9]; + + // 0 1 2 3 + // 1 4 5 6 + // 2 5 7 8 + // 3 6 8 9 + + // TODO + float x1, y1, z1, w1; + for (uint32_t i = 0; i < 4; i++) + { + x1 = cov4[0] * wx + cov4[1] * wy + cov4[2] * wz + cov4[3] * wa; + y1 = cov4[1] * wx + cov4[4] * wy + cov4[5] * wz + cov4[6] * wa; + z1 = cov4[2] * wx + cov4[5] * wy + cov4[7] * wz + cov4[8] * wa; + w1 = cov4[3] * wx + cov4[6] * wy + cov4[8] * wz + cov4[9] * wa; + + float t = sqrtf(x1 * x1 + y1 * y1 + z1 * z1 + w1 * w1); + if (t > basisu::SMALL_FLOAT_VAL) + { + t = 1.0f / t; + x1 *= t; y1 *= t; z1 *= t; w1 *= t; + } + else + { + x1 = y1 = z1 = w1 = .25f; + } + + wx = x1; wy = y1; wz = z1; wa = w1; + } + + const int spans[4] = { max_r - min_r, max_g - min_g, max_b - min_b, max_a - min_a }; + + float mode6_ortho_ratio; + const float mode6_slam_to_line_sse_est = estimate_slam_to_line_sse_4D(cov4, x1, y1, z1, w1, &mode6_ortho_ratio); + const float mode6_sse_est = (mode6_slam_to_line_sse_est + analytical_quant_est_sse(128, 16, 4, spans, nullptr, 1.0f, 16)); + + float mode45_sse_est = 1e+9f, mode7_sse_est = 1e+9f; + + uint8_t mode45_block[sizeof(basist::bc7_block)]; + const bool mode45_valid_flag = (desired_dp_chan >= 0) ? pack_mode4_or_5(mode45_block, pPixels, desired_dp_chan, mode6_sse_est, flags, &mode45_sse_est) : false; + BASISU_NOTE_UNUSED(mode45_valid_flag); + + uint8_t mode7_block[sizeof(basist::bc7_block)]; + bool mode7_valid_flag = false; + BASISU_NOTE_UNUSED(mode7_valid_flag); + + if ((flags & cPackBC7FlagUse2SubsetsRGBA) && (block_max_var4 >= MIN_BLOCK_MAX_VAR_23SUBSETS_RGBA) && (mode6_ortho_ratio > ORTHO_RATIO_23SUBSET_RATIO_THRESH_RGBA)) + { + const bool high_ortho_energy_flag = (mode6_slam_to_line_sse_est >= HIGH_ORTHO_ENERGY_THRESH_RGBA); + + if (high_ortho_energy_flag) + { +#if BASISU_BC7F_PERF_STATS + g_total_high_ortho_energy++; +#endif + mode7_valid_flag = pack_mode7_rgba(mode7_block, pPixels, x1, y1, z1, w1, mean_r, mean_g, mean_b, mean_a, mode6_sse_est, flags, &mode7_sse_est); + } + } + + if ((mode45_sse_est < mode7_sse_est) && (mode45_sse_est < mode6_sse_est)) + { + assert(mode45_valid_flag); + memcpy(pBlock, mode45_block, sizeof(basist::bc7_block)); + return; + } + else if ((mode7_sse_est < mode45_sse_est) && (mode7_sse_est < mode6_sse_est)) + { + assert(mode7_valid_flag); + memcpy(pBlock, mode7_block, sizeof(basist::bc7_block)); + return; + } + + // Fall back to mode 6 + int saxis_r = 256, saxis_g = 256, saxis_b = 256, saxis_a = 256; + + float k = basisu::maximum(fabsf(x1), fabsf(y1), fabsf(z1), fabs(w1)); + if (fabs(k) >= basisu::SMALL_FLOAT_VAL) + { + float m = 2048.0f / k; + saxis_r = (int)(x1 * m); + saxis_g = (int)(y1 * m); + saxis_b = (int)(z1 * m); + saxis_a = (int)(w1 * m); + } + + saxis_r = (int)((uint32_t)saxis_r << 4U); + saxis_g = (int)((uint32_t)saxis_g << 4U); + saxis_b = (int)((uint32_t)saxis_b << 4U); + saxis_a = (int)((uint32_t)saxis_a << 4U); + + int low_dot = INT_MAX, high_dot = INT_MIN; + + for (uint32_t i = 0; i < 16; i += 4) + { + assert(((pPixels[i].r * saxis_r + pPixels[i].g * saxis_g + pPixels[i].b * saxis_b + pPixels[i].a * saxis_a) & 0xF) == 0); // sanity + assert(((pPixels[i + 1].r * saxis_r + pPixels[i + 1].g * saxis_g + pPixels[i + 1].b * saxis_b + pPixels[i + 1].a * saxis_a) & 0xF) == 0); + assert(((pPixels[i + 2].r * saxis_r + pPixels[i + 2].g * saxis_g + pPixels[i + 2].b * saxis_b + pPixels[i + 2].a * saxis_a) & 0xF) == 0); + assert(((pPixels[i + 3].r * saxis_r + pPixels[i + 3].g * saxis_g + pPixels[i + 3].b * saxis_b + pPixels[i + 3].a * saxis_a) & 0xF) == 0); + + const int dot0 = (pPixels[i].r * saxis_r + pPixels[i].g * saxis_g + pPixels[i].b * saxis_b + pPixels[i].a * saxis_a) + i; + const int dot1 = (pPixels[i + 1].r * saxis_r + pPixels[i + 1].g * saxis_g + pPixels[i + 1].b * saxis_b + pPixels[i + 1].a * saxis_a) + i + 1; + const int dot2 = (pPixels[i + 2].r * saxis_r + pPixels[i + 2].g * saxis_g + pPixels[i + 2].b * saxis_b + pPixels[i + 2].a * saxis_a) + i + 2; + const int dot3 = (pPixels[i + 3].r * saxis_r + pPixels[i + 3].g * saxis_g + pPixels[i + 3].b * saxis_b + pPixels[i + 3].a * saxis_a) + i + 3; + + int min_d01 = basisu::minimum(dot0, dot1); + int max_d01 = basisu::maximum(dot0, dot1); + + int min_d23 = basisu::minimum(dot2, dot3); + int max_d23 = basisu::maximum(dot2, dot3); + + int min_d = basisu::minimum(min_d01, min_d23); + int max_d = basisu::maximum(max_d01, max_d23); + + low_dot = basisu::minimum(low_dot, min_d); + high_dot = basisu::maximum(high_dot, max_d); + } + + const int low_c = low_dot & 15; + const int high_c = high_dot & 15; + + int p0, p1, lr, lg, lb, la, hr, hg, hb, ha; + + if (flags & cPackBC7FlagPBitOptMode6) + { + const float q = 1.0f / 255.0f; + float sxl[4] = { (float)pPixels[low_c].r * q, (float)pPixels[low_c].g * q, (float)pPixels[low_c].b * q, (float)pPixels[low_c].a * q }; + float sxh[4] = { (float)pPixels[high_c].r * q, (float)pPixels[high_c].g * q, (float)pPixels[high_c].b * q, (float)pPixels[high_c].a * q }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_unique_pbits(4, 7, sxl, sxh, bestMinColor, bestMaxColor, best_pbits); + + p0 = best_pbits[0], p1 = best_pbits[1]; + lr = bestMinColor.r, lg = bestMinColor.g, lb = bestMinColor.b, la = bestMinColor.a; + hr = bestMaxColor.r, hg = bestMaxColor.g, hb = bestMaxColor.b, ha = bestMaxColor.a; + } + else + { + p0 = pPixels[low_c].a > 128; + p1 = pPixels[high_c].a > 128; + + lr = to_7(pPixels[low_c].r, p0), lg = to_7(pPixels[low_c].g, p0), lb = to_7(pPixels[low_c].b, p0), la = to_7(pPixels[low_c].a, p0); + hr = to_7(pPixels[high_c].r, p1), hg = to_7(pPixels[high_c].g, p1), hb = to_7(pPixels[high_c].b, p1), ha = to_7(pPixels[high_c].a, p1); + } + + uint8_t cur_weights[16]; + eval_weights_mode6_rgba(pPixels, cur_weights, + lr, lg, lb, la, p0, + hr, hg, hb, ha, p1); + + vec4F xl, xh; + bool res = compute_least_squares_endpoints_4D( + 16, cur_weights, 16, + g_bc7_4bit_ls_tab, + xl, xh, + pPixels, + (float)total_r, (float)total_g, (float)total_b, (float)total_a); + + if (res) + { + if (flags & cPackBC7FlagPBitOptMode6) + { + const float q = 1.0f / 255.0f; + float sxl[4] = { xl[0] * q, xl[1] * q, xl[2] * q, xl[3] * q }; + float sxh[4] = { xh[0] * q, xh[1] * q, xh[2] * q, xh[3] * q }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_unique_pbits(4, 7, sxl, sxh, bestMinColor, bestMaxColor, best_pbits); + + p0 = best_pbits[0], p1 = best_pbits[1]; + lr = bestMinColor.r, lg = bestMinColor.g, lb = bestMinColor.b, la = bestMinColor.a; + hr = bestMaxColor.r, hg = bestMaxColor.g, hb = bestMaxColor.b, ha = bestMaxColor.a; + } + else + { + p0 = (xl[3] >= 129.0f); + lr = to_7(xl[0], p0); + lg = to_7(xl[1], p0); + lb = to_7(xl[2], p0); + la = to_7(xl[3], p0); + + p1 = (xh[3] >= 129.0f); + hr = to_7(xh[0], p1); + hg = to_7(xh[1], p1); + hb = to_7(xh[2], p1); + ha = to_7(xh[3], p1); + } + + eval_weights_mode6_rgba(pPixels, cur_weights, + lr, lg, lb, la, p0, + hr, hg, hb, ha, p1); + } + + encode_mode6_rgba_block(pBlock, + lr, lg, lb, la, p0, + hr, hg, hb, ha, p1, + cur_weights); + } + + uint32_t fast_pack_bc7_rgba_partial_analytical(uint8_t* pBlock, const color_rgba* pPixels, uint32_t flags) + { + assert(g_bc7_4bit_ls_tab[1][0]); + +#if BASISU_BC7F_PERF_STATS + g_total_rgba_calls++; +#endif + + const uint32_t fc = *(const uint32_t*)&pPixels[0]; + if (fc == *(const uint32_t*)&pPixels[15]) + { + int k; + for (k = 1; k < 15; k++) + if (*(const uint32_t*)&pPixels[k] != fc) + break; + + if (k == 15) + { + pack_mode5_solid(pBlock, pPixels[0]); + return 0; + } + } + + int total_r = 0, total_g = 0, total_b = 0, total_a = 0; + int min_r = 255, min_g = 255, min_b = 255, min_a = 255; + int max_r = 0, max_g = 0, max_b = 0, max_a = 0; + + for (uint32_t i = 0; i < 16; i++) + { + int r = pPixels[i].r, g = pPixels[i].g, b = pPixels[i].b, a = pPixels[i].a; + + total_r += r; total_g += g; total_b += b; total_a += a; + + min_r = basisu::minimum(min_r, r); min_g = basisu::minimum(min_g, g); min_b = basisu::minimum(min_b, b); min_a = basisu::minimum(min_a, a); + max_r = basisu::maximum(max_r, r); max_g = basisu::maximum(max_g, g); max_b = basisu::maximum(max_b, b); max_a = basisu::maximum(max_a, a); + } + + assert((min_r != max_r) || (min_g != max_g) || (min_b != max_b) || (min_a != max_a)); + + const int mean_r = (total_r + 8) >> 4, mean_g = (total_g + 8) >> 4, mean_b = (total_b + 8) >> 4, mean_a = (total_a + 8) >> 4; + + // covar rows are: + int icov4[10] = { }; + + // 0=rr + // 1=rg + // 2=rb + // 3=ra + // + // 4=gg + // 5=gb + // 6=ga + // + // 7=bb + // 8=ba + // + // 9=aa + + // 0 1 2 3 + // 4 5 6 + // 7 8 + // 9 + + // 0 1 2 3 + // 1 4 5 6 + // 2 5 7 8 + // 3 6 8 9 + + // trace at 0,4,7,9 + + for (uint32_t i = 0; i < 16; i++) + { + const int r = (int)pPixels[i].r - mean_r, g = (int)pPixels[i].g - mean_g, b = (int)pPixels[i].b - mean_b, a = (int)pPixels[i].a - mean_a; + + icov4[0] += r * r; icov4[1] += r * g; icov4[2] += r * b; icov4[3] += r * a; + icov4[4] += g * g; icov4[5] += g * b; icov4[6] += g * a; + icov4[7] += b * b; icov4[8] += b * a; + icov4[9] += a * a; + } + + const int block_max_var4 = basisu::maximum(icov4[0], icov4[4], icov4[7], icov4[9]); // not divided by 16, i.e. scaled by 16 + assert(block_max_var4); // solid blocks already filtered out + + // check for dual plane, if a single component is very strongly decorrelated then switch to modes 4/5 + int desired_dp_chan = -1; + + const bool non_analytical_flag = (flags & cPackBC7FlagNonAnalyticalRGBA) != 0; + + if ((flags & cPackBC7FlagUseDualPlaneRGBA) && + ((!non_analytical_flag && (block_max_var4 >= DP_BLOCK_VAR_THRESH_RGBA)) || (non_analytical_flag && (block_max_var4 > 16))) + ) + { + // Prefer A, if not strongly decorrelated then check RGB. + const float r_var = (float)icov4[0], g_var = (float)icov4[4], b_var = (float)icov4[7], a_var = (float)icov4[9]; + + const bool has_a = icov4[9] > 0; + + if (has_a) + { + const float p_03 = icov4[0] ? fabs((float)icov4[3] / sqrtf(r_var * a_var)) : 1.0f; + const float p_13 = icov4[4] ? fabs((float)icov4[6] / sqrtf(g_var * a_var)) : 1.0f; + const float p_23 = icov4[7] ? fabs((float)icov4[8] / sqrtf(b_var * a_var)) : 1.0f; + + const float min_p = basisu::minimum(p_03, p_13, p_23); + if (min_p < ALPHA_DECORR_THRESHOLD) + { + desired_dp_chan = 3; +#if BASISU_BC7F_PERF_STATS + g_total_dp_valid_chans_a++; +#endif + } + } + + if (desired_dp_chan < 0) + { + const bool has_r = icov4[0] > 16, has_g = icov4[4] > 16, has_b = icov4[7] > 16; + const uint32_t total_active_chans_rgb = has_r + has_g + has_b; + + if (total_active_chans_rgb >= 2) + { + const float rg_corr = (has_r && has_g) ? fabs((float)icov4[1] / sqrtf(r_var * g_var)) : 1.0f; + const float rb_corr = (has_r && has_b) ? fabs((float)icov4[2] / sqrtf(r_var * b_var)) : 1.0f; + const float gb_corr = (has_g && has_b) ? fabs((float)icov4[5] / sqrtf(g_var * b_var)) : 1.0f; + + float min_p = basisu::minimum(rg_corr, rb_corr, gb_corr); + + const float decorr_thresh = non_analytical_flag ? .999f : STRONG_DECORR_THRESH_RGBA; + if (min_p < decorr_thresh) + { + if (total_active_chans_rgb == 2) + { + if (!has_r) + desired_dp_chan = 1; + else if (!has_g) + desired_dp_chan = 0; + else + desired_dp_chan = 0; + } + else + { + // see if rg/rb is weakly correlated vs. gb + if ((rg_corr < gb_corr) && (rb_corr < gb_corr)) + desired_dp_chan = 0; + // see if gr/gb is weakly correlated vs. rb + else if ((rg_corr < rb_corr) && (gb_corr < rb_corr)) + desired_dp_chan = 1; + // assume b is weakest + else + desired_dp_chan = 2; + } +#if BASISU_BC7F_PERF_STATS + g_total_dp_valid_chans_rgb++; +#endif + } + } + } + } + + if ((flags & cPackBC7FlagUseTrivialMode6) && ((desired_dp_chan == -1) && (block_max_var4 < TRIVIAL_BLOCK_THRESH_RGBA))) + { + int low_c = INT_MAX, high_c = 0; + + for (uint32_t i = 0; i < 16; i++) + { + int y = ((16 * 2) * pPixels[i].r + (16 * 4) * pPixels[i].g + 16 * pPixels[i].b + (16 * 4) * pPixels[i].a); + assert((y & 0xF) == 0); + y += i; + low_c = basisu::minimum(low_c, y); + high_c = basisu::maximum(high_c, y); + } + + low_c &= 0xF; + high_c &= 0xF; + + int p0, p1, lr, lg, lb, la, hr, hg, hb, ha; + + if (flags & cPackBC7FlagPBitOptMode6) + { + const float q = 1.0f / 255.0f; + float sxl[4] = { (float)pPixels[low_c].r * q, (float)pPixels[low_c].g * q, (float)pPixels[low_c].b * q, (float)pPixels[low_c].a * q }; + float sxh[4] = { (float)pPixels[high_c].r * q, (float)pPixels[high_c].g * q, (float)pPixels[high_c].b * q, (float)pPixels[high_c].a * q }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_unique_pbits(4, 7, sxl, sxh, bestMinColor, bestMaxColor, best_pbits); + + p0 = best_pbits[0], p1 = best_pbits[1]; + lr = bestMinColor.r, lg = bestMinColor.g, lb = bestMinColor.b, la = bestMinColor.a; + hr = bestMaxColor.r, hg = bestMaxColor.g, hb = bestMaxColor.b, ha = bestMaxColor.a; + } + else + { + p0 = pPixels[low_c].a > 128; + p1 = pPixels[high_c].a > 128; + + lr = to_7(pPixels[low_c].r, p0), lg = to_7(pPixels[low_c].g, p0), lb = to_7(pPixels[low_c].b, p0), la = to_7(pPixels[low_c].a, p0); + hr = to_7(pPixels[high_c].r, p1), hg = to_7(pPixels[high_c].g, p1), hb = to_7(pPixels[high_c].b, p1), ha = to_7(pPixels[high_c].a, p1); + } + + uint8_t cur_weights[16]; + uint32_t mode6_actual_sse = eval_weights_mode6_rgba_sse(pPixels, cur_weights, + lr, lg, lb, la, p0, + hr, hg, hb, ha, p1); + + encode_mode6_rgba_block(pBlock, + lr, lg, lb, la, p0, + hr, hg, hb, ha, p1, + cur_weights); + +#if BASISU_BC7F_PERF_STATS + g_total_trivial_mode6_blocks++; +#endif + +#ifdef _DEBUG + { + // Final sanity checking. + uint32_t expected_actual_sse = calc_sse(pBlock, pPixels); + assert(expected_actual_sse == mode6_actual_sse); + } +#endif + + return mode6_actual_sse; + } + + float cov4[10]; + for (uint32_t i = 0; i < 10; i++) + cov4[i] = (float)icov4[i]; + + // all channel pairs: + // 0,1=1 + // 0,2=2 + // 0,3=3 + // 1,2=5 + // 1,3=6 + // 2,3=8 + //const float r_var = cov4[0], g_var = cov4[4], b_var = cov4[7], a_var = cov4[9]; + + const float sc4 = block_max_var4 ? (1.0f / (float)block_max_var4) : 0; + float wx = sc4 * cov4[0], wy = sc4 * cov4[4], wz = sc4 * cov4[7], wa = sc4 * cov4[9]; + + // 0 1 2 3 + // 1 4 5 6 + // 2 5 7 8 + // 3 6 8 9 + + // TODO + float x1, y1, z1, w1; + for (uint32_t i = 0; i < 4; i++) + { + x1 = cov4[0] * wx + cov4[1] * wy + cov4[2] * wz + cov4[3] * wa; + y1 = cov4[1] * wx + cov4[4] * wy + cov4[5] * wz + cov4[6] * wa; + z1 = cov4[2] * wx + cov4[5] * wy + cov4[7] * wz + cov4[8] * wa; + w1 = cov4[3] * wx + cov4[6] * wy + cov4[8] * wz + cov4[9] * wa; + + float t = sqrtf(x1 * x1 + y1 * y1 + z1 * z1 + w1 * w1); + if (t > basisu::SMALL_FLOAT_VAL) + { + t = 1.0f / t; + x1 *= t; y1 *= t; z1 *= t; w1 *= t; + } + else + { + x1 = y1 = z1 = w1 = .25f; + } + + wx = x1; wy = y1; wz = z1; wa = w1; + } + + // Fall back to mode 6 + int saxis_r = 256, saxis_g = 256, saxis_b = 256, saxis_a = 256; + + float k = basisu::maximum(fabsf(x1), fabsf(y1), fabsf(z1), fabs(w1)); + if (fabs(k) >= basisu::SMALL_FLOAT_VAL) + { + float m = 2048.0f / k; + saxis_r = (int)(x1 * m); + saxis_g = (int)(y1 * m); + saxis_b = (int)(z1 * m); + saxis_a = (int)(w1 * m); + } + + saxis_r = (int)((uint32_t)saxis_r << 4U); + saxis_g = (int)((uint32_t)saxis_g << 4U); + saxis_b = (int)((uint32_t)saxis_b << 4U); + saxis_a = (int)((uint32_t)saxis_a << 4U); + + int low_dot = INT_MAX, high_dot = INT_MIN; + + for (uint32_t i = 0; i < 16; i += 4) + { + assert(((pPixels[i].r * saxis_r + pPixels[i].g * saxis_g + pPixels[i].b * saxis_b + pPixels[i].a * saxis_a) & 0xF) == 0); // sanity + assert(((pPixels[i + 1].r * saxis_r + pPixels[i + 1].g * saxis_g + pPixels[i + 1].b * saxis_b + pPixels[i + 1].a * saxis_a) & 0xF) == 0); + assert(((pPixels[i + 2].r * saxis_r + pPixels[i + 2].g * saxis_g + pPixels[i + 2].b * saxis_b + pPixels[i + 2].a * saxis_a) & 0xF) == 0); + assert(((pPixels[i + 3].r * saxis_r + pPixels[i + 3].g * saxis_g + pPixels[i + 3].b * saxis_b + pPixels[i + 3].a * saxis_a) & 0xF) == 0); + + const int dot0 = (pPixels[i].r * saxis_r + pPixels[i].g * saxis_g + pPixels[i].b * saxis_b + pPixels[i].a * saxis_a) + i; + const int dot1 = (pPixels[i + 1].r * saxis_r + pPixels[i + 1].g * saxis_g + pPixels[i + 1].b * saxis_b + pPixels[i + 1].a * saxis_a) + i + 1; + const int dot2 = (pPixels[i + 2].r * saxis_r + pPixels[i + 2].g * saxis_g + pPixels[i + 2].b * saxis_b + pPixels[i + 2].a * saxis_a) + i + 2; + const int dot3 = (pPixels[i + 3].r * saxis_r + pPixels[i + 3].g * saxis_g + pPixels[i + 3].b * saxis_b + pPixels[i + 3].a * saxis_a) + i + 3; + + int min_d01 = basisu::minimum(dot0, dot1); + int max_d01 = basisu::maximum(dot0, dot1); + + int min_d23 = basisu::minimum(dot2, dot3); + int max_d23 = basisu::maximum(dot2, dot3); + + int min_d = basisu::minimum(min_d01, min_d23); + int max_d = basisu::maximum(max_d01, max_d23); + + low_dot = basisu::minimum(low_dot, min_d); + high_dot = basisu::maximum(high_dot, max_d); + } + + const int low_c = low_dot & 15; + const int high_c = high_dot & 15; + + int p0, p1, lr, lg, lb, la, hr, hg, hb, ha; + + if (flags & cPackBC7FlagPBitOptMode6) + { + const float q = 1.0f / 255.0f; + float sxl[4] = { (float)pPixels[low_c].r * q, (float)pPixels[low_c].g * q, (float)pPixels[low_c].b * q, (float)pPixels[low_c].a * q }; + float sxh[4] = { (float)pPixels[high_c].r * q, (float)pPixels[high_c].g * q, (float)pPixels[high_c].b * q, (float)pPixels[high_c].a * q }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_unique_pbits(4, 7, sxl, sxh, bestMinColor, bestMaxColor, best_pbits); + + p0 = best_pbits[0], p1 = best_pbits[1]; + lr = bestMinColor.r, lg = bestMinColor.g, lb = bestMinColor.b, la = bestMinColor.a; + hr = bestMaxColor.r, hg = bestMaxColor.g, hb = bestMaxColor.b, ha = bestMaxColor.a; + } + else + { + p0 = pPixels[low_c].a > 128; + p1 = pPixels[high_c].a > 128; + + lr = to_7(pPixels[low_c].r, p0), lg = to_7(pPixels[low_c].g, p0), lb = to_7(pPixels[low_c].b, p0), la = to_7(pPixels[low_c].a, p0); + hr = to_7(pPixels[high_c].r, p1), hg = to_7(pPixels[high_c].g, p1), hb = to_7(pPixels[high_c].b, p1), ha = to_7(pPixels[high_c].a, p1); + } + + uint8_t cur_weights[16]; + uint32_t mode6_actual_sse = eval_weights_mode6_rgba_sse(pPixels, cur_weights, + lr, lg, lb, la, p0, + hr, hg, hb, ha, p1); + + if (mode6_actual_sse) + { + vec4F xl, xh; + bool res = compute_least_squares_endpoints_4D( + 16, cur_weights, 16, + g_bc7_4bit_ls_tab, + xl, xh, + pPixels, + (float)total_r, (float)total_g, (float)total_b, (float)total_a); + + if (res) + { + int trial_p0, trial_p1, trial_lr, trial_lg, trial_lb, trial_la, trial_hr, trial_hg, trial_hb, trial_ha; + + if (flags & cPackBC7FlagPBitOptMode6) + { + const float q = 1.0f / 255.0f; + float sxl[4] = { xl[0] * q, xl[1] * q, xl[2] * q, xl[3] * q }; + float sxh[4] = { xh[0] * q, xh[1] * q, xh[2] * q, xh[3] * q }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_unique_pbits(4, 7, sxl, sxh, bestMinColor, bestMaxColor, best_pbits); + + trial_p0 = best_pbits[0], trial_p1 = best_pbits[1]; + trial_lr = bestMinColor.r, trial_lg = bestMinColor.g, trial_lb = bestMinColor.b, trial_la = bestMinColor.a; + trial_hr = bestMaxColor.r, trial_hg = bestMaxColor.g, trial_hb = bestMaxColor.b, trial_ha = bestMaxColor.a; + } + else + { + trial_p0 = (xl[3] >= 129.0f); + trial_lr = to_7(xl[0], trial_p0); + trial_lg = to_7(xl[1], trial_p0); + trial_lb = to_7(xl[2], trial_p0); + trial_la = to_7(xl[3], trial_p0); + + trial_p1 = (xh[3] >= 129.0f); + trial_hr = to_7(xh[0], trial_p1); + trial_hg = to_7(xh[1], trial_p1); + trial_hb = to_7(xh[2], trial_p1); + trial_ha = to_7(xh[3], trial_p1); + } + + uint8_t trial_weights[16]; + uint32_t mode6_ls_actual_sse = eval_weights_mode6_rgba_sse(pPixels, trial_weights, + trial_lr, trial_lg, trial_lb, trial_la, trial_p0, + trial_hr, trial_hg, trial_hb, trial_ha, trial_p1); + + if (mode6_ls_actual_sse < mode6_actual_sse) + { + mode6_actual_sse = mode6_ls_actual_sse; + memcpy(cur_weights, trial_weights, 16); + p0 = trial_p0; p1 = trial_p1; + lr = trial_lr; lg = trial_lg; lb = trial_lb; la = trial_la; + hr = trial_hr; hg = trial_hg; hb = trial_hb; ha = trial_ha; + } + } + } + + uint32_t mode7_actual_sse = UINT32_MAX; + uint8_t mode7_candidate_block[sizeof(basist::bc7_block)]; + + uint32_t mode45_actual_sse = UINT32_MAX; + uint8_t mode45_candidate_block[sizeof(basist::bc7_block)]; + + if (mode6_actual_sse) + { + if (non_analytical_flag) + { + // No gates: very expensive. + if (flags & cPackBC7FlagUse2SubsetsRGBA) + { + pack_mode7_rgba(mode7_candidate_block, pPixels, x1, y1, z1, w1, mean_r, mean_g, mean_b, mean_a, 1e+9f, flags, nullptr, &mode7_actual_sse); + } + + if (flags & cPackBC7FlagUseDualPlaneRGBA) + pack_mode4_or_5(mode45_candidate_block, pPixels, (desired_dp_chan >= 0) ? desired_dp_chan : 3, 1e+9f, flags, nullptr, &mode45_actual_sse); // todo: determine best def channel here + } + else + { + float mode6_ortho_ratio; + const float mode6_slam_to_line_sse_est = estimate_slam_to_line_sse_4D(cov4, x1, y1, z1, w1, &mode6_ortho_ratio); + + if ((flags & cPackBC7FlagUse2SubsetsRGBA) && (block_max_var4 >= MIN_BLOCK_MAX_VAR_23SUBSETS_RGBA) && (mode6_ortho_ratio > ORTHO_RATIO_23SUBSET_RATIO_THRESH_RGBA)) + { + const bool high_ortho_energy_flag = (mode6_slam_to_line_sse_est >= HIGH_ORTHO_ENERGY_THRESH_RGBA); + + if (high_ortho_energy_flag) + { +#if BASISU_BC7F_PERF_STATS + g_total_high_ortho_energy++; +#endif + pack_mode7_rgba(mode7_candidate_block, pPixels, x1, y1, z1, w1, mean_r, mean_g, mean_b, mean_a, 1e+9f, flags, nullptr, &mode7_actual_sse); + } + } + + if (desired_dp_chan >= 0) + pack_mode4_or_5(mode45_candidate_block, pPixels, desired_dp_chan, 1e+9f, flags, nullptr, &mode45_actual_sse); + } + } + + const uint32_t best_actual_sse = basisu::minimum(mode6_actual_sse, mode45_actual_sse, mode7_actual_sse); + + if ((mode45_actual_sse != UINT32_MAX) && (best_actual_sse == mode45_actual_sse)) + { + memcpy(pBlock, mode45_candidate_block, sizeof(basist::bc7_block)); + } + else if ((mode7_actual_sse != UINT32_MAX) && (best_actual_sse == mode7_actual_sse)) + { + memcpy(pBlock, mode7_candidate_block, sizeof(basist::bc7_block)); + } + else + { + assert(mode6_actual_sse == best_actual_sse); + + encode_mode6_rgba_block(pBlock, + lr, lg, lb, la, p0, + hr, hg, hb, ha, p1, + cur_weights); + } + +#ifdef _DEBUG + { + // Final sanity checking. + uint32_t expected_actual_sse = calc_sse(pBlock, pPixels); + assert(expected_actual_sse == best_actual_sse); + } +#endif + + return best_actual_sse; + } + + // Routes to either rgb or rgba automatically + uint32_t fast_pack_bc7_auto_rgba(uint8_t* pBlock, const color_rgba* pPixels, uint32_t flags) + { + for (uint32_t i = 0; i < 16; i += 4) + { + if ((pPixels[i].a < 255) || (pPixels[i + 1].a < 255) || (pPixels[i + 2].a < 255) || (pPixels[i + 3].a < 255)) + { + if (flags & cPackBC7FlagPartiallyAnalyticalRGBA) + return bc7f::fast_pack_bc7_rgba_partial_analytical(pBlock, pPixels, flags); + else + { + bc7f::fast_pack_bc7_rgba_analytical(pBlock, pPixels, flags); + return 0; + } + } + } + + if (flags & cPackBC7FlagPartiallyAnalyticalRGB) + return bc7f::fast_pack_bc7_rgb_partial_analytical(pBlock, pPixels, flags); + else + { + bc7f::fast_pack_bc7_rgb_analytical(pBlock, pPixels, flags); + return 0; + } + } + + // Source block cannot have alpha. + uint32_t fast_pack_bc7_auto_rgb(uint8_t* pBlock, const color_rgba* pPixels, uint32_t flags) + { +// Disabling this check here, because during fuzzing the ktx2 file may lie. In this case it's harmless to output an opaque block. +#if 0 +#if defined(DEBUG) || defined(_DEBUG) + for (uint32_t i = 0; i < 16; i += 4) + { + if ((pPixels[i].a < 255) || (pPixels[i + 1].a < 255) || (pPixels[i + 2].a < 255) || (pPixels[i + 3].a < 255)) + { + // Block can't have alpha here, or the solid color detectors may misfire. + assert(0); + } + } +#endif +#endif + + if (flags & cPackBC7FlagPartiallyAnalyticalRGB) + return bc7f::fast_pack_bc7_rgb_partial_analytical(pBlock, pPixels, flags); + else + { + bc7f::fast_pack_bc7_rgb_analytical(pBlock, pPixels, flags); + return 0; + } + } + + void clear_perf_stats() + { +#if BASISU_BC7F_PERF_STATS +#define BU_CLEAR_BLOCK_STAT(x) x = 0; + BU_CLEAR_BLOCK_STAT(g_total_rgb_calls); + BU_CLEAR_BLOCK_STAT(g_total_rgba_calls); + BU_CLEAR_BLOCK_STAT(g_total_solid_blocks); + BU_CLEAR_BLOCK_STAT(g_total_trivial_mode6_blocks); + BU_CLEAR_BLOCK_STAT(g_total_dp_valid_chans_rgb); + BU_CLEAR_BLOCK_STAT(g_total_dp_valid_chans_a); + BU_CLEAR_BLOCK_STAT(g_total_high_ortho_energy); + BU_CLEAR_BLOCK_STAT(g_total_mode02_evals); + BU_CLEAR_BLOCK_STAT(g_total_mode02_bailouts); + BU_CLEAR_BLOCK_STAT(g_total_mode13_evals); + BU_CLEAR_BLOCK_STAT(g_total_mode13_bailouts); + BU_CLEAR_BLOCK_STAT(g_total_mode45_evals); + BU_CLEAR_BLOCK_STAT(g_total_mode45_bailouts); + BU_CLEAR_BLOCK_STAT(g_total_mode7_evals); + BU_CLEAR_BLOCK_STAT(g_total_mode7_bailouts); +#undef BU_CLEAR_BLOCK_STAT +#endif + } + + void print_perf_stats() + { +#if BASISU_BC7F_PERF_STATS + const uint32_t total_bc7_blocks = g_total_rgb_calls + g_total_rgba_calls; + if (!total_bc7_blocks) + return; + +#define BU_PRINT_BLOCK_STAT(x) printf(#x ": %u %3.2f%%\n", (uint32_t)x, static_cast(x) * 100.0f / (float)total_bc7_blocks); + BU_PRINT_BLOCK_STAT(g_total_rgb_calls); + BU_PRINT_BLOCK_STAT(g_total_rgba_calls); + BU_PRINT_BLOCK_STAT(g_total_solid_blocks); + BU_PRINT_BLOCK_STAT(g_total_trivial_mode6_blocks); + BU_PRINT_BLOCK_STAT(g_total_dp_valid_chans_rgb); + BU_PRINT_BLOCK_STAT(g_total_dp_valid_chans_a); + BU_PRINT_BLOCK_STAT(g_total_high_ortho_energy); + BU_PRINT_BLOCK_STAT(g_total_mode02_evals); + BU_PRINT_BLOCK_STAT(g_total_mode02_bailouts); + BU_PRINT_BLOCK_STAT(g_total_mode13_evals); + BU_PRINT_BLOCK_STAT(g_total_mode13_bailouts); + BU_PRINT_BLOCK_STAT(g_total_mode45_evals); + BU_PRINT_BLOCK_STAT(g_total_mode45_bailouts); + BU_PRINT_BLOCK_STAT(g_total_mode7_evals); + BU_PRINT_BLOCK_STAT(g_total_mode7_bailouts); +#undef BU_PRINT_BLOCK_STAT + +#endif + } + +#if 0 + struct bc7_mode_6 + { + struct + { + uint64_t m_mode : 7; + uint64_t m_r0 : 7; + uint64_t m_r1 : 7; + uint64_t m_g0 : 7; + uint64_t m_g1 : 7; + uint64_t m_b0 : 7; + uint64_t m_b1 : 7; + uint64_t m_a0 : 7; + uint64_t m_a1 : 7; + uint64_t m_p0 : 1; + } m_lo; + + union + { + struct + { + uint64_t m_p1 : 1; + uint64_t m_s00 : 3; + uint64_t m_s10 : 4; + uint64_t m_s20 : 4; + uint64_t m_s30 : 4; + + uint64_t m_s01 : 4; + uint64_t m_s11 : 4; + uint64_t m_s21 : 4; + uint64_t m_s31 : 4; + + uint64_t m_s02 : 4; + uint64_t m_s12 : 4; + uint64_t m_s22 : 4; + uint64_t m_s32 : 4; + + uint64_t m_s03 : 4; + uint64_t m_s13 : 4; + uint64_t m_s23 : 4; + uint64_t m_s33 : 4; + + } m_hi; + + uint64_t m_hi_bits; + }; + }; +#endif + +#if 0 + // Very basic ASTC LDR 4x4 packer which transcodes BC7 mode 6 RGB/RGBA only to ASTC LDR 4x4. + void fast_pack_astc(void* pBlock, const color_rgba* pPixels) + { + astc_helpers::astc_block* pDst_block = (astc_helpers::astc_block*)pBlock; + + astc_helpers::log_astc_block log_blk; + log_blk.clear(); + log_blk.m_grid_width = 4; + log_blk.m_grid_height = 4; + + const uint32_t fc = *(const uint32_t*)&pPixels[0]; + if (fc == *(const uint32_t*)&pPixels[15]) + { + int k; + for (k = 1; k < 15; k++) + if (*(const uint32_t*)&pPixels[k] != fc) + break; + + if (k == 15) + { + const uint32_t r = pPixels[0].r, g = pPixels[0].g, b = pPixels[0].b, a = pPixels[0].a; + + log_blk.m_solid_color_flag_ldr = true; + log_blk.m_solid_color[0] = (uint16_t)(r | ((uint32_t)r << 8)); + log_blk.m_solid_color[1] = (uint16_t)(g | ((uint32_t)g << 8)); + log_blk.m_solid_color[2] = (uint16_t)(b | ((uint32_t)b << 8)); + log_blk.m_solid_color[3] = (uint16_t)(a | ((uint32_t)a << 8)); + + bool pack_status = astc_helpers::pack_astc_block(*pDst_block, log_blk); + assert(pack_status); + BASISU_NOTE_UNUSED(pack_status); + + return; + } + } + + basist::bc7_block bc7_block; + fast_pack_bc7_auto_rgba((uint8_t*)&bc7_block, pPixels, cPackBC7FlagPBitOpt | cPackBC7FlagPBitOptMode6 | cPackBC7FlagUseTrivialMode6); + + assert(bc7u::determine_bc7_mode(&bc7_block) == 6); + + bc7u::bc7_mode_6& mode6 = *(bc7u::bc7_mode_6*)&bc7_block; + + log_blk.m_num_partitions = 1; + + if ((mode6.m_lo.m_a0 < 127) || (mode6.m_lo.m_a1 < 127)) + { + log_blk.m_color_endpoint_modes[0] = 12; + log_blk.m_endpoint_ise_range = astc_helpers::BISE_96_LEVELS; + log_blk.m_weight_ise_range = astc_helpers::BISE_12_LEVELS; + + const auto& endpoint_to_ise = astc_helpers::g_dequant_tables.get_endpoint_tab(log_blk.m_endpoint_ise_range).m_val_to_ise; + const auto& endpoint_from_ise = astc_helpers::g_dequant_tables.get_endpoint_tab(log_blk.m_endpoint_ise_range).m_ISE_to_val; + //const auto& weight_to_ise = astc_helpers::g_dequant_tables.get_weight_tab(log_blk.m_weight_ise_range).m_rank_to_ISE; + + int p0 = mode6.m_lo.m_p0; + int r0 = bc7f::from_7(mode6.m_lo.m_r0, p0); + int g0 = bc7f::from_7(mode6.m_lo.m_g0, p0); + int b0 = bc7f::from_7(mode6.m_lo.m_b0, p0); + int a0 = bc7f::from_7(mode6.m_lo.m_a0, p0); + + int p1 = mode6.m_hi.m_p1; + int r1 = bc7f::from_7(mode6.m_lo.m_r1, p1); + int g1 = bc7f::from_7(mode6.m_lo.m_g1, p1); + int b1 = bc7f::from_7(mode6.m_lo.m_b1, p1); + int a1 = bc7f::from_7(mode6.m_lo.m_a1, p1); + + log_blk.m_endpoints[0] = endpoint_to_ise[r0]; + log_blk.m_endpoints[1] = endpoint_to_ise[r1]; + + log_blk.m_endpoints[2] = endpoint_to_ise[g0]; + log_blk.m_endpoints[3] = endpoint_to_ise[g1]; + + log_blk.m_endpoints[4] = endpoint_to_ise[b0]; + log_blk.m_endpoints[5] = endpoint_to_ise[b1]; + + log_blk.m_endpoints[6] = endpoint_to_ise[a0]; + log_blk.m_endpoints[7] = endpoint_to_ise[a1]; + + int s0 = endpoint_from_ise[log_blk.m_endpoints[0]] + endpoint_from_ise[log_blk.m_endpoints[2]] + endpoint_from_ise[log_blk.m_endpoints[4]]; + int s1 = endpoint_from_ise[log_blk.m_endpoints[1]] + endpoint_from_ise[log_blk.m_endpoints[3]] + endpoint_from_ise[log_blk.m_endpoints[5]]; + + int invw = 0; + if (s1 < s0) + { + std::swap(log_blk.m_endpoints[0], log_blk.m_endpoints[1]); + std::swap(log_blk.m_endpoints[2], log_blk.m_endpoints[3]); + std::swap(log_blk.m_endpoints[4], log_blk.m_endpoints[5]); + std::swap(log_blk.m_endpoints[6], log_blk.m_endpoints[7]); + std::swap(g0, g1); + std::swap(b0, b1); + invw = 15; + } + + static const uint8_t s_pWeight_to_ise[16] = { 0, 4, 8, 8, 2, 6, 10, 10, 11, 11, 7, 3, 9, 9, 5, 1 }; + + log_blk.m_weights[0] = s_pWeight_to_ise[mode6.m_hi.m_s00 ^ invw]; + log_blk.m_weights[1] = s_pWeight_to_ise[mode6.m_hi.m_s10 ^ invw]; + log_blk.m_weights[2] = s_pWeight_to_ise[mode6.m_hi.m_s20 ^ invw]; + log_blk.m_weights[3] = s_pWeight_to_ise[mode6.m_hi.m_s30 ^ invw]; + + log_blk.m_weights[4] = s_pWeight_to_ise[mode6.m_hi.m_s01 ^ invw]; + log_blk.m_weights[5] = s_pWeight_to_ise[mode6.m_hi.m_s11 ^ invw]; + log_blk.m_weights[6] = s_pWeight_to_ise[mode6.m_hi.m_s21 ^ invw]; + log_blk.m_weights[7] = s_pWeight_to_ise[mode6.m_hi.m_s31 ^ invw]; + + log_blk.m_weights[8] = s_pWeight_to_ise[mode6.m_hi.m_s02 ^ invw]; + log_blk.m_weights[9] = s_pWeight_to_ise[mode6.m_hi.m_s12 ^ invw]; + log_blk.m_weights[10] = s_pWeight_to_ise[mode6.m_hi.m_s22 ^ invw]; + log_blk.m_weights[11] = s_pWeight_to_ise[mode6.m_hi.m_s32 ^ invw]; + + log_blk.m_weights[12] = s_pWeight_to_ise[mode6.m_hi.m_s03 ^ invw]; + log_blk.m_weights[13] = s_pWeight_to_ise[mode6.m_hi.m_s13 ^ invw]; + log_blk.m_weights[14] = s_pWeight_to_ise[mode6.m_hi.m_s23 ^ invw]; + log_blk.m_weights[15] = s_pWeight_to_ise[mode6.m_hi.m_s33 ^ invw]; + } + else + { + log_blk.m_color_endpoint_modes[0] = 8; + log_blk.m_endpoint_ise_range = astc_helpers::BISE_192_LEVELS; + log_blk.m_weight_ise_range = astc_helpers::BISE_16_LEVELS; + + const auto& endpoint_to_ise = astc_helpers::g_dequant_tables.get_endpoint_tab(log_blk.m_endpoint_ise_range).m_val_to_ise; + const auto& endpoint_from_ise = astc_helpers::g_dequant_tables.get_endpoint_tab(log_blk.m_endpoint_ise_range).m_ISE_to_val; + const auto& weight_to_ise = astc_helpers::g_dequant_tables.get_weight_tab(log_blk.m_weight_ise_range).m_rank_to_ISE; + + int p0 = mode6.m_lo.m_p0; + int r0 = bc7f::from_7(mode6.m_lo.m_r0, p0); + int g0 = bc7f::from_7(mode6.m_lo.m_g0, p0); + int b0 = bc7f::from_7(mode6.m_lo.m_b0, p0); + + int p1 = mode6.m_hi.m_p1; + int r1 = bc7f::from_7(mode6.m_lo.m_r1, p1); + int g1 = bc7f::from_7(mode6.m_lo.m_g1, p1); + int b1 = bc7f::from_7(mode6.m_lo.m_b1, p1); + + log_blk.m_endpoints[0] = endpoint_to_ise[r0]; + log_blk.m_endpoints[1] = endpoint_to_ise[r1]; + + log_blk.m_endpoints[2] = endpoint_to_ise[g0]; + log_blk.m_endpoints[3] = endpoint_to_ise[g1]; + + log_blk.m_endpoints[4] = endpoint_to_ise[b0]; + log_blk.m_endpoints[5] = endpoint_to_ise[b1]; + + int s0 = endpoint_from_ise[log_blk.m_endpoints[0]] + endpoint_from_ise[log_blk.m_endpoints[2]] + endpoint_from_ise[log_blk.m_endpoints[4]]; + int s1 = endpoint_from_ise[log_blk.m_endpoints[1]] + endpoint_from_ise[log_blk.m_endpoints[3]] + endpoint_from_ise[log_blk.m_endpoints[5]]; + + int invw = 0; + if (s1 < s0) + { + std::swap(log_blk.m_endpoints[0], log_blk.m_endpoints[1]); + std::swap(log_blk.m_endpoints[2], log_blk.m_endpoints[3]); + std::swap(log_blk.m_endpoints[4], log_blk.m_endpoints[5]); + invw = 15; + } + + log_blk.m_weights[0] = weight_to_ise[(size_t)(mode6.m_hi.m_s00 ^ invw)]; + log_blk.m_weights[1] = weight_to_ise[(size_t)(mode6.m_hi.m_s10 ^ invw)]; + log_blk.m_weights[2] = weight_to_ise[(size_t)(mode6.m_hi.m_s20 ^ invw)]; + log_blk.m_weights[3] = weight_to_ise[(size_t)(mode6.m_hi.m_s30 ^ invw)]; + + log_blk.m_weights[4] = weight_to_ise[(size_t)(mode6.m_hi.m_s01 ^ invw)]; + log_blk.m_weights[5] = weight_to_ise[(size_t)(mode6.m_hi.m_s11 ^ invw)]; + log_blk.m_weights[6] = weight_to_ise[(size_t)(mode6.m_hi.m_s21 ^ invw)]; + log_blk.m_weights[7] = weight_to_ise[(size_t)(mode6.m_hi.m_s31 ^ invw)]; + + log_blk.m_weights[8] = weight_to_ise[(size_t)(mode6.m_hi.m_s02 ^ invw)]; + log_blk.m_weights[9] = weight_to_ise[(size_t)(mode6.m_hi.m_s12 ^ invw)]; + log_blk.m_weights[10] = weight_to_ise[(size_t)(mode6.m_hi.m_s22 ^ invw)]; + log_blk.m_weights[11] = weight_to_ise[(size_t)(mode6.m_hi.m_s32 ^ invw)]; + + log_blk.m_weights[12] = weight_to_ise[(size_t)(mode6.m_hi.m_s03 ^ invw)]; + log_blk.m_weights[13] = weight_to_ise[(size_t)(mode6.m_hi.m_s13 ^ invw)]; + log_blk.m_weights[14] = weight_to_ise[(size_t)(mode6.m_hi.m_s23 ^ invw)]; + log_blk.m_weights[15] = weight_to_ise[(size_t)(mode6.m_hi.m_s33 ^ invw)]; + } + + bool pack_status = astc_helpers::pack_astc_block(*pDst_block, log_blk); + assert(pack_status); + BASISU_NOTE_UNUSED(pack_status); + } +#endif + +} // namespace bc7f + +namespace etc1f +{ +#include "basisu_etc1_mods.inl" + + // flip 0: + // 0011 + // 0011 + // 0011 + // 0011 + + // flip 1: + // 0000 + // 0000 + // 1111 + // 1111 + + uint8_t g_nearest5[256], g_nearest4[256]; + + const uint32_t NUM_SOLID_MODS = 4; + + uint8_t g_solid8_5_base[256][NUM_SOLID_MODS][4]; // [desired8][mod][sel] + uint8_t g_solid8_5_err[256][NUM_SOLID_MODS][4]; + uint8_t g_solid8_4_base[256][NUM_SOLID_MODS][4]; + uint8_t g_solid8_4_err[256][NUM_SOLID_MODS][4]; + + uint8_t g_solid_grayscale_etc1_blocks[256][8]; + + inline int expand5(int v5) + { + return (v5 << 3) | (v5 >> 2); + } + + inline int expand4(int v4) + { + return (v4 << 4) | v4; + } + + static inline int dequant4(uint32_t v) + { + assert(v < 16); + return (v << 4) | v; + } + + static inline int dequant5(uint32_t v) + { + assert(v < 32); + return (v << 3) | (v >> 2); + } + + void init() + { + for (int i = 0; i < 256; i++) + { + int best_e = INT_MAX, best_idx = 0; + + for (int s = 0; s < 32; s++) + { + int recovered = (s << 3) | (s >> 2); + int e = basisu::iabs(recovered - i); + if (e < best_e) + { + best_e = e; + best_idx = s; + } + } + + g_nearest5[i] = (uint8_t)best_idx; + } + + for (int i = 0; i < 256; i++) + { + int best_e = INT_MAX, best_idx = 0; + + for (int s = 0; s < 16; s++) + { + int recovered = (s << 4) | s; + int e = basisu::iabs(recovered - i); + if (e < best_e) + { + best_e = e; + best_idx = s; + } + } + + g_nearest4[i] = (uint8_t)best_idx; + } + + for (uint32_t desired8 = 0; desired8 < 256; desired8++) + { + for (uint32_t mod = 0; mod < NUM_SOLID_MODS; mod++) + { + for (uint32_t sel = 0; sel < 4; sel++) + { + int32_t best_err = INT32_MAX; + uint32_t best_base = 0; + + for (uint32_t b = 0; b < 32; b++) + { + int val = basisu::clamp(dequant5(b) + g_etc1_inten_tables[mod][sel], 0, 255); + int err = basisu::iabs(val - desired8); + + if (err < best_err) + { + best_err = err; + best_base = b; + if (!best_err) + break; + } + + } // b + + g_solid8_5_base[desired8][mod][sel] = (uint8_t)best_base; + g_solid8_5_err[desired8][mod][sel] = (uint8_t)basisu::minimum(255, best_err * best_err); + + } // sel + + } // mod + + } // desired8 + + for (uint32_t desired8 = 0; desired8 < 256; desired8++) + { + for (uint32_t mod = 0; mod < NUM_SOLID_MODS; mod++) + { + for (uint32_t sel = 0; sel < 4; sel++) + { + int32_t best_err = INT32_MAX; + uint32_t best_base = 0; + + for (uint32_t b = 0; b < 16; b++) + { + int val = basisu::clamp(dequant4(b) + g_etc1_inten_tables[mod][sel], 0, 255); + int err = basisu::iabs(val - desired8); + + if (err < best_err) + { + best_err = err; + best_base = b; + if (!best_err) + break; + } + + } // b + + g_solid8_4_base[desired8][mod][sel] = (uint8_t)best_base; + g_solid8_4_err[desired8][mod][sel] = (uint8_t)basisu::minimum(255, best_err * best_err); + + } // sel + + } // mod + + } // desired8 + + pack_etc1_state pack_state; + + for (uint32_t i = 0; i <= 255; i++) + { + etc1f::pack_etc1_solid(&g_solid_grayscale_etc1_blocks[i][0], color_rgba(i, i, i, 255), pack_state, true); + } + } + + inline int dequant_d3(int8_t v) + { + assert(v <= 7); + return (int8_t(v << 5) >> 5); + } + + void get_block_colors(uint8_t* pBlock, color_rgba* pColors0, color_rgba* pColors1) + { + const uint32_t b0 = pBlock[0], b1 = pBlock[1], b2 = pBlock[2], b3 = pBlock[3]; + + int base8_r[2], base8_g[2], base8_b[2]; + + if (b3 & 2) + { + // diff mode + base8_r[0] = dequant5(b0 >> 3); + base8_r[1] = dequant5(basisu::clamp((b0 >> 3) + dequant_d3(b0 & 7), 0, 31)); + + base8_g[0] = dequant5(b1 >> 3); + base8_g[1] = dequant5(basisu::clamp((b1 >> 3) + dequant_d3(b1 & 7), 0, 31)); + + base8_b[0] = dequant5(b2 >> 3); + base8_b[1] = dequant5(basisu::clamp((b2 >> 3) + dequant_d3(b2 & 7), 0, 31)); + } + else + { + // abs mode + base8_r[0] = dequant4(b0 >> 4); + base8_r[1] = dequant4(b0 & 15); + + base8_g[0] = dequant4(b1 >> 4); + base8_g[1] = dequant4(b1 & 15); + + base8_b[0] = dequant4(b2 >> 4); + base8_b[1] = dequant4(b2 & 15); + } + + const int* pInten_table0 = &g_etc1_inten_tables[b3 >> 5][0]; + const int* pInten_table1 = &g_etc1_inten_tables[(b3 >> 2) & 7][0]; + + for (uint32_t i = 0; i < 4; i++) + { + const int d = pInten_table0[i]; + pColors0[i].r = (uint8_t)clamp255(base8_r[0] + d); + pColors0[i].g = (uint8_t)clamp255(base8_g[0] + d); + pColors0[i].b = (uint8_t)clamp255(base8_b[0] + d); + pColors0[i].a = 0; + } + + for (uint32_t i = 0; i < 4; i++) + { + const int d = pInten_table1[i]; + pColors1[i].r = (uint8_t)clamp255(base8_r[1] + d); + pColors1[i].g = (uint8_t)clamp255(base8_g[1] + d); + pColors1[i].b = (uint8_t)clamp255(base8_b[1] + d); + pColors1[i].a = 0; + } + } + + void get_block_colors_y(uint8_t* pBlock, uint8_t* pColors0, uint8_t* pColors1) + { + //const uint32_t b0 = pBlock[0], b1 = pBlock[1], b2 = pBlock[2], b3 = pBlock[3]; + const uint32_t b0 = pBlock[0], b3 = pBlock[3]; + + int base8_y[2]; + + if (b3 & 2) + { + // diff mode + base8_y[0] = dequant5(b0 >> 3); + base8_y[1] = dequant5(basisu::clamp((b0 >> 3) + dequant_d3(b0 & 7), 0, 31)); + } + else + { + // abs mode + base8_y[0] = dequant4(b0 >> 4); + base8_y[1] = dequant4(b0 & 15); + } + + const int* pInten_table0 = g_etc1_inten_tables[b3 >> 5]; + const int* pInten_table1 = g_etc1_inten_tables[(b3 >> 2) & 7]; + + for (uint32_t i = 0; i < 4; i++) + { + const int d = pInten_table0[i]; + pColors0[i] = (uint8_t)clamp255(base8_y[0] + d); + } + + for (uint32_t i = 0; i < 4; i++) + { + const int d = pInten_table1[i]; + pColors1[i] = (uint8_t)clamp255(base8_y[1] + d); + } + } + + static inline int q4_floor(int x) { return (x * 15) / 255; } + static inline int q5_floor(int x) { return (x * 31) / 255; } + + void corr_round_555(int R, int G, int B, int& qR, int& qG, int& qB) + { + int rL = q5_floor(R), gL = q5_floor(G), bL = q5_floor(B); + int rH = (rL < 31) ? (rL + 1) : 31; + int gH = (gL < 31) ? (gL + 1) : 31; + int bH = (bL < 31) ? (bL + 1) : 31; + + int r8[2] = { expand5(rL), expand5(rH) }; + int g8[2] = { expand5(gL), expand5(gH) }; + int b8[2] = { expand5(bL), expand5(bH) }; + + int tr = r8[0], tg = g8[0], tb = b8[0]; + int eR = R - tr, eG = G - tg, eB = B - tb; + int bestJ = basisu::squarei(eR - eG) + basisu::squarei(eG - eB) + basisu::squarei(eB - eR); + int br = tr, bg = tg, bb = tb; + + for (int m = 1; m < 8; ++m) + { + tr = r8[m & 1], tg = g8[(m >> 1) & 1], tb = b8[(m >> 2) & 1]; + eR = R - tr, eG = G - tg, eB = B - tb; + + int J = basisu::squarei(eR - eG) + basisu::squarei(eG - eB) + basisu::squarei(eB - eR); + if (J < bestJ) + { + bestJ = J; + br = tr; + bg = tg; + bb = tb; + } + } + + qR = br >> 3; qG = bg >> 3; qB = bb >> 3; + } + + void corr_round_444(int R, int G, int B, int& qR, int& qG, int& qB) + { + int rL = q4_floor(R), gL = q4_floor(G), bL = q4_floor(B); + int rH = (rL < 15) ? (rL + 1) : 15; + int gH = (gL < 15) ? (gL + 1) : 15; + int bH = (bL < 15) ? (bL + 1) : 15; + + int r8[2] = { expand4(rL), expand4(rH) }; + int g8[2] = { expand4(gL), expand4(gH) }; + int b8[2] = { expand4(bL), expand4(bH) }; + + int tr = r8[0], tg = g8[0], tb = b8[0]; + int eR = R - tr, eG = G - tg, eB = B - tb; + int bestJ = basisu::squarei(eR - eG) + basisu::squarei(eG - eB) + basisu::squarei(eB - eR); + int br = tr, bg = tg, bb = tb; + + for (int m = 1; m < 8; ++m) + { + tr = r8[m & 1], tg = g8[(m >> 1) & 1], tb = b8[(m >> 2) & 1]; + eR = R - tr, eG = G - tg, eB = B - tb; + + int J = basisu::squarei(eR - eG) + basisu::squarei(eG - eB) + basisu::squarei(eB - eR); + if (J < bestJ) + { + bestJ = J; + br = tr; + bg = tg; + bb = tb; + } + } + + qR = br >> 4; qG = bg >> 4; qB = bb >> 4; + } + + inline bool quantize_444_color_correlated(int mean8_r, int mean8_g, int mean8_b, int enc_color[3], bool early_out = true) + { + // Floor to low 4-bit + int r4_low = (mean8_r * 15) / 255; + int g4_low = (mean8_g * 15) / 255; + int b4_low = (mean8_b * 15) / 255; + + // High = +1, clamped + int r4_high = basisu::clamp(r4_low + 1, 0, 15); + int g4_high = basisu::clamp(g4_low + 1, 0, 15); + int b4_high = basisu::clamp(b4_low + 1, 0, 15); + + const int r8_low = expand4(r4_low); + const int g8_low = expand4(g4_low); + const int b8_low = expand4(b4_low); + const int r8_high = expand4(r4_high); + const int g8_high = expand4(g4_high); + const int b8_high = expand4(b4_high); + + // Errors if we pick "low" + const float dr = float(r8_low) - mean8_r; + const float dg = float(g8_low) - mean8_g; + const float db = float(b8_low) - mean8_b; + + const float dRG = fabsf(dr - dg); + const float dRB = fabsf(dr - db); + const float dGB = fabsf(dg - db); + const float maxSpread = basisu::maximum(dRG, dRB, dGB); + + if ((early_out) && (maxSpread <= 1.0f)) + return false; + + // Step sizes low->high + const float kr = float(r8_high - r8_low); + const float kg = float(g8_high - g8_low); + const float kb = float(b8_high - b8_low); + + // Precompute constants for cost(mask) = S0 + Lsum - Ksum^2 + const float D = dr + dg + db; + const float S0 = 3.0f * (dr * dr + dg * dg + db * db) - D * D; + + const float Lr = 6.0f * dr * kr + 3.0f * kr * kr - 2.0f * D * kr; + const float Lg = 6.0f * dg * kg + 3.0f * kg * kg - 2.0f * D * kg; + const float Lb = 6.0f * db * kb + 3.0f * kb * kb - 2.0f * D * kb; + + float bestCost = basisu::BIG_FLOAT_VAL; + int bestMask = 0; + + for (int mask = 0; mask < 8; ++mask) + { + const float Ksum = ((mask & 1) ? kr : 0.0f) + ((mask & 2) ? kg : 0.0f) + ((mask & 4) ? kb : 0.0f); + const float Lsum = ((mask & 1) ? Lr : 0.0f) + ((mask & 2) ? Lg : 0.0f) + ((mask & 4) ? Lb : 0.0f); + + const float cost = S0 + Lsum - Ksum * Ksum; + + if (cost < bestCost) + { + bestCost = cost; + bestMask = mask; + } + } + + enc_color[0] = (bestMask & 1) ? r4_high : r4_low; + enc_color[1] = (bestMask & 2) ? g4_high : g4_low; + enc_color[2] = (bestMask & 4) ? b4_high : b4_low; + + assert((enc_color[0]) >= 0 && (enc_color[0] <= 15)); + assert((enc_color[1]) >= 0 && (enc_color[1] <= 15)); + assert((enc_color[2]) >= 0 && (enc_color[2] <= 15)); + + return true; + } + + void pack_etc1_solid(uint8_t* pBlock, const color_rgba& color, pack_etc1_state& state, bool init_flag) + { + uint32_t r8 = color[0], g8 = color[1], b8 = color[2]; + //const uint32_t r8 = 0, g8 = 0, b8 = 0; + + if (!init_flag) + { + if ((r8 == g8) && (r8 == b8)) + { + memcpy(pBlock, &g_solid_grayscale_etc1_blocks[r8][0], sizeof(decoder_etc_block)); + return; + } + + if ((state.m_prev_solid_r8 == (int)r8) && (state.m_prev_solid_g8 == (int)g8) && (state.m_prev_solid_b8 == (int)b8)) + { + memcpy(pBlock, &state.m_prev_solid_block, sizeof(decoder_etc_block)); + return; + } + } + + uint32_t best_err = UINT32_MAX; + uint32_t best_mod = 0, best_sel = 0; + uint32_t best4_flag = false; + + const int RW = 2, GW = 4; + + for (uint32_t mod = 0; mod < NUM_SOLID_MODS; mod++) + { + for (uint32_t sel = 0; sel < 4; sel++) + { + uint32_t total_err5 = RW * g_solid8_5_err[r8][mod][sel] + GW * g_solid8_5_err[g8][mod][sel] + g_solid8_5_err[b8][mod][sel]; + if (total_err5 < best_err) + { + best_err = total_err5; + best_mod = mod; + best_sel = sel; + best4_flag = false; + if (!best_err) + goto etc1_solid_done; + } + + uint32_t total_err4 = RW * g_solid8_4_err[r8][mod][sel] + GW * g_solid8_4_err[g8][mod][sel] + g_solid8_4_err[b8][mod][sel]; + if (total_err4 < best_err) + { + best_err = total_err4; + best_mod = mod; + best_sel = sel; + best4_flag = true; + } + + } // sel + } // mod + + etc1_solid_done: + + if (best4_flag) + { + const uint32_t best_base_r4 = g_solid8_4_base[r8][best_mod][best_sel]; + const uint32_t best_base_g4 = g_solid8_4_base[g8][best_mod][best_sel]; + const uint32_t best_base_b4 = g_solid8_4_base[b8][best_mod][best_sel]; + + pBlock[0] = (uint8_t)(best_base_r4 | (best_base_r4 << 4)); + pBlock[1] = (uint8_t)(best_base_g4 | (best_base_g4 << 4)); + pBlock[2] = (uint8_t)(best_base_b4 | (best_base_b4 << 4)); + } + else + { + pBlock[0] = (uint8_t)(g_solid8_5_base[r8][best_mod][best_sel] << 3); + pBlock[1] = (uint8_t)(g_solid8_5_base[g8][best_mod][best_sel] << 3); + pBlock[2] = (uint8_t)(g_solid8_5_base[b8][best_mod][best_sel] << 3); + } + + const uint32_t flip = 0; + const uint32_t diff = (best4_flag == 0); + pBlock[3] = (uint8_t)(flip | (diff << 1) | (best_mod << 5) | (best_mod << 2)); + + const uint32_t etc1_sels = g_selector_index_to_etc1[best_sel]; + + const uint8_t lb = (etc1_sels & 2) ? 0xFF : 0; + pBlock[4] = lb; + pBlock[5] = lb; + + const uint8_t hb = (etc1_sels & 1) ? 0xFF : 0; + pBlock[6] = hb; + pBlock[7] = hb; + + state.m_prev_solid_r8 = r8; + state.m_prev_solid_g8 = g8; + state.m_prev_solid_b8 = b8; + memcpy(&state.m_prev_solid_block, pBlock, sizeof(decoder_etc_block)); + } + + static const uint8_t s_vi[16] = { 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1 }; + static const uint8_t s_hi[16] = { 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3 }; + + static const uint8_t s_subsets[2][16] = + { + { 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1 }, + { 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1 } + }; + + // [flip][subblock][sel][l/h] + static const uint16_t s_sel_bitmasks[2 * 2 * 4][2] = + { + // flip=0, subblock=0, sels=0-3 + { 0xff, 0xff }, + { 0x0, 0xff }, + { 0x0, 0x0 }, + { 0xff, 0x0 }, + + // flip=0, subblock=1, sels=0-3 + { 0xff00, 0xff00 }, + { 0x0, 0xff00 }, + { 0x0, 0x0 }, + { 0xff00, 0x0 }, + + // flip=1, subblock=0, sels=0-3 + { 0x3333, 0x3333 }, + { 0x0, 0x3333 }, + { 0x0, 0x0 }, + { 0x3333, 0x0 }, + + // flip=1, subblock=1, sels=0-3 + { 0xcccc, 0xcccc }, + { 0x0, 0xcccc }, + { 0x0, 0x0 }, + { 0xcccc, 0x0 } + }; + + void pack_etc1_solid_subblocks(uint8_t* pBlock, const color_rgba* pPixels, const color_rgba subblock_means[2], uint32_t flip) + { + (void)pPixels; + + uint32_t best_mod5[2] = {}, best_sel5[2] = {}; + uint32_t best_base5[2][3] = {}; + uint32_t best_err5[2] = { UINT32_MAX, UINT32_MAX }; + + uint32_t best_mod4[2] = {}, best_sel4[2] = {}; + uint32_t best_base4[2][3] = {}; + uint32_t best_err4[2] = { UINT32_MAX, UINT32_MAX }; + + const int RW = 2, GW = 4; + + for (uint32_t t = 0; t < 2; t++) + { + const uint32_t r8 = subblock_means[t][0]; + const uint32_t g8 = subblock_means[t][1]; + const uint32_t b8 = subblock_means[t][2]; + + const uint8_t* pR5 = &g_solid8_5_err[r8][0][0]; + const uint8_t* pG5 = &g_solid8_5_err[g8][0][0]; + const uint8_t* pB5 = &g_solid8_5_err[b8][0][0]; + + const uint8_t* pR4 = &g_solid8_4_err[r8][0][0]; + const uint8_t* pG4 = &g_solid8_4_err[g8][0][0]; + const uint8_t* pB4 = &g_solid8_4_err[b8][0][0]; + + for (uint32_t mod = 0; mod < NUM_SOLID_MODS; mod++) + { + const uint32_t mod4 = mod << 2; + + const uint32_t total_err5_0 = ((RW * pR5[0] + GW * pG5[0] + pB5[0]) << 5) + (mod4 + 0); + const uint32_t total_err5_1 = ((RW * pR5[1] + GW * pG5[1] + pB5[1]) << 5) + (mod4 + 1); + const uint32_t total_err5_2 = ((RW * pR5[2] + GW * pG5[2] + pB5[2]) << 5) + (mod4 + 2); + const uint32_t total_err5_3 = ((RW * pR5[3] + GW * pG5[3] + pB5[3]) << 5) + (mod4 + 3); + + best_err5[t] = basisu::minimum(best_err5[t], basisu::minimum(total_err5_0, total_err5_1), basisu::minimum(total_err5_2, total_err5_3)); + + const uint32_t total_err4_0 = ((RW * pR4[0] + GW * pG4[0] + pB4[0]) << 5) + (mod4 + 0); + const uint32_t total_err4_1 = ((RW * pR4[1] + GW * pG4[1] + pB4[1]) << 5) + (mod4 + 1); + const uint32_t total_err4_2 = ((RW * pR4[2] + GW * pG4[2] + pB4[2]) << 5) + (mod4 + 2); + const uint32_t total_err4_3 = ((RW * pR4[3] + GW * pG4[3] + pB4[3]) << 5) + (mod4 + 3); + + best_err4[t] = basisu::minimum(best_err4[t], basisu::minimum(total_err4_0, total_err4_1), basisu::minimum(total_err4_2, total_err4_3)); + + pR5 += 4; pG5 += 4; pB5 += 4; + pR4 += 4; pG4 += 4; pB4 += 4; + } // mod + + best_mod5[t] = (best_err5[t] >> 2) & 7; + best_sel5[t] = best_err5[t] & 3; + best_err5[t] >>= 5; + + best_mod4[t] = (best_err4[t] >> 2) & 7; + best_sel4[t] = best_err4[t] & 3; + best_err4[t] >>= 5; + + best_base5[t][0] = g_solid8_5_base[r8][best_mod5[t]][best_sel5[t]]; + best_base5[t][1] = g_solid8_5_base[g8][best_mod5[t]][best_sel5[t]]; + best_base5[t][2] = g_solid8_5_base[b8][best_mod5[t]][best_sel5[t]]; + + best_base4[t][0] = g_solid8_4_base[r8][best_mod4[t]][best_sel4[t]]; + best_base4[t][1] = g_solid8_4_base[g8][best_mod4[t]][best_sel4[t]]; + best_base4[t][2] = g_solid8_4_base[b8][best_mod4[t]][best_sel4[t]]; + + } // t + + uint32_t total_err4 = best_err4[0] + best_err4[1]; + uint32_t total_err5 = best_err5[0] + best_err5[1]; + + bool use_abs = false; + if (total_err4 < total_err5) + { + use_abs = true; + } + else + { + int delta_r = best_base5[1][0] - best_base5[0][0]; + int delta_g = best_base5[1][1] - best_base5[0][1]; + int delta_b = best_base5[1][2] - best_base5[0][2]; + + if ((delta_r < -4) || (delta_r > 3) || + (delta_g < -4) || (delta_g > 3) || + (delta_b < -4) || (delta_b > 3)) + { + use_abs = true; + } + } + + uint32_t* pBest_sels; + + if (use_abs) + { + pBlock[0] = (uint8_t)(best_base4[1][0] | (best_base4[0][0] << 4)); + pBlock[1] = (uint8_t)(best_base4[1][1] | (best_base4[0][1] << 4)); + pBlock[2] = (uint8_t)(best_base4[1][2] | (best_base4[0][2] << 4)); + + const uint32_t diff = false; + pBlock[3] = (uint8_t)(flip | (diff << 1) | (best_mod4[0] << 5) | (best_mod4[1] << 2)); + + pBest_sels = best_sel4; + } + else + { + const int delta_r = (best_base5[1][0] - best_base5[0][0]) & 7; + const int delta_g = (best_base5[1][1] - best_base5[0][1]) & 7; + const int delta_b = (best_base5[1][2] - best_base5[0][2]) & 7; + + pBlock[0] = (uint8_t)(delta_r | (best_base5[0][0] << 3)); + pBlock[1] = (uint8_t)(delta_g | (best_base5[0][1] << 3)); + pBlock[2] = (uint8_t)(delta_b | (best_base5[0][2] << 3)); + + const uint32_t diff = 1; + pBlock[3] = (uint8_t)(flip | (diff << 1) | (best_mod5[0] << 5) | (best_mod5[1] << 2)); + + pBest_sels = best_sel5; + } + + uint16_t l_bitmask = 0, h_bitmask = 0; + + for (uint32_t subblock = 0; subblock < 2; subblock++) + { + uint32_t best_etc1_sel = pBest_sels[subblock]; + + l_bitmask |= s_sel_bitmasks[flip * 8 + subblock * 4 + best_etc1_sel][0]; + h_bitmask |= s_sel_bitmasks[flip * 8 + subblock * 4 + best_etc1_sel][1]; + } + + pBlock[7] = (uint8_t)(l_bitmask); + pBlock[6] = (uint8_t)(l_bitmask >> 8); + pBlock[5] = (uint8_t)(h_bitmask); + pBlock[4] = (uint8_t)(h_bitmask >> 8); + } + + //------------------------------------------ + + void pack_etc1(uint8_t* pBlock, const color_rgba* pPixels, pack_etc1_state& state) + { + { + // Solid block check, ignoring alpha. + const uint32_t fc = *(const uint32_t*)&pPixels[0] & BASISD_COLOR_RGBA_RGB_MASK; + + if (fc == (*(const uint32_t*)&pPixels[15] & BASISD_COLOR_RGBA_RGB_MASK)) + { + int k; + for (k = 1; k < 15; k++) + if ((*(const uint32_t*)&pPixels[k] & BASISD_COLOR_RGBA_RGB_MASK) != fc) + break; + + if (k == 15) + { + pack_etc1_solid(pBlock, pPixels[0], state, false); + return; + } + } + } + + // [0]=left, [1]=right, [2]=top, [3]=bottom + int accum_y[4] = { 0 }, accum_y2[4] = { 0 }, accum_c2[4] = { 0 }; + int total_c2 = 0, max_c2 = 0; + + for (uint32_t i = 0; i < 16; i++) + { + int r = pPixels[i].r, g = pPixels[i].g, b = pPixels[i].b; + int rg = r - g, bg = b - g; + + int y = (r + g + b + 1) / 3; + int y2 = y * y, c2 = rg * rg + bg * bg; + + total_c2 += c2; + max_c2 = basisu::maximum(max_c2, c2); + + const int vi = s_vi[i], hi = s_hi[i]; + + accum_y[vi] += y; + accum_y2[vi] += y2; + accum_c2[vi] += c2; + + accum_y[hi] += y; + accum_y2[hi] += y2; + accum_c2[hi] += c2; + + } // i + +#if 1 + // sqrt(300/16)=~4.33 + const int CHROMA_ENERGY_SUM_THRESH = 300; + const int CHROMA_ENERGY_MAX_THRESH = 32; + if ((total_c2 < CHROMA_ENERGY_SUM_THRESH) && (max_c2 < CHROMA_ENERGY_MAX_THRESH)) + { + //memset(pBlock, 0, 8); + //return; + + uint8_t y_pixels[16]; + if (total_c2 == 0) + { + for (uint32_t i = 0; i < 16; i++) + y_pixels[i] = pPixels[i].r; + } + else + { + for (uint32_t i = 0; i < 16; i++) + y_pixels[i] = (uint8_t)pPixels[i].get_709_luma(); + } + pack_etc1_grayscale(pBlock, y_pixels, state); + return; + } +#endif + + int var_y_scaled[4]; // scaled by x64 (8*8) + for (uint32_t i = 0; i < 4; i++) + var_y_scaled[i] = basisu::maximum(0, (accum_y2[i] << 3) - (accum_y[i] * accum_y[i])); // max not needed + + float std_luma[4], std_chroma[4]; + for (uint32_t i = 0; i < 4; i++) + { + std_luma[i] = sqrtf((float)var_y_scaled[i] * (1.0f / 64.0f)); + std_chroma[i] = sqrtf((float)accum_c2[i] * (1.0f / 8.0f)); + } + + const float LUMA_SCALE = 2, CHROMA_SCALE = 1; + float flip0_score = (std_luma[0] + std_luma[1]) * LUMA_SCALE + (std_chroma[0] + std_chroma[1]) * CHROMA_SCALE; + float flip1_score = (std_luma[2] + std_luma[3]) * LUMA_SCALE + (std_chroma[2] + std_chroma[3]) * CHROMA_SCALE; + + const uint32_t flip = flip1_score < flip0_score; + + int var8_y[2] = {}, mean8_y[2] = {}, mean8_r[2] = {}, mean8_g[2] = {}, mean8_b[2] = {}; + int min_y[2] = { INT_MAX, INT_MAX }, max_y[2] = { INT_MIN, INT_MIN }; + + for (uint32_t i = 0; i < 16; i++) + { + const int r = pPixels[i].r, g = pPixels[i].g, b = pPixels[i].b; + const int y = (r + g + b + 1) / 3; + + const uint32_t s = s_subsets[flip][i]; + + var8_y[s] += y * y; + mean8_y[s] += y; + + mean8_r[s] += r; + mean8_g[s] += g; + mean8_b[s] += b; + + min_y[s] = basisu::minimum(min_y[s], y); + max_y[s] = basisu::maximum(max_y[s], y); + } + + //memset(pBlock, 0, sizeof(etc_block)); + //return; + + if (((max_y[0] - min_y[0]) < 8) && ((max_y[1] - min_y[1]) < 8)) + { + color_rgba subblock_means[2] = { + color_rgba((mean8_r[0] + 4) / 8, (mean8_g[0] + 4) / 8, (mean8_b[0] + 4) / 8, 255), + color_rgba((mean8_r[1] + 4) / 8, (mean8_g[1] + 4) / 8, (mean8_b[1] + 4) / 8, 255), + }; + + if (subblock_means[0] == subblock_means[1]) + pack_etc1_solid(pBlock, subblock_means[0], state, false); + else + pack_etc1_solid_subblocks(pBlock, pPixels, subblock_means, flip); + + return; + } + + //memset(pBlock, 0, sizeof(etc_block)); + //return; + + int half_span8_y[2]; + float stddev_y[2]; + + for (uint32_t i = 0; i < 2; i++) + { + var8_y[i] = basisu::maximum(0, (var8_y[i] << 3) - mean8_y[i] * mean8_y[i]); + stddev_y[i] = std::sqrt(static_cast(var8_y[i])) * (1.0f / 8.0f); + + mean8_y[i] = (mean8_y[i] + 4) >> 3; + + mean8_r[i] = (mean8_r[i] + 4) >> 3; + mean8_g[i] = (mean8_g[i] + 4) >> 3; + mean8_b[i] = (mean8_b[i] + 4) >> 3; + + half_span8_y[i] = basisu::maximum(max_y[i] - mean8_y[i], mean8_y[i] - min_y[i]); + } + + int stddev[2] = + { + basisu::clamp((int)ceilf(9.0f * (stddev_y[0] / (float)basisu::maximum(1, half_span8_y[0]))) - 1, 0, 7), + basisu::clamp((int)ceilf(9.0f * (stddev_y[1] / (float)basisu::maximum(1, half_span8_y[1]))) - 1, 0, 7) + }; + + uint32_t mod_tab[2]; + for (uint32_t i = 0; i < 2; i++) + mod_tab[i] = g_etc1_mod_tabs[basisu::clamp(half_span8_y[i], 1, 255)][stddev[i]]; + + int mean5_r[2], mean5_g[2], mean5_b[2]; + + for (uint32_t i = 0; i < 2; i++) + { +#if 1 + corr_round_555(mean8_r[i], mean8_g[i], mean8_b[i], mean5_r[i], mean5_g[i], mean5_b[i]); +#else + mean5_r[i] = g_nearest5[mean8_r[i]]; + mean5_g[i] = g_nearest5[mean8_g[i]]; + mean5_b[i] = g_nearest5[mean8_b[i]]; +#endif + } + + int delta5_r = mean5_r[1] - mean5_r[0]; + int delta5_g = mean5_g[1] - mean5_g[0]; + int delta5_b = mean5_b[1] - mean5_b[0]; + + const uint32_t z = (delta5_r + 4) | (delta5_g + 4) | (delta5_b + 4); + bool use_abs_colors4 = z > 7; + + if (!use_abs_colors4) + { + assert((delta5_r >= -4) && (delta5_r <= 3)); + assert((delta5_g >= -4) && (delta5_g <= 3)); + assert((delta5_b >= -4) && (delta5_b <= 3)); + } + + if (use_abs_colors4) + { + int mean4_r[2], mean4_g[2], mean4_b[2]; + + for (uint32_t i = 0; i < 2; i++) + { +#if 1 + corr_round_444(mean8_r[i], mean8_g[i], mean8_b[i], mean4_r[i], mean4_g[i], mean4_b[i]); +#else + mean4_r[i] = g_nearest4[mean8_r[i]]; + mean4_g[i] = g_nearest4[mean8_g[i]]; + mean4_b[i] = g_nearest4[mean8_b[i]]; +#endif + } // i + + pBlock[0] = (uint8_t)(mean4_r[1] | (mean4_r[0] << 4)); + pBlock[1] = (uint8_t)(mean4_g[1] | (mean4_g[0] << 4)); + pBlock[2] = (uint8_t)(mean4_b[1] | (mean4_b[0] << 4)); + + const uint32_t diff = 0; + pBlock[3] = (uint8_t)(flip | (diff << 1) | (mod_tab[0] << 5) | (mod_tab[1] << 2)); + } + else + { + pBlock[0] = (uint8_t)((delta5_r & 7) | (mean5_r[0] << 3)); + pBlock[1] = (uint8_t)((delta5_g & 7) | (mean5_g[0] << 3)); + pBlock[2] = (uint8_t)((delta5_b & 7) | (mean5_b[0] << 3)); + + const uint32_t diff = 1; + pBlock[3] = (uint8_t)(flip | (diff << 1) | (mod_tab[0] << 5) | (mod_tab[1] << 2)); + } + + uint16_t l_bitmask = 0; + uint16_t h_bitmask = 0; + + static const uint8_t s_tran[4] = { 1, 0, 2, 3 }; + + color_rgba subblock_colors[2][4]; + get_block_colors(pBlock, &subblock_colors[0][0], &subblock_colors[1][0]); + + for (uint32_t subblock = 0; subblock < 2; subblock++) + { + const color_rgba* block_colors = &subblock_colors[subblock][0]; + + uint32_t block_y[4]; + for (uint32_t i = 0; i < 4; i++) + block_y[i] = block_colors[i][0] * 54 + block_colors[i][1] * 183 + block_colors[i][2] * 19; + + const uint32_t block_y01 = block_y[0] + block_y[1]; + const uint32_t block_y12 = block_y[1] + block_y[2]; + const uint32_t block_y23 = block_y[2] + block_y[3]; + + if (flip) + { + uint32_t ofs = subblock * 2; + + for (uint32_t y = 0; y < 2; y++) + { + for (uint32_t x = 0; x < 4; x++) + { + const color_rgba& c = pPixels[x + (subblock * 2 + y) * 4]; + const uint32_t l = c[0] * 108 + c[1] * 366 + c[2] * 38; + + uint32_t t = s_tran[(l < block_y01) + (l < block_y12) + (l < block_y23)]; + + assert(ofs < 16); + l_bitmask |= ((t & 1) << ofs); + h_bitmask |= ((t >> 1) << ofs); + ofs += 4; + } + + ofs = (int)ofs + 1 - 4 * 4; + } + } + else + { + uint32_t ofs = (subblock * 2) * 4; + for (uint32_t x = 0; x < 2; x++) + { + for (uint32_t y = 0; y < 4; y++) + { + const color_rgba& c = pPixels[subblock * 2 + x + y * 4]; + const uint32_t l = c[0] * 108 + c[1] * 366 + c[2] * 38; + + uint32_t t = s_tran[(l < block_y01) + (l < block_y12) + (l < block_y23)]; + + assert(ofs < 16); + l_bitmask |= ((t & 1) << ofs); + h_bitmask |= ((t >> 1) << ofs); + ++ofs; + } + } + } + + pBlock[7] = (uint8_t)(l_bitmask); + pBlock[6] = (uint8_t)(l_bitmask >> 8); + pBlock[5] = (uint8_t)(h_bitmask); + pBlock[4] = (uint8_t)(h_bitmask >> 8); + + } // subblock + } + + void pack_etc1_grayscale_solid_subblocks(uint8_t* pBlock, const uint8_t* pPixels, const uint8_t subblock_means[2], uint32_t flip) + { + (void)pPixels; + + uint32_t best_mod5[2] = {}, best_sel5[2] = {}; + uint32_t best_base5[2] = {}; + uint32_t best_err5[2] = { UINT32_MAX, UINT32_MAX }; + + uint32_t best_mod4[2] = {}, best_sel4[2] = {}; + uint32_t best_base4[2] = {}; + uint32_t best_err4[2] = { UINT32_MAX, UINT32_MAX }; + + for (uint32_t t = 0; t < 2; t++) + { + const uint32_t y8 = subblock_means[t]; + + const uint8_t* pY5 = &g_solid8_5_err[y8][0][0]; + const uint8_t* pY4 = &g_solid8_4_err[y8][0][0]; + + for (uint32_t mod = 0; mod < NUM_SOLID_MODS; mod++) + { + const uint32_t mod4 = mod << 2; + + const uint32_t total_err5_0 = (pY5[0] << 5) + (mod4 + 0); + const uint32_t total_err5_1 = (pY5[1] << 5) + (mod4 + 1); + const uint32_t total_err5_2 = (pY5[2] << 5) + (mod4 + 2); + const uint32_t total_err5_3 = (pY5[3] << 5) + (mod4 + 3); + + best_err5[t] = basisu::minimum(best_err5[t], basisu::minimum(total_err5_0, total_err5_1), basisu::minimum(total_err5_2, total_err5_3)); + + const uint32_t total_err4_0 = (pY4[0] << 5) + (mod4 + 0); + const uint32_t total_err4_1 = (pY4[1] << 5) + (mod4 + 1); + const uint32_t total_err4_2 = (pY4[2] << 5) + (mod4 + 2); + const uint32_t total_err4_3 = (pY4[3] << 5) + (mod4 + 3); + + best_err4[t] = basisu::minimum(best_err4[t], basisu::minimum(total_err4_0, total_err4_1), basisu::minimum(total_err4_2, total_err4_3)); + + pY5 += 4; + pY4 += 4; + } // mod + + best_mod5[t] = (best_err5[t] >> 2) & 7; + best_sel5[t] = best_err5[t] & 3; + best_err5[t] >>= 5; + + best_mod4[t] = (best_err4[t] >> 2) & 7; + best_sel4[t] = best_err4[t] & 3; + best_err4[t] >>= 5; + + best_base5[t] = g_solid8_5_base[y8][best_mod5[t]][best_sel5[t]]; + + best_base4[t] = g_solid8_4_base[y8][best_mod4[t]][best_sel4[t]]; + + } // t + + uint32_t total_err4 = best_err4[0] + best_err4[1]; + uint32_t total_err5 = best_err5[0] + best_err5[1]; + + bool use_abs = false; + if (total_err4 < total_err5) + { + use_abs = true; + } + else + { + int delta_y = best_base5[1] - best_base5[0]; + + if ((delta_y < -4) || (delta_y > 3)) + { + use_abs = true; + } + } + + uint32_t* pBest_sels; + + if (use_abs) + { + pBlock[0] = pBlock[1] = pBlock[2] = (uint8_t)(best_base4[1] | (best_base4[0] << 4)); + + const uint32_t diff = false; + pBlock[3] = (uint8_t)(flip | (diff << 1) | (best_mod4[0] << 5) | (best_mod4[1] << 2)); + + pBest_sels = best_sel4; + } + else + { + const int delta_y = (best_base5[1] - best_base5[0]) & 7; + + pBlock[0] = pBlock[1] = pBlock[2] = (uint8_t)(delta_y | (best_base5[0] << 3)); + + const uint32_t diff = 1; + pBlock[3] = (uint8_t)(flip | (diff << 1) | (best_mod5[0] << 5) | (best_mod5[1] << 2)); + + pBest_sels = best_sel5; + } + + uint16_t l_bitmask = 0, h_bitmask = 0; + + for (uint32_t subblock = 0; subblock < 2; subblock++) + { + uint32_t best_etc1_sel = pBest_sels[subblock]; + + l_bitmask |= s_sel_bitmasks[flip * 8 + subblock * 4 + best_etc1_sel][0]; + h_bitmask |= s_sel_bitmasks[flip * 8 + subblock * 4 + best_etc1_sel][1]; + } + + pBlock[7] = (uint8_t)(l_bitmask); + pBlock[6] = (uint8_t)(l_bitmask >> 8); + pBlock[5] = (uint8_t)(h_bitmask); + pBlock[4] = (uint8_t)(h_bitmask >> 8); + } + + void pack_etc1_grayscale(uint8_t* pBlock, const uint8_t* pPixels, pack_etc1_state& state) + { + (void)state; + + const uint8_t fc = pPixels[0]; + + if (fc == pPixels[15]) + { + int k; + for (k = 1; k < 15; k++) + if (pPixels[k] != fc) + break; + + if (k == 15) + { + memcpy(pBlock, &g_solid_grayscale_etc1_blocks[fc][0], sizeof(decoder_etc_block)); + return; + } + } + + int accum_y[4] = { 0 }, accum_y2[4] = { 0 }; + + for (uint32_t i = 0; i < 16; i++) + { + int y = pPixels[i]; + int y2 = y * y; + + const int vi = s_vi[i], hi = s_hi[i]; + + accum_y[vi] += y; + accum_y2[vi] += y2; + + accum_y[hi] += y; + accum_y2[hi] += y2; + + } // i + + int var_y_scaled[4]; // scaled by x64 (8*8) + for (uint32_t i = 0; i < 4; i++) + var_y_scaled[i] = basisu::maximum(0, (accum_y2[i] << 3) - (accum_y[i] * accum_y[i])); // max not needed + + float std_luma[4]; + for (uint32_t i = 0; i < 4; i++) + std_luma[i] = sqrtf((float)var_y_scaled[i] * (1.0f / 64.0f)); + + float flip0_score = std_luma[0] + std_luma[1]; + float flip1_score = std_luma[2] + std_luma[3]; + + const uint32_t flip = flip1_score < flip0_score; + + int var8_y[2] = {}, mean8_y[2] = {}; + int min_y[2] = { INT_MAX, INT_MAX }, max_y[2] = { INT_MIN, INT_MIN }; + + for (uint32_t i = 0; i < 16; i++) + { + const int y = pPixels[i]; + + const uint32_t s = s_subsets[flip][i]; + + var8_y[s] += y * y; + mean8_y[s] += y; + + min_y[s] = basisu::minimum(min_y[s], y); + max_y[s] = basisu::maximum(max_y[s], y); + } + + if (((max_y[0] - min_y[0]) < 8) && ((max_y[1] - min_y[1]) < 8)) + { + uint8_t subblock_means[2] = { + (uint8_t)((mean8_y[0] + 4) / 8), + (uint8_t)((mean8_y[1] + 4) / 8), + }; + + if (subblock_means[0] == subblock_means[1]) + memcpy(pBlock, &g_solid_grayscale_etc1_blocks[subblock_means[0]][0], sizeof(decoder_etc_block)); + else + pack_etc1_grayscale_solid_subblocks(pBlock, pPixels, subblock_means, flip); + + return; + } + + int half_span8_y[2]; + float stddev_y[2]; + + for (uint32_t i = 0; i < 2; i++) + { + var8_y[i] = basisu::maximum(0, (var8_y[i] << 3) - mean8_y[i] * mean8_y[i]); + stddev_y[i] = std::sqrt(static_cast(var8_y[i])) * (1.0f / 8.0f); + + mean8_y[i] = (mean8_y[i] + 4) >> 3; + + half_span8_y[i] = basisu::maximum(max_y[i] - mean8_y[i], mean8_y[i] - min_y[i]); + } + + int stddev[2] = + { + basisu::clamp((int)ceilf(9.0f * (stddev_y[0] / (float)basisu::maximum(1, half_span8_y[0]))) - 1, 0, 7), + basisu::clamp((int)ceilf(9.0f * (stddev_y[1] / (float)basisu::maximum(1, half_span8_y[1]))) - 1, 0, 7) + }; + + uint32_t mod_tab[2]; + for (uint32_t i = 0; i < 2; i++) + mod_tab[i] = etc1f::g_etc1_mod_tabs[basisu::clamp(half_span8_y[i], 1, 255)][stddev[i]]; + + int mean5_y[2]; + + for (uint32_t i = 0; i < 2; i++) + mean5_y[i] = g_nearest5[mean8_y[i]]; + + int delta5_y = mean5_y[1] - mean5_y[0]; + + const uint32_t z = delta5_y + 4; + bool use_abs_colors4 = z > 7; + + if (!use_abs_colors4) + { + assert((delta5_y >= -4) && (delta5_y <= 3)); + } + + if (use_abs_colors4) + { + int mean4_y[2]; + + for (uint32_t i = 0; i < 2; i++) + { + mean4_y[i] = g_nearest4[mean8_y[i]]; + } // i + + pBlock[0] = pBlock[1] = pBlock[2] = (uint8_t)(mean4_y[1] | (mean4_y[0] << 4)); + + const uint32_t diff = 0; + pBlock[3] = (uint8_t)(flip | (diff << 1) | (mod_tab[0] << 5) | (mod_tab[1] << 2)); + } + else + { + pBlock[0] = pBlock[1] = pBlock[2] = (uint8_t)((delta5_y & 7) | (mean5_y[0] << 3)); + + const uint32_t diff = 1; + pBlock[3] = (uint8_t)(flip | (diff << 1) | (mod_tab[0] << 5) | (mod_tab[1] << 2)); + } + + //decoder_etc_block& blk = *(decoder_etc_block*)pBlock; + + uint16_t l_bitmask = 0; + uint16_t h_bitmask = 0; + + static const uint8_t s_tran[4] = { 1, 0, 2, 3 }; + + uint8_t subblock_colors[2][4]; + get_block_colors_y(pBlock, &subblock_colors[0][0], &subblock_colors[1][0]); + + for (uint32_t subblock = 0; subblock < 2; subblock++) + { + const uint8_t* block_y = &subblock_colors[subblock][0]; + + //color_rgba block_colors[4]; + //blk.get_block_colors(block_colors, subblock); + + //uint32_t block_y[4]; + //for (uint32_t i = 0; i < 4; i++) + //block_y[i] = block_colors[i]; + + const uint32_t block_y01 = block_y[0] + block_y[1]; + const uint32_t block_y12 = block_y[1] + block_y[2]; + const uint32_t block_y23 = block_y[2] + block_y[3]; + + if (flip) + { + uint32_t ofs = subblock * 2; + + for (uint32_t y = 0; y < 2; y++) + { + for (uint32_t x = 0; x < 4; x++) + { + const uint8_t c = pPixels[x + (subblock * 2 + y) * 4]; + const uint32_t l = c * 2; + + uint32_t t = s_tran[(l < block_y01) + (l < block_y12) + (l < block_y23)]; + + assert(ofs < 16); + l_bitmask |= ((t & 1) << ofs); + h_bitmask |= ((t >> 1) << ofs); + ofs += 4; + } + + ofs = (int)ofs + 1 - 4 * 4; + } + } + else + { + uint32_t ofs = (subblock * 2) * 4; + for (uint32_t x = 0; x < 2; x++) + { + for (uint32_t y = 0; y < 4; y++) + { + const uint8_t c = pPixels[subblock * 2 + x + y * 4]; + const uint32_t l = c * 2; + + uint32_t t = s_tran[(l < block_y01) + (l < block_y12) + (l < block_y23)]; + + assert(ofs < 16); + l_bitmask |= ((t & 1) << ofs); + h_bitmask |= ((t >> 1) << ofs); + ++ofs; + } + } + } + + pBlock[7] = (uint8_t)(l_bitmask); + pBlock[6] = (uint8_t)(l_bitmask >> 8); + pBlock[5] = (uint8_t)(h_bitmask); + pBlock[4] = (uint8_t)(h_bitmask >> 8); + + } // subblock + } + +} // namespace etc1f + +#endif // BASISD_SUPPORT_XUASTC + +//------------------------------------------------------------------------------------------------ +// XUASTC LDR transcoding +//------------------------------------------------------------------------------------------------ +// XUASTC adaptive deblocking threshold +const int XUASTC_LDR_DEBLOCK_SKIP_THRESH = 24; + +block_format xuastc_get_block_format(transcoder_texture_format tex_fmt) +{ + switch (tex_fmt) + { + case transcoder_texture_format::cTFASTC_LDR_4x4_RGBA: return block_format::cASTC_LDR_4x4; + case transcoder_texture_format::cTFASTC_LDR_5x4_RGBA: return block_format::cASTC_LDR_5x4; + case transcoder_texture_format::cTFASTC_LDR_5x5_RGBA: return block_format::cASTC_LDR_5x5; + case transcoder_texture_format::cTFASTC_LDR_6x5_RGBA: return block_format::cASTC_LDR_6x5; + case transcoder_texture_format::cTFASTC_LDR_6x6_RGBA: return block_format::cASTC_LDR_6x6; + case transcoder_texture_format::cTFASTC_LDR_8x5_RGBA: return block_format::cASTC_LDR_8x5; + case transcoder_texture_format::cTFASTC_LDR_8x6_RGBA: return block_format::cASTC_LDR_8x6; + case transcoder_texture_format::cTFASTC_LDR_10x5_RGBA: return block_format::cASTC_LDR_10x5; + case transcoder_texture_format::cTFASTC_LDR_10x6_RGBA: return block_format::cASTC_LDR_10x6; + case transcoder_texture_format::cTFASTC_LDR_8x8_RGBA: return block_format::cASTC_LDR_8x8; + case transcoder_texture_format::cTFASTC_LDR_10x8_RGBA: return block_format::cASTC_LDR_10x8; + case transcoder_texture_format::cTFASTC_LDR_10x10_RGBA: return block_format::cASTC_LDR_10x10; + case transcoder_texture_format::cTFASTC_LDR_12x10_RGBA: return block_format::cASTC_LDR_12x10; + case transcoder_texture_format::cTFASTC_LDR_12x12_RGBA: return block_format::cASTC_LDR_12x12; + default: + break; + } + + assert(0); + return block_format::cASTC_LDR_4x4; +} + +basisu_lowlevel_xuastc_ldr_transcoder::basisu_lowlevel_xuastc_ldr_transcoder() +{ +} + +#if BASISD_SUPPORT_XUASTC +void transcode_4x4_block( + block_format fmt, + uint32_t block_x, uint32_t block_y, + void *pDst_blocks, uint8_t* pDst_block_u8, + const color32* block_pixels, + uint32_t output_block_or_pixel_stride_in_bytes, uint32_t output_row_pitch_in_blocks_or_pixels, uint32_t output_rows_in_pixels, + int channel0, int channel1, + bool high_quality, bool from_alpha, + uint32_t bc7f_flags, + etc1f::pack_etc1_state& etc1_pack_state, + int has_alpha) // has_alpha = -1 unknown, 0=definitely no (a all 255's), 1=potentially yes +{ + BASISU_NOTE_UNUSED(output_block_or_pixel_stride_in_bytes); + + switch (fmt) + { + case block_format::cETC1: + { + assert(output_block_or_pixel_stride_in_bytes == 8); + if (from_alpha) + { + // Annoying overhead + uint8_t alpha_pixels[16]; + for (uint32_t i = 0; i < 16; i++) + alpha_pixels[i] = block_pixels[i].a; + + etc1f::pack_etc1_grayscale(pDst_block_u8, alpha_pixels, etc1_pack_state); + } + else + { + etc1f::pack_etc1(pDst_block_u8, (color_rgba *)block_pixels, etc1_pack_state); + } + break; + } + case block_format::cETC2_RGBA: + { + assert(output_block_or_pixel_stride_in_bytes == 16); + + (high_quality ? pack_eac_high_quality : pack_eac)(reinterpret_cast(pDst_block_u8)[0], &block_pixels[0].c[3], sizeof(color32)); + etc1f::pack_etc1(pDst_block_u8 + 8, (color_rgba*)block_pixels, etc1_pack_state); + + break; + } + case block_format::cETC2_EAC_R11: + { + assert(output_block_or_pixel_stride_in_bytes == 8); + + // Pack R by default + if (channel0 < 0) + channel0 = 0; + + (high_quality ? pack_eac_high_quality : pack_eac)(reinterpret_cast(pDst_block_u8)[0], &block_pixels[0].c[channel0], sizeof(color32)); + + break; + } + case block_format::cETC2_EAC_RG11: + { + assert(output_block_or_pixel_stride_in_bytes == 16); + + // Pack RA by default + if (channel0 < 0) + channel0 = 0; + if (channel1 < 0) + channel1 = 3; + + (high_quality ? pack_eac_high_quality : pack_eac)(reinterpret_cast(pDst_block_u8)[0], &block_pixels[0].c[channel0], sizeof(color32)); + (high_quality ? pack_eac_high_quality : pack_eac)(reinterpret_cast(pDst_block_u8)[1], &block_pixels[0].c[channel1], sizeof(color32)); + + break; + } + case block_format::cBC1: + { + assert(output_block_or_pixel_stride_in_bytes == 8); + + encode_bc1(pDst_block_u8, (const uint8_t *)block_pixels, high_quality ? cEncodeBC1HighQuality : 0); + break; + } + case block_format::cBC3: + { + assert(output_block_or_pixel_stride_in_bytes == 16); + + encode_bc4(pDst_block_u8, &block_pixels[0].c[3], sizeof(color32)); + encode_bc1(pDst_block_u8 + 8, (const uint8_t *)block_pixels, high_quality ? cEncodeBC1HighQuality : 0); + break; + } + case block_format::cBC4: + { + assert(output_block_or_pixel_stride_in_bytes == 8); + + // Pack R by default + if (channel0 < 0) + channel0 = 0; + + encode_bc4(pDst_block_u8, &block_pixels[0].c[channel0], sizeof(color32)); + break; + } + case block_format::cBC5: + { + assert(output_block_or_pixel_stride_in_bytes == 16); + + // Pack RA by default + if (channel0 < 0) + channel0 = 0; + if (channel1 < 0) + channel1 = 3; + + encode_bc4(pDst_block_u8, &block_pixels[0].c[channel0], sizeof(color32)); + encode_bc4(pDst_block_u8 + 8, &block_pixels[0].c[channel1], sizeof(color32)); + + break; + } + case block_format::cBC7: + { + assert(output_block_or_pixel_stride_in_bytes == 16); + + // 0=definitely no alpha, so skip alpha checks + if (has_alpha == 0) + bc7f::fast_pack_bc7_auto_rgb(pDst_block_u8, (const basist::color_rgba*)block_pixels, bc7f_flags); + else + bc7f::fast_pack_bc7_auto_rgba(pDst_block_u8, (const basist::color_rgba*)block_pixels, bc7f_flags); + + break; + } + case block_format::cRGBA32: + { + assert(sizeof(uint32_t) == output_block_or_pixel_stride_in_bytes); + uint8_t* pDst_pixels = static_cast(pDst_blocks) + (block_x * 4 + block_y * 4 * output_row_pitch_in_blocks_or_pixels) * sizeof(uint32_t); + + const uint32_t max_x = basisu::minimum(4, (int)output_row_pitch_in_blocks_or_pixels - (int)block_x * 4); + const uint32_t max_y = basisu::minimum(4, (int)output_rows_in_pixels - (int)block_y * 4); + + if ((max_x == 4) && (max_y == 4)) + { + memcpy(pDst_pixels, block_pixels, 4 * sizeof(color32)); + memcpy(pDst_pixels + output_row_pitch_in_blocks_or_pixels * sizeof(uint32_t) * 1, &block_pixels[1 * 4], 4 * sizeof(color32)); + memcpy(pDst_pixels + output_row_pitch_in_blocks_or_pixels * sizeof(uint32_t) * 2, &block_pixels[2 * 4], 4 * sizeof(color32)); + memcpy(pDst_pixels + output_row_pitch_in_blocks_or_pixels * sizeof(uint32_t) * 3, &block_pixels[3 * 4], 4 * sizeof(color32)); + } + else + { + for (uint32_t y = 0; y < max_y; y++) + { + memcpy(pDst_pixels, &block_pixels[y * 4], max_x * sizeof(color32)); + pDst_pixels += output_row_pitch_in_blocks_or_pixels * sizeof(uint32_t); + } + } + + break; + } + case block_format::cRGB565: + case block_format::cBGR565: + { + // This writes little endian data always. + assert(sizeof(uint16_t) == output_block_or_pixel_stride_in_bytes); + uint8_t* pDst_pixels = static_cast(pDst_blocks) + (block_x * 4 + block_y * 4 * output_row_pitch_in_blocks_or_pixels) * sizeof(uint16_t); + + const uint32_t max_x = basisu::minimum(4, (int)output_row_pitch_in_blocks_or_pixels - (int)block_x * 4); + const uint32_t max_y = basisu::minimum(4, (int)output_rows_in_pixels - (int)block_y * 4); + + for (uint32_t y = 0; y < max_y; y++) + { + for (uint32_t x = 0; x < max_x; x++) + { + const color32& c = block_pixels[y * 4 + x]; + + const uint16_t packed = (fmt == block_format::cRGB565) ? static_cast((mul_8(c.r, 31) << 11) | (mul_8(c.g, 63) << 5) | mul_8(c.b, 31)) : + static_cast((mul_8(c.b, 31) << 11) | (mul_8(c.g, 63) << 5) | mul_8(c.r, 31)); + + pDst_pixels[x * 2 + 0] = (uint8_t)(packed & 0xFF); + pDst_pixels[x * 2 + 1] = (uint8_t)((packed >> 8) & 0xFF); + } + + pDst_pixels += output_row_pitch_in_blocks_or_pixels * sizeof(uint16_t); + } + + break; + } + case block_format::cRGBA4444: + { + // This writes little endian data always. + assert(sizeof(uint16_t) == output_block_or_pixel_stride_in_bytes); + uint8_t* pDst_pixels = static_cast(pDst_blocks) + (block_x * 4 + block_y * 4 * output_row_pitch_in_blocks_or_pixels) * sizeof(uint16_t); + + const uint32_t max_x = basisu::minimum(4, (int)output_row_pitch_in_blocks_or_pixels - (int)block_x * 4); + const uint32_t max_y = basisu::minimum(4, (int)output_rows_in_pixels - (int)block_y * 4); + + for (uint32_t y = 0; y < max_y; y++) + { + for (uint32_t x = 0; x < max_x; x++) + { + const color32& c = block_pixels[y * 4 + x]; + + const uint16_t packed = static_cast((mul_8(c.r, 15) << 12) | (mul_8(c.g, 15) << 8) | (mul_8(c.b, 15) << 4) | mul_8(c.a, 15)); + + pDst_pixels[x * 2 + 0] = (uint8_t)(packed & 0xFF); + pDst_pixels[x * 2 + 1] = (uint8_t)((packed >> 8) & 0xFF); + } + + pDst_pixels += output_row_pitch_in_blocks_or_pixels * sizeof(uint16_t); + } + break; + } + default: + // Unsupported or invalid format + assert(0); + break; + } +} + +static bool xuastc_deblock_filter( + uint32_t filter_block_width, uint32_t filter_block_height, + const basisu::vector2D &src_img, + basisu::vector2D &dst_img, + bool stronger_filtering, int skip_thresh) +{ + basisu::vector2D temp_img; + if (!temp_img.try_resize(src_img.get_width(), src_img.get_height())) + return false; + + if (stronger_filtering) + skip_thresh *= 2; + + //basisu::fmt_printf("stronger filtering: {}, skip_thread: {}\n", stronger_filtering, skip_thresh); + + temp_img = src_img; + + for (int y = 0; y < (int)src_img.get_height(); y++) + { + for (int x = filter_block_width; x < (int)src_img.get_width(); x += filter_block_width) + { + const color32 &ll = src_img.at_clamped(x - 2, y); + const color32 &l = src_img.at_clamped(x - 1, y); + const color32 &r =src_img.at_clamped(x, y); + const color32 &rr = src_img.at_clamped(x + 1, y); + + if (skip_thresh < 256) + { + bool skip_flag = false; + for (uint32_t c = 0; c < 4; c++) + { + int delta = basisu::iabs((int)l[c] - (int)r[c]); + if (delta > skip_thresh) + { + skip_flag = true; + break; + } + } + + if (skip_flag) + continue; + } + + color32 ml, mr; + for (uint32_t c = 0; c < 4; c++) + { + if (stronger_filtering) + { + ml[c] = (3 * l[c] + 2 * r[c] + ll[c] + 3) / 6; + mr[c] = (3 * r[c] + 2 * l[c] + rr[c] + 3) / 6; + } + else + { + ml[c] = (5 * l[c] + 2 * r[c] + ll[c] + 4) / 8; + mr[c] = (5 * r[c] + 2 * l[c] + rr[c] + 4) / 8; + } + } + + temp_img.set_clipped(x - 1, y, ml); + temp_img.set_clipped(x, y, mr); + + } // x + + } // y + + dst_img = temp_img; + + for (int x = 0; x < (int)temp_img.get_width(); x++) + { + for (int y = filter_block_height; y < (int)temp_img.get_height(); y += filter_block_height) + { + const color32 &uu = temp_img.at_clamped(x, y - 2); + const color32 &u = temp_img.at_clamped(x, y - 1); + const color32 &d = temp_img.at_clamped(x, y); + const color32 &dd = temp_img.at_clamped(x, y + 1); + + if (skip_thresh < 256) + { + bool skip_flag = false; + for (uint32_t c = 0; c < 4; c++) + { + int delta = basisu::iabs((int)u[c] - (int)d[c]); + if (delta > skip_thresh) + { + skip_flag = true; + break; + } + } + + if (skip_flag) + continue; + } + + color32 mu, md; + for (uint32_t c = 0; c < 4; c++) + { + if (stronger_filtering) + { + mu[c] = (3 * u[c] + 2 * d[c] + uu[c] + 3) / 6; + md[c] = (3 * d[c] + 2 * u[c] + dd[c] + 3) / 6; + } + else + { + mu[c] = (5 * u[c] + 2 * d[c] + uu[c] + 4) / 8; + md[c] = (5 * d[c] + 2 * u[c] + dd[c] + 4) / 8; + } + } + + dst_img.set_clipped(x, y - 1, mu); + dst_img.set_clipped(x, y, md); + + } // x + + } // y + + return true; +} + +static void xuastc_fixup_pvrtc1_4_modulation_rgb( + const basisu::vector2D& temp_image, + const uint32_t* pPVRTC_endpoints, + void* pDst_blocks, + uint32_t num_blocks_x, uint32_t num_blocks_y, bool from_alpha) +{ + const uint32_t x_mask = num_blocks_x - 1; + const uint32_t y_mask = num_blocks_y - 1; + const uint32_t x_bits = basisu::total_bits(x_mask); + const uint32_t y_bits = basisu::total_bits(y_mask); + const uint32_t min_bits = basisu::minimum(x_bits, y_bits); + //const uint32_t max_bits = basisu::maximum(x_bits, y_bits); + const uint32_t swizzle_mask = (1 << (min_bits * 2)) - 1; + + uint32_t block_index = 0; + + // really 3x3 + int e0[4][4], e1[4][4]; + + for (int y = 0; y < static_cast(num_blocks_y); y++) + { + const uint32_t* pE_rows[3]; + + for (int ey = 0; ey < 3; ey++) + { + int by = y + ey - 1; + + const uint32_t* pE = &pPVRTC_endpoints[(by & y_mask) * num_blocks_x]; + + pE_rows[ey] = pE; + + for (int ex = 0; ex < 3; ex++) + { + int bx = 0 + ex - 1; + + const uint32_t e = pE[bx & x_mask]; + + e0[ex][ey] = (get_opaque_endpoint_l0(e) * 255) / 31; + e1[ex][ey] = (get_opaque_endpoint_l1(e) * 255) / 31; + } + } + + const uint32_t y_swizzle = (g_pvrtc_swizzle_table[y >> 8] << 16) | g_pvrtc_swizzle_table[y & 0xFF]; + + for (int x = 0; x < static_cast(num_blocks_x); x++, block_index++) + { + color32 block_pixels[4][4]; + temp_image.extract_block_clamped(&block_pixels[0][0], x * 4, y * 4, 4, 4); + + if (from_alpha) + { + // Just set RGB to alpha to avoid adding complexity below. + for (uint32_t i = 0; i < 16; i++) + { + const uint8_t a = ((color32*)block_pixels)[i].a; + ((color32*)block_pixels)[i].set(a, a, a, 255); + } + } + + const uint32_t x_swizzle = (g_pvrtc_swizzle_table[x >> 8] << 17) | (g_pvrtc_swizzle_table[x & 0xFF] << 1); + + uint32_t swizzled = x_swizzle | y_swizzle; + if (num_blocks_x != num_blocks_y) + { + swizzled &= swizzle_mask; + + if (num_blocks_x > num_blocks_y) + swizzled |= ((x >> min_bits) << (min_bits * 2)); + else + swizzled |= ((y >> min_bits) << (min_bits * 2)); + } + + pvrtc4_block* pDst_block = static_cast(pDst_blocks) + swizzled; + pDst_block->m_endpoints = pPVRTC_endpoints[block_index]; + + { + const uint32_t ex = 2; + int bx = x + ex - 1; + bx &= x_mask; + +#define BUT_DO_ROW(ey) \ + { \ + const uint32_t e = pE_rows[ey][bx]; \ + e0[ex][ey] = (get_opaque_endpoint_l0(e) * 255) / 31; \ + e1[ex][ey] = (get_opaque_endpoint_l1(e) * 255) / 31; \ + } + + BUT_DO_ROW(0); + BUT_DO_ROW(1); + BUT_DO_ROW(2); +#undef BUT_DO_ROW + } + + uint32_t mod = 0; + +#define BUT_DO_PIX(lx, ly, w0, w1, w2, w3) \ + { \ + int ca_l = a0 * w0 + a1 * w1 + a2 * w2 + a3 * w3; \ + int cb_l = b0 * w0 + b1 * w1 + b2 * w2 + b3 * w3; \ + int cl = (block_pixels[ly][lx].r + block_pixels[ly][lx].g + block_pixels[ly][lx].b) * 16; \ + int dl = cb_l - ca_l; \ + int vl = cl - ca_l; \ + int p = vl * 16; \ + if (ca_l > cb_l) { p = -p; dl = -dl; } \ + uint32_t m = 0; \ + if (p > 3 * dl) m = (uint32_t)(1 << ((ly) * 8 + (lx) * 2)); \ + if (p > 8 * dl) m = (uint32_t)(2 << ((ly) * 8 + (lx) * 2)); \ + if (p > 13 * dl) m = (uint32_t)(3 << ((ly) * 8 + (lx) * 2)); \ + mod |= m; \ + } + + { + const uint32_t ex = 0, ey = 0; + const int a0 = e0[ex][ey], a1 = e0[ex + 1][ey], a2 = e0[ex][ey + 1], a3 = e0[ex + 1][ey + 1]; + const int b0 = e1[ex][ey], b1 = e1[ex + 1][ey], b2 = e1[ex][ey + 1], b3 = e1[ex + 1][ey + 1]; + BUT_DO_PIX(0, 0, 4, 4, 4, 4); + BUT_DO_PIX(1, 0, 2, 6, 2, 6); + BUT_DO_PIX(0, 1, 2, 2, 6, 6); + BUT_DO_PIX(1, 1, 1, 3, 3, 9); + } + + { + const uint32_t ex = 1, ey = 0; + const int a0 = e0[ex][ey], a1 = e0[ex + 1][ey], a2 = e0[ex][ey + 1], a3 = e0[ex + 1][ey + 1]; + const int b0 = e1[ex][ey], b1 = e1[ex + 1][ey], b2 = e1[ex][ey + 1], b3 = e1[ex + 1][ey + 1]; + BUT_DO_PIX(2, 0, 8, 0, 8, 0); + BUT_DO_PIX(3, 0, 6, 2, 6, 2); + BUT_DO_PIX(2, 1, 4, 0, 12, 0); + BUT_DO_PIX(3, 1, 3, 1, 9, 3); + } + + { + const uint32_t ex = 0, ey = 1; + const int a0 = e0[ex][ey], a1 = e0[ex + 1][ey], a2 = e0[ex][ey + 1], a3 = e0[ex + 1][ey + 1]; + const int b0 = e1[ex][ey], b1 = e1[ex + 1][ey], b2 = e1[ex][ey + 1], b3 = e1[ex + 1][ey + 1]; + BUT_DO_PIX(0, 2, 8, 8, 0, 0); + BUT_DO_PIX(1, 2, 4, 12, 0, 0); + BUT_DO_PIX(0, 3, 6, 6, 2, 2); + BUT_DO_PIX(1, 3, 3, 9, 1, 3); + } + + { + const uint32_t ex = 1, ey = 1; + const int a0 = e0[ex][ey], a1 = e0[ex + 1][ey], a2 = e0[ex][ey + 1], a3 = e0[ex + 1][ey + 1]; + const int b0 = e1[ex][ey], b1 = e1[ex + 1][ey], b2 = e1[ex][ey + 1], b3 = e1[ex + 1][ey + 1]; + BUT_DO_PIX(2, 2, 16, 0, 0, 0); + BUT_DO_PIX(3, 2, 12, 4, 0, 0); + BUT_DO_PIX(2, 3, 12, 0, 4, 0); + BUT_DO_PIX(3, 3, 9, 3, 3, 1); + } +#undef BUT_DO_PIX + + pDst_block->m_modulation = mod; + + e0[0][0] = e0[1][0]; e0[1][0] = e0[2][0]; + e0[0][1] = e0[1][1]; e0[1][1] = e0[2][1]; + e0[0][2] = e0[1][2]; e0[1][2] = e0[2][2]; + + e1[0][0] = e1[1][0]; e1[1][0] = e1[2][0]; + e1[0][1] = e1[1][1]; e1[1][1] = e1[2][1]; + e1[0][2] = e1[1][2]; e1[1][2] = e1[2][2]; + + } // x + } // y +} + +static void xuastc_fixup_pvrtc1_4_modulation_rgba( + const basisu::vector2D& temp_image, + const uint32_t* pPVRTC_endpoints, + void* pDst_blocks, uint32_t num_blocks_x, uint32_t num_blocks_y) +{ + const uint32_t x_mask = num_blocks_x - 1; + const uint32_t y_mask = num_blocks_y - 1; + const uint32_t x_bits = basisu::total_bits(x_mask); + const uint32_t y_bits = basisu::total_bits(y_mask); + const uint32_t min_bits = basisu::minimum(x_bits, y_bits); + //const uint32_t max_bits = basisu::maximum(x_bits, y_bits); + const uint32_t swizzle_mask = (1 << (min_bits * 2)) - 1; + + uint32_t block_index = 0; + + // really 3x3 + int e0[4][4], e1[4][4]; + + for (int y = 0; y < static_cast(num_blocks_y); y++) + { + const uint32_t* pE_rows[3]; + + for (int ey = 0; ey < 3; ey++) + { + int by = y + ey - 1; + + const uint32_t* pE = &pPVRTC_endpoints[(by & y_mask) * num_blocks_x]; + + pE_rows[ey] = pE; + + for (int ex = 0; ex < 3; ex++) + { + int bx = 0 + ex - 1; + + const uint32_t e = pE[bx & x_mask]; + + e0[ex][ey] = get_endpoint_l8(e, 0); + e1[ex][ey] = get_endpoint_l8(e, 1); + } + } + + const uint32_t y_swizzle = (g_pvrtc_swizzle_table[y >> 8] << 16) | g_pvrtc_swizzle_table[y & 0xFF]; + + for (int x = 0; x < static_cast(num_blocks_x); x++, block_index++) + { + color32 block_pixels[4][4]; + temp_image.extract_block_clamped(&block_pixels[0][0], x * 4, y * 4, 4, 4); + + const uint32_t x_swizzle = (g_pvrtc_swizzle_table[x >> 8] << 17) | (g_pvrtc_swizzle_table[x & 0xFF] << 1); + + uint32_t swizzled = x_swizzle | y_swizzle; + if (num_blocks_x != num_blocks_y) + { + swizzled &= swizzle_mask; + + if (num_blocks_x > num_blocks_y) + swizzled |= ((x >> min_bits) << (min_bits * 2)); + else + swizzled |= ((y >> min_bits) << (min_bits * 2)); + } + + pvrtc4_block* pDst_block = static_cast(pDst_blocks) + swizzled; + pDst_block->m_endpoints = pPVRTC_endpoints[block_index]; + + { + const uint32_t ex = 2; + int bx = x + ex - 1; + bx &= x_mask; + +#define DO_ROW(ey) \ + { \ + const uint32_t e = pE_rows[ey][bx]; \ + e0[ex][ey] = get_endpoint_l8(e, 0); \ + e1[ex][ey] = get_endpoint_l8(e, 1); \ + } + + DO_ROW(0); + DO_ROW(1); + DO_ROW(2); +#undef DO_ROW + } + + uint32_t mod = 0; + +#define DO_PIX(lx, ly, w0, w1, w2, w3) \ + { \ + int ca_l = a0 * w0 + a1 * w1 + a2 * w2 + a3 * w3; \ + int cb_l = b0 * w0 + b1 * w1 + b2 * w2 + b3 * w3; \ + int cl = 16 * (block_pixels[ly][lx].r + block_pixels[ly][lx].g + block_pixels[ly][lx].b + block_pixels[ly][lx].a); \ + int dl = cb_l - ca_l; \ + int vl = cl - ca_l; \ + int p = vl * 16; \ + if (ca_l > cb_l) { p = -p; dl = -dl; } \ + uint32_t m = 0; \ + if (p > 3 * dl) m = (uint32_t)(1 << ((ly) * 8 + (lx) * 2)); \ + if (p > 8 * dl) m = (uint32_t)(2 << ((ly) * 8 + (lx) * 2)); \ + if (p > 13 * dl) m = (uint32_t)(3 << ((ly) * 8 + (lx) * 2)); \ + mod |= m; \ + } + + { + const uint32_t ex = 0, ey = 0; + const int a0 = e0[ex][ey], a1 = e0[ex + 1][ey], a2 = e0[ex][ey + 1], a3 = e0[ex + 1][ey + 1]; + const int b0 = e1[ex][ey], b1 = e1[ex + 1][ey], b2 = e1[ex][ey + 1], b3 = e1[ex + 1][ey + 1]; + DO_PIX(0, 0, 4, 4, 4, 4); + DO_PIX(1, 0, 2, 6, 2, 6); + DO_PIX(0, 1, 2, 2, 6, 6); + DO_PIX(1, 1, 1, 3, 3, 9); + } + + { + const uint32_t ex = 1, ey = 0; + const int a0 = e0[ex][ey], a1 = e0[ex + 1][ey], a2 = e0[ex][ey + 1], a3 = e0[ex + 1][ey + 1]; + const int b0 = e1[ex][ey], b1 = e1[ex + 1][ey], b2 = e1[ex][ey + 1], b3 = e1[ex + 1][ey + 1]; + DO_PIX(2, 0, 8, 0, 8, 0); + DO_PIX(3, 0, 6, 2, 6, 2); + DO_PIX(2, 1, 4, 0, 12, 0); + DO_PIX(3, 1, 3, 1, 9, 3); + } + + { + const uint32_t ex = 0, ey = 1; + const int a0 = e0[ex][ey], a1 = e0[ex + 1][ey], a2 = e0[ex][ey + 1], a3 = e0[ex + 1][ey + 1]; + const int b0 = e1[ex][ey], b1 = e1[ex + 1][ey], b2 = e1[ex][ey + 1], b3 = e1[ex + 1][ey + 1]; + DO_PIX(0, 2, 8, 8, 0, 0); + DO_PIX(1, 2, 4, 12, 0, 0); + DO_PIX(0, 3, 6, 6, 2, 2); + DO_PIX(1, 3, 3, 9, 1, 3); + } + + { + const uint32_t ex = 1, ey = 1; + const int a0 = e0[ex][ey], a1 = e0[ex + 1][ey], a2 = e0[ex][ey + 1], a3 = e0[ex + 1][ey + 1]; + const int b0 = e1[ex][ey], b1 = e1[ex + 1][ey], b2 = e1[ex][ey + 1], b3 = e1[ex + 1][ey + 1]; + DO_PIX(2, 2, 16, 0, 0, 0); + DO_PIX(3, 2, 12, 4, 0, 0); + DO_PIX(2, 3, 12, 0, 4, 0); + DO_PIX(3, 3, 9, 3, 3, 1); + } +#undef DO_PIX + + pDst_block->m_modulation = mod; + + e0[0][0] = e0[1][0]; e0[1][0] = e0[2][0]; + e0[0][1] = e0[1][1]; e0[1][1] = e0[2][1]; + e0[0][2] = e0[1][2]; e0[1][2] = e0[2][2]; + + e1[0][0] = e1[1][0]; e1[1][0] = e1[2][0]; + e1[0][1] = e1[1][1]; e1[1][1] = e1[2][1]; + e1[0][2] = e1[1][2]; e1[1][2] = e1[2][2]; + + } // x + } // y +} + +void encode_pvrtc1( + block_format fmt, void* pDst_blocks, + const basisu::vector2D &temp_image, + uint32_t dst_num_blocks_x, uint32_t dst_num_blocks_y, bool from_alpha) +{ + assert((fmt == block_format::cPVRTC1_4_RGB) || (fmt == block_format::cPVRTC1_4_RGBA)); + + basisu::vector2D pvrtc1_endpoints(dst_num_blocks_x, dst_num_blocks_y); + + // Determine block endpoints + for (uint32_t dst_by = 0; dst_by < dst_num_blocks_y; dst_by++) + { + for (uint32_t dst_bx = 0; dst_bx < dst_num_blocks_x; dst_bx++) + { + color32 block_pixels[4 * 4]; + + temp_image.extract_block_clamped(block_pixels, dst_bx * 4, dst_by * 4, 4, 4); + + color32 low_color(255, 255, 255, 255), high_color(0, 0, 0, 0); + + for (uint32_t i = 0; i < 16; i++) + { + low_color = color32::comp_min(low_color, block_pixels[i]); + high_color = color32::comp_max(high_color, block_pixels[i]); + } + + if ((fmt == block_format::cPVRTC1_4_RGB) && (from_alpha)) + { + low_color.set(low_color.a, low_color.a, low_color.a, 255); + high_color.set(high_color.a, high_color.a, high_color.a, 255); + } + + pvrtc4_block temp; + if (fmt == block_format::cPVRTC1_4_RGBA) + { + temp.set_endpoint_floor(0, low_color); + temp.set_endpoint_ceil(1, high_color); + } + else + { + temp.set_opaque_endpoint_floor(0, low_color); + temp.set_opaque_endpoint_ceil(1, high_color); + } + + pvrtc1_endpoints(dst_bx, dst_by) = temp.m_endpoints; + } // dst_bx + + } // dst_by + + // Create PVRTC1 texture data. + if (fmt == block_format::cPVRTC1_4_RGBA) + xuastc_fixup_pvrtc1_4_modulation_rgba(temp_image, pvrtc1_endpoints.get_ptr(), pDst_blocks, dst_num_blocks_x, dst_num_blocks_y); + else + xuastc_fixup_pvrtc1_4_modulation_rgb(temp_image, pvrtc1_endpoints.get_ptr(), pDst_blocks, dst_num_blocks_x, dst_num_blocks_y, from_alpha); +} + +#endif // BASISD_SUPPORT_XUASTC + +static inline bool blocks_same_solid_colors(const astc_helpers::log_astc_block& a, const astc_helpers::log_astc_block& b, uint32_t tol) +{ + if ((!a.m_solid_color_flag_ldr) || (!b.m_solid_color_flag_ldr)) + return false; + + if (tol == 0) + { + return (a.m_solid_color[0] == b.m_solid_color[0]) && (a.m_solid_color[1] == b.m_solid_color[1]) && + (a.m_solid_color[2] == b.m_solid_color[2]) && (a.m_solid_color[3] == b.m_solid_color[3]); + } + + for (uint32_t i = 0; i < 4; i++) + { + int ac = a.m_solid_color[i] >> 8; + int bc = b.m_solid_color[i] >> 8; + + const int dl = basisu::iabs((int)ac - (int)bc); + if (dl > (int)tol) + return false; + } + + return true; +} + +static inline bool blocks_same_single_subset_endpoints(const astc_helpers::log_astc_block& a, const astc_helpers::log_astc_block& b, uint32_t tol) +{ + if (a.m_solid_color_flag_ldr || b.m_solid_color_flag_ldr) + return false; + + if (a.m_dual_plane || b.m_dual_plane) + return false; + + if ((a.m_num_partitions > 1) || (b.m_num_partitions > 1)) + return false; + + if (a.m_color_endpoint_modes[0] != b.m_color_endpoint_modes[0]) + return false; + + if (a.m_endpoint_ise_range != b.m_endpoint_ise_range) + return false; + + if (tol > 0) + { + // Compare endpoints with tolerance + color_rgba al, ah; + astc_ldr_t::decode_endpoints(a.m_color_endpoint_modes[0], a.m_endpoints, a.m_endpoint_ise_range, al, ah); + + color_rgba bl, bh; + astc_ldr_t::decode_endpoints(b.m_color_endpoint_modes[0], b.m_endpoints, b.m_endpoint_ise_range, bl, bh); + + for (uint32_t i = 0; i < 4; i++) + { + const int dl = basisu::iabs((int)al[i] - (int)bl[i]); + if (dl > (int)tol) + return false; + + const int dh = basisu::iabs((int)ah[i] - (int)bh[i]); + if (dh > (int)tol) + return false; + } + } + else + { + uint32_t total_endpoint_vals = astc_helpers::get_num_cem_values(a.m_color_endpoint_modes[0]); + if (memcmp(a.m_endpoints, b.m_endpoints, total_endpoint_vals) != 0) + return false; + } + + return true; +} + +static inline bool block_has_alpha(const astc_helpers::log_astc_block& a) +{ + if (a.m_solid_color_flag_ldr) + { + return (a.m_solid_color[3] >> 8) != 255; + } + + assert(a.m_num_partitions == 1); + + return astc_helpers::does_cem_have_alpha(a.m_color_endpoint_modes[0]); +} + +static void astc_upsample_grid_weights(const astc_helpers::log_astc_block& log_blk, uint8_t* pDst_weights, uint32_t block_width, uint32_t block_height) +{ + // Skip if solid (which is fine) + if (log_blk.m_solid_color_flag_ldr) + { +#if defined(DEBUG) || defined(_DEBUG) + memset(pDst_weights, 0xFF, block_width * block_height); +#endif + return; + } + + assert((log_blk.m_grid_width <= block_width) && (log_blk.m_grid_height <= block_height)); + + uint8_t dequantized_weights[astc_helpers::MAX_BLOCK_PIXELS]; + + const uint32_t total_weight_vals = log_blk.m_grid_width * log_blk.m_grid_height; + + const astc_helpers::dequant_table& weight_dequant_tab = astc_helpers::g_dequant_tables.get_weight_tab(log_blk.m_weight_ise_range); + const uint8_t* pWeight_dequant = weight_dequant_tab.m_ISE_to_val.data(); + + for (uint32_t i = 0; i < total_weight_vals; i++) + { + assert(log_blk.m_weights[i] < weight_dequant_tab.m_ISE_to_val.size_u32()); + + dequantized_weights[i] = pWeight_dequant[log_blk.m_weights[i]]; + } + + if ((log_blk.m_grid_width < block_width) || (log_blk.m_grid_height < block_height)) + { + astc_helpers::upsample_weight_grid_xuastc_ldr(block_width, block_height, log_blk.m_grid_width, log_blk.m_grid_height, dequantized_weights, pDst_weights, nullptr, nullptr); + } + else + { + memcpy(pDst_weights, dequantized_weights, block_width * block_height); + } +} + +bool basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice( + basis_tex_format src_format, bool use_astc_srgb_decode_profile, + void* pDst_blocks, + uint32_t src_num_blocks_x, uint32_t src_num_blocks_y, + const uint8_t* pImage_data, uint32_t image_data_size, block_format fmt, + uint32_t output_block_or_pixel_stride_in_bytes, bool bc1_allow_threecolor_blocks, bool has_alpha, + const uint32_t orig_width, const uint32_t orig_height, uint32_t output_row_pitch_in_blocks_or_pixels, + basisu_transcoder_state* pState, uint32_t output_rows_in_pixels, int channel0, int channel1, uint32_t decode_flags) +{ + BASISU_NOTE_UNUSED(pState); + BASISU_NOTE_UNUSED(bc1_allow_threecolor_blocks); + +#if BASISD_SUPPORT_XUASTC + + assert(g_transcoder_initialized); + if (!g_transcoder_initialized) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: Transcoder not globally initialized.\n"); + return false; + } + + if (block_format_is_hdr(fmt)) + { + assert(0); + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: Invalid fmt argument\n"); + return false; + } + + //const uint32_t total_blocks = num_blocks_x * num_blocks_y; + + const uint32_t src_block_width = basis_tex_format_get_block_width(src_format), src_block_height = basis_tex_format_get_block_height(src_format); + + const uint32_t dst_fmt_block_width = get_block_width(fmt), dst_fmt_block_height = get_block_height(fmt); + const bool dst_fmt_is_astc = block_format_is_astc(fmt); + const bool dst_fmt_is_pvrtc1 = (fmt == block_format::cPVRTC1_4_RGB) || (fmt == block_format::cPVRTC1_4_RGBA); + + if (dst_fmt_is_pvrtc1) + { + if (!basisu::is_pow2(orig_width) || !basisu::is_pow2(orig_height)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: PVRTC1 requires power of 2 texture dimensions\n"); + return false; + } + } + + const bool is_uncompressed_fmt = basis_block_format_is_uncompressed(fmt); + if (!output_row_pitch_in_blocks_or_pixels) + { + if (is_uncompressed_fmt) + output_row_pitch_in_blocks_or_pixels = orig_width; + else + output_row_pitch_in_blocks_or_pixels = (orig_width + dst_fmt_block_width - 1) / dst_fmt_block_width; + } + + if (is_uncompressed_fmt) + { + if (!output_rows_in_pixels) + output_rows_in_pixels = orig_height; + } + + const bool high_quality = (decode_flags & cDecodeFlagsHighQuality) != 0; + const bool enable_fast_bc7_transcoding = (decode_flags & cDecodeFlagXUASTCLDRDisableFastBC7Transcoding) == 0; + const bool from_alpha = has_alpha && (decode_flags & cDecodeFlagsTranscodeAlphaDataToOpaqueFormats) != 0; + const bool disable_deblocking = (decode_flags & cDecodeFlagsNoDeblockFiltering) != 0; + const bool stronger_deblocking = ((decode_flags & cDecodeFlagsStrongerDeblockFiltering) != 0) || ((src_block_width > 8) || (src_block_height > 8)); + const bool force_deblocking = (decode_flags & cDecodeFlagsForceDeblockFiltering) != 0; + const bool deblock_filtering = !disable_deblocking && (force_deblocking || ((src_block_width > 8) || (src_block_height > 6))); + + const uint32_t bc7f_flags = high_quality ? bc7f::cPackBC7FlagDefaultPartiallyAnalytical : bc7f::cPackBC7FlagDefault; + etc1f::pack_etc1_state etc1_pack_state; + + if (basis_tex_format_is_astc_ldr(src_format)) + { + // Plain ASTC LDR 4x4-12x12 - note it could be ANY valid/standard ASTC written by any ASTC encoder, so we cannot trust this ASTC data. + // It must be fully validated. + if (dst_fmt_is_astc) + { + assert(output_block_or_pixel_stride_in_bytes == sizeof(astc_helpers::astc_block)); + + if ((dst_fmt_block_width != src_block_width) || (dst_fmt_block_height != src_block_height)) + { + // ASTC block dimensions must match, i.e. we can't change the ASTC block size during transcoding. + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: fmt's ASTC block dimensions don't match the content's block dimensions\n"); + return false; + } + + // No transcoding needed, it's ASTC in->ASTC out. + memcpy(pDst_blocks, pImage_data, src_num_blocks_x * src_num_blocks_y * sizeof(astc_helpers::astc_block)); + } + else if (((src_block_width == 4) && (src_block_height == 4)) && (!dst_fmt_is_pvrtc1) && (!deblock_filtering)) + { + // Block dimensions aren't changing, no pvrtc1, no deblock filtering + if ((dst_fmt_block_width != 4) || (dst_fmt_block_height != 4)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: fmt's ASTC block dimensions don't match the content's block dimensions\n"); + return false; + } + + const astc_helpers::astc_block* pSrc_phys_blk = (const astc_helpers::astc_block*)pImage_data; + + astc_helpers::log_astc_block log_blk; + + for (uint32_t block_y = 0; block_y < src_num_blocks_y; block_y++) + { + uint8_t* pDst_block_u8 = (uint8_t*)pDst_blocks + block_y * output_row_pitch_in_blocks_or_pixels * output_block_or_pixel_stride_in_bytes; + + for (uint32_t block_x = 0; block_x < src_num_blocks_x; block_x++) + { + color32 block_pixels[4 * 4]; + + bool unpack_status = astc_helpers::unpack_block(pSrc_phys_blk, log_blk, 4, 4); + if (!unpack_status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_helpers::unpack_block() failed\n"); + return false; + } + + // TODO: Specially handle solid block case + bool decode_status = astc_helpers::decode_block(log_blk, block_pixels, 4, 4, use_astc_srgb_decode_profile ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8); + if (!decode_status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_helpers::decode_block() failed\n"); + return false; + } + + transcode_4x4_block( + fmt, + block_x, block_y, + pDst_blocks, pDst_block_u8, + block_pixels, + output_block_or_pixel_stride_in_bytes, output_row_pitch_in_blocks_or_pixels, output_rows_in_pixels, + channel0, channel1, + high_quality, from_alpha, + bc7f_flags, + etc1_pack_state); + + pDst_block_u8 += output_block_or_pixel_stride_in_bytes; + ++pSrc_phys_blk; + + } // block_x + + } // block_y + } + else if ((!deblock_filtering) && (!dst_fmt_is_pvrtc1)) + { + assert((dst_fmt_block_width == 4) && (dst_fmt_block_height == 4)); + + // Compute how many source block rows we need to buffer so we have a multiple of 4 scanlines. The max # of scanlines is 20. + uint32_t num_src_block_rows_to_buffer = 1; + while ((num_src_block_rows_to_buffer * src_block_height) & 3) + num_src_block_rows_to_buffer++; + assert((num_src_block_rows_to_buffer >= 1) && (num_src_block_rows_to_buffer <= 4)); + + // Compute how many 4x4 dest blocks fit into these many source rows. + assert(((num_src_block_rows_to_buffer * src_block_height) & 3) == 0); + //const uint32_t num_dst_block_rows_to_buffer = (num_src_block_rows_to_buffer * src_block_height) >> 2; + + const uint32_t dst_num_blocks_x = (orig_width + dst_fmt_block_width - 1) / dst_fmt_block_width; + const uint32_t dst_num_blocks_y = (orig_height + dst_fmt_block_height - 1) / dst_fmt_block_height; + + basisu::vector2D buffered_rows(src_num_blocks_x * src_block_width, num_src_block_rows_to_buffer * src_block_height); + + const astc_helpers::astc_block* pSrc_phys_blk = (const astc_helpers::astc_block*)pImage_data; + + astc_helpers::log_astc_block log_blk; + + for (uint32_t by = 0; by < src_num_blocks_y; by++) + { + const uint32_t buffered_src_block_row_y = (by % num_src_block_rows_to_buffer); + + for (uint32_t bx = 0; bx < src_num_blocks_x; bx++) + { + color32 block_pixels[astc_helpers::MAX_BLOCK_PIXELS]; + + bool unpack_status = astc_helpers::unpack_block(pSrc_phys_blk, log_blk, src_block_width, src_block_height); + if (!unpack_status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_helpers::unpack_block() failed\n"); + return false; + } + + bool decode_status = astc_helpers::decode_block(log_blk, block_pixels, src_block_width, src_block_height, use_astc_srgb_decode_profile ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8); + if (!decode_status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_helpers::decode_block() failed\n"); + return false; + } + + color32* pSrc_pixels = block_pixels; + color32* pDst_pixels = &buffered_rows(bx * src_block_width, buffered_src_block_row_y * src_block_height); + + for (uint32_t y = 0; y < src_block_height; y++) + { + memcpy(pDst_pixels, pSrc_pixels, src_block_width * sizeof(color32)); + + pSrc_pixels += src_block_width; + pDst_pixels += buffered_rows.get_width(); + } // y + + ++pSrc_phys_blk; + + } // block_x + + const bool final_src_block_row = (by == (src_num_blocks_y - 1)); + + if ((buffered_src_block_row_y != (num_src_block_rows_to_buffer - 1)) && (!final_src_block_row)) + continue; + + // src/destination image Y coordinate of the top of the buffered rows + const uint32_t buffered_src_pixel_y = ((by / num_src_block_rows_to_buffer) * num_src_block_rows_to_buffer) * src_block_height; + assert((buffered_src_pixel_y & 3) == 0); + + // The total # of valid src block rows we can read. + const uint32_t num_buffered_src_block_rows = buffered_src_block_row_y + 1; + + assert((num_buffered_src_block_rows == num_src_block_rows_to_buffer) || (final_src_block_row)); + + // The maximum number of valid buffer scanlines we can fetch from, taking into account the original texture's actual (unpadded) height. + const uint32_t override_buffer_height = basisu::minimum(orig_height - buffered_src_pixel_y, num_buffered_src_block_rows * src_block_height); + assert(override_buffer_height); + + // total_dst_block_rows_to_emit=really an upper bound for the final row of src ASTC blocks + const uint32_t total_dst_block_rows_to_emit = (num_buffered_src_block_rows * src_block_height + 3) >> 2; + + for (uint32_t dst_ofs_by = 0; dst_ofs_by < total_dst_block_rows_to_emit; dst_ofs_by++) + { + const uint32_t dst_by = (buffered_src_pixel_y >> 2) + dst_ofs_by; + if (dst_by >= dst_num_blocks_y) + break; + + for (uint32_t dst_bx = 0; dst_bx < dst_num_blocks_x; dst_bx++) + { + color32 block_pixels[4 * 4]; + + // Extract the 4x4 block pixels from our buffered rows, taking into account the actual # of valid scanlines inside the buffer. + buffered_rows.extract_block_clamped(block_pixels, dst_bx * 4, dst_ofs_by * 4, 4, 4, override_buffer_height); + + uint8_t* pDst_block_u8 = (uint8_t*)pDst_blocks + (dst_by * output_row_pitch_in_blocks_or_pixels + dst_bx) * output_block_or_pixel_stride_in_bytes; + + transcode_4x4_block( + fmt, + dst_bx, dst_by, + pDst_blocks, pDst_block_u8, + block_pixels, + output_block_or_pixel_stride_in_bytes, output_row_pitch_in_blocks_or_pixels, output_rows_in_pixels, + channel0, channel1, + high_quality, from_alpha, + bc7f_flags, + etc1_pack_state); + + } // dst_bx + + } // dst_ofs_by + + } // block_y + + } + else + { + // unpack entire 32bpp image into memory (needed for deblocking and PVRTC1) + // TODO: Add more memory efficient non-deblocking code path + basisu::vector2D temp_image; + + const uint32_t actual_width = src_block_width * src_num_blocks_x, actual_height = src_block_height * src_num_blocks_y; + + if (!temp_image.try_resize(actual_width, actual_height)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: out of memory\n"); + return false; + } + + const astc_helpers::astc_block* pSrc_phys_blk = (const astc_helpers::astc_block*)pImage_data; + + astc_helpers::log_astc_block log_blk; + color32 block_pixels[astc_helpers::MAX_BLOCK_PIXELS]; + + for (uint32_t src_by = 0; src_by < src_num_blocks_y; src_by++) + { + const uint32_t img_y = src_by * src_block_height; + + for (uint32_t src_bx = 0; src_bx < src_num_blocks_x; src_bx++) + { + bool unpack_status = astc_helpers::unpack_block(pSrc_phys_blk, log_blk, src_block_width, src_block_height); + if (!unpack_status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_helpers::unpack_block() failed\n"); + return false; + } + + bool decode_status = astc_helpers::decode_block(log_blk, block_pixels, src_block_width, src_block_height, use_astc_srgb_decode_profile ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8); + if (!decode_status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_helpers::decode_block() failed\n"); + return false; + } + + color32* pSrc_pixels = (color32*)block_pixels; + color32* pDst_pixels = &temp_image(src_bx * src_block_width, img_y); + + for (uint32_t y = 0; y < src_block_height; y++) + { + memcpy(pDst_pixels, pSrc_pixels, src_block_width * sizeof(color32)); + + pSrc_pixels += src_block_width; + pDst_pixels += temp_image.get_width(); + } // y + + ++pSrc_phys_blk; + + } // src_bx + } // src_by + + if (deblock_filtering) + { + if (!xuastc_deblock_filter( + src_block_width, src_block_height, + temp_image, temp_image, + stronger_deblocking, XUASTC_LDR_DEBLOCK_SKIP_THRESH)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: out of memory\n"); + return false; + } + } + + const uint32_t dst_num_blocks_x = (orig_width + dst_fmt_block_width - 1) / dst_fmt_block_width; + const uint32_t dst_num_blocks_y = (orig_height + dst_fmt_block_height - 1) / dst_fmt_block_height; + + if (dst_fmt_is_pvrtc1) + { + assert((dst_fmt_block_width == 4) && (dst_fmt_block_height == 4)); + + encode_pvrtc1(fmt, pDst_blocks, temp_image, dst_num_blocks_x, dst_num_blocks_y, from_alpha); + } + else + { + for (uint32_t dst_by = 0; dst_by < dst_num_blocks_y; dst_by++) + { + uint8_t* pDst_block_u8 = (uint8_t*)pDst_blocks + dst_by * output_row_pitch_in_blocks_or_pixels * output_block_or_pixel_stride_in_bytes; + + for (uint32_t dst_bx = 0; dst_bx < dst_num_blocks_x; dst_bx++) + { + temp_image.extract_block_clamped(block_pixels, dst_bx * 4, dst_by * 4, 4, 4); + + transcode_4x4_block( + fmt, + dst_bx, dst_by, + pDst_blocks, pDst_block_u8, + block_pixels, + output_block_or_pixel_stride_in_bytes, output_row_pitch_in_blocks_or_pixels, output_rows_in_pixels, + channel0, channel1, + high_quality, from_alpha, + bc7f_flags, + etc1_pack_state); + + pDst_block_u8 += output_block_or_pixel_stride_in_bytes; + + } // dst_bx + + } // dst_by + + } // if (dst_fmt_is_pvrtc1) + } + } + else + { + // Supercompressed XUASTC LDR 4x4-12x12 + // note use_astc_srgb_decode_profile can be ignored here, we'll use the decoded sRGB profile bit from the compressed stream. + + if (dst_fmt_is_astc) + { + // src and dst are ASTC - ideal case, just pack physical ASTC blocks to output buffer during transcoding. + struct decode_state + { + uint32_t m_src_num_blocks_x; + uint32_t m_src_num_blocks_y; + uint32_t m_dst_format_block_width; + uint32_t m_dst_format_block_height; + + void* m_pDst_blocks; + uint32_t m_output_row_pitch_in_blocks_or_pixels; + uint32_t m_output_block_or_pixel_stride_in_bytes; + }; + + decode_state dec_state; + dec_state.m_src_num_blocks_x = src_num_blocks_x; + dec_state.m_src_num_blocks_y = src_num_blocks_y; + dec_state.m_dst_format_block_width = dst_fmt_block_width; + dec_state.m_dst_format_block_height = dst_fmt_block_height; + dec_state.m_pDst_blocks = pDst_blocks; + dec_state.m_output_row_pitch_in_blocks_or_pixels = output_row_pitch_in_blocks_or_pixels; + dec_state.m_output_block_or_pixel_stride_in_bytes = output_block_or_pixel_stride_in_bytes; + + auto init_func = [](uint32_t num_blocks_x, uint32_t num_blocks_y, uint32_t block_width, uint32_t block_height, bool srgb_decode_profile, float dct_q, bool has_alpha, void* pData) + { + BASISU_NOTE_UNUSED(srgb_decode_profile); + BASISU_NOTE_UNUSED(dct_q); + BASISU_NOTE_UNUSED(has_alpha); + + if (basisu::g_debug_printf) + basisu::debug_printf("init_func: %u %u %u %u %u %f %u\n", num_blocks_x, num_blocks_y, block_width, block_height, srgb_decode_profile, dct_q, has_alpha); + + decode_state& state = *(decode_state*)pData; + if ((block_width != state.m_dst_format_block_width) || (block_height != state.m_dst_format_block_height)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: header validation failed (1)\n"); + return false; + } + if ((num_blocks_x != state.m_src_num_blocks_x) || (num_blocks_y != state.m_src_num_blocks_y)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: header validation failed (2)\n"); + return false; + } + return true; + }; + + auto src_block_func = [](uint32_t bx, uint32_t by, const astc_helpers::log_astc_block& log_blk, void* pData) + { + decode_state& state = *(decode_state*)pData; + assert((bx < state.m_src_num_blocks_x) && (by < state.m_src_num_blocks_y)); + + astc_helpers::astc_block* pDst_astc_block = (astc_helpers::astc_block*)((uint8_t*)state.m_pDst_blocks + (by * state.m_output_row_pitch_in_blocks_or_pixels + bx) * state.m_output_block_or_pixel_stride_in_bytes); + + bool pack_status = astc_helpers::pack_astc_block(*pDst_astc_block, log_blk); + if (!pack_status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_helpers::pack_astc_block() failed\n"); + return false; + } + + return true; + }; + + xuastc_decoded_image decoded_image; + + const bool decomp_flag = decoded_image.decode(pImage_data, image_data_size, init_func, &dec_state, src_block_func, &dec_state); + if (!decomp_flag) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_ldr_t::decompress_image() failed\n"); + return false; + } + } + else if ((fmt == block_format::cBC7) && (src_block_width == 8) && (src_block_height == 6) && + (enable_fast_bc7_transcoding) && (!high_quality) && (!deblock_filtering)) + { + // src is ASTC LDR 8x6, destination is BC7, no deblocking: buffer 2 rows of ASTC logical blocks, favor fast pure transcode to BC7 whenever possible. + // transcodes 2 ASTC 8x6 blocks (a tile of 1x2 or 2*48=96 pixels) to 6 BC7 blocks (a tile of 2x3 of 6*16=96 pixels) + // no BC7 block crosses more than 2 ASTC blocks making this easy if the source blocks are only solid or 1 subset + + const uint32_t num_src_block_rows_to_buffer = 2; + + assert(((num_src_block_rows_to_buffer* src_block_height) & 3) == 0); + + const uint32_t dst_num_blocks_x = (orig_width + dst_fmt_block_width - 1) / dst_fmt_block_width; + const uint32_t dst_num_blocks_y = (orig_height + dst_fmt_block_height - 1) / dst_fmt_block_height; + + basisu::vector2D buffered_rows(src_num_blocks_x, num_src_block_rows_to_buffer); + + struct decode_state + { + uint32_t m_orig_height; + + uint32_t m_src_num_blocks_x; + uint32_t m_src_num_blocks_y; + uint32_t m_src_block_width; + uint32_t m_src_block_height; + + uint32_t m_dst_num_blocks_x; + uint32_t m_dst_num_blocks_y; + + void* m_pDst_blocks; + uint32_t m_output_row_pitch_in_blocks_or_pixels; + uint32_t m_output_block_or_pixel_stride_in_bytes; + uint32_t m_output_rows_in_pixels; + + uint32_t m_num_src_block_rows_to_buffer; + //uint32_t m_num_dst_block_rows_to_buffer; + + basisu::vector2D* m_pBuffered_rows; + + bool m_used_srgb_astc_decode_mode; + bool m_has_alpha; + + uint32_t m_total_src_blocks_unpacked; + uint32_t m_total_src_blocks_partial_unpacked; + uint32_t m_total_blocks_transcoded; + uint32_t m_total_blocks_encoded; + + block_format m_fmt; + int m_channel0, m_channel1; + bool m_high_quality; + bool m_from_alpha; + uint32_t m_bc7f_flags; + etc1f::pack_etc1_state* m_pEtc1_pack_state; + }; + + decode_state dec_state; + dec_state.m_orig_height = orig_height; + dec_state.m_src_num_blocks_x = src_num_blocks_x; + dec_state.m_src_num_blocks_y = src_num_blocks_y; + dec_state.m_src_block_width = src_block_width; + dec_state.m_src_block_height = src_block_height; + dec_state.m_dst_num_blocks_x = dst_num_blocks_x; + dec_state.m_dst_num_blocks_y = dst_num_blocks_y; + dec_state.m_pDst_blocks = pDst_blocks; + dec_state.m_output_row_pitch_in_blocks_or_pixels = output_row_pitch_in_blocks_or_pixels; + dec_state.m_output_block_or_pixel_stride_in_bytes = output_block_or_pixel_stride_in_bytes; + dec_state.m_output_rows_in_pixels = output_rows_in_pixels; + + dec_state.m_num_src_block_rows_to_buffer = num_src_block_rows_to_buffer; + //dec_state.m_num_dst_block_rows_to_buffer = num_dst_block_rows_to_buffer; + + dec_state.m_pBuffered_rows = &buffered_rows; + dec_state.m_used_srgb_astc_decode_mode = false; // will be set by init from the compressed stream's header + dec_state.m_has_alpha = true; // will be set by init from the compressed stream's header + + dec_state.m_total_src_blocks_unpacked = 0; + dec_state.m_total_src_blocks_partial_unpacked = 0; + dec_state.m_total_blocks_transcoded = 0; + dec_state.m_total_blocks_encoded = 0; + + dec_state.m_fmt = fmt; + dec_state.m_channel0 = channel0; + dec_state.m_channel1 = channel1; + dec_state.m_high_quality = high_quality; + dec_state.m_from_alpha = from_alpha; + dec_state.m_bc7f_flags = bc7f_flags; + dec_state.m_pEtc1_pack_state = &etc1_pack_state; + + auto init_func = [](uint32_t num_blocks_x, uint32_t num_blocks_y, uint32_t block_width, uint32_t block_height, bool srgb_decode_profile, float dct_q, bool has_alpha, void* pData) + { + BASISU_NOTE_UNUSED(srgb_decode_profile); + BASISU_NOTE_UNUSED(dct_q); + + if (basisu::g_debug_printf) + basisu::debug_printf("init_func: %u %u %u %u %u %f %u\n", num_blocks_x, num_blocks_y, block_width, block_height, srgb_decode_profile, dct_q, has_alpha); + + decode_state& state = *(decode_state*)pData; + if ((block_width != state.m_src_block_width) || (block_height != state.m_src_block_height)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: header validation failed (3)\n"); + return false; + } + if ((num_blocks_x != state.m_src_num_blocks_x) || (num_blocks_y != state.m_src_num_blocks_y)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: header validation failed (4)\n"); + return false; + } + + state.m_used_srgb_astc_decode_mode = srgb_decode_profile; + state.m_has_alpha = has_alpha; + + return true; + }; + + auto src_block_func = [](uint32_t bx, uint32_t by, const astc_helpers::log_astc_block& log_blk, void* pData) + { + decode_state& state = *(decode_state*)pData; + assert((bx < state.m_src_num_blocks_x) && (by < state.m_src_num_blocks_y)); + assert(state.m_num_src_block_rows_to_buffer == 2); // hardcoded for 6x6 + const astc_helpers::decode_mode astc_dec_mode = state.m_used_srgb_astc_decode_mode ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8; + + const uint32_t buffered_src_block_row_y = (by & 1); + + memcpy(&(*state.m_pBuffered_rows)(bx, buffered_src_block_row_y), &log_blk, sizeof(log_blk)); + + // Last block on this src row? If not, exit. + if (bx != (state.m_src_num_blocks_x - 1)) + return true; + + // We've written the final src block for this ASTC src row. + // See if we have enough source rows to create full 4x4 destination blocks. + const bool final_src_block_row = (by == (state.m_src_num_blocks_y - 1)); + + if ((buffered_src_block_row_y != (state.m_num_src_block_rows_to_buffer - 1)) && (!final_src_block_row)) + return true; + + // We have a full 1-2 rows of ASTC 6x6 blocks to process to BC7. + + // src/destination image Y coordinate of the top of the buffered rows + const uint32_t buffered_src_pixel_y = ((by / state.m_num_src_block_rows_to_buffer) * state.m_num_src_block_rows_to_buffer) * state.m_src_block_height; + assert((buffered_src_pixel_y & 3) == 0); + + // The total # of valid src block rows we can read. + const uint32_t num_buffered_src_block_rows = buffered_src_block_row_y + 1; + + assert((num_buffered_src_block_rows == state.m_num_src_block_rows_to_buffer) || (final_src_block_row)); + +#if defined(DEBUG) || defined(_DEBUG) + // The maximum number of valid buffer pixel scanlines we can fetch from, taking into account the original texture's actual (unpadded) height. + const uint32_t override_buffer_height = basisu::minimum(state.m_orig_height - buffered_src_pixel_y, num_buffered_src_block_rows * state.m_src_block_height); + assert(override_buffer_height); +#endif + + // total_dst_block_rows_to_emit=really an upper bound for the final row of src ASTC blocks + const uint32_t total_dst_block_rows_to_emit = (num_buffered_src_block_rows * state.m_src_block_height + 3) >> 2; + + color_rgba unpacked_src_blocks[2][8 * 6]; // [astc_by][pixel] + uint8_t upsampled_src_weights[2][8 * 6]; // [astc_by][pixel] + color_rgba temp_pixels_16[16]; + + // Process each source ASTC 8x6 block group, 1x2 ASTC blocks at a time + for (uint32_t src_bx = 0; src_bx < state.m_src_num_blocks_x; src_bx++) + { + bool has_unpacked_src_blocks[2] = { }; // [astc_by] + + // Grab pointers to the 1x2 src ASTC blocks we'll be transcoding + const astc_helpers::log_astc_block* pSrc_log_blocks[2]; // [astc_by] + + pSrc_log_blocks[0] = &state.m_pBuffered_rows->at(src_bx, 0); + pSrc_log_blocks[1] = &state.m_pBuffered_rows->at(src_bx, basisu::minimum(1, num_buffered_src_block_rows - 1)); + + // From here we can always assume 2x2 src ASTC 6x6 blocks, with ASTC block pointers duplicated at the borders if needed. + + const astc_helpers::log_astc_block* pU = pSrc_log_blocks[0]; + const astc_helpers::log_astc_block* pL = pSrc_log_blocks[1]; + + if (blocks_same_solid_colors(*pU, *pL, 0)) + { + // Easy and fast case: All 2 ASTC blocks are solid and the same color, so all 6 BC7 blocks are solid too and the same color. + color_rgba sc; + sc.r = (uint8_t)(pU->m_solid_color[0] >> 8); + sc.g = (uint8_t)(pU->m_solid_color[1] >> 8); + sc.b = (uint8_t)(pU->m_solid_color[2] >> 8); + sc.a = (uint8_t)(pU->m_solid_color[3] >> 8); + + bc7_block temp_blk; + bc7f::pack_mode5_solid((uint8_t*)&temp_blk, sc); + + for (uint32_t dy = 0; dy < total_dst_block_rows_to_emit; dy++) // up to 3 dst block rows + { + const uint32_t dst_by = (buffered_src_pixel_y >> 2) + dy; + if (dst_by >= state.m_dst_num_blocks_y) + break; + + for (uint32_t dx = 0; dx < 2; dx++) + { + const uint32_t dst_bx = (src_bx << 1) + dx; + if (dst_bx >= state.m_dst_num_blocks_x) + break; + + uint8_t* pDst_block_u8 = (uint8_t*)state.m_pDst_blocks + (dst_by * state.m_output_row_pitch_in_blocks_or_pixels + dst_bx) * state.m_output_block_or_pixel_stride_in_bytes; + + memcpy(pDst_block_u8, &temp_blk, sizeof(bc7_block)); + } // dx + } // dy + + continue; + } + + uint32_t num_hard_src_blocks = 0; + + for (uint32_t y = 0; y < 2; y++) + num_hard_src_blocks += (pSrc_log_blocks[y]->m_dual_plane || (pSrc_log_blocks[y]->m_num_partitions > 1)); + + if (num_hard_src_blocks == 2) + { + // All src blocks hard, no easy optimizations, so unpack and encode pixels analytically + for (uint32_t y = 0; y < 2; y++) + { + bool status = astc_helpers::decode_block_xuastc_ldr(*pSrc_log_blocks[y], &unpacked_src_blocks[y][0], 8, 6, astc_dec_mode); + if (!status) + { + return false; + } + state.m_total_src_blocks_unpacked++; + } + + for (uint32_t dy = 0; dy < total_dst_block_rows_to_emit; dy++) // up to 3 dst block rows + { + const uint32_t dst_by = (buffered_src_pixel_y >> 2) + dy; + if (dst_by >= state.m_dst_num_blocks_y) + break; + + for (uint32_t dx = 0; dx < 2; dx++) + { + const uint32_t dst_bx = (src_bx << 1) + dx; + if (dst_bx >= state.m_dst_num_blocks_x) + break; + + for (uint32_t y = 0; y < 4; y++) + { + const uint32_t sy = dy * 4 + y; + + const uint32_t sy_div6 = sy / 6; + const uint32_t src_row_ofs = (sy % 6) * 8; + + for (uint32_t x = 0; x < 4; x++) + { + const uint32_t sx = dx * 4 + x; + temp_pixels_16[x + y * 4] = unpacked_src_blocks[sy_div6][sx + src_row_ofs]; + } // x + } // y + + uint8_t* pDst_block_u8 = (uint8_t*)state.m_pDst_blocks + (dst_by * state.m_output_row_pitch_in_blocks_or_pixels + dst_bx) * state.m_output_block_or_pixel_stride_in_bytes; + + if (state.m_has_alpha) + bc7f::fast_pack_bc7_auto_rgba(pDst_block_u8, temp_pixels_16, state.m_bc7f_flags); + else + bc7f::fast_pack_bc7_auto_rgb(pDst_block_u8, temp_pixels_16, state.m_bc7f_flags); + + state.m_total_blocks_encoded++; + } // dx + } // dy + + continue; + } + + // One or both of the 2 source blocks is solid or 1 subset. + + // For simplicity: First unpack the first plane weight grids (unless solid) + for (uint32_t y = 0; y < 2; y++) + astc_upsample_grid_weights(*pSrc_log_blocks[y], &upsampled_src_weights[y][0], 8, 6); + + const uint32_t ENDPOINT_TOL = 0; + + // Process each of the up to 6 destination BC7 blocks. + for (uint32_t dy = 0; dy < total_dst_block_rows_to_emit; dy++) // up to 3 dst block rows + { + const uint32_t dst_by = (buffered_src_pixel_y >> 2) + dy; + if (dst_by >= state.m_dst_num_blocks_y) + break; + + for (uint32_t dx = 0; dx < 2; dx++) + { + const uint32_t dst_bx = (src_bx << 1) + dx; + if (dst_bx >= state.m_dst_num_blocks_x) + break; + + uint8_t* pDst_block_u8 = (uint8_t*)state.m_pDst_blocks + (dst_by * state.m_output_row_pitch_in_blocks_or_pixels + dst_bx) * state.m_output_block_or_pixel_stride_in_bytes; + + // BC7 block only overlaps 1 or 2 ASTC blocks. + const int top_dy = dy * 4; + const int bot_dy = top_dy + 3; + + const int top_by = top_dy / 6; + const int bot_by = bot_dy / 6; + + const astc_helpers::log_astc_block* pB0 = pSrc_log_blocks[top_by]; + const astc_helpers::log_astc_block* pB1 = pSrc_log_blocks[bot_by]; + + const bool single_src_block = (top_by == bot_by); + + bool full_encode_flag = false; + + if (pB0->m_dual_plane || pB1->m_dual_plane || + (pB0->m_num_partitions > 1) || (pB1->m_num_partitions > 1)) + { + // Either block is complex, fall back to reencoding + full_encode_flag = true; + } + else if (single_src_block) + { + assert(pB0 == pB1); + + // BC7 block is at a corner, and only overlaps a single ASTC block - output solid or single subset BC7 + if (pB0->m_solid_color_flag_ldr) + { + color_rgba sc; + sc.r = (uint8_t)(pB0->m_solid_color[0] >> 8); + sc.g = (uint8_t)(pB0->m_solid_color[1] >> 8); + sc.b = (uint8_t)(pB0->m_solid_color[2] >> 8); + sc.a = (uint8_t)(pB0->m_solid_color[3] >> 8); + + bc7f::pack_mode5_solid(pDst_block_u8, sc); + } + else + { + // Output mode 6 BC7 + bc7f::pack_from_astc_single_subset(pDst_block_u8, *pB0, &upsampled_src_weights[top_by][0], dx * 4, (dy * 4) % 6, 8, 6); + } + + state.m_total_blocks_transcoded++; + } + // must be overlapping 2 ASTC blocks, can't be both solid as we've already checked + else if (blocks_same_single_subset_endpoints(*pB0, *pB1, ENDPOINT_TOL)) + { + // BC7 blocks overlaps 2 single subset ASTC blocks, both have the same or very similar endpoints, output mode 6 BC7 + bc7f::pack_from_astc_to_single_subset_same_endpoints( + pDst_block_u8, + *pB0, &upsampled_src_weights[top_by][0], + *pB1, &upsampled_src_weights[bot_by][0], + dx, dy, + 8, 6); + + state.m_total_blocks_transcoded++; + } + else if (!block_has_alpha(*pB0) && !block_has_alpha(*pB1)) + { + bool fallback_encode_flag = false; + + // BC7 block overlaps 2 ASTC blocks with different endpoints (or solid colors) - output 2 subset mode 1 BC7 + if (!bc7f::pack_from_astc_8x6_to_two_subsets_different_endpoints_hq( + pDst_block_u8, + *pB0, &upsampled_src_weights[top_by][0], + *pB1, &upsampled_src_weights[bot_by][0], + dx, dy, state.m_used_srgb_astc_decode_mode, fallback_encode_flag)) + { + full_encode_flag = true; + } + else + { + if (fallback_encode_flag) + state.m_total_src_blocks_partial_unpacked++; + else + state.m_total_blocks_transcoded++; + } + } + else + { + full_encode_flag = true; + } + + if (full_encode_flag) + { + // one or both ASTC blocks are just too complex, unpack and reencode + if (!has_unpacked_src_blocks[top_by]) + { + bool status = astc_helpers::decode_block_xuastc_ldr(*pB0, &unpacked_src_blocks[top_by][0], 8, 6, astc_dec_mode, &upsampled_src_weights[top_by][0]); + if (!status) + { + return false; + } + state.m_total_src_blocks_unpacked++; + has_unpacked_src_blocks[top_by] = true; + } + + if (!has_unpacked_src_blocks[bot_by]) + { + bool status = astc_helpers::decode_block_xuastc_ldr(*pB1, &unpacked_src_blocks[bot_by][0], 8, 6, astc_dec_mode, &upsampled_src_weights[bot_by][0]); + if (!status) + { + return false; + } + state.m_total_src_blocks_unpacked++; + has_unpacked_src_blocks[bot_by] = true; + } + + for (uint32_t y = 0; y < 4; y++) + { + const uint32_t sy = dy * 4 + y; + + assert(has_unpacked_src_blocks[sy / 6]); + + const uint32_t sy_div6 = sy / 6; + const uint32_t src_row_ofs = (sy % 6) * 8; + + for (uint32_t x = 0; x < 4; x++) + { + const uint32_t sx = dx * 4 + x; + + temp_pixels_16[x + y * 4] = unpacked_src_blocks[sy_div6][sx + src_row_ofs]; + } // x + } // y + + if (state.m_has_alpha) + bc7f::fast_pack_bc7_auto_rgba(pDst_block_u8, temp_pixels_16, state.m_bc7f_flags); + else + bc7f::fast_pack_bc7_auto_rgb(pDst_block_u8, temp_pixels_16, state.m_bc7f_flags); + + state.m_total_blocks_encoded++; + } + + } // dx + } // dy + + } // src_bx + + return true; + }; + + xuastc_decoded_image decoded_image; + + const bool decomp_flag = decoded_image.decode(pImage_data, image_data_size, init_func, &dec_state, src_block_func, &dec_state); + if (!decomp_flag) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_ldr_t::decompress_image() failed\n"); + return false; + } + + if (basisu::g_debug_printf) + { + basisu::fmt_debug_printf("Total src blocks: {}, Total src blocks fully unpacked to pixels: {}\n", + dec_state.m_src_num_blocks_x * dec_state.m_src_num_blocks_y, + dec_state.m_total_src_blocks_unpacked); + + basisu::fmt_debug_printf("Total dst blocks: {}, Total blocks transcoded: {}, fully encoded: {}, total partially unpacked/fast mode 6 encoded: {}\n", + dec_state.m_dst_num_blocks_x * dec_state.m_dst_num_blocks_y, + dec_state.m_total_blocks_transcoded, dec_state.m_total_blocks_encoded, + dec_state.m_total_src_blocks_partial_unpacked); + } + + // end of 8x6->4x4 transcoder + } + else if ((fmt == block_format::cBC7) && (src_block_width == 6) && (src_block_height == 6) && + (enable_fast_bc7_transcoding) && (!high_quality) && (!deblock_filtering)) + { + // src is ASTC LDR 6x6, destination is BC7, no deblocking: buffer 2 rows of ASTC logical blocks, favor fast pure transcode to BC7 whenever possible. + // This path is maddenningly tricky, but the speed gains in certain use cases are worth it. + + const uint32_t num_src_block_rows_to_buffer = 2; + + assert(((num_src_block_rows_to_buffer * src_block_height) & 3) == 0); + + const uint32_t dst_num_blocks_x = (orig_width + dst_fmt_block_width - 1) / dst_fmt_block_width; + const uint32_t dst_num_blocks_y = (orig_height + dst_fmt_block_height - 1) / dst_fmt_block_height; + + basisu::vector2D buffered_rows(src_num_blocks_x, num_src_block_rows_to_buffer); + + struct decode_state + { + uint32_t m_orig_height; + + uint32_t m_src_num_blocks_x; + uint32_t m_src_num_blocks_y; + uint32_t m_src_block_width; + uint32_t m_src_block_height; + + uint32_t m_dst_num_blocks_x; + uint32_t m_dst_num_blocks_y; + + void* m_pDst_blocks; + uint32_t m_output_row_pitch_in_blocks_or_pixels; + uint32_t m_output_block_or_pixel_stride_in_bytes; + uint32_t m_output_rows_in_pixels; + + uint32_t m_num_src_block_rows_to_buffer; + //uint32_t m_num_dst_block_rows_to_buffer; + + basisu::vector2D *m_pBuffered_rows; + + bool m_used_srgb_astc_decode_mode; + bool m_has_alpha; + + uint32_t m_total_src_blocks_unpacked; + uint32_t m_total_src_blocks_partial_unpacked; + uint32_t m_total_blocks_transcoded; + uint32_t m_total_blocks_encoded; + + block_format m_fmt; + int m_channel0, m_channel1; + bool m_high_quality; + bool m_from_alpha; + uint32_t m_bc7f_flags; + etc1f::pack_etc1_state* m_pEtc1_pack_state; + }; + + decode_state dec_state; + dec_state.m_orig_height = orig_height; + dec_state.m_src_num_blocks_x = src_num_blocks_x; + dec_state.m_src_num_blocks_y = src_num_blocks_y; + dec_state.m_src_block_width = src_block_width; + dec_state.m_src_block_height = src_block_height; + dec_state.m_dst_num_blocks_x = dst_num_blocks_x; + dec_state.m_dst_num_blocks_y = dst_num_blocks_y; + dec_state.m_pDst_blocks = pDst_blocks; + dec_state.m_output_row_pitch_in_blocks_or_pixels = output_row_pitch_in_blocks_or_pixels; + dec_state.m_output_block_or_pixel_stride_in_bytes = output_block_or_pixel_stride_in_bytes; + dec_state.m_output_rows_in_pixels = output_rows_in_pixels; + + dec_state.m_num_src_block_rows_to_buffer = num_src_block_rows_to_buffer; + //dec_state.m_num_dst_block_rows_to_buffer = num_dst_block_rows_to_buffer; + + dec_state.m_pBuffered_rows = &buffered_rows; + dec_state.m_used_srgb_astc_decode_mode = false; // will be set by init from the compressed stream's header + dec_state.m_has_alpha = true; // will be set by init from the compressed stream's header + + dec_state.m_total_src_blocks_unpacked = 0; + dec_state.m_total_src_blocks_partial_unpacked = 0; + dec_state.m_total_blocks_transcoded = 0; + dec_state.m_total_blocks_encoded = 0; + + dec_state.m_fmt = fmt; + dec_state.m_channel0 = channel0; + dec_state.m_channel1 = channel1; + dec_state.m_high_quality = high_quality; + dec_state.m_from_alpha = from_alpha; + dec_state.m_bc7f_flags = bc7f_flags; + dec_state.m_pEtc1_pack_state = &etc1_pack_state; + + auto init_func = [](uint32_t num_blocks_x, uint32_t num_blocks_y, uint32_t block_width, uint32_t block_height, bool srgb_decode_profile, float dct_q, bool has_alpha, void* pData) + { + BASISU_NOTE_UNUSED(srgb_decode_profile); + BASISU_NOTE_UNUSED(dct_q); + + if (basisu::g_debug_printf) + basisu::debug_printf("init_func: %u %u %u %u %u %f %u\n", num_blocks_x, num_blocks_y, block_width, block_height, srgb_decode_profile, dct_q, has_alpha); + + decode_state& state = *(decode_state*)pData; + if ((block_width != state.m_src_block_width) || (block_height != state.m_src_block_height)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: header validation failed (3)\n"); + return false; + } + if ((num_blocks_x != state.m_src_num_blocks_x) || (num_blocks_y != state.m_src_num_blocks_y)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: header validation failed (4)\n"); + return false; + } + + state.m_used_srgb_astc_decode_mode = srgb_decode_profile; + state.m_has_alpha = has_alpha; + + return true; + }; + + auto src_block_func = [](uint32_t bx, uint32_t by, const astc_helpers::log_astc_block& log_blk, void* pData) + { + decode_state& state = *(decode_state*)pData; + assert((bx < state.m_src_num_blocks_x) && (by < state.m_src_num_blocks_y)); + assert(state.m_num_src_block_rows_to_buffer == 2); // hardcoded for 6x6 + const astc_helpers::decode_mode astc_dec_mode = state.m_used_srgb_astc_decode_mode ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8; + + const uint32_t buffered_src_block_row_y = (by & 1); + + memcpy(&(*state.m_pBuffered_rows)(bx, buffered_src_block_row_y), &log_blk, sizeof(log_blk)); + + // Last block on this src row? If not, exit. + if (bx != (state.m_src_num_blocks_x - 1)) + return true; + + // We've written the final src block for this ASTC src row. + // See if we have enough source rows to create full 4x4 destination blocks. + const bool final_src_block_row = (by == (state.m_src_num_blocks_y - 1)); + + if ((buffered_src_block_row_y != (state.m_num_src_block_rows_to_buffer - 1)) && (!final_src_block_row)) + return true; + + // We have a full 1-2 rows of ASTC 6x6 blocks to process to BC7. + + // src/destination image Y coordinate of the top of the buffered rows + const uint32_t buffered_src_pixel_y = ((by / state.m_num_src_block_rows_to_buffer) * state.m_num_src_block_rows_to_buffer) * state.m_src_block_height; + assert((buffered_src_pixel_y & 3) == 0); + + // The total # of valid src block rows we can read. + const uint32_t num_buffered_src_block_rows = buffered_src_block_row_y + 1; + + assert((num_buffered_src_block_rows == state.m_num_src_block_rows_to_buffer) || (final_src_block_row)); + +#if defined(DEBUG) || defined(_DEBUG) + // The maximum number of valid buffer pixel scanlines we can fetch from, taking into account the original texture's actual (unpadded) height. + const uint32_t override_buffer_height = basisu::minimum(state.m_orig_height - buffered_src_pixel_y, num_buffered_src_block_rows * state.m_src_block_height); + assert(override_buffer_height); +#endif + + // total_dst_block_rows_to_emit=really an upper bound for the final row of src ASTC blocks + const uint32_t total_dst_block_rows_to_emit = (num_buffered_src_block_rows * state.m_src_block_height + 3) >> 2; + + color_rgba unpacked_src_blocks[2][2][6 * 6]; // [x][y][pixel] + uint8_t upsampled_src_weights[2][2][6 * 6]; // [x][y][pixel] + color_rgba temp_pixels_16[16]; + + // Process each source ASTC 6x6 block group, 2x2 ASTC blocks at a time (12x12 pixels, or 3x3 BC7 blocks) + for (uint32_t src_bx = 0; src_bx < state.m_src_num_blocks_x; src_bx += 2) + { + bool has_unpacked_src_blocks[2][2] = { }; // [x][y] + + // Grab pointers to the 2x2 src ASTC blocks we'll be transcoding + const astc_helpers::log_astc_block *pSrc_log_blocks[2][2]; // [x][y] + + pSrc_log_blocks[0][0] = &state.m_pBuffered_rows->at(src_bx, 0); + pSrc_log_blocks[1][0] = &state.m_pBuffered_rows->at(basisu::minimum(src_bx + 1, state.m_src_num_blocks_x - 1), 0); + + pSrc_log_blocks[0][1] = &state.m_pBuffered_rows->at(src_bx, basisu::minimum(1, num_buffered_src_block_rows - 1)); + + pSrc_log_blocks[1][1] = &state.m_pBuffered_rows->at( + basisu::minimum(src_bx + 1, state.m_src_num_blocks_x - 1), + basisu::minimum(1, num_buffered_src_block_rows - 1)); + + // From here we can always assume 2x2 src ASTC 6x6 blocks, with ASTC block pointers duplicated at the borders if needed. + + const astc_helpers::log_astc_block* pUL = pSrc_log_blocks[0][0]; + + // First see if all the astc blocks are the same solid color. Likely a common case on some images. + bool all_solid = true; + for (uint32_t y = 0; (y < 2) && all_solid; y++) + { + for (uint32_t x = 0; x < 2; x++) + { + if (!pSrc_log_blocks[x][y]->m_solid_color_flag_ldr) + { + all_solid = false; + break; + } + + if ((pSrc_log_blocks[x][y]->m_solid_color[0] != pUL->m_solid_color[0]) || (pSrc_log_blocks[x][y]->m_solid_color[1] != pUL->m_solid_color[1]) || + (pSrc_log_blocks[x][y]->m_solid_color[2] != pUL->m_solid_color[2]) || (pSrc_log_blocks[x][y]->m_solid_color[3] != pUL->m_solid_color[3])) + { + all_solid = false; + break; + } + } + } + + if (all_solid) + { + // Easy and fast case: All 4 ASTC blocks are solid and the same color, so all 9 BC7 blocks are solid too and the same color. + color_rgba sc; + sc.r = (uint8_t)(pUL->m_solid_color[0] >> 8); + sc.g = (uint8_t)(pUL->m_solid_color[1] >> 8); + sc.b = (uint8_t)(pUL->m_solid_color[2] >> 8); + sc.a = (uint8_t)(pUL->m_solid_color[3] >> 8); + + bc7_block temp_blk; + bc7f::pack_mode5_solid((uint8_t *)&temp_blk, sc); + + for (uint32_t dy = 0; dy < total_dst_block_rows_to_emit; dy++) // up to 3 dst block rows + { + const uint32_t dst_by = (buffered_src_pixel_y >> 2) + dy; + if (dst_by >= state.m_dst_num_blocks_y) + break; + + for (uint32_t dx = 0; dx < 3; dx++) + { + const uint32_t dst_bx = ((src_bx * 6) >> 2) + dx; + if (dst_bx >= state.m_dst_num_blocks_x) + break; + + uint8_t* pDst_block_u8 = (uint8_t*)state.m_pDst_blocks + (dst_by * state.m_output_row_pitch_in_blocks_or_pixels + dst_bx) * state.m_output_block_or_pixel_stride_in_bytes; + + memcpy(pDst_block_u8, &temp_blk, sizeof(bc7_block)); + } + } + + continue; + } + + uint32_t num_hard_src_blocks = 0; + + for (uint32_t y = 0; y < 2; y++) + for (uint32_t x = 0; x < 2; x++) + num_hard_src_blocks += (pSrc_log_blocks[x][y]->m_dual_plane || (pSrc_log_blocks[x][y]->m_num_partitions > 1)); + + if (num_hard_src_blocks == 4) + { + // All src blocks hard, no easy optimizations, so unpack and encode pixels analytically + for (uint32_t y = 0; y < 2; y++) + { + for (uint32_t x = 0; x < 2; x++) + { + bool status = astc_helpers::decode_block_xuastc_ldr(*pSrc_log_blocks[x][y], &unpacked_src_blocks[x][y][0], 6, 6, astc_dec_mode); + if (!status) + { + return false; + } + state.m_total_src_blocks_unpacked++; + } + } + + for (uint32_t dy = 0; dy < total_dst_block_rows_to_emit; dy++) // up to 3 dst block rows + { + const uint32_t dst_by = (buffered_src_pixel_y >> 2) + dy; + if (dst_by >= state.m_dst_num_blocks_y) + break; + + for (uint32_t dx = 0; dx < 3; dx++) + { + const uint32_t dst_bx = ((src_bx * 6) >> 2) + dx; + if (dst_bx >= state.m_dst_num_blocks_x) + break; + + // TODO: Optimize + for (uint32_t y = 0; y < 4; y++) + { + const uint32_t sy = dy * 4 + y; + for (uint32_t x = 0; x < 4; x++) + { + const uint32_t sx = dx * 4 + x; + temp_pixels_16[x + y * 4] = unpacked_src_blocks[sx / 6][sy / 6][(sx % 6) + (sy % 6) * 6]; + } // x + } // y + + uint8_t* pDst_block_u8 = (uint8_t*)state.m_pDst_blocks + (dst_by * state.m_output_row_pitch_in_blocks_or_pixels + dst_bx) * state.m_output_block_or_pixel_stride_in_bytes; + + if (state.m_has_alpha) + bc7f::fast_pack_bc7_auto_rgba(pDst_block_u8, temp_pixels_16, state.m_bc7f_flags); + else + bc7f::fast_pack_bc7_auto_rgb(pDst_block_u8, temp_pixels_16, state.m_bc7f_flags); + + state.m_total_blocks_encoded++; + } // dx + } // dy + + continue; + } + + // One or more of the 4 source blocks is solid or 1 subset. + + // For simplicity: First unpack the first plane weight grids (unless solid) + for (uint32_t y = 0; y < 2; y++) + for (uint32_t x = 0; x < 2; x++) + astc_upsample_grid_weights(*pSrc_log_blocks[x][y], &upsampled_src_weights[x][y][0], 6, 6); + + //const uint32_t ENDPOINT_TOL = 3; + const uint32_t ENDPOINT_TOL = 1; + + // Process each of the 9 destination BC7 blocks. + for (uint32_t dy = 0; dy < total_dst_block_rows_to_emit; dy++) // up to 3 dst block rows + { + const uint32_t dst_by = (buffered_src_pixel_y >> 2) + dy; + if (dst_by >= state.m_dst_num_blocks_y) + break; + + for (uint32_t dx = 0; dx < 3; dx++) + { + // skip the central block, which we'll processed last + if ((dx == 1) && (dy == 1)) + continue; + + const uint32_t dst_bx = ((src_bx * 6) >> 2) + dx; + if (dst_bx >= state.m_dst_num_blocks_x) + break; + + uint8_t* pDst_block_u8 = (uint8_t*)state.m_pDst_blocks + (dst_by * state.m_output_row_pitch_in_blocks_or_pixels + dst_bx) * state.m_output_block_or_pixel_stride_in_bytes; + + // We're NOT at the center BC7 dst block. + // BC7 block only overlaps 1 or 2 ASTC blocks. + const int top_dx = dx * 4, top_dy = dy * 4; + const int bot_dx = top_dx + 3, bot_dy = top_dy + 3; + + const int top_bx = top_dx / 6, top_by = top_dy / 6; + const int bot_bx = bot_dx / 6, bot_by = bot_dy / 6; + + const astc_helpers::log_astc_block* pB0 = pSrc_log_blocks[top_bx][top_by]; + const astc_helpers::log_astc_block* pB1 = pSrc_log_blocks[bot_bx][bot_by]; + + // Note: because of row/col duplication at the edges of images, pB0 could equal pB1 even though we're not at a BC7 corner block. + + const bool single_src_block = (top_bx == bot_bx) && (top_by == bot_by); + + bool full_encode_flag = false; + + if (pB0->m_dual_plane || pB1->m_dual_plane || + (pB0->m_num_partitions > 1) || (pB1->m_num_partitions > 1)) + { + // Either block is complex, fall back to reencoding + full_encode_flag = true; + } + else if (single_src_block) + { + assert(pB0 == pB1); + + // BC7 block is at a corner, and only overlaps a single ASTC block - output solid or single subset BC7 + if (pB0->m_solid_color_flag_ldr) + { + color_rgba sc; + sc.r = (uint8_t)(pB0->m_solid_color[0] >> 8); + sc.g = (uint8_t)(pB0->m_solid_color[1] >> 8); + sc.b = (uint8_t)(pB0->m_solid_color[2] >> 8); + sc.a = (uint8_t)(pB0->m_solid_color[3] >> 8); + + bc7f::pack_mode5_solid(pDst_block_u8, sc); + } + else + { + // Output mode 6 BC7 + bc7f::pack_from_astc_single_subset(pDst_block_u8, *pB0, &upsampled_src_weights[top_bx][top_by][0], (dx * 4) % 6, (dy * 4) % 6, 6, 6); + } + + state.m_total_blocks_transcoded++; + } + // below here BC7 block always overlaps 2 ASTC 6x6 blocks (1 block case just ruled out) + else if (blocks_same_solid_colors(*pB0, *pB1, 0)) + { + // BC7 block overlaps 2 ASTC blocks, both solid colors, both same colors + + color_rgba sc; + sc.r = (uint8_t)(pB0->m_solid_color[0] >> 8); + sc.g = (uint8_t)(pB0->m_solid_color[1] >> 8); + sc.b = (uint8_t)(pB0->m_solid_color[2] >> 8); + sc.a = (uint8_t)(pB0->m_solid_color[3] >> 8); + bc7f::pack_mode5_solid(pDst_block_u8, sc); + + state.m_total_blocks_transcoded++; + } + else if (blocks_same_single_subset_endpoints(*pB0, *pB1, ENDPOINT_TOL)) + { + // BC7 blocks overlaps 2 single subset ASTC blocks, both have the same or very similar endpoints, output mode 6 BC7 + bc7f::pack_from_astc_to_single_subset_same_endpoints( + pDst_block_u8, + *pB0, &upsampled_src_weights[top_bx][top_by][0], + *pB1, &upsampled_src_weights[bot_bx][bot_by][0], + dx, dy, + 6, 6); + + state.m_total_blocks_transcoded++; + } + else if (!block_has_alpha(*pB0) && !block_has_alpha(*pB1)) + { + // BC7 block overlaps 2 ASTC blocks with different endpoints (or solid colors) - output 2 subset mode 1 BC7 + if (!bc7f::pack_from_astc_6x6_to_two_subsets_different_endpoints( + pDst_block_u8, + *pB0, &upsampled_src_weights[top_bx][top_by][0], + *pB1, &upsampled_src_weights[bot_bx][bot_by][0], + dx, dy)) + { + full_encode_flag = true; + } + else + { + state.m_total_blocks_transcoded++; + } + } + else + { + full_encode_flag = true; + } + + if (full_encode_flag) + { + // one or both ASTC blocks are just too complex, unpack and reencode + if (!has_unpacked_src_blocks[top_bx][top_by]) + { + bool status = astc_helpers::decode_block_xuastc_ldr(*pB0, &unpacked_src_blocks[top_bx][top_by][0], 6, 6, astc_dec_mode, &upsampled_src_weights[top_bx][top_by][0]); + if (!status) + { + return false; + } + state.m_total_src_blocks_unpacked++; + has_unpacked_src_blocks[top_bx][top_by] = true; + } + + if (!has_unpacked_src_blocks[bot_bx][bot_by]) + { + bool status = astc_helpers::decode_block_xuastc_ldr(*pB1, &unpacked_src_blocks[bot_bx][bot_by][0], 6, 6, astc_dec_mode, &upsampled_src_weights[bot_bx][bot_by][0]); + if (!status) + { + return false; + } + state.m_total_src_blocks_unpacked++; + has_unpacked_src_blocks[bot_bx][bot_by] = true; + } + + // TODO: Optimize + for (uint32_t y = 0; y < 4; y++) + { + const uint32_t sy = dy * 4 + y; + for (uint32_t x = 0; x < 4; x++) + { + const uint32_t sx = dx * 4 + x; + + assert(has_unpacked_src_blocks[sx / 6][sy / 6]); + + temp_pixels_16[x + y * 4] = unpacked_src_blocks[sx / 6][sy / 6][(sx % 6) + (sy % 6) * 6]; + } // x + } // y + + if (state.m_has_alpha) + bc7f::fast_pack_bc7_auto_rgba(pDst_block_u8, temp_pixels_16, state.m_bc7f_flags); + else + bc7f::fast_pack_bc7_auto_rgb(pDst_block_u8, temp_pixels_16, state.m_bc7f_flags); + + state.m_total_blocks_encoded++; + } + + } // dx + } // dy + + // Now handle the center BC7 block - by this point we may have already decoded 1 or more ASTC 6x6 blocks. + const uint32_t dx = 1, dy = 1; + const uint32_t dst_bx = ((src_bx * 6) >> 2) + dx; + const uint32_t dst_by = (buffered_src_pixel_y >> 2) + dy; + + if ((dst_bx < state.m_dst_num_blocks_x) && (dst_by < state.m_dst_num_blocks_y)) + { + uint8_t* pDst_block_u8 = (uint8_t*)state.m_pDst_blocks + (dst_by * state.m_output_row_pitch_in_blocks_or_pixels + dst_bx) * state.m_output_block_or_pixel_stride_in_bytes; + + bool skip_full_encode = false; + + // the unfortunate middle BC7 block, overlaps all 4 ASTC blocks + // The 4 ASTC blocks cannot be all solid, and cannot be all hard. + if (num_hard_src_blocks == 0) + { + // all blocks are simple, see if we can find a simple case to handle quickly + bool top_same_solid_color = blocks_same_solid_colors(*pSrc_log_blocks[0][0], *pSrc_log_blocks[1][0], 1); + bool bot_same_solid_color = blocks_same_solid_colors(*pSrc_log_blocks[0][1], *pSrc_log_blocks[1][1], 1); + + bool left_same_solid_color = blocks_same_solid_colors(*pSrc_log_blocks[0][0], *pSrc_log_blocks[0][1], 1); + bool right_same_solid_color = blocks_same_solid_colors(*pSrc_log_blocks[1][0], *pSrc_log_blocks[1][1], 1); + + bool top_same_endpoints = blocks_same_single_subset_endpoints(*pSrc_log_blocks[0][0], *pSrc_log_blocks[1][0], ENDPOINT_TOL); + bool bot_same_endpoints = blocks_same_single_subset_endpoints(*pSrc_log_blocks[0][1], *pSrc_log_blocks[1][1], ENDPOINT_TOL); + + bool left_same_endpoints = blocks_same_single_subset_endpoints(*pSrc_log_blocks[0][0], *pSrc_log_blocks[0][1], ENDPOINT_TOL); + bool right_same_endpoints = blocks_same_single_subset_endpoints(*pSrc_log_blocks[1][0], *pSrc_log_blocks[1][1], ENDPOINT_TOL); + + bool top_no_alpha = !block_has_alpha(*pSrc_log_blocks[0][0]) && !block_has_alpha(*pSrc_log_blocks[1][0]); + bool bot_no_alpha = !block_has_alpha(*pSrc_log_blocks[0][1]) && !block_has_alpha(*pSrc_log_blocks[1][1]); + + bool left_no_alpha = !block_has_alpha(*pSrc_log_blocks[0][0]) && !block_has_alpha(*pSrc_log_blocks[0][1]); + bool right_no_alpha = !block_has_alpha(*pSrc_log_blocks[1][0]) && !block_has_alpha(*pSrc_log_blocks[1][1]); + + bool top_transcodable = (top_same_solid_color || top_same_endpoints) && top_no_alpha; + bool bot_transcodable = (bot_same_solid_color || bot_same_endpoints) && bot_no_alpha; + + bool left_transcodable = (left_same_solid_color || left_same_endpoints) && left_no_alpha; + bool right_transcodable = (right_same_solid_color || right_same_endpoints) && right_no_alpha; + + if (top_transcodable && bot_transcodable) + { + // BC7 mode 1 + bc7f::pack_astc_6x6_to_two_subsets_middle_block( + pDst_block_u8, + pSrc_log_blocks, upsampled_src_weights, + false); + state.m_total_blocks_transcoded++; + skip_full_encode = true; + } + else if (left_transcodable && right_transcodable) + { + // BC7 mode 1 + bc7f::pack_astc_6x6_to_two_subsets_middle_block( + pDst_block_u8, + pSrc_log_blocks, upsampled_src_weights, + true); + state.m_total_blocks_transcoded++; + skip_full_encode = true; + } + + // TODO: Handle non-exact cases + } + + if (!skip_full_encode) + { + // Center BC7 block scenario is complex, requires full decode+analytical BC7 encode. + if (has_unpacked_src_blocks[0][0] && has_unpacked_src_blocks[1][0] && has_unpacked_src_blocks[0][1] && has_unpacked_src_blocks[1][1]) + { + // We've already decoded all the ASTC 6x6 blocks fully, so grab central pixels + for (uint32_t y = 0; y < 4; y++) + { + const uint32_t sy = dy * 4 + y; + for (uint32_t x = 0; x < 4; x++) + { + const uint32_t sx = dx * 4 + x; + + temp_pixels_16[x + y * 4] = unpacked_src_blocks[sx / 6][sy / 6][(sx % 6) + (sy % 6) * 6]; + } // x + } // y + + if (state.m_has_alpha) + bc7f::fast_pack_bc7_auto_rgba(pDst_block_u8, temp_pixels_16, state.m_bc7f_flags); + else + bc7f::fast_pack_bc7_auto_rgb(pDst_block_u8, temp_pixels_16, state.m_bc7f_flags); + + state.m_total_blocks_encoded++; + } + else + { + for (uint32_t iby = 0; iby < 2; iby++) + { + for (uint32_t ibx = 0; ibx < 2; ibx++) + { + // Do a partial decode of the ASTC block, just to get those central pixels (big savings) + uint32_t start_x, start_y, end_x, end_y; + + switch (ibx + iby * 2) + { + case 0: start_x = 4; end_x = 6; start_y = 4; end_y = 6; break; + case 1: start_x = 0; end_x = 2; start_y = 4; end_y = 6; break; + case 2: start_x = 4; end_x = 6; start_y = 0; end_y = 2; break; + default: + case 3: start_x = 0; end_x = 2; start_y = 0; end_y = 2; break; + } + + // See if we've already decoded the block + if (has_unpacked_src_blocks[ibx][iby]) + { + for (uint32_t py = 0; py < 2; py++) + { + const uint32_t sy = start_y + py; + + for (uint32_t px = 0; px < 2; px++) + { + const uint32_t sx = start_x + px; + + temp_pixels_16[(px + ibx * 2) + (py + iby * 2) * 4] = unpacked_src_blocks[ibx][iby][(sx % 6) + (sy % 6) * 6]; + } // x + } // y + } + else + { + // Partial decode + color_rgba temp_pixels[6 * 6]; + + bool status = astc_helpers::decode_block_xuastc_ldr(*pSrc_log_blocks[ibx][iby], temp_pixels, 6, 6, astc_dec_mode, + &upsampled_src_weights[ibx][iby][0], start_x, start_y, end_x, end_y); + if (!status) + { + return false; + } + state.m_total_src_blocks_partial_unpacked++; + + for (uint32_t py = 0; py < 2; py++) + { + const uint32_t ey = py + iby * 2; + + for (uint32_t px = 0; px < 2; px++) + { + const uint32_t ex = px + ibx * 2; + + temp_pixels_16[ex + ey * 4] = temp_pixels[(start_x + px) + (start_y + py) * 6]; + } // x + } // y + } + } // x + } // y + + if (state.m_has_alpha) + bc7f::fast_pack_bc7_auto_rgba(pDst_block_u8, temp_pixels_16, state.m_bc7f_flags); + else + bc7f::fast_pack_bc7_auto_rgb(pDst_block_u8, temp_pixels_16, state.m_bc7f_flags); + + state.m_total_blocks_encoded++; + } + + } // skip_full_encode + + } // if ((dst_bx < state.m_dst_num_blocks_x) && (dst_by < state.m_dst_num_blocks_y)) + + } // src_bx + + return true; + }; + + xuastc_decoded_image decoded_image; + + const bool decomp_flag = decoded_image.decode(pImage_data, image_data_size, init_func, &dec_state, src_block_func, &dec_state); + if (!decomp_flag) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_ldr_t::decompress_image() failed\n"); + return false; + } + + if (basisu::g_debug_printf) + { + basisu::fmt_debug_printf("Total src blocks: {}, Total src blocks unpacked to pixels: {}, total partial unpacks: {}\n", + dec_state.m_src_num_blocks_x * dec_state.m_src_num_blocks_y, + dec_state.m_total_src_blocks_unpacked, dec_state.m_total_src_blocks_partial_unpacked); + + basisu::fmt_debug_printf("Total dst blocks: {}, Total blocks transcoded: {}, encoded: {}\n", + dec_state.m_dst_num_blocks_x * dec_state.m_dst_num_blocks_y, + dec_state.m_total_blocks_transcoded, dec_state.m_total_blocks_encoded); + } + } + else if (((src_block_width == 4) && (src_block_height == 4)) && (!dst_fmt_is_pvrtc1) && (!deblock_filtering)) + { + // src is ASTC LDR 4x4, destination block size must be 4x4, no PVRTC1, no deblocking. Directly pack to target format during transcoding. + struct decode_state + { + uint32_t m_src_num_blocks_x; + uint32_t m_src_num_blocks_y; + + void* m_pDst_blocks; + uint32_t m_output_row_pitch_in_blocks_or_pixels; + uint32_t m_output_block_or_pixel_stride_in_bytes; + uint32_t m_output_rows_in_pixels; + + block_format m_fmt; + bool m_used_srgb_astc_decode_mode; + bool m_has_alpha; + + int m_channel0, m_channel1; + bool m_high_quality; + bool m_enable_fast_bc7_transcoding; + bool m_from_alpha; + uint32_t m_bc7f_flags; + etc1f::pack_etc1_state* m_pEtc1_pack_state; + }; + + decode_state dec_state; + dec_state.m_src_num_blocks_x = src_num_blocks_x; + dec_state.m_src_num_blocks_y = src_num_blocks_y; + dec_state.m_pDst_blocks = pDst_blocks; + dec_state.m_output_row_pitch_in_blocks_or_pixels = output_row_pitch_in_blocks_or_pixels; + dec_state.m_output_block_or_pixel_stride_in_bytes = output_block_or_pixel_stride_in_bytes; + dec_state.m_output_rows_in_pixels = output_rows_in_pixels; + dec_state.m_fmt = fmt; + dec_state.m_used_srgb_astc_decode_mode = false; // will be set by init from the compressed stream's header + dec_state.m_has_alpha = true; // will be set by init from the compressed stream's header + + dec_state.m_channel0 = channel0; + dec_state.m_channel1 = channel1; + dec_state.m_high_quality = high_quality; + dec_state.m_enable_fast_bc7_transcoding = enable_fast_bc7_transcoding; + dec_state.m_from_alpha = from_alpha; + dec_state.m_bc7f_flags = bc7f_flags; + dec_state.m_pEtc1_pack_state = &etc1_pack_state; + + auto init_func = [](uint32_t num_blocks_x, uint32_t num_blocks_y, uint32_t block_width, uint32_t block_height, bool srgb_decode_profile, float dct_q, bool has_alpha, void* pData) + { + BASISU_NOTE_UNUSED(srgb_decode_profile); + BASISU_NOTE_UNUSED(dct_q); + + if (basisu::g_debug_printf) + basisu::debug_printf("init_func: %u %u %u %u %u %f %u\n", num_blocks_x, num_blocks_y, block_width, block_height, srgb_decode_profile, dct_q, has_alpha); + + decode_state& state = *(decode_state*)pData; + if ((block_width != 4) || (block_height != 4)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: header validation failed (3)\n"); + return false; + } + if ((num_blocks_x != state.m_src_num_blocks_x) || (num_blocks_y != state.m_src_num_blocks_y)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: header validation failed (4)\n"); + return false; + } + + state.m_used_srgb_astc_decode_mode = srgb_decode_profile; + state.m_has_alpha = has_alpha; + + return true; + }; + + auto src_block_func = [](uint32_t bx, uint32_t by, const astc_helpers::log_astc_block& log_blk, void* pData) + { + decode_state& state = *(decode_state*)pData; + assert((bx < state.m_src_num_blocks_x) && (by < state.m_src_num_blocks_y)); + + uint8_t* pDst_block_u8 = (uint8_t*)state.m_pDst_blocks + (by * state.m_output_row_pitch_in_blocks_or_pixels + bx) * state.m_output_block_or_pixel_stride_in_bytes; + + // Special fast cases for BC7 transcode target given common ASTC configs + if (state.m_fmt == block_format::cBC7) + { + if (log_blk.m_solid_color_flag_ldr) + { + color_rgba sc; + sc.r = (uint8_t)(log_blk.m_solid_color[0] >> 8); + sc.g = (uint8_t)(log_blk.m_solid_color[1] >> 8); + sc.b = (uint8_t)(log_blk.m_solid_color[2] >> 8); + sc.a = (uint8_t)(log_blk.m_solid_color[3] >> 8); + + bc7f::pack_mode5_solid(pDst_block_u8, sc); + return true; + } + else if (!log_blk.m_dual_plane && (log_blk.m_num_partitions == 1) && !state.m_high_quality && state.m_enable_fast_bc7_transcoding) + { + // TODO: This does cost a tiny amount of PSNR (.1-25 dB or so), but is way faster. + bc7f::pack_from_astc_4x4_single_subset(pDst_block_u8, log_blk); + return true; + } + } + + // Fall back to block pixel unpack then analytical encode. + color32 block_pixels[4 * 4]; + bool decode_status = astc_helpers::decode_block_xuastc_ldr(log_blk, block_pixels, 4, 4, state.m_used_srgb_astc_decode_mode ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8); + if (!decode_status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_helpers::decode_block_xuastc_ldr() failed\n"); + return false; + } + +#if defined(_DEBUG) || defined(DEBUG) + color32 alt_block_pixels[4 * 4]; + if (!astc_helpers::decode_block(log_blk, alt_block_pixels, 4, 4, state.m_used_srgb_astc_decode_mode ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_helpers::decode_block() failed\n"); + return false; + } + + for (uint32_t i = 0; i < 16; i++) + { + assert(block_pixels[i][0] == alt_block_pixels[i][0]); + assert(block_pixels[i][1] == alt_block_pixels[i][1]); + assert(block_pixels[i][2] == alt_block_pixels[i][2]); + assert(block_pixels[i][3] == alt_block_pixels[i][3]); + } +#endif + + transcode_4x4_block( + state.m_fmt, + bx, by, + state.m_pDst_blocks, pDst_block_u8, + block_pixels, + state.m_output_block_or_pixel_stride_in_bytes, state.m_output_row_pitch_in_blocks_or_pixels, state.m_output_rows_in_pixels, + state.m_channel0, state.m_channel1, + state.m_high_quality, state.m_from_alpha, + state.m_bc7f_flags, + *state.m_pEtc1_pack_state, + state.m_has_alpha ? 1 : 0); + + return true; + }; + + xuastc_decoded_image decoded_image; + + const bool decomp_flag = decoded_image.decode(pImage_data, image_data_size, init_func, &dec_state, src_block_func, &dec_state); + if (!decomp_flag) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_ldr_t::decompress_image() failed\n"); + return false; + } + } + else if ((deblock_filtering) || (dst_fmt_is_pvrtc1)) + { + // Completely general case. Unpack entire 32bpp image into memory (needed for deblocking and PVRTC1). + assert((dst_fmt_block_width == 4) && (dst_fmt_block_height == 4)); + basisu::vector2D temp_image; + + if (!temp_image.try_resize(src_num_blocks_x * src_block_width, src_num_blocks_y * src_block_height)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: out of memory\n"); + return false; + } + + struct decode_state + { + uint32_t m_src_num_blocks_x; + uint32_t m_src_num_blocks_y; + uint32_t m_src_block_width; + uint32_t m_src_block_height; + + void* m_pDst_blocks; + uint32_t m_output_row_pitch_in_blocks_or_pixels; + uint32_t m_output_block_or_pixel_stride_in_bytes; + uint32_t m_output_rows_in_pixels; + + basisu::vector2D* m_pTemp_image; + + bool m_used_srgb_astc_decode_mode; + bool m_has_alpha; + }; + + decode_state dec_state; + dec_state.m_src_num_blocks_x = src_num_blocks_x; + dec_state.m_src_num_blocks_y = src_num_blocks_y; + dec_state.m_src_block_width = src_block_width; + dec_state.m_src_block_height = src_block_height; + dec_state.m_pDst_blocks = pDst_blocks; + dec_state.m_output_row_pitch_in_blocks_or_pixels = output_row_pitch_in_blocks_or_pixels; + dec_state.m_output_block_or_pixel_stride_in_bytes = output_block_or_pixel_stride_in_bytes; + dec_state.m_output_rows_in_pixels = output_rows_in_pixels; + dec_state.m_pTemp_image = &temp_image; + dec_state.m_used_srgb_astc_decode_mode = false; // will be set by init from the compressed stream's header + dec_state.m_has_alpha = true; // will be set by init from the compressed stream's header + + auto init_func = [](uint32_t num_blocks_x, uint32_t num_blocks_y, uint32_t block_width, uint32_t block_height, bool srgb_decode_profile, float dct_q, bool has_alpha, void* pData) + { + BASISU_NOTE_UNUSED(srgb_decode_profile); + BASISU_NOTE_UNUSED(dct_q); + + if (basisu::g_debug_printf) + basisu::debug_printf("init_func: %u %u %u %u %u %f %u\n", num_blocks_x, num_blocks_y, block_width, block_height, srgb_decode_profile, dct_q, has_alpha); + + decode_state& state = *(decode_state*)pData; + if ((block_width != state.m_src_block_width) || (block_height != state.m_src_block_height)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: header validation failed (3)\n"); + return false; + } + if ((num_blocks_x != state.m_src_num_blocks_x) || (num_blocks_y != state.m_src_num_blocks_y)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: header validation failed (4)\n"); + return false; + } + + state.m_used_srgb_astc_decode_mode = srgb_decode_profile; + state.m_has_alpha = has_alpha; + + return true; + }; + + auto src_block_func = [](uint32_t bx, uint32_t by, const astc_helpers::log_astc_block& log_blk, void* pData) + { + decode_state& state = *(decode_state*)pData; + assert((bx < state.m_src_num_blocks_x) && (by < state.m_src_num_blocks_y)); + + color32 block_pixels[astc_helpers::MAX_BLOCK_PIXELS]; + bool decode_status = astc_helpers::decode_block_xuastc_ldr(log_blk, block_pixels, state.m_src_block_width, state.m_src_block_height, state.m_used_srgb_astc_decode_mode ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8); + if (!decode_status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_helpers::decode_block_xuastc_ldr() failed\n"); + return false; + } + +#if defined(_DEBUG) || defined(DEBUG) + // sanity check vs. our vanilla/full-featured ASTC decoder + color32 alt_block_pixels[astc_helpers::MAX_BLOCK_PIXELS]; + if (!astc_helpers::decode_block(log_blk, alt_block_pixels, state.m_src_block_width, state.m_src_block_height, state.m_used_srgb_astc_decode_mode ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_helpers::decode_block() failed\n"); + return false; + } + + for (uint32_t i = 0; i < state.m_src_block_width * state.m_src_block_height; i++) + { + assert(block_pixels[i][0] == alt_block_pixels[i][0]); + assert(block_pixels[i][1] == alt_block_pixels[i][1]); + assert(block_pixels[i][2] == alt_block_pixels[i][2]); + assert(block_pixels[i][3] == alt_block_pixels[i][3]); + } +#endif + + color32* pSrc_pixels = block_pixels; + color32* pDst_pixels = &(*state.m_pTemp_image)(bx * state.m_src_block_width, by * state.m_src_block_height); + + for (uint32_t y = 0; y < state.m_src_block_height; y++) + { + memcpy(pDst_pixels, pSrc_pixels, state.m_src_block_width * sizeof(color32)); + + pSrc_pixels += state.m_src_block_width; + pDst_pixels += state.m_pTemp_image->get_width(); + } // y + + return true; + }; + + xuastc_decoded_image decoded_image; + + const bool decomp_flag = decoded_image.decode(pImage_data, image_data_size, init_func, &dec_state, src_block_func, &dec_state); + if (!decomp_flag) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_ldr_t::decompress_image() failed\n"); + return false; + } + + if (deblock_filtering) + { + if (!xuastc_deblock_filter( + decoded_image.m_actual_block_width, decoded_image.m_actual_block_height, + temp_image, temp_image, + stronger_deblocking, XUASTC_LDR_DEBLOCK_SKIP_THRESH)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: out of memory\n"); + return false; + } + } + + const uint32_t dst_num_blocks_x = (orig_width + dst_fmt_block_width - 1) / dst_fmt_block_width; + const uint32_t dst_num_blocks_y = (orig_height + dst_fmt_block_height - 1) / dst_fmt_block_height; + + if (dst_fmt_is_pvrtc1) + { + assert((dst_fmt_block_width == 4) && (dst_fmt_block_height == 4)); + + encode_pvrtc1(fmt, pDst_blocks, temp_image, dst_num_blocks_x, dst_num_blocks_y, from_alpha); + } + else + { + color32 block_pixels[astc_helpers::MAX_BLOCK_PIXELS]; + + for (uint32_t dst_by = 0; dst_by < dst_num_blocks_y; dst_by++) + { + uint8_t* pDst_block_u8 = (uint8_t*)pDst_blocks + dst_by * output_row_pitch_in_blocks_or_pixels * output_block_or_pixel_stride_in_bytes; + + for (uint32_t dst_bx = 0; dst_bx < dst_num_blocks_x; dst_bx++) + { + temp_image.extract_block_clamped(block_pixels, dst_bx * 4, dst_by * 4, 4, 4); + + transcode_4x4_block( + fmt, + dst_bx, dst_by, + pDst_blocks, pDst_block_u8, + block_pixels, + output_block_or_pixel_stride_in_bytes, output_row_pitch_in_blocks_or_pixels, output_rows_in_pixels, + channel0, channel1, + high_quality, from_alpha, + bc7f_flags, + etc1_pack_state, + dec_state.m_has_alpha ? 1 : 0); + + pDst_block_u8 += output_block_or_pixel_stride_in_bytes; + + } // dst_bx + + } // dst_by + + } // if (dst_fmt_is_pvrtc1) + } + else + { + // No PVRTC1/ASTC, no deblocking. Unpack as few source row blocks into memory as possible needed for transcoding to 4x4. Output block size must be 4x4. + assert((dst_fmt_block_width == 4) && (dst_fmt_block_height == 4)); + + // Compute how many source block rows we need to buffer so we have a multiple of 4 scanlines. The max # of scanlines is 20. + uint32_t num_src_block_rows_to_buffer = 1; + while ((num_src_block_rows_to_buffer * src_block_height) & 3) + num_src_block_rows_to_buffer++; + assert((num_src_block_rows_to_buffer >= 1) && (num_src_block_rows_to_buffer <= 4)); + + // Compute how many 4x4 dest blocks fit into these many source rows. + assert(((num_src_block_rows_to_buffer * src_block_height) & 3) == 0); + //const uint32_t num_dst_block_rows_to_buffer = (num_src_block_rows_to_buffer * src_block_height) >> 2; + + const uint32_t dst_num_blocks_x = (orig_width + dst_fmt_block_width - 1) / dst_fmt_block_width; + const uint32_t dst_num_blocks_y = (orig_height + dst_fmt_block_height - 1) / dst_fmt_block_height; + + basisu::vector2D buffered_rows(src_num_blocks_x * src_block_width, num_src_block_rows_to_buffer * src_block_height); + + struct decode_state + { + uint32_t m_orig_height; + + uint32_t m_src_num_blocks_x; + uint32_t m_src_num_blocks_y; + uint32_t m_src_block_width; + uint32_t m_src_block_height; + + uint32_t m_dst_num_blocks_x; + uint32_t m_dst_num_blocks_y; + + void* m_pDst_blocks; + uint32_t m_output_row_pitch_in_blocks_or_pixels; + uint32_t m_output_block_or_pixel_stride_in_bytes; + uint32_t m_output_rows_in_pixels; + + uint32_t m_num_src_block_rows_to_buffer; + //uint32_t m_num_dst_block_rows_to_buffer; + + basisu::vector2D* m_pBuffered_rows; + + bool m_used_srgb_astc_decode_mode; + bool m_has_alpha; + + block_format m_fmt; + int m_channel0, m_channel1; + bool m_high_quality; + bool m_from_alpha; + uint32_t m_bc7f_flags; + etc1f::pack_etc1_state* m_pEtc1_pack_state; + }; + + decode_state dec_state; + dec_state.m_orig_height = orig_height; + dec_state.m_src_num_blocks_x = src_num_blocks_x; + dec_state.m_src_num_blocks_y = src_num_blocks_y; + dec_state.m_src_block_width = src_block_width; + dec_state.m_src_block_height = src_block_height; + dec_state.m_dst_num_blocks_x = dst_num_blocks_x; + dec_state.m_dst_num_blocks_y = dst_num_blocks_y; + dec_state.m_pDst_blocks = pDst_blocks; + dec_state.m_output_row_pitch_in_blocks_or_pixels = output_row_pitch_in_blocks_or_pixels; + dec_state.m_output_block_or_pixel_stride_in_bytes = output_block_or_pixel_stride_in_bytes; + dec_state.m_output_rows_in_pixels = output_rows_in_pixels; + + dec_state.m_num_src_block_rows_to_buffer = num_src_block_rows_to_buffer; + //dec_state.m_num_dst_block_rows_to_buffer = num_dst_block_rows_to_buffer; + + dec_state.m_pBuffered_rows = &buffered_rows; + dec_state.m_used_srgb_astc_decode_mode = false; // will be set by init from the compressed stream's header + dec_state.m_has_alpha = true; // will be set by init from the compressed stream's header + + dec_state.m_fmt = fmt; + dec_state.m_channel0 = channel0; + dec_state.m_channel1 = channel1; + dec_state.m_high_quality = high_quality; + dec_state.m_from_alpha = from_alpha; + dec_state.m_bc7f_flags = bc7f_flags; + dec_state.m_pEtc1_pack_state = &etc1_pack_state; + + auto init_func = [](uint32_t num_blocks_x, uint32_t num_blocks_y, uint32_t block_width, uint32_t block_height, bool srgb_decode_profile, float dct_q, bool has_alpha, void* pData) + { + BASISU_NOTE_UNUSED(srgb_decode_profile); + BASISU_NOTE_UNUSED(dct_q); + + if (basisu::g_debug_printf) + basisu::debug_printf("init_func: %u %u %u %u %u %f %u\n", num_blocks_x, num_blocks_y, block_width, block_height, srgb_decode_profile, dct_q, has_alpha); + + decode_state& state = *(decode_state*)pData; + if ((block_width != state.m_src_block_width) || (block_height != state.m_src_block_height)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: header validation failed (3)\n"); + return false; + } + if ((num_blocks_x != state.m_src_num_blocks_x) || (num_blocks_y != state.m_src_num_blocks_y)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: header validation failed (4)\n"); + return false; + } + + state.m_used_srgb_astc_decode_mode = srgb_decode_profile; + state.m_has_alpha = has_alpha; + + return true; + }; + + auto src_block_func = [](uint32_t bx, uint32_t by, const astc_helpers::log_astc_block& log_blk, void* pData) + { + decode_state& state = *(decode_state*)pData; + assert((bx < state.m_src_num_blocks_x) && (by < state.m_src_num_blocks_y)); + + // Unpack ASTC block, distribute to temp output buffer. + color32 block_pixels[astc_helpers::MAX_BLOCK_PIXELS]; + bool decode_status = astc_helpers::decode_block_xuastc_ldr(log_blk, block_pixels, state.m_src_block_width, state.m_src_block_height, state.m_used_srgb_astc_decode_mode ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8); + if (!decode_status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_helpers::decode_block_xuastc_ldr() failed\n"); + return false; + } + +#if defined(_DEBUG) || defined(DEBUG) + color32 alt_block_pixels[astc_helpers::MAX_BLOCK_PIXELS]; + if (!astc_helpers::decode_block(log_blk, alt_block_pixels, state.m_src_block_width, state.m_src_block_height, state.m_used_srgb_astc_decode_mode ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_helpers::decode_block() failed\n"); + return false; + } + + for (uint32_t i = 0; i < state.m_src_block_width * state.m_src_block_height; i++) + { + assert(block_pixels[i][0] == alt_block_pixels[i][0]); + assert(block_pixels[i][1] == alt_block_pixels[i][1]); + assert(block_pixels[i][2] == alt_block_pixels[i][2]); + assert(block_pixels[i][3] == alt_block_pixels[i][3]); + } +#endif + + // TODO: For uncompressed outputs, we could write directly to the output buffer, skipping buffering. + const uint32_t buffered_src_block_row_y = (by % state.m_num_src_block_rows_to_buffer); + + color32* pSrc_pixels = block_pixels; + color32* pDst_pixels = &(*state.m_pBuffered_rows)(bx * state.m_src_block_width, buffered_src_block_row_y * state.m_src_block_height); + + for (uint32_t y = 0; y < state.m_src_block_height; y++) + { + memcpy(pDst_pixels, pSrc_pixels, state.m_src_block_width * sizeof(color32)); + + pSrc_pixels += state.m_src_block_width; + pDst_pixels += state.m_pBuffered_rows->get_width(); + } // y + + // Last block on this src row? If not, exit. + if (bx != (state.m_src_num_blocks_x - 1)) + return true; + + // We've written the final src block for this ASTC src row. + // See if we have enough source rows to create full 4x4 destination blocks. + const bool final_src_block_row = (by == (state.m_src_num_blocks_y - 1)); + + if ( (buffered_src_block_row_y != (state.m_num_src_block_rows_to_buffer - 1)) && (!final_src_block_row) ) + return true; + + // src/destination image Y coordinate of the top of the buffered rows + const uint32_t buffered_src_pixel_y = ((by / state.m_num_src_block_rows_to_buffer) * state.m_num_src_block_rows_to_buffer) * state.m_src_block_height; + assert((buffered_src_pixel_y & 3) == 0); + + // The total # of valid src block rows we can read. + const uint32_t num_buffered_src_block_rows = buffered_src_block_row_y + 1; + + assert((num_buffered_src_block_rows == state.m_num_src_block_rows_to_buffer) || (final_src_block_row)); + + // The maximum number of valid buffer scanlines we can fetch from, taking into account the original texture's actual (unpadded) height. + const uint32_t override_buffer_height = basisu::minimum(state.m_orig_height - buffered_src_pixel_y, num_buffered_src_block_rows * state.m_src_block_height); + assert(override_buffer_height); + + // total_dst_block_rows_to_emit=really an upper bound for the final row of src ASTC blocks + const uint32_t total_dst_block_rows_to_emit = (num_buffered_src_block_rows * state.m_src_block_height + 3) >> 2; + + for (uint32_t dst_ofs_by = 0; dst_ofs_by < total_dst_block_rows_to_emit; dst_ofs_by++) + { + const uint32_t dst_by = (buffered_src_pixel_y >> 2) + dst_ofs_by; + if (dst_by >= state.m_dst_num_blocks_y) + break; + + for (uint32_t dst_bx = 0; dst_bx < state.m_dst_num_blocks_x; dst_bx++) + { + // Extract the 4x4 block pixels from our buffered rows, taking into account the actual # of valid scanlines inside the buffer. + state.m_pBuffered_rows->extract_block_clamped(block_pixels, dst_bx * 4, dst_ofs_by * 4, 4, 4, override_buffer_height); + + uint8_t* pDst_block_u8 = (uint8_t*)state.m_pDst_blocks + (dst_by * state.m_output_row_pitch_in_blocks_or_pixels + dst_bx) * state.m_output_block_or_pixel_stride_in_bytes; + + transcode_4x4_block( + state.m_fmt, + dst_bx, dst_by, + state.m_pDst_blocks, pDst_block_u8, + block_pixels, + state.m_output_block_or_pixel_stride_in_bytes, state.m_output_row_pitch_in_blocks_or_pixels, state.m_output_rows_in_pixels, + state.m_channel0, state.m_channel1, + state.m_high_quality, state.m_from_alpha, + state.m_bc7f_flags, + *state.m_pEtc1_pack_state, + state.m_has_alpha ? 1 : 0); + + } // dst_bx + + } // dst_ofs_by + + return true; + }; + + xuastc_decoded_image decoded_image; + + const bool decomp_flag = decoded_image.decode(pImage_data, image_data_size, init_func, &dec_state, src_block_func, &dec_state); + if (!decomp_flag) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_ldr_t::decompress_image() failed\n"); + return false; + } + } + + } // if (basis_tex_format_is_astc_ldr(src_format)) + + return true; +#else + assert(0); + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: XUASTC support disabled\n"); + return false; +#endif // BASISD_SUPPORT_XUASTC +} + +// Container independent transcoding +bool basisu_lowlevel_xuastc_ldr_transcoder::transcode_image( + basis_tex_format src_format, bool use_astc_srgb_decode_profile, + transcoder_texture_format target_format, + void* pOutput_blocks, uint32_t output_blocks_buf_size_in_blocks_or_pixels, + const uint8_t* pCompressed_data, uint32_t compressed_data_length, + uint32_t src_num_blocks_x, uint32_t src_num_blocks_y, uint32_t orig_width, uint32_t orig_height, uint32_t level_index, + uint64_t slice_offset, uint32_t slice_length, + uint32_t decode_flags, + bool has_alpha, + bool is_video, + uint32_t output_row_pitch_in_blocks_or_pixels, + basisu_transcoder_state* pState, + uint32_t output_rows_in_pixels, + int channel0, int channel1) +{ + BASISU_NOTE_UNUSED(is_video); + BASISU_NOTE_UNUSED(level_index); + +#if BASISD_SUPPORT_XUASTC + + if (((uint64_t)slice_offset + slice_length) > (uint64_t)compressed_data_length) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: source data buffer too small\n"); + return false; + } + + if ((target_format == transcoder_texture_format::cTFPVRTC1_4_RGB) || (target_format == transcoder_texture_format::cTFPVRTC1_4_RGBA)) + { + if ((!basisu::is_pow2(orig_width)) || (!basisu::is_pow2(orig_height))) + { + // PVRTC1 only supports power of 2 dimensions + BASISU_DEVEL_ERROR("basisu_lowlevel_etc1s_transcoder::transcode_image: PVRTC1 only supports power of 2 dimensions\n"); + return false; + } + } + + const bool transcode_alpha_data_to_opaque_formats = (decode_flags & cDecodeFlagsTranscodeAlphaDataToOpaqueFormats) != 0; + const uint32_t bytes_per_block_or_pixel = basis_get_bytes_per_block_or_pixel(target_format); + + if (!basis_validate_output_buffer_size(target_format, output_blocks_buf_size_in_blocks_or_pixels, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, output_rows_in_pixels)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: output buffer size too small\n"); + return false; + } + + bool status = false; + + switch (target_format) + { + case transcoder_texture_format::cTFETC1_RGB: + { + status = transcode_slice(src_format, use_astc_srgb_decode_profile, pOutput_blocks, src_num_blocks_x, src_num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cETC1, + bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, channel0, channel1, decode_flags); + + if (!status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: transcode_slice() to ETC1 failed\n"); + } + break; + } + case transcoder_texture_format::cTFETC2_RGBA: + { + status = transcode_slice(src_format, use_astc_srgb_decode_profile, pOutput_blocks, src_num_blocks_x, src_num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cETC2_RGBA, + bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, channel0, channel1, decode_flags); + if (!status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: transcode_slice() to ETC2 failed\n"); + } + break; + } + case transcoder_texture_format::cTFBC1_RGB: + { + // TODO: ETC1S allows BC1 from alpha channel. That doesn't seem actually useful, though. + status = transcode_slice(src_format, use_astc_srgb_decode_profile, pOutput_blocks, src_num_blocks_x, src_num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cBC1, + bytes_per_block_or_pixel, true, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, channel0, channel1, decode_flags); + if (!status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: transcode_slice() to BC1 failed\n"); + } + break; + } + case transcoder_texture_format::cTFBC3_RGBA: + { + status = transcode_slice(src_format, use_astc_srgb_decode_profile, pOutput_blocks, src_num_blocks_x, src_num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cBC3, + bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, channel0, channel1, decode_flags); + if (!status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: transcode_slice() to BC3 failed\n"); + } + break; + } + case transcoder_texture_format::cTFBC4_R: + { + status = transcode_slice(src_format, use_astc_srgb_decode_profile, pOutput_blocks, src_num_blocks_x, src_num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cBC4, + bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, + ((has_alpha) && (transcode_alpha_data_to_opaque_formats)) ? 3 : 0, -1, decode_flags); + if (!status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: transcode_slice() to BC4 failed\n"); + } + break; + } + case transcoder_texture_format::cTFBC5_RG: + { + status = transcode_slice(src_format, use_astc_srgb_decode_profile, pOutput_blocks, src_num_blocks_x, src_num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cBC5, + bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, + 0, 3, decode_flags); + if (!status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: transcode_slice() to BC5 failed\n"); + } + break; + } + case transcoder_texture_format::cTFBC7_RGBA: + case transcoder_texture_format::cTFBC7_ALT: + { + status = transcode_slice(src_format, use_astc_srgb_decode_profile, pOutput_blocks, src_num_blocks_x, src_num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cBC7, + bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, -1, -1, decode_flags); + if (!status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: transcode_slice() to BC7 failed\n"); + } + break; + } + case transcoder_texture_format::cTFPVRTC1_4_RGB: + { + status = transcode_slice(src_format, use_astc_srgb_decode_profile, pOutput_blocks, src_num_blocks_x, src_num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cPVRTC1_4_RGB, + bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, channel0, channel1, decode_flags); + + if (!status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: transcode_slice() to PVRTC1_RGB failed\n"); + } + break; + } + case transcoder_texture_format::cTFPVRTC1_4_RGBA: + { + status = transcode_slice(src_format, use_astc_srgb_decode_profile, pOutput_blocks, src_num_blocks_x, src_num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cPVRTC1_4_RGBA, + bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, channel0, channel1, decode_flags); + + if (!status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: transcode_slice() to PVRTC1_RGBA failed\n"); + } + break; + } + case transcoder_texture_format::cTFASTC_LDR_4x4_RGBA: + case transcoder_texture_format::cTFASTC_LDR_5x4_RGBA: + case transcoder_texture_format::cTFASTC_LDR_5x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_6x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_6x6_RGBA: + case transcoder_texture_format::cTFASTC_LDR_8x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_8x6_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x6_RGBA: + case transcoder_texture_format::cTFASTC_LDR_8x8_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x8_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x10_RGBA: + case transcoder_texture_format::cTFASTC_LDR_12x10_RGBA: + case transcoder_texture_format::cTFASTC_LDR_12x12_RGBA: + { + status = transcode_slice(src_format, use_astc_srgb_decode_profile, pOutput_blocks, src_num_blocks_x, src_num_blocks_y, pCompressed_data + slice_offset, slice_length, xuastc_get_block_format(target_format), + bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, -1, -1, decode_flags); + if (!status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: transcode_slice() to ASTC 4x4 failed\n"); + } + break; + } + case transcoder_texture_format::cTFATC_RGB: + case transcoder_texture_format::cTFATC_RGBA: + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: UASTC LDR 4x4->ATC currently unsupported\n"); + return false; + } + case transcoder_texture_format::cTFFXT1_RGB: + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: UASTC LDR 4x4->FXT1 currently unsupported\n"); + return false; + } + case transcoder_texture_format::cTFPVRTC2_4_RGB: + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: UASTC LDR 4x4->PVRTC2 currently unsupported\n"); + return false; + } + case transcoder_texture_format::cTFPVRTC2_4_RGBA: + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: UASTC LDR 4x4->PVRTC2 currently unsupported\n"); + return false; + } + case transcoder_texture_format::cTFETC2_EAC_R11: + { + status = transcode_slice(src_format, use_astc_srgb_decode_profile, pOutput_blocks, src_num_blocks_x, src_num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cETC2_EAC_R11, + bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, + ((has_alpha) && (transcode_alpha_data_to_opaque_formats)) ? 3 : 0, -1, decode_flags); + if (!status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: transcode_slice() to EAC R11 failed\n"); + } + break; + } + case transcoder_texture_format::cTFETC2_EAC_RG11: + { + status = transcode_slice(src_format, use_astc_srgb_decode_profile, pOutput_blocks, src_num_blocks_x, src_num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cETC2_EAC_RG11, + bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, + 0, 3, decode_flags); + if (!status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: transcode_slice() to EAC RG11 failed\n"); + } + break; + } + case transcoder_texture_format::cTFRGBA32: + { + status = transcode_slice(src_format, use_astc_srgb_decode_profile, pOutput_blocks, src_num_blocks_x, src_num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cRGBA32, + bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, -1, -1, decode_flags); + if (!status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: transcode_slice() to RGBA32 failed\n"); + } + break; + } + case transcoder_texture_format::cTFRGB565: + { + status = transcode_slice(src_format, use_astc_srgb_decode_profile, pOutput_blocks, src_num_blocks_x, src_num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cRGB565, + bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, -1, -1, decode_flags); + if (!status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: transcode_slice() to RGB565 failed\n"); + } + break; + } + case transcoder_texture_format::cTFBGR565: + { + status = transcode_slice(src_format, use_astc_srgb_decode_profile, pOutput_blocks, src_num_blocks_x, src_num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cBGR565, + bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, -1, -1, decode_flags); + if (!status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: transcode_slice() to RGB565 failed\n"); + } + break; + } + case transcoder_texture_format::cTFRGBA4444: + { + status = transcode_slice(src_format, use_astc_srgb_decode_profile, pOutput_blocks, src_num_blocks_x, src_num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cRGBA4444, + bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, -1, -1, decode_flags); + if (!status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: transcode_slice() to RGBA4444 failed\n"); + } + break; + } + default: + { + assert(0); + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: Invalid format\n"); + break; + } + } + + return status; +#else + assert(0); + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: XUASTC support disabled\n"); + return false; +#endif // BASISD_SUPPORT_XUASTC +} + +} // namespace basist + diff --git a/external/basis_universal/transcoder/basisu_transcoder.h b/external/basis_universal/transcoder/basisu_transcoder.h index dc9329c342..a2fc5b35d8 100644 --- a/external/basis_universal/transcoder/basisu_transcoder.h +++ b/external/basis_universal/transcoder/basisu_transcoder.h @@ -1,5 +1,5 @@ // basisu_transcoder.h -// Copyright (C) 2019-2024 Binomial LLC. All Rights Reserved. +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. // Important: If compiling with gcc, be sure strict aliasing is disabled: -fno-strict-aliasing // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,6 +13,8 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +// +// Also see basis_tex_format in basisu_file_headers.h (TODO: Perhaps move key definitions into here.) #pragma once // By default KTX2 support is enabled to simplify compilation. This implies the need for the Zstandard library (which we distribute as a single source file in the "zstd" directory) by default. @@ -22,17 +24,11 @@ #define BASISD_SUPPORT_KTX2 1 #endif -// Set BASISD_SUPPORT_KTX2_ZSTD to 0 to disable Zstd usage and KTX2 UASTC Zstd supercompression support +// Set BASISD_SUPPORT_KTX2_ZSTD to 0 to disable Zstd usage and KTX2 UASTC Zstd supercompression support #ifndef BASISD_SUPPORT_KTX2_ZSTD #define BASISD_SUPPORT_KTX2_ZSTD 1 #endif -// Set BASISU_FORCE_DEVEL_MESSAGES to 1 to enable debug printf()'s whenever an error occurs, for easier debugging during development. -#ifndef BASISU_FORCE_DEVEL_MESSAGES - // TODO - disable before checking in - #define BASISU_FORCE_DEVEL_MESSAGES 0 -#endif - #include "basisu_transcoder_internal.h" #include "basisu_transcoder_uastc.h" #include "basisu_file_headers.h" @@ -42,7 +38,7 @@ namespace basist // High-level composite texture formats supported by the transcoder. // Each of these texture formats directly correspond to OpenGL/D3D/Vulkan etc. texture formats. // Notes: - // - If you specify a texture format that supports alpha, but the .basis file doesn't have alpha, the transcoder will automatically output a + // - If you specify a texture format that supports alpha, but the .basis file doesn't have alpha, the transcoder will automatically output a // fully opaque (255) alpha channel. // - The PVRTC1 texture formats only support power of 2 dimension .basis files, but this may be relaxed in a future version. // - The PVRTC1 transcoders are real-time encoders, so don't expect the highest quality. We may add a slower encoder with improved quality. @@ -66,13 +62,13 @@ namespace basist cTFPVRTC1_4_RGB = 8, // Opaque only, RGB or alpha if cDecodeFlagsTranscodeAlphaDataToOpaqueFormats flag is specified, nearly lowest quality of any texture format. cTFPVRTC1_4_RGBA = 9, // Opaque+alpha, most useful for simple opacity maps. If .basis file doesn't have alpha cTFPVRTC1_4_RGB will be used instead. Lowest quality of any supported texture format. - // ASTC (mobile, Intel devices, hopefully all desktop GPU's one day) - cTFASTC_4x4_RGBA = 10, // LDR. Opaque+alpha, ASTC 4x4, alpha channel will be opaque for opaque .basis files. + // ASTC (mobile, some Intel CPU's, hopefully all desktop GPU's one day) + cTFASTC_LDR_4x4_RGBA = 10, // LDR. Opaque+alpha, ASTC 4x4, alpha channel will be opaque for opaque .basis files. // LDR: Transcoder uses RGB/RGBA/L/LA modes, void extent, and up to two ([0,47] and [0,255]) endpoint precisions. // ATC (mobile, Adreno devices, this is a niche format) cTFATC_RGB = 11, // Opaque, RGB or alpha if cDecodeFlagsTranscodeAlphaDataToOpaqueFormats flag is specified. ATI ATC (GL_ATC_RGB_AMD) - cTFATC_RGBA = 12, // Opaque+alpha, alpha channel will be opaque for opaque .basis files. ATI ATC (GL_ATC_RGBA_INTERPOLATED_ALPHA_AMD) + cTFATC_RGBA = 12, // Opaque+alpha, alpha channel will be opaque for opaque .basis files. ATI ATC (GL_ATC_RGBA_INTERPOLATED_ALPHA_AMD) // FXT1 (desktop, Intel devices, this is a super obscure format) cTFFXT1_RGB = 17, // Opaque only, uses exclusively CC_MIXED blocks. Notable for having a 8x4 block size. GL_3DFX_texture_compression_FXT1 is supported on Intel integrated GPU's (such as HD 630). @@ -94,7 +90,7 @@ namespace basist cTFRGB565 = 14, // 16bpp RGB image stored in raster (not block) order in memory, R at bit position 11 cTFBGR565 = 15, // 16bpp RGB image stored in raster (not block) order in memory, R at bit position 0 cTFRGBA4444 = 16, // 16bpp RGBA image stored in raster (not block) order in memory, R at bit position 12, A at bit position 0 - + // Note these uncompressed formats (HALF and 9E5) can only be transcoded to from HDR input files (UASTC HDR 4x4 or ASTC HDR 6x6). cTFRGB_HALF = 24, // 48bpp RGB half (16-bits/component, 3 components) cTFRGBA_HALF = 25, // 64bpp RGBA half (16-bits/component, 4 components) (A will always currently 1.0, UASTC_HDR doesn't support alpha) @@ -102,7 +98,23 @@ namespace basist cTFASTC_HDR_6x6_RGBA = 27, // HDR, RGBA (currently our ASTC HDR 6x6 encodes are only RGB), unsigned - cTFTotalTextureFormats = 28, + + // The remaining LDR ASTC block sizes, excluding 4x4 (which is above). There are 14 total valid ASTC LDR/HDR block sizes. + cTFASTC_LDR_5x4_RGBA = 28, + cTFASTC_LDR_5x5_RGBA = 29, + cTFASTC_LDR_6x5_RGBA = 30, + cTFASTC_LDR_6x6_RGBA = 31, + cTFASTC_LDR_8x5_RGBA = 32, + cTFASTC_LDR_8x6_RGBA = 33, + cTFASTC_LDR_10x5_RGBA = 34, + cTFASTC_LDR_10x6_RGBA = 35, + cTFASTC_LDR_8x8_RGBA = 36, + cTFASTC_LDR_10x8_RGBA = 37, + cTFASTC_LDR_10x10_RGBA = 38, + cTFASTC_LDR_12x10_RGBA = 39, + cTFASTC_LDR_12x12_RGBA = 40, + + cTFTotalTextureFormats = 41, // ----- The following are old/legacy enums for compatibility with code compiled against previous versions cTFETC1 = cTFETC1_RGB, @@ -112,25 +124,30 @@ namespace basist cTFBC4 = cTFBC4_R, cTFBC5 = cTFBC5_RG, - // Previously, the caller had some control over which BC7 mode the transcoder output. We've simplified this due to UASTC, which supports numerous modes. + // Previously, the caller had some control over which BC7 mode the transcoder output. We've simplified this due to UASTC LDR 4x4, which supports numerous modes. cTFBC7_M6_RGB = cTFBC7_RGBA, // Opaque only, RGB or alpha if cDecodeFlagsTranscodeAlphaDataToOpaqueFormats flag is specified. Highest quality of all the non-ETC1 formats. cTFBC7_M5_RGBA = cTFBC7_RGBA, // Opaque+alpha, alpha channel will be opaque for opaque .basis files cTFBC7_M6_OPAQUE_ONLY = cTFBC7_RGBA, cTFBC7_M5 = cTFBC7_RGBA, cTFBC7_ALT = 7, - cTFASTC_4x4 = cTFASTC_4x4_RGBA, + cTFASTC_4x4 = cTFASTC_LDR_4x4_RGBA, cTFATC_RGBA_INTERPOLATED_ALPHA = cTFATC_RGBA, + + cTFASTC_4x4_RGBA = cTFASTC_LDR_4x4_RGBA }; // For compressed texture formats, this returns the # of bytes per block. For uncompressed, it returns the # of bytes per pixel. // NOTE: Previously, this function was called basis_get_bytes_per_block(), and it always returned 16*bytes_per_pixel for uncompressed formats which was confusing. uint32_t basis_get_bytes_per_block_or_pixel(transcoder_texture_format fmt); - // Returns format's name in ASCII + // Returns the transcoder texture format's name in ASCII const char* basis_get_format_name(transcoder_texture_format fmt); + // Returns basis texture format name in ASCII + const char* basis_get_tex_format_name(basis_tex_format fmt); + // Returns block format name in ASCII const char* basis_get_block_format_name(block_format fmt); @@ -143,6 +160,9 @@ namespace basist // Returns true if the format is LDR. inline bool basis_transcoder_format_is_ldr(transcoder_texture_format fmt) { return !basis_transcoder_format_is_hdr(fmt); } + // Returns true if the format is an LDR or HDR ASTC format. + bool basis_is_transcoder_texture_format_astc(transcoder_texture_format fmt); + // Returns the basisu::texture_format corresponding to the specified transcoder_texture_format. basisu::texture_format basis_get_basisu_texture_format(transcoder_texture_format fmt); @@ -156,23 +176,32 @@ namespace basist uint32_t basis_get_uncompressed_bytes_per_pixel(transcoder_texture_format fmt); // Returns the block width for the specified texture format, which is currently either 4 or 8 for FXT1. - uint32_t basis_get_block_width(transcoder_texture_format tex_type); + uint32_t basis_get_block_width(transcoder_texture_format fmt); // Returns the block height for the specified texture format, which is currently always 4. - uint32_t basis_get_block_height(transcoder_texture_format tex_type); - - // Returns true if the specified format was enabled at compile time, and is supported for the specific basis/ktx2 texture format (ETC1S, UASTC, or UASTC HDR). + uint32_t basis_get_block_height(transcoder_texture_format fmt); + + // ASTC/XUASTC LDR formats only: Given a basis_tex_format (mode or codec), return the corresponding ASTC basisu::texture_format with the proper block size from 4x4-12x12. + basisu::texture_format basis_get_texture_format_from_xuastc_or_astc_ldr_basis_tex_format(basis_tex_format fmt); + + // For any given basis_tex_format (mode or codec), return the LDR/HDR ASTC transcoder texture format with the proper block size. + transcoder_texture_format basis_get_transcoder_texture_format_from_basis_tex_format(basis_tex_format fmt); + // basis_get_transcoder_texture_format_from_xuastc_or_astc_ldr_basis_tex_format: same as basis_get_transcoder_texture_format_from_basis_tex_format (TODO: remove) + transcoder_texture_format basis_get_transcoder_texture_format_from_xuastc_or_astc_ldr_basis_tex_format(basis_tex_format fmt); + + // Returns true if the specified format was enabled at compile time, and is supported for the specific basis/ktx2 texture format (ETC1S, UASTC, or UASTC HDR, or XUASTC LDR 4x4-12x12). + // For XUASTC the ASTC block size must match the transcoder_texture_format's ASTC block size. bool basis_is_format_supported(transcoder_texture_format tex_type, basis_tex_format fmt = basis_tex_format::cETC1S); // Returns the block width/height for the specified basis texture file format. uint32_t basis_tex_format_get_block_width(basis_tex_format fmt); uint32_t basis_tex_format_get_block_height(basis_tex_format fmt); - + bool basis_tex_format_is_hdr(basis_tex_format fmt); inline bool basis_tex_format_is_ldr(basis_tex_format fmt) { return !basis_tex_format_is_hdr(fmt); } - + // Validates that the output buffer is large enough to hold the entire transcoded texture. - // For uncompressed texture formats, most input parameters are in pixels, not blocks. Blocks are 4x4 pixels. + // For uncompressed texture formats, most input parameters are in pixels, not blocks. bool basis_validate_output_buffer_size(transcoder_texture_format target_format, uint32_t output_blocks_buf_size_in_blocks_or_pixels, uint32_t orig_width, uint32_t orig_height, @@ -199,7 +228,7 @@ namespace basist basisu::vector m_block_endpoint_preds[2]; enum { cMaxPrevFrameLevels = 16 }; - basisu::vector m_prev_frame_indices[2][cMaxPrevFrameLevels]; // [alpha_flag][level_index] + basisu::vector m_prev_frame_indices[2][cMaxPrevFrameLevels]; // [alpha_flag][level_index] void clear() { @@ -214,7 +243,46 @@ namespace basist }; // Low-level helper classes that do the actual transcoding. + enum basisu_decode_flags + { + // PVRTC1: decode non-pow2 ETC1S texture level to the next larger power of 2 (not implemented yet, but we're going to support it). Ignored if the slice's dimensions are already a power of 2. + cDecodeFlagsPVRTCDecodeToNextPow2 = 2, + // When decoding to an opaque texture format, if the basis file has alpha, decode the alpha slice instead of the color slice to the output texture format. + // This is primarily to allow decoding of textures with alpha to multiple ETC1 textures (one for color, another for alpha). + cDecodeFlagsTranscodeAlphaDataToOpaqueFormats = 4, + + // Forbid usage of BC1 3 color blocks (we don't support BC1 punchthrough alpha yet). + // This flag is used internally when decoding to BC3. + cDecodeFlagsBC1ForbidThreeColorBlocks = 8, + + // The output buffer contains alpha endpoint/selector indices. + // Used internally when decoding formats like ASTC that require both color and alpha data to be available when transcoding to the output format. + cDecodeFlagsOutputHasAlphaIndices = 16, + + // Enable slower, but higher quality transcoding for some formats. + // For ASTC/XUASTC->BC7, this enables partially analytical encoding vs. fully analytical. + cDecodeFlagsHighQuality = 32, + + // Disable ETC1S->BC7 adaptive chroma filtering, for much faster transcoding to BC7. + cDecodeFlagsNoETC1SChromaFiltering = 64, + + // Disable deblock filtering for XUASTC LDR transcoding to non-ASTC formats. + // For ASTC 8x6 or smaller block sizes, deblocking is always disabled unless you force it on using cDecodeFlagsForceDeblockFiltering. + cDecodeFlagsNoDeblockFiltering = 128, + + // More aggressive deblock filtering (only used when it's enabled) + cDecodeFlagsStrongerDeblockFiltering = 256, + + // Always apply deblocking, even for smaller ASTC block sizes (4x4-8x6). + cDecodeFlagsForceDeblockFiltering = 512, + + // By default XUASTC LDR 4x4, 6x6 and 8x6 are directly transcoded to BC7 without always requiring a full ASTC block unpack and analytical BC7 encode. This is 1.4x up to 3x faster in WASM. + // This trade offs some quality. The largest transcoding speed gain is achieved when the source XUASTC data isn't dual plane and only uses 1 subset. Otherwise the actual perf. gain is variable. + // To disable this optimization for all XUASTC block sizes and always use the fallback encoder, specify cDecodeFlagXUASTCLDRDisableFastBC7Transcoding. + cDecodeFlagXUASTCLDRDisableFastBC7Transcoding = 1024 + }; + // ETC1S class basisu_lowlevel_etc1s_transcoder { @@ -279,69 +347,117 @@ namespace basist typedef basisu::vector selector_vec; const selector_vec& get_selectors() const { return m_local_selectors; } - + private: const basisu_lowlevel_etc1s_transcoder* m_pGlobal_codebook; endpoint_vec m_local_endpoints; selector_vec m_local_selectors; - + huffman_decoding_table m_endpoint_pred_model, m_delta_endpoint_model, m_selector_model, m_selector_history_buf_rle_model; uint32_t m_selector_history_buf_size; basisu_transcoder_state m_def_state; }; - - enum basisu_decode_flags + + // UASTC LDR 4x4 + class basisu_lowlevel_uastc_ldr_4x4_transcoder { - // PVRTC1: decode non-pow2 ETC1S texture level to the next larger power of 2 (not implemented yet, but we're going to support it). Ignored if the slice's dimensions are already a power of 2. - cDecodeFlagsPVRTCDecodeToNextPow2 = 2, + friend class basisu_transcoder; - // When decoding to an opaque texture format, if the basis file has alpha, decode the alpha slice instead of the color slice to the output texture format. - // This is primarily to allow decoding of textures with alpha to multiple ETC1 textures (one for color, another for alpha). - cDecodeFlagsTranscodeAlphaDataToOpaqueFormats = 4, + public: + basisu_lowlevel_uastc_ldr_4x4_transcoder(); - // Forbid usage of BC1 3 color blocks (we don't support BC1 punchthrough alpha yet). - // This flag is used internally when decoding to BC3. - cDecodeFlagsBC1ForbidThreeColorBlocks = 8, + bool transcode_slice(void* pDst_blocks, uint32_t num_blocks_x, uint32_t num_blocks_y, const uint8_t* pImage_data, uint32_t image_data_size, block_format fmt, + uint32_t output_block_or_pixel_stride_in_bytes, bool bc1_allow_threecolor_blocks, bool has_alpha, const uint32_t orig_width, const uint32_t orig_height, uint32_t output_row_pitch_in_blocks_or_pixels = 0, + basisu_transcoder_state* pState = nullptr, uint32_t output_rows_in_pixels = 0, int channel0 = -1, int channel1 = -1, uint32_t decode_flags = 0); - // The output buffer contains alpha endpoint/selector indices. - // Used internally when decoding formats like ASTC that require both color and alpha data to be available when transcoding to the output format. - cDecodeFlagsOutputHasAlphaIndices = 16, + bool transcode_slice(void* pDst_blocks, uint32_t num_blocks_x, uint32_t num_blocks_y, const uint8_t* pImage_data, uint32_t image_data_size, block_format fmt, + uint32_t output_block_or_pixel_stride_in_bytes, bool bc1_allow_threecolor_blocks, const basis_file_header& header, const basis_slice_desc& slice_desc, uint32_t output_row_pitch_in_blocks_or_pixels = 0, + basisu_transcoder_state* pState = nullptr, uint32_t output_rows_in_pixels = 0, int channel0 = -1, int channel1 = -1, uint32_t decode_flags = 0) + { + return transcode_slice(pDst_blocks, num_blocks_x, num_blocks_y, pImage_data, image_data_size, fmt, + output_block_or_pixel_stride_in_bytes, bc1_allow_threecolor_blocks, (header.m_flags & cBASISHeaderFlagHasAlphaSlices) != 0, slice_desc.m_orig_width, slice_desc.m_orig_height, output_row_pitch_in_blocks_or_pixels, + pState, output_rows_in_pixels, channel0, channel1, decode_flags); + } - cDecodeFlagsHighQuality = 32, + // Container independent transcoding + bool transcode_image( + transcoder_texture_format target_format, + void* pOutput_blocks, uint32_t output_blocks_buf_size_in_blocks_or_pixels, + const uint8_t* pCompressed_data, uint32_t compressed_data_length, + uint32_t num_blocks_x, uint32_t num_blocks_y, uint32_t orig_width, uint32_t orig_height, uint32_t level_index, + uint64_t slice_offset, uint32_t slice_length, + uint32_t decode_flags = 0, + bool has_alpha = false, + bool is_video = false, + uint32_t output_row_pitch_in_blocks_or_pixels = 0, + basisu_transcoder_state* pState = nullptr, + uint32_t output_rows_in_pixels = 0, + int channel0 = -1, int channel1 = -1); + }; - cDecodeFlagsNoETC1SChromaFiltering = 64 +#if BASISD_SUPPORT_XUASTC + // XUASTC LDR 4x4-12x12 or ASTC LDR 4x4-12x12 + struct xuastc_decoded_image + { + uint32_t m_actual_block_width = 0, m_actual_block_height = 0, m_actual_width = 0, m_actual_height = 0; + bool m_actual_has_alpha = false, m_uses_srgb_astc_decode_mode = false; + + bool decode(const uint8_t* pImage_data, uint32_t image_data_size, + astc_ldr_t::xuastc_decomp_image_init_callback_ptr pInit_callback, void* pInit_callback_data, + astc_ldr_t::xuastc_decomp_image_block_callback_ptr pBlock_callback, void* pBlock_callback_data) + { + const bool decomp_flag = astc_ldr_t::xuastc_ldr_decompress_image(pImage_data, image_data_size, + m_actual_block_width, m_actual_block_height, + m_actual_width, m_actual_height, + m_actual_has_alpha, m_uses_srgb_astc_decode_mode, basisu::g_debug_printf, + pInit_callback, pInit_callback_data, + pBlock_callback, pBlock_callback_data); + + return decomp_flag; + } + + void clear() + { + m_actual_block_width = 0; + m_actual_block_height = 0; + m_actual_width = 0; + m_actual_height = 0; + m_actual_has_alpha = false; + m_uses_srgb_astc_decode_mode = false; + } }; +#endif - // UASTC LDR 4x4 - class basisu_lowlevel_uastc_ldr_4x4_transcoder + class basisu_lowlevel_xuastc_ldr_transcoder { friend class basisu_transcoder; public: - basisu_lowlevel_uastc_ldr_4x4_transcoder(); + basisu_lowlevel_xuastc_ldr_transcoder(); - bool transcode_slice(void* pDst_blocks, uint32_t num_blocks_x, uint32_t num_blocks_y, const uint8_t* pImage_data, uint32_t image_data_size, block_format fmt, + bool transcode_slice(basis_tex_format src_format, bool use_astc_srgb_decode_profile, void* pDst_blocks, uint32_t src_num_blocks_x, uint32_t src_num_blocks_y, const uint8_t* pImage_data, uint32_t image_data_size, block_format fmt, uint32_t output_block_or_pixel_stride_in_bytes, bool bc1_allow_threecolor_blocks, bool has_alpha, const uint32_t orig_width, const uint32_t orig_height, uint32_t output_row_pitch_in_blocks_or_pixels = 0, basisu_transcoder_state* pState = nullptr, uint32_t output_rows_in_pixels = 0, int channel0 = -1, int channel1 = -1, uint32_t decode_flags = 0); - bool transcode_slice(void* pDst_blocks, uint32_t num_blocks_x, uint32_t num_blocks_y, const uint8_t* pImage_data, uint32_t image_data_size, block_format fmt, + bool transcode_slice(basis_tex_format src_format, bool use_astc_srgb_decode_profile, void* pDst_blocks, uint32_t src_num_blocks_x, uint32_t src_num_blocks_y, const uint8_t* pImage_data, uint32_t image_data_size, block_format fmt, uint32_t output_block_or_pixel_stride_in_bytes, bool bc1_allow_threecolor_blocks, const basis_file_header& header, const basis_slice_desc& slice_desc, uint32_t output_row_pitch_in_blocks_or_pixels = 0, basisu_transcoder_state* pState = nullptr, uint32_t output_rows_in_pixels = 0, int channel0 = -1, int channel1 = -1, uint32_t decode_flags = 0) { - return transcode_slice(pDst_blocks, num_blocks_x, num_blocks_y, pImage_data, image_data_size, fmt, + return transcode_slice(src_format, use_astc_srgb_decode_profile, pDst_blocks, src_num_blocks_x, src_num_blocks_y, pImage_data, image_data_size, fmt, output_block_or_pixel_stride_in_bytes, bc1_allow_threecolor_blocks, (header.m_flags & cBASISHeaderFlagHasAlphaSlices) != 0, slice_desc.m_orig_width, slice_desc.m_orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, channel0, channel1, decode_flags); } // Container independent transcoding bool transcode_image( + basis_tex_format src_format, bool use_astc_srgb_decode_profile, transcoder_texture_format target_format, void* pOutput_blocks, uint32_t output_blocks_buf_size_in_blocks_or_pixels, const uint8_t* pCompressed_data, uint32_t compressed_data_length, - uint32_t num_blocks_x, uint32_t num_blocks_y, uint32_t orig_width, uint32_t orig_height, uint32_t level_index, + uint32_t src_num_blocks_x, uint32_t src_num_blocks_y, uint32_t orig_width, uint32_t orig_height, uint32_t level_index, uint64_t slice_offset, uint32_t slice_length, uint32_t decode_flags = 0, bool has_alpha = false, @@ -426,13 +542,13 @@ namespace basist int channel0 = -1, int channel1 = -1); }; - // ASTC HDR 6x6 intermediate - class basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder + // UASTC HDR 6x6 intermediate + class basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder { friend class basisu_transcoder; public: - basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder(); + basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder(); bool transcode_slice(void* pDst_blocks, uint32_t num_blocks_x, uint32_t num_blocks_y, const uint8_t* pImage_data, uint32_t image_data_size, block_format fmt, uint32_t output_block_or_pixel_stride_in_bytes, bool bc1_allow_threecolor_blocks, bool has_alpha, const uint32_t orig_width, const uint32_t orig_height, uint32_t output_row_pitch_in_blocks_or_pixels = 0, @@ -465,9 +581,11 @@ namespace basist struct basisu_slice_info { + // The image's ACTUAL dimensions in texels. uint32_t m_orig_width; uint32_t m_orig_height; + // The texture's dimensions in texels - always a multiple of the texture's underlying block size (4x4-12x12). uint32_t m_width; uint32_t m_height; @@ -497,9 +615,11 @@ namespace basist uint32_t m_image_index; uint32_t m_total_levels; + // The image's ACTUAL dimensions in texels. uint32_t m_orig_width; uint32_t m_orig_height; - + + // The texture's dimensions in texels - always a multiple of the texture's underlying block size (4x4-12x12). uint32_t m_width; uint32_t m_height; @@ -583,12 +703,13 @@ namespace basist uint32_t m_block_height; bool m_y_flipped; // true if the image was Y flipped + bool m_srgb; // true if the image is sRGB, false if linear bool m_etc1s; // true if the file is ETC1S bool m_has_alpha_slices; // true if the texture has alpha slices (for ETC1S: even slices RGB, odd slices alpha) }; // High-level transcoder class which accepts .basis file data and allows the caller to query information about the file and transcode image levels to various texture formats. - // If you're just starting out this is the class you care about. + // If you're just starting out this is the class you care about (or see the KTX2 transcoder below). class basisu_transcoder { basisu_transcoder(basisu_transcoder&); @@ -639,11 +760,11 @@ namespace basist // transcode_image_level() decodes a single mipmap level from the .basis file to any of the supported output texture formats. // It'll first find the slice(s) to transcode, then call transcode_slice() one or two times to decode both the color and alpha texture data (or RG texture data from two slices for BC5). // If the .basis file doesn't have alpha slices, the output alpha blocks will be set to fully opaque (all 255's). - // Currently, to decode to PVRTC1 the basis texture's dimensions in pixels must be a power of 2, due to PVRTC1 format requirements. + // Currently, to decode to PVRTC1 the basis texture's dimensions in pixels must be a power of 2, due to PVRTC1 format requirements. // output_blocks_buf_size_in_blocks_or_pixels should be at least the image level's total_blocks (num_blocks_x * num_blocks_y), or the total number of output pixels if fmt==cTFRGBA32 etc. // output_row_pitch_in_blocks_or_pixels: Number of blocks or pixels per row. If 0, the transcoder uses the slice's num_blocks_x or orig_width (NOT num_blocks_x * 4). Ignored for PVRTC1 (due to texture swizzling). // output_rows_in_pixels: Ignored unless fmt is uncompressed (cRGBA32, etc.). The total number of output rows in the output buffer. If 0, the transcoder assumes the slice's orig_height (NOT num_blocks_y * 4). - // Notes: + // Notes: // - basisu_transcoder_init() must have been called first to initialize the transcoder lookup tables before calling this function. // - This method assumes the output texture buffer is readable. In some cases to handle alpha, the transcoder will write temporary data to the output texture in // a first pass, which will be read in a second pass. @@ -682,15 +803,16 @@ namespace basist const basisu_lowlevel_etc1s_transcoder& get_lowlevel_etc1s_decoder() const { return m_lowlevel_etc1s_decoder; } basisu_lowlevel_etc1s_transcoder& get_lowlevel_etc1s_decoder() { return m_lowlevel_etc1s_decoder; } - const basisu_lowlevel_uastc_ldr_4x4_transcoder& get_lowlevel_uastc_decoder() const { return m_lowlevel_uastc_decoder; } - basisu_lowlevel_uastc_ldr_4x4_transcoder& get_lowlevel_uastc_decoder() { return m_lowlevel_uastc_decoder; } + const basisu_lowlevel_uastc_ldr_4x4_transcoder& get_lowlevel_uastc_decoder() const { return m_lowlevel_uastc_ldr_4x4_decoder; } + basisu_lowlevel_uastc_ldr_4x4_transcoder& get_lowlevel_uastc_decoder() { return m_lowlevel_uastc_ldr_4x4_decoder; } private: mutable basisu_lowlevel_etc1s_transcoder m_lowlevel_etc1s_decoder; - mutable basisu_lowlevel_uastc_ldr_4x4_transcoder m_lowlevel_uastc_decoder; + mutable basisu_lowlevel_uastc_ldr_4x4_transcoder m_lowlevel_uastc_ldr_4x4_decoder; + mutable basisu_lowlevel_xuastc_ldr_transcoder m_lowlevel_xuastc_ldr_decoder; mutable basisu_lowlevel_uastc_hdr_4x4_transcoder m_lowlevel_uastc_4x4_hdr_decoder; mutable basisu_lowlevel_astc_hdr_6x6_transcoder m_lowlevel_astc_6x6_hdr_decoder; - mutable basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder m_lowlevel_astc_6x6_hdr_intermediate_decoder; + mutable basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder m_lowlevel_astc_6x6_hdr_intermediate_decoder; bool m_ready_to_transcode; @@ -701,7 +823,7 @@ namespace basist // basisu_transcoder_init() MUST be called before a .basis file can be transcoded. void basisu_transcoder_init(); - + enum debug_flags_t { cDebugFlagVisCRs = 1, @@ -711,10 +833,10 @@ namespace basist uint32_t get_debug_flags(); void set_debug_flags(uint32_t f); - // ------------------------------------------------------------------------------------------------------ + // ------------------------------------------------------------------------------------------------------ // Optional .KTX2 file format support // KTX2 reading optionally requires miniz or Zstd decompressors for supercompressed UASTC files. - // ------------------------------------------------------------------------------------------------------ + // ------------------------------------------------------------------------------------------------------ #if BASISD_SUPPORT_KTX2 #pragma pack(push) #pragma pack(1) @@ -764,10 +886,10 @@ namespace basist basisu::packed_uint<4> m_alpha_slice_byte_length; }; - struct ktx2_astc_hdr_6x6_intermediate_image_desc + struct ktx2_slice_offset_len_desc { - basisu::packed_uint<4> m_rgb_slice_byte_offset; - basisu::packed_uint<4> m_rgb_slice_byte_length; + basisu::packed_uint<4> m_slice_byte_offset; + basisu::packed_uint<4> m_slice_byte_length; }; struct ktx2_animdata @@ -779,7 +901,7 @@ namespace basist #pragma pack(pop) const uint32_t KTX2_VK_FORMAT_UNDEFINED = 0; - + // These are standard Vulkan texture VkFormat ID's, see https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VkFormat.html const uint32_t KTX2_FORMAT_ASTC_4x4_SFLOAT_BLOCK = 1000066000; const uint32_t KTX2_FORMAT_ASTC_5x4_SFLOAT_BLOCK = 1000066001; @@ -789,12 +911,28 @@ namespace basist const uint32_t KTX2_FORMAT_ASTC_8x5_SFLOAT_BLOCK = 1000066005; const uint32_t KTX2_FORMAT_ASTC_8x6_SFLOAT_BLOCK = 1000066006; + const uint32_t KTX2_FORMAT_ASTC_4x4_UNORM_BLOCK = 157, KTX2_FORMAT_ASTC_4x4_SRGB_BLOCK = 158; + const uint32_t KTX2_FORMAT_ASTC_5x4_UNORM_BLOCK = 159, KTX2_FORMAT_ASTC_5x4_SRGB_BLOCK = 160; + const uint32_t KTX2_FORMAT_ASTC_5x5_UNORM_BLOCK = 161, KTX2_FORMAT_ASTC_5x5_SRGB_BLOCK = 162; + const uint32_t KTX2_FORMAT_ASTC_6x5_UNORM_BLOCK = 163, KTX2_FORMAT_ASTC_6x5_SRGB_BLOCK = 164; + const uint32_t KTX2_FORMAT_ASTC_6x6_UNORM_BLOCK = 165, KTX2_FORMAT_ASTC_6x6_SRGB_BLOCK = 166; + const uint32_t KTX2_FORMAT_ASTC_8x5_UNORM_BLOCK = 167, KTX2_FORMAT_ASTC_8x5_SRGB_BLOCK = 168; + const uint32_t KTX2_FORMAT_ASTC_8x6_UNORM_BLOCK = 169, KTX2_FORMAT_ASTC_8x6_SRGB_BLOCK = 170; + const uint32_t KTX2_FORMAT_ASTC_10x5_UNORM_BLOCK = 173, KTX2_FORMAT_ASTC_10x5_SRGB_BLOCK = 174; + const uint32_t KTX2_FORMAT_ASTC_10x6_UNORM_BLOCK = 175, KTX2_FORMAT_ASTC_10x6_SRGB_BLOCK = 176; + const uint32_t KTX2_FORMAT_ASTC_8x8_UNORM_BLOCK = 171, KTX2_FORMAT_ASTC_8x8_SRGB_BLOCK = 172; // note the ASTC block size order is off in the vkFormat definitions + const uint32_t KTX2_FORMAT_ASTC_10x8_UNORM_BLOCK = 177, KTX2_FORMAT_ASTC_10x8_SRGB_BLOCK = 178; + const uint32_t KTX2_FORMAT_ASTC_10x10_UNORM_BLOCK = 179, KTX2_FORMAT_ASTC_10x10_SRGB_BLOCK = 180; + const uint32_t KTX2_FORMAT_ASTC_12x10_UNORM_BLOCK = 181, KTX2_FORMAT_ASTC_12x10_SRGB_BLOCK = 182; + const uint32_t KTX2_FORMAT_ASTC_12x12_UNORM_BLOCK = 183, KTX2_FORMAT_ASTC_12x12_SRGB_BLOCK = 184; + const uint32_t KTX2_KDF_DF_MODEL_ASTC = 162; // 0xA2 const uint32_t KTX2_KDF_DF_MODEL_ETC1S = 163; // 0xA3 const uint32_t KTX2_KDF_DF_MODEL_UASTC_LDR_4X4 = 166; // 0xA6 const uint32_t KTX2_KDF_DF_MODEL_UASTC_HDR_4X4 = 167; // 0xA7 - const uint32_t KTX2_KDF_DF_MODEL_ASTC_HDR_6X6_INTERMEDIATE = 168; // 0xA8, TODO - coordinate with Khronos on this - + const uint32_t KTX2_KDF_DF_MODEL_UASTC_HDR_6X6_INTERMEDIATE = 168; // 0xA8, TODO - coordinate with Khronos on this + const uint32_t KTX2_KDF_DF_MODEL_XUASTC_LDR_INTERMEDIATE = 169; // 0xA9, TODO - coordinate with Khronos on this + const uint32_t KTX2_IMAGE_IS_P_FRAME = 2; const uint32_t KTX2_UASTC_BLOCK_SIZE = 16; // also the block size for UASTC_HDR const uint32_t KTX2_MAX_SUPPORTED_LEVEL_COUNT = 16; // this is an implementation specific constraint and can be increased @@ -878,12 +1016,12 @@ namespace basist { case KTX2_DF_PRIMARIES_UNSPECIFIED: return "UNSPECIFIED"; case KTX2_DF_PRIMARIES_BT709: return "BT709"; - case KTX2_DF_PRIMARIES_BT601_EBU: return "EBU"; + case KTX2_DF_PRIMARIES_BT601_EBU: return "EBU"; case KTX2_DF_PRIMARIES_BT601_SMPTE: return "SMPTE"; case KTX2_DF_PRIMARIES_BT2020: return "BT2020"; case KTX2_DF_PRIMARIES_CIEXYZ: return "CIEXYZ"; case KTX2_DF_PRIMARIES_ACES: return "ACES"; - case KTX2_DF_PRIMARIES_ACESCC: return "ACESCC"; + case KTX2_DF_PRIMARIES_ACESCC: return "ACESCC"; case KTX2_DF_PRIMARIES_NTSC1953: return "NTSC1953"; case KTX2_DF_PRIMARIES_PAL525: return "PAL525"; case KTX2_DF_PRIMARIES_DISPLAYP3: return "DISPLAYP3"; @@ -891,7 +1029,7 @@ namespace basist default: break; } return "?"; - } + } // Information about a single 2D texture "image" in a KTX2 file. struct ktx2_image_level_info @@ -901,19 +1039,19 @@ namespace basist uint32_t m_layer_index; uint32_t m_face_index; - // The image's actual (or the original source image's) width/height in pixels, which may not be divisible by 4 pixels. + // The image's ACTUAL (or the original source image's) width/height in pixels, which may not be divisible by the block size (4-12 pixels). uint32_t m_orig_width; uint32_t m_orig_height; - // The image's physical width/height, which will always be divisible by 4 pixels. + // The image's physical width/height, which will always be divisible by the format's block size (4-12 pixels). uint32_t m_width; uint32_t m_height; - - // The texture's dimensions in 4x4 or 6x6 texel blocks. + + // The texture's dimensions in 4x4-12x12 texel blocks. uint32_t m_num_blocks_x; uint32_t m_num_blocks_y; - // The format's block width/height (currently either 4 or 6). + // The format's block width/height (4-12). uint32_t m_block_width; uint32_t m_block_height; @@ -926,7 +1064,7 @@ namespace basist // true if the image is an I-Frame. Currently, for ETC1S textures, the first frame will always be an I-Frame, and subsequent frames will always be P-Frames. bool m_iframe_flag; }; - + // Thread-specific ETC1S/supercompressed UASTC transcoder state. (If you're not doing multithreading transcoding you can ignore this.) struct ktx2_transcoder_state { @@ -944,9 +1082,9 @@ namespace basist // This class is quite similar to basisu_transcoder. It treats KTX2 files as a simple container for ETC1S/UASTC texture data. // It does not support 1D or 3D textures. - // It only supports 2D and cubemap textures, with or without mipmaps, texture arrays of 2D/cubemap textures, and texture video files. + // It only supports 2D and cubemap textures, with or without mipmaps, texture arrays of 2D/cubemap textures, and texture video files. // It only supports raw non-supercompressed UASTC, ETC1S, UASTC+Zstd, or UASTC+zlib compressed files. - // DFD (Data Format Descriptor) parsing is purposely as simple as possible. + // DFD (Data Format Descriptor) parsing is purposely as simple as possible. // If you need to know how to interpret the texture channels you'll need to parse the DFD yourself after calling get_dfd(). class ktx2_transcoder { @@ -971,10 +1109,10 @@ namespace basist // Returns the KTX2 level index array. There will be one entry for each mipmap level. Valid after init(). const basisu::vector& get_level_index() const { return m_levels; } - // Returns the texture's width in texels. Always non-zero, might not be divisible by 4. Valid after init(). + // Returns the texture's width in texels. Always non-zero, might not be divisible by the block size. Valid after init(). uint32_t get_width() const { return m_header.m_pixel_width; } - // Returns the texture's height in texels. Always non-zero, might not be divisible by 4. Valid after init(). + // Returns the texture's height in texels. Always non-zero, might not be divisible by the block size. Valid after init(). uint32_t get_height() const { return m_header.m_pixel_height; } // Returns the texture's number of mipmap levels. Always returns 1 or higher. Valid after init(). @@ -986,15 +1124,15 @@ namespace basist // Returns 0 or the number of layers in the texture array or texture video. Valid after init(). uint32_t get_layers() const { return m_header.m_layer_count; } - // Returns cETC1S, cUASTC4x4, cUASTC_HDR_4x4, cASTC_HDR_6x6, cASTC_HDR_6x6_INTERMEDIATE. Valid after init(). + // Returns cETC1S, cUASTC4x4, cUASTC_HDR_4x4, cASTC_HDR_6x6, cUASTC_HDR_6x6_INTERMEDIATE, etc. Valid after init(). basist::basis_tex_format get_basis_tex_format() const { return m_format; } // ETC1S LDR 4x4 bool is_etc1s() const { return get_basis_tex_format() == basist::basis_tex_format::cETC1S; } // UASTC LDR 4x4 (only) - bool is_uastc() const { return get_basis_tex_format() == basist::basis_tex_format::cUASTC4x4; } - + bool is_uastc() const { return get_basis_tex_format() == basist::basis_tex_format::cUASTC_LDR_4x4; } + // Is ASTC HDR 4x4 or 6x6 bool is_hdr() const { @@ -1006,18 +1144,26 @@ namespace basist return !is_hdr(); } + // is UASTC HDR 4x4 (which is also standard ASTC HDR 4x4 data) bool is_hdr_4x4() const { return (get_basis_tex_format() == basist::basis_tex_format::cUASTC_HDR_4x4); } + // is ASTC HDR 6x6 or UASTC HDR 6x6 intermediate (only) bool is_hdr_6x6() const { - return (get_basis_tex_format() == basist::basis_tex_format::cASTC_HDR_6x6) || (get_basis_tex_format() == basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE); + return (get_basis_tex_format() == basist::basis_tex_format::cASTC_HDR_6x6) || (get_basis_tex_format() == basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE); } + + // is ASTC LDR 4x4-12x12 (only) + bool is_astc_ldr() const { return basis_tex_format_is_astc_ldr(get_basis_tex_format()); } + + // is XUASTC LDR 4x4-12x12 (only) + bool is_xuastc_ldr() const { return basis_tex_format_is_xuastc_ldr(get_basis_tex_format()); } uint32_t get_block_width() const { return basis_tex_format_get_block_width(get_basis_tex_format()); } - uint32_t get_block_height() const { return basis_tex_format_get_block_height(get_basis_tex_format()); } + uint32_t get_block_height() const { return basis_tex_format_get_block_height(get_basis_tex_format()); } // Returns true if the ETC1S file has two planes (typically RGBA, or RRRG), or true if the UASTC file has alpha data. Valid after init(). uint32_t get_has_alpha() const { return m_has_alpha; } @@ -1032,17 +1178,19 @@ namespace basist // Returns the DFD color primary. // We do not validate the color primaries, so the returned value may not be in the ktx2_df_color_primaries enum. ktx2_df_color_primaries get_dfd_color_primaries() const { return m_dfd_color_prims; } - + // Returns KTX2_KHR_DF_TRANSFER_LINEAR or KTX2_KHR_DF_TRANSFER_SRGB. uint32_t get_dfd_transfer_func() const { return m_dfd_transfer_func; } + bool is_srgb() const { return (get_dfd_transfer_func() == KTX2_KHR_DF_TRANSFER_SRGB); } + uint32_t get_dfd_flags() const { return m_dfd_flags; } // Returns 1 (ETC1S/UASTC) or 2 (ETC1S with an internal alpha channel). uint32_t get_dfd_total_samples() const { return m_dfd_samples; } - - // Returns the channel mapping for each DFD "sample". UASTC always has 1 sample, ETC1S can have one or two. - // Note the returned value SHOULD be one of the ktx2_df_channel_id enums, but we don't validate that. + + // Returns the channel mapping for each DFD "sample". UASTC always has 1 sample, ETC1S can have one or two. + // Note the returned value SHOULD be one of the ktx2_df_channel_id enums, but we don't validate that. // It's up to the caller to decide what to do if the value isn't in the enum. ktx2_df_channel_id get_dfd_channel_id0() const { return m_dfd_chan0; } ktx2_df_channel_id get_dfd_channel_id1() const { return m_dfd_chan1; } @@ -1050,11 +1198,11 @@ namespace basist // Key value field data. struct key_value { - // The key field is UTF8 and always zero terminated. + // The key field is UTF8 and always zero terminated. // In memory we always append a zero terminator to the key. basisu::uint8_vec m_key; - // The value may be empty. In the KTX2 file it consists of raw bytes which may or may not be zero terminated. + // The value may be empty. In the KTX2 file it consists of raw bytes which may or may not be zero terminated. // In memory we always append a zero terminator to the value. basisu::uint8_vec m_value; @@ -1076,7 +1224,7 @@ namespace basist // Returns the array of ETC1S image descriptors, which is only valid after get_etc1s_image_descs() is called. const basisu::vector& get_etc1s_image_descs() const { return m_etc1s_image_descs; } - const basisu::vector& get_astc_hdr_6x6_intermediate_image_descs() const { return m_astc_6x6_intermediate_image_descs; } + const basisu::vector& get_slice_offset_len_descs() const { return m_slice_offset_len_descs; } // Must have called startTranscoding() first uint32_t get_etc1s_image_descs_image_flags(uint32_t level_index, uint32_t layer_index, uint32_t face_index) const; @@ -1084,21 +1232,21 @@ namespace basist // is_video() is only valid after start_transcoding() is called. // For ETC1S data, if this returns true you must currently transcode the file from first to last frame, in order, without skipping any frames. bool is_video() const { return m_is_video; } - + // Defaults to 0, only non-zero if the key existed in the source KTX2 file. float get_ldr_hdr_upconversion_nit_multiplier() const { return m_ldr_hdr_upconversion_nit_multiplier; } - - // start_transcoding() MUST be called before calling transcode_image(). + + // start_transcoding() MUST be called before calling transcode_image_level(). // This method decompresses the ETC1S global endpoint/selector codebooks, which is not free, so try to avoid calling it excessively. bool start_transcoding(); - + // get_image_level_info() be called after init(), but the m_iframe_flag's won't be valid until start_transcoding() is called. // You can call this method before calling transcode_image_level() to retrieve basic information about the mipmap level's dimensions, etc. bool get_image_level_info(ktx2_image_level_info& level_info, uint32_t level_index, uint32_t layer_index, uint32_t face_index) const; // transcode_image_level() transcodes a single 2D texture or cubemap face from the KTX2 file. // Internally it uses the same low-level transcode API's as basisu_transcoder::transcode_image_level(). - // If the file is UASTC and is supercompressed with Zstandard, and the file is a texture array or cubemap, it's highly recommended that each mipmap level is + // If the file is UASTC and is supercompressed with Zstandard, and the file is a texture array or cubemap, it's highly recommended that each mipmap level is // completely transcoded before switching to another level. Every time the mipmap level is changed all supercompressed level data must be decompressed using Zstandard as a single unit. // Currently ETC1S videos must always be transcoded from first to last frame (or KTX2 "layer"), in order, with no skipping of frames. // By default this method is not thread safe unless you specify a pointer to a user allocated thread-specific transcoder_state struct. @@ -1108,7 +1256,7 @@ namespace basist basist::transcoder_texture_format fmt, uint32_t decode_flags = 0, uint32_t output_row_pitch_in_blocks_or_pixels = 0, uint32_t output_rows_in_pixels = 0, int channel0 = -1, int channel1 = -1, ktx2_transcoder_state *pState = nullptr); - + private: const uint8_t* m_pData; uint32_t m_data_size; @@ -1117,26 +1265,30 @@ namespace basist basisu::vector m_levels; basisu::uint8_vec m_dfd; key_value_vec m_key_values; - + ktx2_etc1s_global_data_header m_etc1s_header; basisu::vector m_etc1s_image_descs; - basisu::vector m_astc_6x6_intermediate_image_descs; + basisu::vector m_slice_offset_len_descs; basist::basis_tex_format m_format; - + uint32_t m_dfd_color_model; ktx2_df_color_primaries m_dfd_color_prims; - uint32_t m_dfd_transfer_func; + + // KTX2_KHR_DF_TRANSFER_LINEAR vs. KTX2_KHR_DF_TRANSFER_SRGB (for XUASTC LDR: which profile was used during encoding) + uint32_t m_dfd_transfer_func; + uint32_t m_dfd_flags; uint32_t m_dfd_samples; ktx2_df_channel_id m_dfd_chan0, m_dfd_chan1; - + basist::basisu_lowlevel_etc1s_transcoder m_etc1s_transcoder; - basist::basisu_lowlevel_uastc_ldr_4x4_transcoder m_uastc_transcoder; + basist::basisu_lowlevel_uastc_ldr_4x4_transcoder m_uastc_ldr_transcoder; + basist::basisu_lowlevel_xuastc_ldr_transcoder m_xuastc_ldr_transcoder; basist::basisu_lowlevel_uastc_hdr_4x4_transcoder m_uastc_hdr_transcoder; basist::basisu_lowlevel_astc_hdr_6x6_transcoder m_astc_hdr_6x6_transcoder; - basist::basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder m_astc_hdr_6x6_intermediate_transcoder; - + basist::basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder m_astc_hdr_6x6_intermediate_transcoder; + ktx2_transcoder_state m_def_transcoder_state; bool m_has_alpha; @@ -1144,7 +1296,7 @@ namespace basist float m_ldr_hdr_upconversion_nit_multiplier; bool decompress_level_data(uint32_t level_index, basisu::uint8_vec& uncomp_data); - bool read_astc_6x6_hdr_intermediate_global_data(); + bool read_slice_offset_len_global_data(); bool decompress_etc1s_global_data(); bool read_key_values(); }; @@ -1165,7 +1317,7 @@ namespace basist break; } } - + if (!p) p = key_values.enlarge(1); @@ -1189,3 +1341,4 @@ namespace basist bool basisu_transcoder_supports_ktx2_zstd(); } // namespace basisu + diff --git a/external/basis_universal/transcoder/basisu_transcoder_internal.h b/external/basis_universal/transcoder/basisu_transcoder_internal.h index 5480ba32f6..178448973a 100644 --- a/external/basis_universal/transcoder/basisu_transcoder_internal.h +++ b/external/basis_universal/transcoder/basisu_transcoder_internal.h @@ -1,5 +1,5 @@ // basisu_transcoder_internal.h - Universal texture format transcoder library. -// Copyright (C) 2019-2024 Binomial LLC. All Rights Reserved. +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. // // Important: If compiling with gcc, be sure strict aliasing is disabled: -fno-strict-aliasing // @@ -22,8 +22,10 @@ // v1.50: Added UASTC HDR 4x4 support // v1.60: Added RDO ASTC HDR 6x6 and intermediate support -#define BASISD_LIB_VERSION 160 -#define BASISD_VERSION_STRING "01.60" +// v1.65: Added ASTC LDR 4x4-12x12 and XUASTC LDR 4x4-12x12 +// v2.00: Added unified effort/quality options across all formats, fast direct transcoding of XUASTC 4x4/6x6/8x6 to BC7, adaptive deblocking, ZStd or arithmetic profiles, weight grid DCT +#define BASISD_LIB_VERSION 200 +#define BASISD_VERSION_STRING "02.00" #ifdef _DEBUG #define BASISD_BUILD_DEBUG @@ -32,6 +34,7 @@ #endif #include "basisu.h" +#include "basisu_astc_helpers.h" #define BASISD_znew (z = 36969 * (z & 65535) + (z >> 16)) @@ -46,9 +49,9 @@ namespace basist // You probably don't care about these enum's unless you are going pretty low-level and calling the transcoder to decode individual slices. enum class block_format { - cETC1, // ETC1S RGB + cETC1, // ETC1S RGB cETC2_RGBA, // full ETC2 EAC RGBA8 block - cBC1, // DXT1 RGB + cBC1, // DXT1 RGB cBC3, // BC4 block followed by a four color BC1 block cBC4, // DXT5A (alpha block only) cBC5, // two BC4 blocks @@ -58,9 +61,9 @@ namespace basist cBC7_M5_COLOR, // RGB BC7 mode 5 color (writes an opaque mode 5 block) cBC7_M5_ALPHA, // alpha portion of BC7 mode 5 (cBC7_M5_COLOR output data must have been written to the output buffer first to set the mode/rot fields etc.) cETC2_EAC_A8, // alpha block of ETC2 EAC (first 8 bytes of the 16-bit ETC2 EAC RGBA format) - cASTC_4x4, // ASTC 4x4 (either color-only or color+alpha). Note that the transcoder always currently assumes sRGB is not enabled when outputting ASTC + cASTC_LDR_4x4, // ASTC LDR 4x4 (either color-only or color+alpha). Note that the transcoder always currently assumes sRGB decode mode is not enabled when outputting ASTC LDR for ETC1S/UASTC LDR 4x4. // data. If you use a sRGB ASTC format you'll get ~1 LSB of additional error, because of the different way ASTC decoders scale 8-bit endpoints to 16-bits during unpacking. - + cATC_RGB, cATC_RGBA_INTERPOLATED_ALPHA, cFXT1_RGB, // Opaque-only, has oddball 8x4 pixel block size @@ -70,16 +73,16 @@ namespace basist cETC2_EAC_R11, cETC2_EAC_RG11, - + cIndices, // Used internally: Write 16-bit endpoint and selector indices directly to output (output block must be at least 32-bits) cRGB32, // Writes RGB components to 32bpp output pixels cRGBA32, // Writes RGB255 components to 32bpp output pixels cA32, // Writes alpha component to 32bpp output pixels - + cRGB565, cBGR565, - + cRGBA4444_COLOR, cRGBA4444_ALPHA, cRGBA4444_COLOR_OPAQUE, @@ -91,12 +94,73 @@ namespace basist cUASTC_4x4, // LDR, universal cUASTC_HDR_4x4, // HDR, transcodes only to 4x4 HDR ASTC, BC6H, or uncompressed cBC6H, + cASTC_HDR_4x4, cASTC_HDR_6x6, + // The remaining ASTC LDR block sizes. + cASTC_LDR_5x4, + cASTC_LDR_5x5, + cASTC_LDR_6x5, + cASTC_LDR_6x6, + cASTC_LDR_8x5, + cASTC_LDR_8x6, + cASTC_LDR_10x5, + cASTC_LDR_10x6, + cASTC_LDR_8x8, + cASTC_LDR_10x8, + cASTC_LDR_10x10, + cASTC_LDR_12x10, + cASTC_LDR_12x12, + cTotalBlockFormats }; + inline bool block_format_is_hdr(block_format fmt) + { + switch (fmt) + { + case block_format::cUASTC_HDR_4x4: + case block_format::cBC6H: + case block_format::cASTC_HDR_4x4: + case block_format::cASTC_HDR_6x6: + return true; + default: + break; + } + + return false; + } + + // LDR or HDR ASTC? + inline bool block_format_is_astc(block_format fmt) + { + switch (fmt) + { + case block_format::cASTC_LDR_4x4: + case block_format::cASTC_LDR_5x4: + case block_format::cASTC_LDR_5x5: + case block_format::cASTC_LDR_6x5: + case block_format::cASTC_LDR_6x6: + case block_format::cASTC_LDR_8x5: + case block_format::cASTC_LDR_8x6: + case block_format::cASTC_LDR_10x5: + case block_format::cASTC_LDR_10x6: + case block_format::cASTC_LDR_8x8: + case block_format::cASTC_LDR_10x8: + case block_format::cASTC_LDR_10x10: + case block_format::cASTC_LDR_12x10: + case block_format::cASTC_LDR_12x12: + case block_format::cASTC_HDR_4x4: + case block_format::cASTC_HDR_6x6: + return true; + default: + break; + } + + return false; + } + inline uint32_t get_block_width(block_format fmt) { switch (fmt) @@ -105,6 +169,21 @@ namespace basist return 8; case block_format::cASTC_HDR_6x6: return 6; + + case block_format::cASTC_LDR_5x4: return 5; + case block_format::cASTC_LDR_5x5: return 5; + case block_format::cASTC_LDR_6x5: return 6; + case block_format::cASTC_LDR_6x6: return 6; + case block_format::cASTC_LDR_8x5: return 8; + case block_format::cASTC_LDR_8x6: return 8; + case block_format::cASTC_LDR_10x5: return 10; + case block_format::cASTC_LDR_10x6: return 10; + case block_format::cASTC_LDR_8x8: return 8; + case block_format::cASTC_LDR_10x8: return 10; + case block_format::cASTC_LDR_10x10: return 10; + case block_format::cASTC_LDR_12x10: return 12; + case block_format::cASTC_LDR_12x12: return 12; + default: break; } @@ -117,6 +196,20 @@ namespace basist { case block_format::cASTC_HDR_6x6: return 6; + + case block_format::cASTC_LDR_5x5: return 5; + case block_format::cASTC_LDR_6x5: return 5; + case block_format::cASTC_LDR_6x6: return 6; + case block_format::cASTC_LDR_8x5: return 5; + case block_format::cASTC_LDR_8x6: return 6; + case block_format::cASTC_LDR_10x5: return 5; + case block_format::cASTC_LDR_10x6: return 6; + case block_format::cASTC_LDR_8x8: return 8; + case block_format::cASTC_LDR_10x8: return 8; + case block_format::cASTC_LDR_10x10: return 10; + case block_format::cASTC_LDR_12x10: return 10; + case block_format::cASTC_LDR_12x12: return 12; + default: break; } @@ -140,9 +233,31 @@ namespace basist const uint32_t SELECTOR_HISTORY_BUF_RLE_COUNT_THRESH = 3; const uint32_t SELECTOR_HISTORY_BUF_RLE_COUNT_BITS = 6; const uint32_t SELECTOR_HISTORY_BUF_RLE_COUNT_TOTAL = (1 << SELECTOR_HISTORY_BUF_RLE_COUNT_BITS); - + uint16_t crc16(const void *r, size_t size, uint16_t crc); + uint32_t hash_hsieh(const uint8_t* pBuf, size_t len); + + template + struct bit_hasher + { + inline std::size_t operator()(const Key& k) const + { + return hash_hsieh(reinterpret_cast(&k), sizeof(k)); + } + }; + + struct string_hasher + { + inline std::size_t operator()(const std::string& k) const + { + size_t l = k.size(); + if (!l) + return 0; + return hash_hsieh(reinterpret_cast(k.c_str()), l); + } + }; + class huffman_decoding_table { friend class bitwise_decoder; @@ -260,7 +375,7 @@ namespace basist return false; else if (idx >= (int)m_tree.size()) m_tree.resize(idx + 1); - + if (!m_tree[idx]) { m_tree[idx] = (int16_t)tree_next; @@ -350,7 +465,7 @@ namespace basist void stop() { } - + inline uint32_t peek_bits(uint32_t num_bits) { if (!num_bits) @@ -429,14 +544,14 @@ namespace basist for (;;) { uint32_t k = peek_bits(16); - + uint32_t l = 0; while (k & 1) { l++; k >>= 1; } - + q += l; remove_bits(l); @@ -454,7 +569,7 @@ namespace basist const uint32_t chunk_size = 1 << chunk_bits; const uint32_t chunk_mask = chunk_size - 1; - + uint32_t v = 0; uint32_t ofs = 0; @@ -466,7 +581,7 @@ namespace basist if ((s & chunk_size) == 0) break; - + if (ofs >= 32) { assert(0); @@ -482,7 +597,7 @@ namespace basist assert(ct.m_code_sizes.size()); const uint32_t huffman_fast_lookup_size = 1 << fast_lookup_bits; - + while (m_bit_buf_size < 16) { uint32_t c = 0; @@ -493,7 +608,7 @@ namespace basist m_bit_buf_size += 8; assert(m_bit_buf_size <= 32); } - + int code_len; int sym; @@ -603,6 +718,100 @@ namespace basist uint32_t m_bit_buf_size; }; + class simplified_bitwise_decoder + { + public: + simplified_bitwise_decoder() : + m_pBuf(nullptr), + m_pBuf_end(nullptr), + m_bit_buf(0) + { + } + + void clear() + { + m_pBuf = nullptr; + m_pBuf_end = nullptr; + m_bit_buf = 0; + } + + bool init(const uint8_t* pBuf, size_t buf_size) + { + if ((!pBuf) && (buf_size)) + return false; + + m_pBuf = pBuf; + m_pBuf_end = pBuf + buf_size; + m_bit_buf = 1; + return true; + } + + bool init(const basisu::uint8_vec& buf) + { + return init(buf.data(), buf.size()); + } + + // num_bits must be 1, 2, 4 or 8 and codes cannot cross bytes + inline uint32_t get_bits(uint32_t num_bits) + { + assert(m_pBuf); + + if (m_bit_buf <= 1) + m_bit_buf = 256 | ((m_pBuf < m_pBuf_end) ? *m_pBuf++ : 0); + + const uint32_t mask = (1 << num_bits) - 1; + const uint32_t res = m_bit_buf & mask; + m_bit_buf >>= num_bits; + assert(m_bit_buf >= 1); + + return res; + } + + inline uint32_t get_bits1() + { + assert(m_pBuf); + if (m_bit_buf <= 1) + m_bit_buf = 256 | ((m_pBuf < m_pBuf_end) ? *m_pBuf++ : 0); + const uint32_t res = m_bit_buf & 1; + m_bit_buf >>= 1; + assert(m_bit_buf >= 1); + return res; + } + + inline uint32_t get_bits2() + { + assert(m_pBuf); + if (m_bit_buf <= 1) + m_bit_buf = 256 | ((m_pBuf < m_pBuf_end) ? *m_pBuf++ : 0); + const uint32_t res = m_bit_buf & 3; + m_bit_buf >>= 2; + assert(m_bit_buf >= 1); + return res; + } + + inline uint32_t get_bits4() + { + assert(m_pBuf); + if (m_bit_buf <= 1) + m_bit_buf = 256 | ((m_pBuf < m_pBuf_end) ? *m_pBuf++ : 0); + const uint32_t res = m_bit_buf & 15; + m_bit_buf >>= 4; + assert(m_bit_buf >= 1); + return res; + } + + // No bitbuffer, can only ever retrieve bytes correctly. + inline uint32_t get_bits8() + { + assert(m_pBuf); + return (m_pBuf < m_pBuf_end) ? *m_pBuf++ : 0; + } + + const uint8_t* m_pBuf; + const uint8_t* m_pBuf_end; + uint32_t m_bit_buf; + }; + inline uint32_t basisd_rand(uint32_t seed) { if (!seed) @@ -684,7 +893,7 @@ namespace basist }; struct decoder_etc_block; - + inline uint8_t clamp255(int32_t i) { return (uint8_t)((i & 0xFFFFFF00U) ? (~(i >> 31)) : i); @@ -708,11 +917,12 @@ namespace basist }; uint8_t c[4]; - + uint32_t m; }; - color32() { } + //color32() { } + color32() = default; color32(uint32_t vr, uint32_t vg, uint32_t vb, uint32_t va) { set(vr, vg, vb, va); } color32(eNoClamp unused, uint32_t vr, uint32_t vg, uint32_t vb, uint32_t va) { (void)unused; set_noclamp_rgba(vr, vg, vb, va); } @@ -744,6 +954,172 @@ namespace basist bool operator!= (const endpoint& rhs) const { return !(*this == rhs); } }; + // This duplicates key functionality in the encoder library's color_rgba class. Porting and retesting code that uses it to color32 is impractical. + class color_rgba + { + public: + union + { + uint8_t m_comps[4]; + + struct + { + uint8_t r; + uint8_t g; + uint8_t b; + uint8_t a; + }; + }; + + inline color_rgba() + { + static_assert(sizeof(*this) == 4, "sizeof(*this) != 4"); + static_assert(sizeof(*this) == sizeof(color32), "sizeof(*this) != sizeof(basist::color32)"); + } + + inline color_rgba(const color32& other) : + r(other.r), + g(other.g), + b(other.b), + a(other.a) + { + } + + color_rgba& operator= (const basist::color32& rhs) + { + r = rhs.r; + g = rhs.g; + b = rhs.b; + a = rhs.a; + return *this; + } + + inline color_rgba(int y) + { + set(y); + } + + inline color_rgba(int y, int na) + { + set(y, na); + } + + inline color_rgba(int sr, int sg, int sb, int sa) + { + set(sr, sg, sb, sa); + } + + inline color_rgba(eNoClamp, int sr, int sg, int sb, int sa) + { + set_noclamp_rgba((uint8_t)sr, (uint8_t)sg, (uint8_t)sb, (uint8_t)sa); + } + + inline color_rgba& set_noclamp_y(int y) + { + m_comps[0] = (uint8_t)y; + m_comps[1] = (uint8_t)y; + m_comps[2] = (uint8_t)y; + m_comps[3] = (uint8_t)255; + return *this; + } + + inline color_rgba& set_noclamp_rgba(int sr, int sg, int sb, int sa) + { + m_comps[0] = (uint8_t)sr; + m_comps[1] = (uint8_t)sg; + m_comps[2] = (uint8_t)sb; + m_comps[3] = (uint8_t)sa; + return *this; + } + + inline color_rgba& set(int y) + { + m_comps[0] = static_cast(basisu::clamp(y, 0, 255)); + m_comps[1] = m_comps[0]; + m_comps[2] = m_comps[0]; + m_comps[3] = 255; + return *this; + } + + inline color_rgba& set(int y, int na) + { + m_comps[0] = static_cast(basisu::clamp(y, 0, 255)); + m_comps[1] = m_comps[0]; + m_comps[2] = m_comps[0]; + m_comps[3] = static_cast(basisu::clamp(na, 0, 255)); + return *this; + } + + inline color_rgba& set(int sr, int sg, int sb, int sa) + { + m_comps[0] = static_cast(basisu::clamp(sr, 0, 255)); + m_comps[1] = static_cast(basisu::clamp(sg, 0, 255)); + m_comps[2] = static_cast(basisu::clamp(sb, 0, 255)); + m_comps[3] = static_cast(basisu::clamp(sa, 0, 255)); + return *this; + } + + inline color_rgba& set_rgb(int sr, int sg, int sb) + { + m_comps[0] = static_cast(basisu::clamp(sr, 0, 255)); + m_comps[1] = static_cast(basisu::clamp(sg, 0, 255)); + m_comps[2] = static_cast(basisu::clamp(sb, 0, 255)); + return *this; + } + + inline color_rgba& set_rgb(const color_rgba& other) + { + r = other.r; + g = other.g; + b = other.b; + return *this; + } + + inline const uint8_t& operator[] (uint32_t index) const { assert(index < 4); return m_comps[index]; } + inline uint8_t& operator[] (uint32_t index) { assert(index < 4); return m_comps[index]; } + + inline void clear() + { + m_comps[0] = 0; + m_comps[1] = 0; + m_comps[2] = 0; + m_comps[3] = 0; + } + + inline bool operator== (const color_rgba& rhs) const + { + if (m_comps[0] != rhs.m_comps[0]) return false; + if (m_comps[1] != rhs.m_comps[1]) return false; + if (m_comps[2] != rhs.m_comps[2]) return false; + if (m_comps[3] != rhs.m_comps[3]) return false; + return true; + } + + inline bool operator!= (const color_rgba& rhs) const + { + return !(*this == rhs); + } + + inline bool operator<(const color_rgba& rhs) const + { + for (int i = 0; i < 4; i++) + { + if (m_comps[i] < rhs.m_comps[i]) + return true; + else if (m_comps[i] != rhs.m_comps[i]) + return false; + } + return false; + } + + inline color32 get_color32() const + { + return color32(r, g, b, a); + } + + inline int get_709_luma() const { return (13938U * m_comps[0] + 46869U * m_comps[1] + 4729U * m_comps[2] + 32768U) >> 16U; } + }; + struct selector { // Plain selectors (2-bits per value) @@ -989,7 +1365,7 @@ namespace basist extern const uint8_t g_bc6h_weight4[16]; extern const int8_t g_bc6h_mode_lookup[32]; - + // Converts b16 to half float inline half_float bc6h_blog16_to_half(uint32_t comp) { @@ -1003,7 +1379,7 @@ namespace basist const uint32_t MAX_BC6H_HALF_FLOAT_AS_UINT = 0x7BFF; // Inverts bc6h_blog16_to_half(). - // Returns the nearest blog16 given a half value. + // Returns the nearest blog16 given a half value. inline uint32_t bc6h_half_to_blog16(half_float h) { assert(h <= MAX_BC6H_HALF_FLOAT_AS_UINT); @@ -1044,10 +1420,1844 @@ namespace basist }; void pack_bc6h_block(bc6h_block& dst_blk, bc6h_logical_block& log_blk); - + namespace bc7_mode_5_encoder { void encode_bc7_mode_5_block(void* pDst_block, color32* pPixels, bool hq_mode); } -} // namespace basist + namespace astc_6x6_hdr + { + extern uint8_t g_quantize_tables_preserve2[21 - 1][256]; // astc_helpers::TOTAL_ISE_RANGES=21 + extern uint8_t g_quantize_tables_preserve3[21 - 1][256]; + } // namespace astc_6x6_hdr + +#if BASISD_SUPPORT_XUASTC + namespace astc_ldr_t + { + const uint32_t ARITH_HEADER_MARKER = 0x01; + const uint32_t ARITH_HEADER_MARKER_BITS = 5; + + const uint32_t FULL_ZSTD_HEADER_MARKER = 0x01; + const uint32_t FULL_ZSTD_HEADER_MARKER_BITS = 5; + + const uint32_t FINAL_SYNC_MARKER = 0xAF; + const uint32_t FINAL_SYNC_MARKER_BITS = 8; + + const uint32_t cMaxConfigReuseNeighbors = 3; + +#pragma pack(push, 1) + struct xuastc_ldr_arith_header + { + uint8_t m_flags; + basisu::packed_uint<4> m_arith_bytes_len; + basisu::packed_uint<4> m_mean0_bits_len; + basisu::packed_uint<4> m_mean1_bytes_len; + basisu::packed_uint<4> m_run_bytes_len; + basisu::packed_uint<4> m_coeff_bytes_len; + basisu::packed_uint<4> m_sign_bits_len; + basisu::packed_uint<4> m_weight2_bits_len; // 2-bit weights (4 per byte), up to BISE_4_LEVELS + basisu::packed_uint<4> m_weight3_bits_len; // 3-bit weights (2 per byte), up to BISE_8_LEVELS + basisu::packed_uint<4> m_weight4_bits_len; // 4-bit weights (2 per byte), up to BISE_16_LEVELS + basisu::packed_uint<4> m_weight8_bytes_len; // 8-bit weights (1 per byte), up to BISE_32_LEVELS + basisu::packed_uint<4> m_unused; // Future expansion + }; + + struct xuastc_ldr_full_zstd_header + { + uint8_t m_flags; + + // Control + basisu::packed_uint<4> m_raw_bits_len; // uncompressed + basisu::packed_uint<4> m_mode_bytes_len; + basisu::packed_uint<4> m_solid_dpcm_bytes_len; + + // Endpoint DPCM + basisu::packed_uint<4> m_endpoint_dpcm_reuse_indices_len; + basisu::packed_uint<4> m_use_bc_bits_len; + basisu::packed_uint<4> m_endpoint_dpcm_3bit_len; + basisu::packed_uint<4> m_endpoint_dpcm_4bit_len; + basisu::packed_uint<4> m_endpoint_dpcm_5bit_len; + basisu::packed_uint<4> m_endpoint_dpcm_6bit_len; + basisu::packed_uint<4> m_endpoint_dpcm_7bit_len; + basisu::packed_uint<4> m_endpoint_dpcm_8bit_len; + + // Weight grid DCT + basisu::packed_uint<4> m_mean0_bits_len; + basisu::packed_uint<4> m_mean1_bytes_len; + basisu::packed_uint<4> m_run_bytes_len; + basisu::packed_uint<4> m_coeff_bytes_len; + basisu::packed_uint<4> m_sign_bits_len; + + // Weight DPCM + basisu::packed_uint<4> m_weight2_bits_len; // 2-bit weights (4 per byte), up to BISE_4_LEVELS + basisu::packed_uint<4> m_weight3_bits_len; // 3-bit weights (4 per byte), up to BISE_8_LEVELS + basisu::packed_uint<4> m_weight4_bits_len; // 4-bit weights (2 per byte), up to BISE_16_LEVELS + basisu::packed_uint<4> m_weight8_bytes_len; // 8-bit weights (1 per byte), up to BISE_32_LEVELS + + basisu::packed_uint<4> m_unused; // Future expansion + }; +#pragma pack(pop) + + const uint32_t DCT_RUN_LEN_EOB_SYM_INDEX = 64; + const uint32_t DCT_MAX_ARITH_COEFF_MAG = 255; + + const uint32_t DCT_MEAN_LEVELS0 = 9, DCT_MEAN_LEVELS1 = 33; + + const uint32_t PART_HASH_BITS = 6u; + const uint32_t PART_HASH_SIZE = 1u << PART_HASH_BITS; + + const uint32_t TM_HASH_BITS = 7u; + const uint32_t TM_HASH_SIZE = 1u << TM_HASH_BITS; + + typedef basisu::vector fvec; + + void init(); + + color_rgba blue_contract_enc(color_rgba orig, bool& did_clamp, int encoded_b); + color_rgba blue_contract_dec(int enc_r, int enc_g, int enc_b, int enc_a); + + struct astc_block_grid_config + { + uint16_t m_block_width, m_block_height; + uint16_t m_grid_width, m_grid_height; + + astc_block_grid_config() {} + + astc_block_grid_config(uint32_t block_width, uint32_t block_height, uint32_t grid_width, uint32_t grid_height) + { + assert((block_width >= 4) && (block_width <= 12)); + assert((block_height >= 4) && (block_height <= 12)); + m_block_width = (uint16_t)block_width; + m_block_height = (uint16_t)block_height; + + assert((grid_width >= 2) && (grid_width <= block_width)); + assert((grid_height >= 2) && (grid_height <= block_height)); + m_grid_width = (uint16_t)grid_width; + m_grid_height = (uint16_t)grid_height; + } + + bool operator==(const astc_block_grid_config& other) const + { + return (m_block_width == other.m_block_width) && (m_block_height == other.m_block_height) && + (m_grid_width == other.m_grid_width) && (m_grid_height == other.m_grid_height); + } + }; + + struct astc_block_grid_data + { + float m_weight_gamma; + + // An unfortunate difference of containers, but in memory these matrices are both addressed as [r][c]. + basisu::vector2D m_upsample_matrix; + + basisu::vector m_downsample_matrix; + + astc_block_grid_data() {} + astc_block_grid_data(float weight_gamma) : m_weight_gamma(weight_gamma) {} + }; + + typedef basisu::hash_map > astc_block_grid_data_hash_t; + + void decode_endpoints_ise20(uint32_t cem_index, const uint8_t* pEndpoint_vals, color32& l, color32& h); + void decode_endpoints(uint32_t cem_index, const uint8_t* pEndpoint_vals, uint32_t endpoint_ise_index, color32& l, color32& h, float* pScale = nullptr); + + void decode_endpoints_ise20(uint32_t cem_index, const uint8_t* pEndpoint_vals, color_rgba& l, color_rgba& h); + void decode_endpoints(uint32_t cem_index, const uint8_t* pEndpoint_vals, uint32_t endpoint_ise_index, color_rgba& l, color_rgba& h, float* pScale = nullptr); + + void compute_adjoint_downsample_matrix(basisu::vector& downsample_matrix, uint32_t block_width, uint32_t block_height, uint32_t grid_width, uint32_t grid_height); + void compute_upsample_matrix(basisu::vector2D& upsample_matrix, uint32_t block_width, uint32_t block_height, uint32_t grid_width, uint32_t grid_height); + + class dct2f + { + enum { cMaxSize = 12 }; + + public: + dct2f() : m_rows(0u), m_cols(0u) {} + + // call with grid_height/grid_width (INVERTED) + bool init(uint32_t rows, uint32_t cols); + + uint32_t rows() const { return m_rows; } + uint32_t cols() const { return m_cols; } + + void forward(const float* pSrc, float* pDst, fvec& work) const; + + void inverse(const float* pSrc, float* pDst, fvec& work) const; + + // check variants use a less optimized implementation, used for sanity checking + void inverse_check(const float* pSrc, float* pDst, fvec& work) const; + + void forward(const float* pSrc, uint32_t src_stride, + float* pDst, uint32_t dst_stride, fvec& work) const; + + void inverse(const float* pSrc, uint32_t src_stride, + float* pDst, uint32_t dst_stride, fvec& work) const; + + void inverse_check(const float* pSrc, uint32_t src_stride, + float* pDst, uint32_t dst_stride, fvec& work) const; + + private: + uint32_t m_rows, m_cols; + fvec m_c_col; // [u*m_rows + x] + fvec m_c_row; // [v*m_cols + y] + fvec m_a_col; // alpha(u) + fvec m_a_row; // alpha(v) + }; + + struct dct_syms + { + dct_syms() + { + clear(); + } + + void clear() + { + m_dc_sym = 0; + m_num_dc_levels = 0; + m_coeffs.resize(0); + m_max_coeff_mag = 0; + m_max_zigzag_index = 0; + } + + uint32_t m_dc_sym; + uint32_t m_num_dc_levels; + + struct coeff + { + uint16_t m_num_zeros; + int16_t m_coeff; // or INT16_MAX if invalid + + coeff() {} + coeff(uint16_t num_zeros, int16_t coeff) : m_num_zeros(num_zeros), m_coeff(coeff) {} + }; + + basisu::static_vector m_coeffs; + + uint32_t m_max_coeff_mag; + uint32_t m_max_zigzag_index; + }; + + struct grid_dim_key + { + int m_grid_width; + int m_grid_height; + + grid_dim_key() {} + + grid_dim_key(int w, int h) : m_grid_width(w), m_grid_height(h) {} + + bool operator== (const grid_dim_key& rhs) const + { + return (m_grid_width == rhs.m_grid_width) && (m_grid_height == rhs.m_grid_height); + } + }; + + struct grid_dim_value + { + basisu::int_vec m_zigzag; + dct2f m_dct; + }; + + typedef basisu::hash_map > grid_dim_hash_map; + + void init_astc_block_grid_data_hash(); + + const astc_block_grid_data* find_astc_block_grid_data(uint32_t block_width, uint32_t block_height, uint32_t grid_width, uint32_t grid_height); + + const float DEADZONE_ALPHA = .5f; + const float SCALED_WEIGHT_BASE_CODING_SCALE = .5f; // typically ~5 bits [0,32], or 3 [0,8] + + struct sample_quant_table_state + { + float m_q, m_sx, m_sy, m_level_scale; + + void init(float q, + uint32_t block_width, uint32_t block_height, + float level_scale) + { + m_q = q; + m_level_scale = level_scale; + + const int Bx = block_width, By = block_height; + + m_sx = (float)8.0f / (float)Bx; + m_sy = (float)8.0f / (float)By; + } + }; + + class grid_weight_dct + { + public: + grid_weight_dct() { } + + void init(uint32_t block_width, uint32_t block_height); + + static uint32_t get_num_weight_dc_levels(uint32_t weight_ise_range) + { + float scaled_weight_coding_scale = SCALED_WEIGHT_BASE_CODING_SCALE; + if (weight_ise_range <= astc_helpers::BISE_8_LEVELS) + scaled_weight_coding_scale = 1.0f / 8.0f; + + return (uint32_t)(64.0f * scaled_weight_coding_scale) + 1; + } + + struct block_stats + { + float m_mean_weight; + uint32_t m_total_coded_acs; + uint32_t m_max_ac_coeff; + }; + + bool decode_block_weights( + float q, uint32_t plane_index, // plane of weights to decode and IDCT from stream + astc_helpers::log_astc_block& log_blk, // must be initialized except for the plane weights which are decoded + basist::bitwise_decoder* pDec, + const astc_block_grid_data* pGrid_data, // grid data for this grid size + block_stats* pS, + fvec& dct_work, // thread local + const dct_syms* pSyms = nullptr) const; + + enum { m_zero_run = 3, m_coeff = 2 }; + + uint32_t m_block_width, m_block_height; + + grid_dim_hash_map m_grid_dim_key_vals; + + // Adaptively compensate for weight level quantization noise being fed into the DCT. + // The more coursely the weight levels are quantized, the more noise injected, and the more noise will be spread between multiple AC coefficients. + // This will cause some previously 0 coefficients to increase in mag, but they're likely noise. So carefully nudge the quant step size to compensate. + static float scale_quant_steps(int Q_astc, float gamma = 0.1f /*.13f*/, float clamp_max = 2.0f) + { + assert(Q_astc >= 2); + float factor = 63.0f / (Q_astc - 1); + // TODO: Approximate powf() + float scaled = powf(factor, gamma); + scaled = basisu::clamp(scaled, 1.0f, clamp_max); + return scaled; + } + + float compute_level_scale(float q, float span_len, float weight_gamma, uint32_t grid_width, uint32_t grid_height, uint32_t weight_ise_range) const; + + int sample_quant_table(sample_quant_table_state& state, uint32_t x, uint32_t y) const; + + void compute_quant_table(float q, + uint32_t grid_width, uint32_t grid_height, + float level_scale, int* dct_quant_tab) const; + + float get_max_span_len(const astc_helpers::log_astc_block& log_blk, uint32_t plane_index) const; + + inline int quantize_deadzone(float d, int L, float alpha, uint32_t x, uint32_t y) const + { + assert((x < m_block_width) && (y < m_block_height)); + + if (((x == 1) && (y == 0)) || + ((x == 0) && (y == 1))) + { + return (int)std::round(d / (float)L); + } + + // L = quant step, alpha in [0,1.2] (typical 0.7-0.85) + if (L <= 0) + return 0; + + float s = fabsf(d); + float tau = alpha * float(L); // half-width of the zero band + + if (s <= tau) + return 0; // inside dead-zone towards zero + + // Quantize the residual outside the dead-zone with mid-tread rounding + float qf = (s - tau) / float(L); + int q = (int)floorf(qf + 0.5f); // ties-nearest + return (d < 0.0f) ? -q : q; + } + + inline float dequant_deadzone(int q, int L, float alpha, uint32_t x, uint32_t y) const + { + assert((x < m_block_width) && (y < m_block_height)); + + if (((x == 1) && (y == 0)) || + ((x == 0) && (y == 1))) + { + return (float)q * (float)L; + } + + if (q == 0 || L <= 0) + return 0.0f; + + float tau = alpha * float(L); + float mag = tau + float(abs(q)) * float(L); // center of the (nonzero) bin + return (q < 0) ? -mag : mag; + } + }; + + struct trial_mode + { + uint32_t m_grid_width; + uint32_t m_grid_height; + uint32_t m_cem; + int m_ccs_index; + uint32_t m_endpoint_ise_range; + uint32_t m_weight_ise_range; + uint32_t m_num_parts; + + bool operator==(const trial_mode& other) const + { +#define BU_COMP(a) if (a != other.a) return false; + BU_COMP(m_grid_width); + BU_COMP(m_grid_height); + BU_COMP(m_cem); + BU_COMP(m_ccs_index); + BU_COMP(m_endpoint_ise_range); + BU_COMP(m_weight_ise_range); + BU_COMP(m_num_parts); +#undef BU_COMP + return true; + } + + bool operator<(const trial_mode& rhs) const + { +#define BU_COMP(a) if (a < rhs.a) return true; else if (a > rhs.a) return false; + BU_COMP(m_grid_width); + BU_COMP(m_grid_height); + BU_COMP(m_cem); + BU_COMP(m_ccs_index); + BU_COMP(m_endpoint_ise_range); + BU_COMP(m_weight_ise_range); + BU_COMP(m_num_parts); +#undef BU_COMP + return false; + } + + operator size_t() const + { + size_t h = 0xABC1F419; +#define BU_FIELD(a) do { h ^= hash_hsieh(reinterpret_cast(&a), sizeof(a)); } while(0) + BU_FIELD(m_grid_width); + BU_FIELD(m_grid_height); + BU_FIELD(m_cem); + BU_FIELD(m_ccs_index); + BU_FIELD(m_endpoint_ise_range); + BU_FIELD(m_weight_ise_range); + BU_FIELD(m_num_parts); +#undef BU_FIELD + return h; + } + }; + + // Organize trial modes for faster initial mode triaging. + const uint32_t OTM_NUM_CEMS = 14; // 0-13 (13=highest valid LDR CEM) + const uint32_t OTM_NUM_SUBSETS = 3; // 1-3 + const uint32_t OTM_NUM_CCS = 5; // -1 to 3 + const uint32_t OTM_NUM_GRID_SIZES = 2; // 0=small or 1=large (grid_w>=block_w-1 and grid_h>=block_h-1) + const uint32_t OTM_NUM_GRID_ANISOS = 3; // 0=W=H, 1=W>H, 2=W 0) && (gh > 0)); + assert((bw > 0) && (bh > 0)); + assert((gw <= 12) && (gh <= 12) && (bw <= 12) && (bh <= 12)); + assert((gw <= bw) && (gh <= bh)); + +#if 0 + // Prev. code: + uint32_t grid_aniso = 0; + if (tm.m_grid_width != tm.m_grid_height) // not optimal for non-square block sizes + { + const float grid_x_fract = (float)tm.m_grid_width / (float)block_width; + const float grid_y_fract = (float)tm.m_grid_height / (float)block_height; + if (grid_x_fract >= grid_y_fract) + grid_aniso = 1; + else if (grid_x_fract < grid_y_fract) + grid_aniso = 2; + } +#endif + // Compare gw/bw vs. gh/bh using integer math: + // gw*bh >= gh*bw -> X-dominant (1), else Y-dominant (2) + const uint32_t lhs = gw * bh; + const uint32_t rhs = gh * bw; + + // Equal (isotropic), X=Y + if (lhs == rhs) + return 0; + + // Anisotropic - 1=X, 2=Y + return (lhs >= rhs) ? 1 : 2; + } + + struct grouped_trial_modes + { + basisu::uint_vec m_tm_groups[OTM_NUM_CEMS][OTM_NUM_SUBSETS][OTM_NUM_CCS][OTM_NUM_GRID_SIZES][OTM_NUM_GRID_ANISOS]; // indices of encoder trial modes in each bucket + + void clear() + { + for (uint32_t cem_iter = 0; cem_iter < OTM_NUM_CEMS; cem_iter++) + for (uint32_t subsets_iter = 0; subsets_iter < OTM_NUM_SUBSETS; subsets_iter++) + for (uint32_t ccs_iter = 0; ccs_iter < OTM_NUM_CCS; ccs_iter++) + for (uint32_t grid_sizes_iter = 0; grid_sizes_iter < OTM_NUM_GRID_SIZES; grid_sizes_iter++) + for (uint32_t grid_anisos_iter = 0; grid_anisos_iter < OTM_NUM_GRID_ANISOS; grid_anisos_iter++) + m_tm_groups[cem_iter][subsets_iter][ccs_iter][grid_sizes_iter][grid_anisos_iter].clear(); + } + + void add(uint32_t block_width, uint32_t block_height, + const trial_mode& tm, uint32_t tm_index) + { + const uint32_t cem_index = tm.m_cem; + assert(cem_index < OTM_NUM_CEMS); + + const uint32_t subset_index = tm.m_num_parts - 1; + assert(subset_index < OTM_NUM_SUBSETS); + + const uint32_t ccs_index = tm.m_ccs_index + 1; + assert(ccs_index < OTM_NUM_CCS); + + const uint32_t grid_size = (tm.m_grid_width >= (block_width - 1)) && (tm.m_grid_height >= (block_height - 1)); + const uint32_t grid_aniso = calc_grid_aniso_val(tm.m_grid_width, tm.m_grid_height, block_width, block_height); + + basisu::uint_vec& v = m_tm_groups[cem_index][subset_index][ccs_index][grid_size][grid_aniso]; + if (!v.capacity()) + v.reserve(64); + + v.push_back(tm_index); + } + + uint32_t count_used_groups() const + { + uint32_t n = 0; + + for (uint32_t cem_iter = 0; cem_iter < OTM_NUM_CEMS; cem_iter++) + for (uint32_t subsets_iter = 0; subsets_iter < OTM_NUM_SUBSETS; subsets_iter++) + for (uint32_t ccs_iter = 0; ccs_iter < OTM_NUM_CCS; ccs_iter++) + for (uint32_t grid_sizes_iter = 0; grid_sizes_iter < OTM_NUM_GRID_SIZES; grid_sizes_iter++) + for (uint32_t grid_anisos_iter = 0; grid_anisos_iter < OTM_NUM_GRID_ANISOS; grid_anisos_iter++) + { + if (m_tm_groups[cem_iter][subsets_iter][ccs_iter][grid_sizes_iter][grid_anisos_iter].size()) + n++; + } + return n; + } + }; + + extern grouped_trial_modes g_grouped_encoder_trial_modes[astc_helpers::cTOTAL_BLOCK_SIZES]; + + inline const basisu::uint_vec& get_tm_candidates(const grouped_trial_modes& grouped_enc_trial_modes, + uint32_t cem_index, uint32_t subset_index, uint32_t ccs_index, uint32_t grid_size, uint32_t grid_aniso) + { + assert(cem_index < OTM_NUM_CEMS); + assert(subset_index < OTM_NUM_SUBSETS); + assert(ccs_index < OTM_NUM_CCS); + assert(grid_size < OTM_NUM_GRID_SIZES); + assert(grid_aniso < OTM_NUM_GRID_ANISOS); + + const basisu::uint_vec& modes = grouped_enc_trial_modes.m_tm_groups[cem_index][subset_index][ccs_index][grid_size][grid_aniso]; + return modes; + } + + const uint32_t CFG_PACK_GRID_BITS = 7; + const uint32_t CFG_PACK_CEM_BITS = 3; + const uint32_t CFG_PACK_CCS_BITS = 3; + const uint32_t CFG_PACK_SUBSETS_BITS = 2; + const uint32_t CFG_PACK_WISE_BITS = 4; + const uint32_t CFG_PACK_EISE_BITS = 5; + + extern const int s_unique_ldr_index_to_astc_cem[6]; + + enum class xuastc_mode + { + cMODE_SOLID = 0, + cMODE_RAW = 1, + + // Full cfg, partition ID, and all endpoint value reuse. + cMODE_REUSE_CFG_ENDPOINTS_LEFT = 2, + cMODE_REUSE_CFG_ENDPOINTS_UP = 3, + cMODE_REUSE_CFG_ENDPOINTS_DIAG = 4, + + cMODE_RUN = 5, + + cMODE_TOTAL, + }; + + enum class xuastc_zstd_mode + { + // len=1 bits + cMODE_RAW = 0b0, + + // len=2 bits + cMODE_RUN = 0b01, + + // len=4 bits + cMODE_SOLID = 0b0011, + cMODE_REUSE_CFG_ENDPOINTS_LEFT = 0b0111, + cMODE_REUSE_CFG_ENDPOINTS_UP = 0b1011, + cMODE_REUSE_CFG_ENDPOINTS_DIAG = 0b1111 + }; + + const uint32_t XUASTC_LDR_MODE_BYTE_IS_BASE_OFS_FLAG = 1 << 3; + const uint32_t XUASTC_LDR_MODE_BYTE_PART_HASH_HIT = 1 << 4; + const uint32_t XUASTC_LDR_MODE_BYTE_DPCM_ENDPOINTS_FLAG = 1 << 5; + const uint32_t XUASTC_LDR_MODE_BYTE_TM_HASH_HIT_FLAG = 1 << 6; + const uint32_t XUASTC_LDR_MODE_BYTE_USE_DCT = 1 << 7; + + enum class xuastc_ldr_syntax + { + cFullArith = 0, + cHybridArithZStd = 1, + cFullZStd = 2, + + cTotal + }; + + void create_encoder_trial_modes_table(uint32_t block_width, uint32_t block_height, + basisu::vector& encoder_trial_modes, grouped_trial_modes& grouped_encoder_trial_modes, + bool print_debug_info, bool print_modes); + + extern basisu::vector g_encoder_trial_modes[astc_helpers::cTOTAL_BLOCK_SIZES]; + + inline uint32_t part_hash_index(uint32_t x) + { + // fib hash + return (x * 2654435769u) & (PART_HASH_SIZE - 1); + } + + // Full ZStd syntax only + inline uint32_t tm_hash_index(uint32_t x) + { + // fib hash + return (x * 2654435769u) & (TM_HASH_SIZE - 1); + } + + // TODO: Some fields are unused during transcoding. + struct prev_block_state + { + bool m_was_solid_color; + bool m_used_weight_dct; + bool m_first_endpoint_uses_bc; + bool m_reused_full_cfg; + bool m_used_part_hash; + + int m_tm_index; // -1 if invalid (solid color block) + uint32_t m_base_cem_index; // doesn't include base+ofs + uint32_t m_subset_index, m_ccs_index, m_grid_size, m_grid_aniso; + + prev_block_state() + { + clear(); + } + + void clear() + { + basisu::clear_obj(*this); + } + }; + + struct prev_block_state_full_zstd + { + int m_tm_index; // -1 if invalid (solid color block) + + bool was_solid_color() const { return m_tm_index < 0; } + + prev_block_state_full_zstd() + { + clear(); + } + + void clear() + { + basisu::clear_obj(*this); + } + }; + + inline uint32_t cem_to_ldrcem_index(uint32_t cem) + { + switch (cem) + { + case astc_helpers::CEM_LDR_LUM_DIRECT: return 0; + case astc_helpers::CEM_LDR_LUM_ALPHA_DIRECT: return 1; + case astc_helpers::CEM_LDR_RGB_BASE_SCALE: return 2; + case astc_helpers::CEM_LDR_RGB_DIRECT: return 3; + case astc_helpers::CEM_LDR_RGB_BASE_PLUS_OFFSET: return 4; + case astc_helpers::CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A: return 5; + case astc_helpers::CEM_LDR_RGBA_DIRECT: return 6; + case astc_helpers::CEM_LDR_RGBA_BASE_PLUS_OFFSET: return 7; + default: + assert(0); + break; + } + + return 0; + } + + bool pack_base_offset( + uint32_t cem_index, uint32_t dst_ise_endpoint_range, uint8_t* pPacked_endpoints, + const color_rgba& l, const color_rgba& h, + bool use_blue_contraction, bool auto_disable_blue_contraction_if_clamped, + bool& blue_contraction_clamped_flag, bool& base_ofs_clamped_flag, bool& endpoints_swapped); + + bool convert_endpoints_across_cems( + uint32_t prev_cem, uint32_t prev_endpoint_ise_range, const uint8_t* pPrev_endpoints, + uint32_t dst_cem, uint32_t dst_endpoint_ise_range, uint8_t* pDst_endpoints, + bool always_repack, + bool use_blue_contraction, bool auto_disable_blue_contraction_if_clamped, + bool& blue_contraction_clamped_flag, bool& base_ofs_clamped_flag); + + uint32_t get_total_unique_patterns(uint32_t astc_block_size_index, uint32_t num_parts); + //uint16_t unique_pat_index_to_part_seed(uint32_t astc_block_size_index, uint32_t num_parts, uint32_t unique_pat_index); + + typedef bool (*xuastc_decomp_image_init_callback_ptr)(uint32_t num_blocks_x, uint32_t num_blocks_y, uint32_t block_width, uint32_t block_height, bool srgb_decode_profile, float dct_q, bool has_alpha, void* pData); + typedef bool (*xuastc_decomp_image_block_callback_ptr)(uint32_t bx, uint32_t by, const astc_helpers::log_astc_block& log_blk, void* pData); + + bool xuastc_ldr_decompress_image( + const uint8_t* pComp_data, size_t comp_data_size, + uint32_t& astc_block_width, uint32_t& astc_block_height, + uint32_t& actual_width, uint32_t& actual_height, bool& has_alpha, bool& uses_srgb_astc_decode_mode, + bool debug_output, + xuastc_decomp_image_init_callback_ptr pInit_callback, void *pInit_callback_data, + xuastc_decomp_image_block_callback_ptr pBlock_callback, void *pBlock_callback_data); + + } // namespace astc_ldr_t + + namespace arith_fastbits_f32 + { + enum { TABLE_BITS = 8 }; // 256..1024 entries typical (8..10) + enum { TABLE_SIZE = 1 << TABLE_BITS }; + enum { MANT_BITS = 23 }; + enum { FRAC_BITS = (int)MANT_BITS - (int)TABLE_BITS }; + enum { FRAC_MASK = (1u << FRAC_BITS) - 1u }; + + extern bool g_initialized; + extern float g_lut_edge[TABLE_SIZE + 1]; // samples at m = 1 + i/TABLE_SIZE (for linear) + + inline void init() + { + if (g_initialized) + return; + + const float inv_ln2 = 1.4426950408889634f; // 1/ln(2) + + for (int i = 0; i <= TABLE_SIZE; ++i) + { + float m = 1.0f + float(i) / float(TABLE_SIZE); // m in [1,2] + g_lut_edge[i] = logf(m) * inv_ln2; // log2(m) + } + + g_initialized = true; + } + + inline void unpack(float p, int& e_unbiased, uint32_t& mant) + { + // kill any denorms + if (p < FLT_MIN) + p = 0; + + union { float f; uint32_t u; } x; + x.f = p; + e_unbiased = int((x.u >> 23) & 0xFF) - 127; + mant = (x.u & 0x7FFFFFu); // 23-bit mantissa + } + + // Returns estimated bits given probability p, approximates -log2f(p). + inline float bits_from_prob_linear(float p) + { + assert((p > 0.0f) && (p <= 1.0f)); + if (!g_initialized) + init(); + + int e; uint32_t mant; + unpack(p, e, mant); + + uint32_t idx = mant >> FRAC_BITS; // 0..TABLE_SIZE-1 + uint32_t frac = mant & FRAC_MASK; // low FRAC_BITS + const float inv_scale = 1.0f / float(1u << FRAC_BITS); + float t = float(frac) * inv_scale; // [0,1) + + float y0 = g_lut_edge[idx]; + float y1 = g_lut_edge[idx + 1]; + float log2m = y0 + t * (y1 - y0); + + return -(float(e) + log2m); + } + + } // namespace arith_fastbits_f32 + + namespace arith + { + // A simple range coder + const uint32_t ArithMaxSyms = 2048; + const uint32_t DMLenShift = 15u; + const uint32_t DMMaxCount = 1u << DMLenShift; + const uint32_t BMLenShift = 13u; + const uint32_t BMMaxCount = 1u << BMLenShift; + const uint32_t ArithMinLen = 1u << 24u; + const uint32_t ArithMaxLen = UINT32_MAX; + const uint32_t ArithMinExpectedDataBufSize = 5; + + class arith_bit_model + { + public: + arith_bit_model() + { + reset(); + } + + void init() + { + reset(); + } + + void reset() + { + m_bit0_count = 1; + m_bit_count = 2; + m_bit0_prob = 1U << (BMLenShift - 1); + m_update_interval = 4; + m_bits_until_update = 4; + } + + float get_price(bool bit) const + { + const float prob_0 = (float)m_bit0_prob / (float)BMMaxCount; + const float prob = bit ? (1.0f - prob_0) : prob_0; + const float bits = arith_fastbits_f32::bits_from_prob_linear(prob); + assert(fabs(bits - (-log2f(prob))) < .00125f); // basic sanity check + return bits; + } + + void update() + { + assert(m_bit_count >= 2); + assert(m_bit0_count < m_bit_count); + + if (m_bit_count >= BMMaxCount) + { + assert(m_bit_count && m_bit0_count); + + m_bit_count = (m_bit_count + 1) >> 1; + m_bit0_count = (m_bit0_count + 1) >> 1; + + if (m_bit0_count == m_bit_count) + ++m_bit_count; + + assert(m_bit0_count < m_bit_count); + } + + const uint32_t scale = 0x80000000U / m_bit_count; + m_bit0_prob = (m_bit0_count * scale) >> (31 - BMLenShift); + + m_update_interval = basisu::clamp((5 * m_update_interval) >> 2, 4u, 128); + + m_bits_until_update = m_update_interval; + } + + void print_prices(const char* pDesc) + { + if (pDesc) + printf("arith_data_model bit prices for model %s:\n", pDesc); + for (uint32_t i = 0; i < 2; i++) + printf("%u: %3.3f bits\n", i, get_price(i)); + printf("\n"); + } + + private: + friend class arith_enc; + friend class arith_dec; + + uint32_t m_bit0_prob; // snapshot made at last update + + uint32_t m_bit0_count; // live + uint32_t m_bit_count; // live + + int m_bits_until_update; + uint32_t m_update_interval; + }; + + enum { cARITH_GAMMA_MAX_TAIL_CTX = 4, cARITH_GAMMA_MAX_PREFIX_CTX = 3 }; + struct arith_gamma_contexts + { + arith_bit_model m_ctx_prefix[cARITH_GAMMA_MAX_PREFIX_CTX]; // for unary continue prefix + arith_bit_model m_ctx_tail[cARITH_GAMMA_MAX_TAIL_CTX]; // for binary suffix bits + }; + + class arith_data_model + { + public: + arith_data_model() : + m_num_data_syms(0), + m_total_sym_freq(0), + m_update_interval(0), + m_num_syms_until_next_update(0) + { + } + + arith_data_model(uint32_t num_syms, bool faster_update = false) : + m_num_data_syms(0), + m_total_sym_freq(0), + m_update_interval(0), + m_num_syms_until_next_update(0) + { + init(num_syms, faster_update); + } + + void clear() + { + m_cum_sym_freqs.clear(); + m_sym_freqs.clear(); + + m_num_data_syms = 0; + m_total_sym_freq = 0; + m_update_interval = 0; + m_num_syms_until_next_update = 0; + } + + void init(uint32_t num_syms, bool faster_update = false) + { + assert((num_syms >= 2) && (num_syms <= ArithMaxSyms)); + + m_num_data_syms = num_syms; + + m_sym_freqs.resize(num_syms); + m_cum_sym_freqs.resize(num_syms + 1); + + reset(faster_update); + } + + void reset(bool faster_update = false) + { + if (!m_num_data_syms) + return; + + m_sym_freqs.set_all(1); + m_total_sym_freq = m_num_data_syms; + + m_update_interval = m_num_data_syms; + m_num_syms_until_next_update = 0; + + update(false); + + if (faster_update) + { + m_update_interval = basisu::clamp((m_num_data_syms + 7) / 8, 4u, (m_num_data_syms + 6) << 3); + m_num_syms_until_next_update = m_update_interval; + } + } + + void update(bool enc_flag) + { + assert(m_num_data_syms); + BASISU_NOTE_UNUSED(enc_flag); + + if (!m_num_data_syms) + return; + + while (m_total_sym_freq >= DMMaxCount) + { + m_total_sym_freq = 0; + + for (uint32_t n = 0; n < m_num_data_syms; n++) + { + m_sym_freqs[n] = (m_sym_freqs[n] + 1u) >> 1u; + m_total_sym_freq += m_sym_freqs[n]; + } + } + + const uint32_t scale = 0x80000000U / m_total_sym_freq; + + uint32_t sum = 0; + for (uint32_t i = 0; i < m_num_data_syms; ++i) + { + assert(((uint64_t)scale * sum) <= UINT32_MAX); + m_cum_sym_freqs[i] = (scale * sum) >> (31 - DMLenShift); + sum += m_sym_freqs[i]; + } + assert(sum == m_total_sym_freq); + + m_cum_sym_freqs[m_num_data_syms] = DMMaxCount; + + m_update_interval = basisu::clamp((5 * m_update_interval) >> 2, 4u, (m_num_data_syms + 6) << 3); + + m_num_syms_until_next_update = m_update_interval; + } + + float get_price(uint32_t sym_index) const + { + assert(sym_index < m_num_data_syms); + + if (sym_index >= m_num_data_syms) + return 0.0f; + + const float prob = (float)(m_cum_sym_freqs[sym_index + 1] - m_cum_sym_freqs[sym_index]) / (float)DMMaxCount; + + const float bits = arith_fastbits_f32::bits_from_prob_linear(prob); + assert(fabs(bits - (-log2f(prob))) < .00125f); // basic sanity check + return bits; + } + + void print_prices(const char* pDesc) + { + if (pDesc) + printf("arith_data_model bit prices for model %s:\n", pDesc); + for (uint32_t i = 0; i < m_num_data_syms; i++) + printf("%u: %3.3f bits\n", i, get_price(i)); + printf("\n"); + } + + uint32_t get_num_data_syms() const { return m_num_data_syms; } + + private: + friend class arith_enc; + friend class arith_dec; + + uint32_t m_num_data_syms; + + basisu::uint_vec m_sym_freqs; // live histogram + uint32_t m_total_sym_freq; // always live vs. m_sym_freqs + + basisu::uint_vec m_cum_sym_freqs; // has 1 extra entry, snapshot from last update + + uint32_t m_update_interval; + int m_num_syms_until_next_update; + + uint32_t get_last_sym_index() const { return m_num_data_syms - 1; } + }; + + class arith_enc + { + public: + arith_enc() + { + clear(); + } + + void clear() + { + m_data_buf.clear(); + + m_base = 0; + m_length = ArithMaxLen; + } + + void init(size_t reserve_size) + { + m_data_buf.reserve(reserve_size); + m_data_buf.resize(0); + + m_base = 0; + m_length = ArithMaxLen; + + // Place 8-bit marker at beginning. + // This virtually always guarantees no backwards carries can be lost at the very beginning of the stream. (Should be impossible with this design.) + // It always pushes out 1 0 byte at the very beginning to absorb future carries. + // Caller does this now, we send a tiny header anyway + //put_bits(0x1, 8); + //assert(m_data_buf[0] != 0xFF); + } + + void put_bit(uint32_t bit) + { + m_length >>= 1; + + if (bit) + { + const uint32_t orig_base = m_base; + + m_base += m_length; + + if (orig_base > m_base) + prop_carry(); + } + + if (m_length < ArithMinLen) + renorm(); + } + + enum { cMaxPutBitsLen = 20 }; + void put_bits(uint32_t val, uint32_t num_bits) + { + assert(num_bits && (num_bits <= cMaxPutBitsLen)); + assert(val < (1u << num_bits)); + + m_length >>= num_bits; + + const uint32_t orig_base = m_base; + + m_base += val * m_length; + + if (orig_base > m_base) + prop_carry(); + + if (m_length < ArithMinLen) + renorm(); + } + + // returns # of bits actually written + inline uint32_t put_truncated_binary(uint32_t v, uint32_t n) + { + assert((n >= 2) && (v < n)); + + uint32_t k = basisu::floor_log2i(n); + uint32_t u = (1 << (k + 1)) - n; + + if (v < u) + { + put_bits(v, k); + return k; + } + + uint32_t x = v + u; + assert((x >> 1) >= u); + + put_bits(x >> 1, k); + put_bits(x & 1, 1); + return k + 1; + } + + static inline uint32_t get_truncated_binary_bits(uint32_t v, uint32_t n) + { + assert((n >= 2) && (v < n)); + + uint32_t k = basisu::floor_log2i(n); + uint32_t u = (1 << (k + 1)) - n; + + if (v < u) + return k; + +#ifdef _DEBUG + uint32_t x = v + u; + assert((x >> 1) >= u); +#endif + + return k + 1; + } + + inline uint32_t put_rice(uint32_t v, uint32_t m) + { + assert(m); + + uint32_t q = v >> m, r = v & ((1 << m) - 1); + + // rice coding sanity check + assert(q <= 64); + + uint32_t total_bits = q; + + // TODO: put_bits the pattern inverted in bit order + while (q) + { + put_bit(1); + q--; + } + + put_bit(0); + + put_bits(r, m); + + total_bits += (m + 1); + + return total_bits; + } + + static inline uint32_t get_rice_price(uint32_t v, uint32_t m) + { + assert(m); + + uint32_t q = v >> m; + + // rice coding sanity check + assert(q <= 64); + + uint32_t total_bits = q + 1 + m; + + return total_bits; + } + + inline void put_gamma(uint32_t n, arith_gamma_contexts& ctxs) + { + assert(n); + if (!n) + return; + + const int k = basisu::floor_log2i(n); + if (k > 16) + { + assert(0); + return; + } + + // prefix: k times '1' then a '0' + for (int i = 0; i < k; ++i) + encode(1, ctxs.m_ctx_prefix[basisu::minimum(i, cARITH_GAMMA_MAX_PREFIX_CTX - 1)]); + + encode(0, ctxs.m_ctx_prefix[basisu::minimum(k, cARITH_GAMMA_MAX_PREFIX_CTX - 1)]); + + // suffix: the k low bits of n + for (int i = k - 1; i >= 0; --i) + { + uint32_t bit = (n >> i) & 1u; + encode(bit, ctxs.m_ctx_tail[basisu::minimum(i, cARITH_GAMMA_MAX_TAIL_CTX - 1)]); + } + } + + inline float put_gamma_and_return_price(uint32_t n, arith_gamma_contexts& ctxs) + { + assert(n); + if (!n) + return 0.0f; + + const int k = basisu::floor_log2i(n); + if (k > 16) + { + assert(0); + return 0.0f; + } + + float total_price = 0.0f; + + // prefix: k times '1' then a '0' + for (int i = 0; i < k; ++i) + { + total_price += ctxs.m_ctx_prefix[basisu::minimum(i, cARITH_GAMMA_MAX_PREFIX_CTX - 1)].get_price(1); + encode(1, ctxs.m_ctx_prefix[basisu::minimum(i, cARITH_GAMMA_MAX_PREFIX_CTX - 1)]); + } + + total_price += ctxs.m_ctx_prefix[basisu::minimum(k, cARITH_GAMMA_MAX_PREFIX_CTX - 1)].get_price(0); + encode(0, ctxs.m_ctx_prefix[basisu::minimum(k, cARITH_GAMMA_MAX_PREFIX_CTX - 1)]); + + // suffix: the k low bits of n + for (int i = k - 1; i >= 0; --i) + { + uint32_t bit = (n >> i) & 1u; + total_price += ctxs.m_ctx_tail[basisu::minimum(i, cARITH_GAMMA_MAX_TAIL_CTX - 1)].get_price(bit); + encode(bit, ctxs.m_ctx_tail[basisu::minimum(i, cARITH_GAMMA_MAX_TAIL_CTX - 1)]); + } + + return total_price; + } + + // prediced price, won't be accurate if a binary arith model decides to update in between + inline float get_gamma_price(uint32_t n, const arith_gamma_contexts& ctxs) + { + assert(n); + if (!n) + return 0.0f; + + const int k = basisu::floor_log2i(n); + if (k > 16) + { + assert(0); + return 0.0f; + } + + float total_price = 0.0f; + + // prefix: k times '1' then a '0' + for (int i = 0; i < k; ++i) + total_price += ctxs.m_ctx_prefix[basisu::minimum(i, cARITH_GAMMA_MAX_PREFIX_CTX - 1)].get_price(1); + + total_price += ctxs.m_ctx_prefix[basisu::minimum(k, cARITH_GAMMA_MAX_PREFIX_CTX - 1)].get_price(0); + + // suffix: the k low bits of n + for (int i = k - 1; i >= 0; --i) + { + uint32_t bit = (n >> i) & 1u; + total_price += ctxs.m_ctx_tail[basisu::minimum(i, cARITH_GAMMA_MAX_TAIL_CTX - 1)].get_price(bit); + } + + return total_price; + } + + void encode(uint32_t bit, arith_bit_model& dm) + { + uint32_t x = dm.m_bit0_prob * (m_length >> BMLenShift); + + if (!bit) + { + m_length = x; + ++dm.m_bit0_count; + } + else + { + const uint32_t orig_base = m_base; + m_base += x; + m_length -= x; + + if (orig_base > m_base) + prop_carry(); + } + ++dm.m_bit_count; + + if (m_length < ArithMinLen) + renorm(); + + if (--dm.m_bits_until_update <= 0) + dm.update(); + } + + float encode_and_return_price(uint32_t bit, arith_bit_model& dm) + { + const float price = dm.get_price(bit); + encode(bit, dm); + return price; + } + + void encode(uint32_t sym, arith_data_model& dm) + { + assert(sym < dm.m_num_data_syms); + + const uint32_t orig_base = m_base; + + if (sym == dm.get_last_sym_index()) + { + uint32_t x = dm.m_cum_sym_freqs[sym] * (m_length >> DMLenShift); + m_base += x; + m_length -= x; + } + else + { + m_length >>= DMLenShift; + uint32_t x = dm.m_cum_sym_freqs[sym] * m_length; + m_base += x; + m_length = dm.m_cum_sym_freqs[sym + 1] * m_length - x; + } + + if (orig_base > m_base) + prop_carry(); + + if (m_length < ArithMinLen) + renorm(); + + ++dm.m_sym_freqs[sym]; + ++dm.m_total_sym_freq; + + if (--dm.m_num_syms_until_next_update <= 0) + dm.update(true); + } + + float encode_and_return_price(uint32_t sym, arith_data_model& dm) + { + const float price = dm.get_price(sym); + encode(sym, dm); + return price; + } + + void flush() + { + const uint32_t orig_base = m_base; + + if (m_length <= (2 * ArithMinLen)) + { + m_base += ArithMinLen >> 1; + m_length = ArithMinLen >> 9; + } + else + { + m_base += ArithMinLen; + m_length = ArithMinLen >> 1; + } + + if (orig_base > m_base) + prop_carry(); + + renorm(); + + // Pad output to min 5 bytes - quite conservative; we're typically compressing large streams so the overhead shouldn't matter. + if (m_data_buf.size() < ArithMinExpectedDataBufSize) + m_data_buf.resize(ArithMinExpectedDataBufSize); + } + + basisu::uint8_vec& get_data_buf() { return m_data_buf; } + const basisu::uint8_vec& get_data_buf() const { return m_data_buf; } + + private: + basisu::uint8_vec m_data_buf; + uint32_t m_base, m_length; + + inline void prop_carry() + { + int64_t ofs = m_data_buf.size() - 1; + + for (; (ofs >= 0) && (m_data_buf[(size_t)ofs] == 0xFF); --ofs) + m_data_buf[(size_t)ofs] = 0; + + if (ofs >= 0) + ++m_data_buf[(size_t)ofs]; + } + + inline void renorm() + { + assert(m_length < ArithMinLen); + do + { + m_data_buf.push_back((uint8_t)(m_base >> 24u)); + m_base <<= 8u; + m_length <<= 8u; + } while (m_length < ArithMinLen); + } + }; + + class arith_dec + { + public: + arith_dec() + { + clear(); + } + + void clear() + { + m_pData_buf = nullptr; + m_pData_buf_last_byte = nullptr; + m_pData_buf_cur = nullptr; + m_data_buf_size = 0; + + m_value = 0; + m_length = 0; + } + + bool init(const uint8_t* pBuf, size_t buf_size) + { + if (buf_size < ArithMinExpectedDataBufSize) + { + assert(0); + return false; + } + + m_pData_buf = pBuf; + m_pData_buf_last_byte = pBuf + buf_size - 1; + m_pData_buf_cur = m_pData_buf + 4; + m_data_buf_size = buf_size; + + m_value = ((uint32_t)(pBuf[0]) << 24u) | ((uint32_t)(pBuf[1]) << 16u) | ((uint32_t)(pBuf[2]) << 8u) | (uint32_t)(pBuf[3]); + m_length = ArithMaxLen; + + // Check for the 8-bit marker we always place at the beginning of the stream. + //uint32_t marker = get_bits(8); + //if (marker != 0x1) + // return false; + + return true; + } + + uint32_t get_bit() + { + assert(m_data_buf_size); + + m_length >>= 1; + + uint32_t bit = (m_value >= m_length); + + if (bit) + m_value -= m_length; + + if (m_length < ArithMinLen) + renorm(); + + return bit; + } + + enum { cMaxGetBitsLen = 20 }; + + uint32_t get_bits(uint32_t num_bits) + { + assert(m_data_buf_size); + + if ((num_bits < 1) || (num_bits > cMaxGetBitsLen)) + { + assert(0); + return 0; + } + + m_length >>= num_bits; + assert(m_length); + + const uint32_t v = m_value / m_length; + + m_value -= m_length * v; + + if (m_length < ArithMinLen) + renorm(); + + return v; + } + + uint32_t decode_truncated_binary(uint32_t n) + { + assert(n >= 2); + + const uint32_t k = basisu::floor_log2i(n); + const uint32_t u = (1 << (k + 1)) - n; + + uint32_t result = get_bits(k); + + if (result >= u) + result = ((result << 1) | get_bits(1)) - u; + + return result; + } + + uint32_t decode_rice(uint32_t m) + { + assert(m); + + uint32_t q = 0; + for (;;) + { + uint32_t k = get_bit(); + if (!k) + break; + + q++; + if (q > 64) + { + assert(0); + return 0; + } + } + + return (q << m) + get_bits(m); + } + + uint32_t decode_bit(arith_bit_model& dm) + { + assert(m_data_buf_size); + + uint32_t x = dm.m_bit0_prob * (m_length >> BMLenShift); + uint32_t bit = (m_value >= x); + + if (bit == 0) + { + m_length = x; + ++dm.m_bit0_count; + } + else + { + m_value -= x; + m_length -= x; + } + ++dm.m_bit_count; + + if (m_length < ArithMinLen) + renorm(); + + if (--dm.m_bits_until_update <= 0) + dm.update(); + + return bit; + } + + inline uint32_t decode_gamma(arith_gamma_contexts& ctxs) + { + int k = 0; + while (decode_bit(ctxs.m_ctx_prefix[basisu::minimum(k, cARITH_GAMMA_MAX_PREFIX_CTX - 1)])) + { + ++k; + + if (k > 16) + { + // something is very wrong + assert(0); + return 0; + } + } + + int n = 1 << k; + for (int i = k - 1; i >= 0; --i) + { + uint32_t bit = decode_bit(ctxs.m_ctx_tail[basisu::minimum(i, cARITH_GAMMA_MAX_TAIL_CTX - 1)]); + n |= (bit << i); + } + + return n; + } + + uint32_t decode_sym(arith_data_model& dm) + { + assert(m_data_buf_size); + assert(dm.m_num_data_syms); + + uint32_t x = 0, y = m_length; + + m_length >>= DMLenShift; + + uint32_t low_idx = 0, hi_idx = dm.m_num_data_syms; + uint32_t mid_idx = hi_idx >> 1; + + do + { + uint32_t z = m_length * dm.m_cum_sym_freqs[mid_idx]; + + if (z > m_value) + { + hi_idx = mid_idx; + y = z; + } + else + { + low_idx = mid_idx; + x = z; + } + mid_idx = (low_idx + hi_idx) >> 1; + + } while (mid_idx != low_idx); + + m_value -= x; + m_length = y - x; + + if (m_length < ArithMinLen) + renorm(); + + ++dm.m_sym_freqs[low_idx]; + ++dm.m_total_sym_freq; + + if (--dm.m_num_syms_until_next_update <= 0) + dm.update(false); + + return low_idx; + } + + private: + const uint8_t* m_pData_buf; + const uint8_t* m_pData_buf_last_byte; + const uint8_t* m_pData_buf_cur; + size_t m_data_buf_size; + + uint32_t m_value, m_length; + + inline void renorm() + { + do + { + const uint32_t next_byte = (m_pData_buf_cur > m_pData_buf_last_byte) ? 0 : *m_pData_buf_cur++; + + m_value = (m_value << 8u) | next_byte; + + } while ((m_length <<= 8u) < ArithMinLen); + } + }; + + } // namespace arith +#endif // BASISD_SUPPORT_XUASTC + +#if BASISD_SUPPORT_XUASTC + namespace bc7u + { + int determine_bc7_mode(const void* pBlock); + int determine_bc7_mode_4_index_mode(const void* pBlock); + int determine_bc7_mode_4_or_5_rotation(const void* pBlock); + bool unpack_bc7_mode6(const void* pBlock_bits, color_rgba* pPixels); + bool unpack_bc7(const void* pBlock, color_rgba* pPixels); + } // namespace bc7u + + namespace bc7f + { + enum + { + // Low-level BC7 encoder configuration flags. + cPackBC7FlagUse2SubsetsRGB = 1, // use mode 1/3 for RGB blocks + cPackBC7FlagUse2SubsetsRGBA = 2, // use mode 7 for RGBA blocks + + cPackBC7FlagUse3SubsetsRGB = 4, // also use mode 0/2, cPackBC7FlagUse2SubsetsRGB MUST be enabled too + + cPackBC7FlagUseDualPlaneRGB = 8, // enable mode 4/5 usage for RGB blocks + cPackBC7FlagUseDualPlaneRGBA = 16, // enable mode 4/5 usage for RGBA blocks + + cPackBC7FlagPBitOpt = 32, // enable to disable usage of fixed p-bits on some modes; slower + cPackBC7FlagPBitOptMode6 = 64, // enable to disable usage of fixed p-bits on mode 6, alpha on fully opaque blocks may be 254 however; slower + + cPackBC7FlagUseTrivialMode6 = 128, // enable trivial fast mode 6 encoder on blocks with very low variances (highly recommended) + + cPackBC7FlagPartiallyAnalyticalRGB = 256, // partially analytical mode for RGB blocks, slower but higher quality, computes actual SSE's on complex blocks to resolve which mode to use vs. predictions + cPackBC7FlagPartiallyAnalyticalRGBA = 512, // partially analytical mode for RGBA blocks, slower but higher quality, computes actual SSE's on complex blocks to resolve which mode to use vs. predictions + + // Non-analytical is really still partially analytical on the mode pairs (0 vs. 2, 1 vs 3, 4 vs. 5). + cPackBC7FlagNonAnalyticalRGB = 1024, // very slow/brute force, totally abuses the encoder, MUST use with cPackBC7FlagPartiallyAnalyticalRGB flag + cPackBC7FlagNonAnalyticalRGBA = 2048, // very slow/brute force, totally abuses the encoder, MUST use with cPackBC7FlagPartiallyAnalyticalRGBA flag + + // Default to use first: + + // Decent analytical BC7 defaults + cPackBC7FlagDefaultFastest = cPackBC7FlagUseTrivialMode6, // very weak particularly on alpha, mode 6 only for RGB/RGBA, + + // Mode 6 with pbits for RGB, Modes 4,5,6 for alpha. + cPackBC7FlagDefaultFaster = cPackBC7FlagPBitOpt | cPackBC7FlagUseDualPlaneRGBA | cPackBC7FlagUseTrivialMode6, + + cPackBC7FlagDefaultFast = cPackBC7FlagUse2SubsetsRGB | cPackBC7FlagUse2SubsetsRGBA | cPackBC7FlagUseDualPlaneRGBA | + cPackBC7FlagPBitOpt | cPackBC7FlagUseTrivialMode6, + + cPackBC7FlagDefault = (cPackBC7FlagUse2SubsetsRGB | cPackBC7FlagUse2SubsetsRGBA | cPackBC7FlagUse3SubsetsRGB) | + (cPackBC7FlagUseDualPlaneRGB | cPackBC7FlagUseDualPlaneRGBA) | + (cPackBC7FlagPBitOpt | cPackBC7FlagPBitOptMode6) | + cPackBC7FlagUseTrivialMode6, + + // Default partially analytical BC7 defaults (slower) + cPackBC7FlagDefaultPartiallyAnalytical = cPackBC7FlagDefault | (cPackBC7FlagPartiallyAnalyticalRGB | cPackBC7FlagPartiallyAnalyticalRGBA), + + // Default non-analytical BC7 defaults (very slow). In reality the encoder is still analytical on the mode pairs, but at the highest level is non-analytical. + cPackBC7FlagDefaultNonAnalytical = (cPackBC7FlagDefaultPartiallyAnalytical | (cPackBC7FlagNonAnalyticalRGB | cPackBC7FlagNonAnalyticalRGBA)) & ~cPackBC7FlagUseTrivialMode6 + }; + + void init(); + + void fast_pack_bc7_rgb_analytical(uint8_t* pBlock, const color_rgba* pPixels, uint32_t flags); + uint32_t fast_pack_bc7_rgb_partial_analytical(uint8_t* pBlock, const color_rgba* pPixels, uint32_t flags); + + void fast_pack_bc7_rgba_analytical(uint8_t* pBlock, const color_rgba* pPixels, uint32_t flags); + uint32_t fast_pack_bc7_rgba_partial_analytical(uint8_t* pBlock, const color_rgba* pPixels, uint32_t flags); + + uint32_t fast_pack_bc7_auto_rgba(uint8_t* pBlock, const color_rgba* pPixels, uint32_t flags); + + void print_perf_stats(); + +#if 0 + // Very basic BC7 mode 6 only to ASTC. + void fast_pack_astc(void* pBlock, const color_rgba* pPixels); +#endif + + uint32_t calc_sse(const uint8_t* pBlock, const color_rgba* pPixels); + + } // namespace bc7f + + namespace etc1f + { + struct pack_etc1_state + { + uint64_t m_prev_solid_block; + //decoder_etc_block m_prev_solid_block; + + int m_prev_solid_r8; + int m_prev_solid_g8; + int m_prev_solid_b8; + + pack_etc1_state() + { + clear(); + } + + void clear() + { + m_prev_solid_r8 = -1; + m_prev_solid_g8 = -1; + m_prev_solid_b8 = -1; + } + }; + + void init(); + + void pack_etc1_solid(uint8_t* pBlock, const color_rgba& color, pack_etc1_state& state, bool init_flag = false); + + void pack_etc1(uint8_t* pBlock, const color_rgba* pPixels, pack_etc1_state& state); + + void pack_etc1_grayscale(uint8_t* pBlock, const uint8_t* pPixels, pack_etc1_state& state); + + } // namespace etc1f +#endif // BASISD_SUPPORT_XUASTC + + // Private/internal XUASTC LDR transcoding helpers + + // XUASTC LDR formats only + enum class transcoder_texture_format; + block_format xuastc_get_block_format(transcoder_texture_format tex_fmt); + +#if BASISD_SUPPORT_XUASTC + // Low-quality, but fast, PVRTC1 RGB/RGBA encoder. Power of 2 texture dimensions required. + // Note: Not yet part of our public API: this API may change! + void encode_pvrtc1( + block_format fmt, void* pDst_blocks, + const basisu::vector2D& temp_image, + uint32_t dst_num_blocks_x, uint32_t dst_num_blocks_y, bool from_alpha); + + void transcode_4x4_block( + block_format fmt, // desired output block format + uint32_t block_x, uint32_t block_y, // 4x4 block being processed + void* pDst_blocks, // base pointer to output buffer/bitmap + uint8_t* pDst_block_u8, // pointer to output block/or first pixel to write + const color32* block_pixels, // pointer to 4x4 (16) 32bpp RGBA pixels + uint32_t output_block_or_pixel_stride_in_bytes, uint32_t output_row_pitch_in_blocks_or_pixels, uint32_t output_rows_in_pixels, // output buffer dimensions + int channel0, int channel1, // channels to process, used by some block formats + bool high_quality, bool from_alpha, // Flags specific to certain block formats + uint32_t bc7f_flags, // Real-time bc7f BC7 encoder flags, see bc7f::cPackBC7FlagDefault etc. + etc1f::pack_etc1_state& etc1_pack_state, // etc1f thread local state + int has_alpha = -1); // has_alpha = -1 unknown, 0=definitely no (a all 255's), 1=potentially yes +#endif // BASISD_SUPPORT_XUASTC + + struct bc7_mode_5 + { + union + { + struct + { + uint64_t m_mode : 6; + uint64_t m_rot : 2; + + uint64_t m_r0 : 7; + uint64_t m_r1 : 7; + uint64_t m_g0 : 7; + uint64_t m_g1 : 7; + uint64_t m_b0 : 7; + uint64_t m_b1 : 7; + uint64_t m_a0 : 8; + uint64_t m_a1_0 : 6; + + } m_lo; + + uint64_t m_lo_bits; + }; + + union + { + struct + { + uint64_t m_a1_1 : 2; + + // bit 2 + uint64_t m_c00 : 1; + uint64_t m_c10 : 2; + uint64_t m_c20 : 2; + uint64_t m_c30 : 2; + + uint64_t m_c01 : 2; + uint64_t m_c11 : 2; + uint64_t m_c21 : 2; + uint64_t m_c31 : 2; + + uint64_t m_c02 : 2; + uint64_t m_c12 : 2; + uint64_t m_c22 : 2; + uint64_t m_c32 : 2; + + uint64_t m_c03 : 2; + uint64_t m_c13 : 2; + uint64_t m_c23 : 2; + uint64_t m_c33 : 2; + + // bit 33 + uint64_t m_a00 : 1; + uint64_t m_a10 : 2; + uint64_t m_a20 : 2; + uint64_t m_a30 : 2; + + uint64_t m_a01 : 2; + uint64_t m_a11 : 2; + uint64_t m_a21 : 2; + uint64_t m_a31 : 2; + + uint64_t m_a02 : 2; + uint64_t m_a12 : 2; + uint64_t m_a22 : 2; + uint64_t m_a32 : 2; + + uint64_t m_a03 : 2; + uint64_t m_a13 : 2; + uint64_t m_a23 : 2; + uint64_t m_a33 : 2; + + } m_hi; + + uint64_t m_hi_bits; + }; + }; + +} // namespace basist + + + diff --git a/external/basis_universal/webgl/README.md b/external/basis_universal/webgl/README.md index 2d655eb9bc..d283e23cd4 100644 --- a/external/basis_universal/webgl/README.md +++ b/external/basis_universal/webgl/README.md @@ -6,7 +6,7 @@ To build the encoder and transcoder WASM libraries using Emscripten, see the var ## Transcoder (texture_test) -Live demo: [webgl/texture_test/index.html](https://subquantumtech.com/uastchdr2/texture_test/) +Live demo: [webgl/texture_test/index.html](https://subquantumtech.com/xu/texture_test/) Renders a single texture, using the transcoder (compiled to WASM with emscripten) to generate one of the following compressed texture formats: @@ -23,7 +23,7 @@ On browsers that don't support any compressed texture format, there's a low-qual ## glTF 3D Model -Live demo: [`gltf/index.html`](https://subquantumtech.com/uastchdr2/gltf/) +Live demo: [`gltf/index.html`](https://subquantumtech.com/xu/gltf/) Renders a glTF 3D model with `.basis` texture files, transcoded into one of the following compressed texture formats: @@ -43,7 +43,7 @@ extension that is [currently in development](https://github.com/KhronosGroup/glT ## Compressor (ktx2_encode_test) -Live demo: [`ktx2_encode_test/index.html'](https://subquantumtech.com/uastchdr2/ktx2_encode_test/) +Live demo: [`ktx2_encode_test/index.html'](https://subquantumtech.com/xu/ktx2_encode_test/) This demo shows how to use the compressor from JavaScript. To use it, select a .PNG file then hit the "Encode!" button. The compressor will dynamically generate a .ktx2 file in memory which will then be immediately transcoded and displayed. Hit the "Download!" button to locally download the generated .ktx2 file. @@ -59,4 +59,5 @@ You can locally host the files under the "webgl" folder. One way is to use [Pyth cd webgl python3 -m http.server 8000 ``` -Note: For WASM multithreading to be available and enabled, the server must be properly configured. + +Note: For WASM multithreading to be available and enabled, the server [must be properly configured](https://unlimited3d.wordpress.com/2021/12/21/webassembly-and-multi-threading/). See the `webgl/start_webserver.sh` and `webgl/webserver_cross_origin.py` example scripts. diff --git a/external/basis_universal/webgl/encoder/CMakeLists.txt b/external/basis_universal/webgl/encoder/CMakeLists.txt index 48ada14bf9..e8e5adb141 100644 --- a/external/basis_universal/webgl/encoder/CMakeLists.txt +++ b/external/basis_universal/webgl/encoder/CMakeLists.txt @@ -1,94 +1,136 @@ cmake_minimum_required(VERSION 3.5) - project(basisu_encoder_js) -# The encoder always supports generating KTX2 files, but Zstandard support is optional. If it's disabled, KTX2 UASTC files will always be uncompressed. -# If you know you'll never be encoding UASTC+Zstd KTX2 files you can set KTX2_ZSTANDARD to 0 to reduce the size of the compiled encoder. -option(KTX2_ZSTANDARD "KTX2_ZSTANDARD" TRUE) - +# Toggle Zstd support for KTX2 (ON by default). +option(KTX2_ZSTANDARD "Enable KTX2 Zstandard support" TRUE) message("KTX2_ZSTANDARD=${KTX2_ZSTANDARD}") -if (EMSCRIPTEN) +# Only for Emscripten builds. +if(EMSCRIPTEN) set(CMAKE_CXX_STANDARD 17) + # ---------- Pick config once (single-config generators) ---------- + # Supports: Release (default), Debug, SAN + if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE "Release") + endif() + string(TOUPPER "${CMAKE_BUILD_TYPE}" BUILD_MODE) + + # Per-config compile/link flags + set(CONFIG_CFLAGS "") + set(CONFIG_DEFS "") + set(CONFIG_LINK "") + + if(BUILD_MODE STREQUAL "RELEASE") + set(CONFIG_CFLAGS "-O3") + set(CONFIG_DEFS "NDEBUG") + set(CONFIG_LINK "-O3 -s ASSERTIONS=0") + elseif(BUILD_MODE STREQUAL "DEBUG") + set(CONFIG_CFLAGS "-g -O0") + set(CONFIG_DEFS "DEBUG") + set(CONFIG_LINK "-g -s ASSERTIONS=2") + elseif(BUILD_MODE STREQUAL "SAN") + set(CONFIG_CFLAGS "-g -O1 -fsanitize=undefined -fsanitize=address") + set(CONFIG_DEFS "DEBUG") + set(CONFIG_LINK "-g -s ASSERTIONS=2 -fsanitize=undefined -fsanitize=address") + else() + message(WARNING "Unknown CMAKE_BUILD_TYPE='${CMAKE_BUILD_TYPE}', defaulting to Release-like flags.") + set(CONFIG_CFLAGS "-O3") + set(CONFIG_DEFS "NDEBUG") + set(CONFIG_LINK "-O3 -s ASSERTIONS=0") + endif() + + # ---------- Sources (shared) ---------- set(SRC_LIST ../transcoder/basis_wrappers.cpp ../../transcoder/basisu_transcoder.cpp - ../../encoder/basisu_backend.cpp - ../../encoder/basisu_basis_file.cpp - ../../encoder/basisu_comp.cpp - ../../encoder/basisu_enc.cpp - ../../encoder/basisu_etc.cpp - ../../encoder/basisu_frontend.cpp - ../../encoder/basisu_gpu_texture.cpp - ../../encoder/basisu_pvrtc1_4.cpp - ../../encoder/basisu_resampler.cpp - ../../encoder/basisu_resample_filters.cpp - ../../encoder/basisu_ssim.cpp - ../../encoder/basisu_uastc_enc.cpp - ../../encoder/basisu_bc7enc.cpp - ../../encoder/basisu_kernels_sse.cpp - ../../encoder/basisu_opencl.cpp - ../../encoder/pvpngreader.cpp - ../../encoder/jpgd.cpp - ../../encoder/3rdparty/android_astc_decomp.cpp - ../../encoder/basisu_uastc_hdr_4x4_enc.cpp - ../../encoder/basisu_astc_hdr_6x6_enc.cpp - ../../encoder/basisu_astc_hdr_common.cpp - ../../encoder/3rdparty/tinyexr.cpp + ../../encoder/basisu_backend.cpp + ../../encoder/basisu_basis_file.cpp + ../../encoder/basisu_comp.cpp + ../../encoder/basisu_enc.cpp + ../../encoder/basisu_etc.cpp + ../../encoder/basisu_frontend.cpp + ../../encoder/basisu_gpu_texture.cpp + ../../encoder/basisu_pvrtc1_4.cpp + ../../encoder/basisu_resampler.cpp + ../../encoder/basisu_resample_filters.cpp + ../../encoder/basisu_ssim.cpp + ../../encoder/basisu_uastc_enc.cpp + ../../encoder/basisu_bc7enc.cpp + ../../encoder/basisu_kernels_sse.cpp + ../../encoder/basisu_opencl.cpp + ../../encoder/pvpngreader.cpp + ../../encoder/jpgd.cpp + ../../encoder/3rdparty/android_astc_decomp.cpp + ../../encoder/basisu_uastc_hdr_4x4_enc.cpp + ../../encoder/basisu_astc_hdr_6x6_enc.cpp + ../../encoder/basisu_astc_hdr_common.cpp + ../../encoder/basisu_astc_ldr_common.cpp + ../../encoder/basisu_astc_ldr_encode.cpp + ../../encoder/3rdparty/tinyexr.cpp ) - - if (KTX2_ZSTANDARD) - set(SRC_LIST ${SRC_LIST} - ../../zstd/zstd.c - ) - set(ZSTD_DEFINITION BASISD_SUPPORT_KTX2_ZSTD=1) + if(KTX2_ZSTANDARD) + list(APPEND SRC_LIST ../../zstd/zstd.c) + set(ZSTD_DEFINITION BASISD_SUPPORT_KTX2_ZSTD=1) else() - set(ZSTD_DEFINITION BASISD_SUPPORT_KTX2_ZSTD=0) + set(ZSTD_DEFINITION BASISD_SUPPORT_KTX2_ZSTD=0) endif() - # No threading version - add_executable(basis_encoder.js ${SRC_LIST}) - - #target_compile_definitions(basis_encoder.js PRIVATE NDEBUG BASISD_SUPPORT_UASTC=1 BASISD_SUPPORT_BC7=1 BASISD_SUPPORT_ATC=0 BASISD_SUPPORT_ASTC_HIGHER_OPAQUE_QUALITY=0 BASISD_SUPPORT_PVRTC2=0 BASISD_SUPPORT_FXT1=0 BASISD_SUPPORT_ETC2_EAC_RG11=0 BASISU_SUPPORT_ENCODING=1 BASISU_SUPPORT_SSE=0 ${ZSTD_DEFINITION} ) - #target_compile_options(basis_encoder.js PRIVATE -fno-strict-aliasing -O3) - - #target_compile_definitions(basis_encoder.js PRIVATE DEBUG BASISD_SUPPORT_UASTC=1 BASISD_SUPPORT_BC7=1 BASISD_SUPPORT_ATC=0 BASISD_SUPPORT_ASTC_HIGHER_OPAQUE_QUALITY=0 BASISD_SUPPORT_PVRTC2=0 BASISD_SUPPORT_FXT1=0 BASISD_SUPPORT_ETC2_EAC_RG11=0 BASISU_SUPPORT_ENCODING=1 BASISU_SUPPORT_SSE=0 ${ZSTD_DEFINITION} ) - #target_compile_options(basis_encoder.js PRIVATE -fno-strict-aliasing -g -O1 -fsanitize=undefined -fsanitize=address) - - # debug options - #target_compile_definitions(basis_encoder.js PRIVATE DEBUG BASISD_SUPPORT_UASTC=1 BASISD_SUPPORT_BC7=1 BASISD_SUPPORT_ATC=0 BASISD_SUPPORT_ASTC_HIGHER_OPAQUE_QUALITY=0 BASISD_SUPPORT_PVRTC2=0 BASISD_SUPPORT_FXT1=0 BASISD_SUPPORT_ETC2_EAC_RG11=0 BASISU_SUPPORT_ENCODING=1 BASISU_SUPPORT_SSE=0 ${ZSTD_DEFINITION} ) - #target_compile_options(basis_encoder.js PRIVATE -fno-strict-aliasing -g -O0) - - # release options - target_compile_definitions(basis_encoder.js PRIVATE NDEBUG BASISD_SUPPORT_UASTC=1 BASISD_SUPPORT_BC7=1 BASISD_SUPPORT_ATC=0 BASISD_SUPPORT_ASTC_HIGHER_OPAQUE_QUALITY=0 BASISD_SUPPORT_PVRTC2=0 BASISD_SUPPORT_FXT1=0 BASISD_SUPPORT_ETC2_EAC_RG11=0 BASISU_SUPPORT_ENCODING=1 BASISU_SUPPORT_SSE=0 ${ZSTD_DEFINITION} ) - target_compile_options(basis_encoder.js PRIVATE -fno-strict-aliasing -O3) - - target_include_directories(basis_encoder.js PRIVATE ../../transcoder) - - set_target_properties(basis_encoder.js PROPERTIES - OUTPUT_NAME "basis_encoder" - SUFFIX ".js" - - #LINK_FLAGS "--bind -s INITIAL_MEMORY=536870912 -s ALLOW_MEMORY_GROWTH=1 -s STACK_SIZE=262144 -s MODULARIZE=1 -s EXPORT_NAME=BASIS -s EXPORTED_RUNTIME_METHODS=['HEAP8']") - #LINK_FLAGS "--bind -s INITIAL_MEMORY=536870912 -g -s STACK_SIZE=262144 -s ALLOW_MEMORY_GROWTH=1 -s MODULARIZE=1 -s EXPORT_NAME=BASIS -fsanitize=undefined -fsanitize=address -s EXPORTED_RUNTIME_METHODS=['HEAP8']") - #LINK_FLAGS "--bind -s INITIAL_MEMORY=536870912 -g -s STACK_SIZE=262144 -s ALLOW_MEMORY_GROWTH=1 -s MODULARIZE=1 -s EXPORT_NAME=BASIS -s ASSERTIONS=2 -s EXPORTED_RUNTIME_METHODS=['HEAP8']") - LINK_FLAGS "--bind -s ALLOW_MEMORY_GROWTH=1 -s INITIAL_MEMORY=536870912 -s STACK_SIZE=262144 -s MODULARIZE=1 -s EXPORT_NAME=BASIS -s ASSERTIONS=0 -s EXPORTED_RUNTIME_METHODS=['HEAP8']") - - add_executable(basis_encoder_threads.js ${SRC_LIST}) - - # Threaded version - target_compile_definitions(basis_encoder_threads.js PRIVATE NDEBUG BASISD_SUPPORT_UASTC=1 BASISD_SUPPORT_BC7=1 BASISD_SUPPORT_ATC=0 BASISD_SUPPORT_ASTC_HIGHER_OPAQUE_QUALITY=0 BASISD_SUPPORT_PVRTC2=0 BASISD_SUPPORT_FXT1=0 BASISD_SUPPORT_ETC2_EAC_RG11=0 BASISU_SUPPORT_ENCODING=1 BASISU_SUPPORT_SSE=0 ${ZSTD_DEFINITION} WASM_THREADS_ENABLED=1 ) - target_include_directories(basis_encoder_threads.js PRIVATE ../../transcoder) - - target_compile_options(basis_encoder_threads.js PRIVATE -fno-strict-aliasing -O3 -matomics -mbulk-memory) - - set_target_properties(basis_encoder_threads.js PROPERTIES - OUTPUT_NAME "basis_encoder_threads" - SUFFIX ".js" - #LINK_FLAGS "--bind -s INITIAL_MEMORY=536870912 -s ALLOW_MEMORY_GROWTH=1 -s STACK_SIZE=262144 -s MODULARIZE=1 -s EXPORT_NAME=BASIS -s EXPORTED_RUNTIME_METHODS=['HEAP8']") - #LINK_FLAGS "--bind -s INITIAL_MEMORY=536870912 -g -s STACK_SIZE=262144 -s ALLOW_MEMORY_GROWTH=1 -s MODULARIZE=1 -s EXPORT_NAME=BASIS -fsanitize=undefined -fsanitize=address -s EXPORTED_RUNTIME_METHODS=['HEAP8']") - #LINK_FLAGS "--bind -s INITIAL_MEMORY=536870912 -g -s STACK_SIZE=262144 -s ALLOW_MEMORY_GROWTH=1 -s MODULARIZE=1 -s EXPORT_NAME=BASIS -s ASSERTIONS=2 -s USE_PTHREADS=1 -s PTHREAD_POOL_SIZE=18 -s ENVIRONMENT=web,worker -s EXPORTED_RUNTIME_METHODS=['HEAP8']") - LINK_FLAGS "--bind -s ALLOW_MEMORY_GROWTH=1 -s INITIAL_MEMORY=536870912 -O3 -s STACK_SIZE=262144 -s MODULARIZE=1 -s EXPORT_NAME=BASIS -s ASSERTIONS=0 -s USE_PTHREADS=1 -s PTHREAD_POOL_SIZE=18 -s ENVIRONMENT=web,worker -s EXPORTED_RUNTIME_METHODS=['HEAP8']") + # Common preprocessor defines (same as your original) + set(COMMON_DEFS + BASISD_SUPPORT_UASTC=1 + BASISD_SUPPORT_BC7=1 + BASISD_SUPPORT_ATC=0 + BASISD_SUPPORT_ASTC_HIGHER_OPAQUE_QUALITY=0 + BASISD_SUPPORT_PVRTC2=0 + BASISD_SUPPORT_FXT1=0 + BASISD_SUPPORT_ETC2_EAC_RG11=0 + BASISU_SUPPORT_ENCODING=1 + BASISU_SUPPORT_SSE=0 + BASISD_SUPPORT_XUASTC=1 + ${ZSTD_DEFINITION} + ) + # Base link flags + set(LINK_BASE "--bind -s ALLOW_MEMORY_GROWTH=1 -s INITIAL_MEMORY=536870912 -s STACK_SIZE=2097152 -s MODULARIZE=1 -s EXPORT_NAME=BASIS -s EXPORTED_RUNTIME_METHODS=['HEAP8']") + set(LINK_THREADS "-s USE_PTHREADS=1 -s PTHREAD_POOL_SIZE=18 -s ENVIRONMENT=web,worker") + set(LINK_WASM64 "-s MEMORY64=1 -sWASM_BIGINT=1 --profiling-funcs") + + # Helper to avoid repetition + function(add_encoder target out_name use_threads use_wasm64) + add_executable(${target} ${SRC_LIST}) + set_target_properties(${target} PROPERTIES OUTPUT_NAME "${out_name}" SUFFIX ".js") + target_include_directories(${target} PRIVATE ../../transcoder) + + # Compile defs and options + target_compile_definitions(${target} PRIVATE ${COMMON_DEFS} ${CONFIG_DEFS}) + target_compile_options(${target} PRIVATE -fno-strict-aliasing ${CONFIG_CFLAGS}) + + if(${use_threads}) + target_compile_options(${target} PRIVATE -matomics -mbulk-memory) + target_compile_definitions(${target} PRIVATE WASM_THREADS_ENABLED=1) + endif() + + if(${use_wasm64}) + target_compile_options(${target} PRIVATE -s MEMORY64=1) + endif() + + # Link flags (no generator expressions) + set(_lf "${LINK_BASE} ${CONFIG_LINK}") + + if(${use_threads}) + set(_lf "${_lf} ${LINK_THREADS}") + endif() + + if(${use_wasm64}) + set(_lf "${_lf} ${LINK_WASM64} -s INITIAL_MEMORY=1073741824 -s MAXIMUM_MEMORY=12884901888") + endif() + + set_target_properties(${target} PROPERTIES LINK_FLAGS "${_lf}") + endfunction() + + # The three outputs (names unchanged) + add_encoder(basis_encoder.js "basis_encoder" OFF OFF) # wasm32 + add_encoder(basis_encoder_threads.js "basis_encoder_threads" ON OFF) # wasm32 + threads + add_encoder(basis_encoder_threads_wasm64.js "basis_encoder_threads_wasm64" ON ON ) # wasm64 + threads endif() diff --git a/external/basis_universal/webgl/encoder/build/basis_encoder.js b/external/basis_universal/webgl/encoder/build/basis_encoder.js index 845fd0660e..81aeb1d6aa 100644 --- a/external/basis_universal/webgl/encoder/build/basis_encoder.js +++ b/external/basis_universal/webgl/encoder/build/basis_encoder.js @@ -1,6048 +1,2 @@ -// This code implements the `-sMODULARIZE` settings by taking the generated -// JS program code (INNER_JS_CODE) and wrapping it in a factory function. - -// Single threaded MINIMAL_RUNTIME programs do not need access to -// document.currentScript, so a simple export declaration is enough. -var BASIS = (() => { - // When MODULARIZE this JS may be executed later, - // after document.currentScript is gone, so we save it. - // In EXPORT_ES6 mode we can just use 'import.meta.url'. - var _scriptName = globalThis.document?.currentScript?.src; - return async function(moduleArg = {}) { - var moduleRtn; - -// include: shell.js -// include: minimum_runtime_check.js -// end include: minimum_runtime_check.js -// The Module object: Our interface to the outside world. We import -// and export values on it. There are various ways Module can be used: -// 1. Not defined. We create it here -// 2. A function parameter, function(moduleArg) => Promise -// 3. pre-run appended it, var Module = {}; ..generated code.. -// 4. External script tag defines var Module. -// We need to check if Module already exists (e.g. case 3 above). -// Substitution will be replaced with actual code on later stage of the build, -// this way Closure Compiler will not mangle it (e.g. case 4. above). -// Note that if you want to run closure, and also to use Module -// after the generated code, you will need to define var Module = {}; -// before the code. Then that object will be used in the code, and you -// can continue to use Module afterwards as well. -var Module = moduleArg; - -// Determine the runtime environment we are in. You can customize this by -// setting the ENVIRONMENT setting at compile time (see settings.js). - -// Attempt to auto-detect the environment -var ENVIRONMENT_IS_WEB = !!globalThis.window; -var ENVIRONMENT_IS_WORKER = !!globalThis.WorkerGlobalScope; -// N.b. Electron.js environment is simultaneously a NODE-environment, but -// also a web environment. -var ENVIRONMENT_IS_NODE = globalThis.process?.versions?.node && globalThis.process?.type != 'renderer'; -var ENVIRONMENT_IS_SHELL = !ENVIRONMENT_IS_WEB && !ENVIRONMENT_IS_NODE && !ENVIRONMENT_IS_WORKER; - -// --pre-jses are emitted after the Module integration code, so that they can -// refer to Module (if they choose; they can also define Module) - - -var arguments_ = []; -var thisProgram = './this.program'; -var quit_ = (status, toThrow) => { - throw toThrow; -}; - -if (typeof __filename != 'undefined') { // Node - _scriptName = __filename; -} else -if (ENVIRONMENT_IS_WORKER) { - _scriptName = self.location.href; -} - -// `/` should be present at the end if `scriptDirectory` is not empty -var scriptDirectory = ''; -function locateFile(path) { - if (Module['locateFile']) { - return Module['locateFile'](path, scriptDirectory); - } - return scriptDirectory + path; -} - -// Hooks that are implemented differently in different runtime environments. -var readAsync, readBinary; - -if (ENVIRONMENT_IS_NODE) { - - // These modules will usually be used on Node.js. Load them eagerly to avoid - // the complexity of lazy-loading. - var fs = require('fs'); - - scriptDirectory = __dirname + '/'; - -// include: node_shell_read.js -readBinary = (filename) => { - // We need to re-wrap `file://` strings to URLs. - filename = isFileURI(filename) ? new URL(filename) : filename; - var ret = fs.readFileSync(filename); - return ret; -}; - -readAsync = async (filename, binary = true) => { - // See the comment in the `readBinary` function. - filename = isFileURI(filename) ? new URL(filename) : filename; - var ret = fs.readFileSync(filename, binary ? undefined : 'utf8'); - return ret; -}; -// end include: node_shell_read.js - if (process.argv.length > 1) { - thisProgram = process.argv[1].replace(/\\/g, '/'); - } - - arguments_ = process.argv.slice(2); - - quit_ = (status, toThrow) => { - process.exitCode = status; - throw toThrow; - }; - -} else - -// Note that this includes Node.js workers when relevant (pthreads is enabled). -// Node.js workers are detected as a combination of ENVIRONMENT_IS_WORKER and -// ENVIRONMENT_IS_NODE. -if (ENVIRONMENT_IS_WEB || ENVIRONMENT_IS_WORKER) { - try { - scriptDirectory = new URL('.', _scriptName).href; // includes trailing slash - } catch { - // Must be a `blob:` or `data:` URL (e.g. `blob:http://site.com/etc/etc`), we cannot - // infer anything from them. - } - - { -// include: web_or_worker_shell_read.js -if (ENVIRONMENT_IS_WORKER) { - readBinary = (url) => { - var xhr = new XMLHttpRequest(); - xhr.open('GET', url, false); - xhr.responseType = 'arraybuffer'; - xhr.send(null); - return new Uint8Array(/** @type{!ArrayBuffer} */(xhr.response)); - }; - } - - readAsync = async (url) => { - // Fetch has some additional restrictions over XHR, like it can't be used on a file:// url. - // See https://github.com/github/fetch/pull/92#issuecomment-140665932 - // Cordova or Electron apps are typically loaded from a file:// url. - // So use XHR on webview if URL is a file URL. - if (isFileURI(url)) { - return new Promise((resolve, reject) => { - var xhr = new XMLHttpRequest(); - xhr.open('GET', url, true); - xhr.responseType = 'arraybuffer'; - xhr.onload = () => { - if (xhr.status == 200 || (xhr.status == 0 && xhr.response)) { // file URLs can return 0 - resolve(xhr.response); - return; - } - reject(xhr.status); - }; - xhr.onerror = reject; - xhr.send(null); - }); - } - var response = await fetch(url, { credentials: 'same-origin' }); - if (response.ok) { - return response.arrayBuffer(); - } - throw new Error(response.status + ' : ' + response.url); - }; -// end include: web_or_worker_shell_read.js - } -} else -{ -} - -var out = console.log.bind(console); -var err = console.error.bind(console); - -// end include: shell.js - -// include: preamble.js -// === Preamble library stuff === - -// Documentation for the public APIs defined in this file must be updated in: -// site/source/docs/api_reference/preamble.js.rst -// A prebuilt local version of the documentation is available at: -// site/build/text/docs/api_reference/preamble.js.txt -// You can also build docs locally as HTML or other formats in site/ -// An online HTML version (which may be of a different version of Emscripten) -// is up at http://kripken.github.io/emscripten-site/docs/api_reference/preamble.js.html - -var wasmBinary; - -// Wasm globals - -//======================================== -// Runtime essentials -//======================================== - -// whether we are quitting the application. no code should run after this. -// set in exit() and abort() -var ABORT = false; - -// set by exit() and abort(). Passed to 'onExit' handler. -// NOTE: This is also used as the process return code code in shell environments -// but only when noExitRuntime is false. -var EXITSTATUS; - -// In STRICT mode, we only define assert() when ASSERTIONS is set. i.e. we -// don't define it at all in release modes. This matches the behaviour of -// MINIMAL_RUNTIME. -// TODO(sbc): Make this the default even without STRICT enabled. -/** @type {function(*, string=)} */ -function assert(condition, text) { - if (!condition) { - // This build was created without ASSERTIONS defined. `assert()` should not - // ever be called in this configuration but in case there are callers in - // the wild leave this simple abort() implementation here for now. - abort(text); - } -} - -/** - * Indicates whether filename is delivered via file protocol (as opposed to http/https) - * @noinline - */ -var isFileURI = (filename) => filename.startsWith('file://'); - -// include: runtime_common.js -// include: runtime_stack_check.js -// end include: runtime_stack_check.js -// include: runtime_exceptions.js -// end include: runtime_exceptions.js -// include: runtime_debug.js -// end include: runtime_debug.js -var readyPromiseResolve, readyPromiseReject; - -// Memory management -var -/** @type {!Int8Array} */ - HEAP8, -/** @type {!Uint8Array} */ - HEAPU8, -/** @type {!Int16Array} */ - HEAP16, -/** @type {!Uint16Array} */ - HEAPU16, -/** @type {!Int32Array} */ - HEAP32, -/** @type {!Uint32Array} */ - HEAPU32, -/** @type {!Float32Array} */ - HEAPF32, -/** @type {!Float64Array} */ - HEAPF64; - -// BigInt64Array type is not correctly defined in closure -var -/** not-@type {!BigInt64Array} */ - HEAP64, -/* BigUint64Array type is not correctly defined in closure -/** not-@type {!BigUint64Array} */ - HEAPU64; - -var runtimeInitialized = false; - - - -function updateMemoryViews() { - var b = wasmMemory.buffer; - Module['HEAP8'] = HEAP8 = new Int8Array(b); - HEAP16 = new Int16Array(b); - HEAPU8 = new Uint8Array(b); - HEAPU16 = new Uint16Array(b); - HEAP32 = new Int32Array(b); - HEAPU32 = new Uint32Array(b); - HEAPF32 = new Float32Array(b); - HEAPF64 = new Float64Array(b); - HEAP64 = new BigInt64Array(b); - HEAPU64 = new BigUint64Array(b); -} - -// include: memoryprofiler.js -// end include: memoryprofiler.js -// end include: runtime_common.js -function preRun() { - if (Module['preRun']) { - if (typeof Module['preRun'] == 'function') Module['preRun'] = [Module['preRun']]; - while (Module['preRun'].length) { - addOnPreRun(Module['preRun'].shift()); - } - } - // Begin ATPRERUNS hooks - callRuntimeCallbacks(onPreRuns); - // End ATPRERUNS hooks -} - -function initRuntime() { - runtimeInitialized = true; - - // Begin ATINITS hooks - if (!Module['noFSInit'] && !FS.initialized) FS.init(); -TTY.init(); - // End ATINITS hooks - - wasmExports['__wasm_call_ctors'](); - - // Begin ATPOSTCTORS hooks - FS.ignorePermissions = false; - // End ATPOSTCTORS hooks -} - -function postRun() { - // PThreads reuse the runtime from the main thread. - - if (Module['postRun']) { - if (typeof Module['postRun'] == 'function') Module['postRun'] = [Module['postRun']]; - while (Module['postRun'].length) { - addOnPostRun(Module['postRun'].shift()); - } - } - - // Begin ATPOSTRUNS hooks - callRuntimeCallbacks(onPostRuns); - // End ATPOSTRUNS hooks -} - -/** @param {string|number=} what */ -function abort(what) { - Module['onAbort']?.(what); - - what = 'Aborted(' + what + ')'; - // TODO(sbc): Should we remove printing and leave it up to whoever - // catches the exception? - err(what); - - ABORT = true; - - what += '. Build with -sASSERTIONS for more info.'; - - // Use a wasm runtime error, because a JS error might be seen as a foreign - // exception, which means we'd run destructors on it. We need the error to - // simply make the program stop. - // FIXME This approach does not work in Wasm EH because it currently does not assume - // all RuntimeErrors are from traps; it decides whether a RuntimeError is from - // a trap or not based on a hidden field within the object. So at the moment - // we don't have a way of throwing a wasm trap from JS. TODO Make a JS API that - // allows this in the wasm spec. - - // Suppress closure compiler warning here. Closure compiler's builtin extern - // definition for WebAssembly.RuntimeError claims it takes no arguments even - // though it can. - // TODO(https://github.com/google/closure-compiler/pull/3913): Remove if/when upstream closure gets fixed. - /** @suppress {checkTypes} */ - var e = new WebAssembly.RuntimeError(what); - - readyPromiseReject?.(e); - // Throw the error whether or not MODULARIZE is set because abort is used - // in code paths apart from instantiation where an exception is expected - // to be thrown when abort is called. - throw e; -} - -var wasmBinaryFile; - -function findWasmBinary() { - return locateFile('basis_encoder.wasm'); -} - -function getBinarySync(file) { - if (file == wasmBinaryFile && wasmBinary) { - return new Uint8Array(wasmBinary); - } - if (readBinary) { - return readBinary(file); - } - // Throwing a plain string here, even though it not normally adviables since - // this gets turning into an `abort` in instantiateArrayBuffer. - throw 'both async and sync fetching of the wasm failed'; -} - -async function getWasmBinary(binaryFile) { - // If we don't have the binary yet, load it asynchronously using readAsync. - if (!wasmBinary) { - // Fetch the binary using readAsync - try { - var response = await readAsync(binaryFile); - return new Uint8Array(response); - } catch { - // Fall back to getBinarySync below; - } - } - - // Otherwise, getBinarySync should be able to get it synchronously - return getBinarySync(binaryFile); -} - -async function instantiateArrayBuffer(binaryFile, imports) { - try { - var binary = await getWasmBinary(binaryFile); - var instance = await WebAssembly.instantiate(binary, imports); - return instance; - } catch (reason) { - err(`failed to asynchronously prepare wasm: ${reason}`); - - abort(reason); - } -} - -async function instantiateAsync(binary, binaryFile, imports) { - if (!binary - // Don't use streaming for file:// delivered objects in a webview, fetch them synchronously. - && !isFileURI(binaryFile) - // Avoid instantiateStreaming() on Node.js environment for now, as while - // Node.js v18.1.0 implements it, it does not have a full fetch() - // implementation yet. - // - // Reference: - // https://github.com/emscripten-core/emscripten/pull/16917 - && !ENVIRONMENT_IS_NODE - ) { - try { - var response = fetch(binaryFile, { credentials: 'same-origin' }); - var instantiationResult = await WebAssembly.instantiateStreaming(response, imports); - return instantiationResult; - } catch (reason) { - // We expect the most common failure cause to be a bad MIME type for the binary, - // in which case falling back to ArrayBuffer instantiation should work. - err(`wasm streaming compile failed: ${reason}`); - err('falling back to ArrayBuffer instantiation'); - // fall back of instantiateArrayBuffer below - }; - } - return instantiateArrayBuffer(binaryFile, imports); -} - -function getWasmImports() { - // prepare imports - var imports = { - 'env': wasmImports, - 'wasi_snapshot_preview1': wasmImports, - }; - return imports; -} - -// Create the wasm instance. -// Receives the wasm imports, returns the exports. -async function createWasm() { - // Load the wasm module and create an instance of using native support in the JS engine. - // handle a generated wasm instance, receiving its exports and - // performing other necessary setup - /** @param {WebAssembly.Module=} module*/ - function receiveInstance(instance, module) { - wasmExports = instance.exports; - - assignWasmExports(wasmExports); - - updateMemoryViews(); - - return wasmExports; - } - - // Prefer streaming instantiation if available. - function receiveInstantiationResult(result) { - // 'result' is a ResultObject object which has both the module and instance. - // receiveInstance() will swap in the exports (to Module.asm) so they can be called - // TODO: Due to Closure regression https://github.com/google/closure-compiler/issues/3193, the above line no longer optimizes out down to the following line. - // When the regression is fixed, can restore the above PTHREADS-enabled path. - return receiveInstance(result['instance']); - } - - var info = getWasmImports(); - - // User shell pages can write their own Module.instantiateWasm = function(imports, successCallback) callback - // to manually instantiate the Wasm module themselves. This allows pages to - // run the instantiation parallel to any other async startup actions they are - // performing. - // Also pthreads and wasm workers initialize the wasm instance through this - // path. - if (Module['instantiateWasm']) { - return new Promise((resolve, reject) => { - Module['instantiateWasm'](info, (inst, mod) => { - resolve(receiveInstance(inst, mod)); - }); - }); - } - - wasmBinaryFile ??= findWasmBinary(); - var result = await instantiateAsync(wasmBinary, wasmBinaryFile, info); - var exports = receiveInstantiationResult(result); - return exports; -} - -// end include: preamble.js - -// Begin JS library code - - - class ExitStatus { - name = 'ExitStatus'; - constructor(status) { - this.message = `Program terminated with exit(${status})`; - this.status = status; - } - } - - var callRuntimeCallbacks = (callbacks) => { - while (callbacks.length > 0) { - // Pass the module as the first argument. - callbacks.shift()(Module); - } - }; - var onPostRuns = []; - var addOnPostRun = (cb) => onPostRuns.push(cb); - - var onPreRuns = []; - var addOnPreRun = (cb) => onPreRuns.push(cb); - - - - /** - * @param {number} ptr - * @param {string} type - */ - function getValue(ptr, type = 'i8') { - if (type.endsWith('*')) type = '*'; - switch (type) { - case 'i1': return HEAP8[ptr]; - case 'i8': return HEAP8[ptr]; - case 'i16': return HEAP16[((ptr)>>1)]; - case 'i32': return HEAP32[((ptr)>>2)]; - case 'i64': return HEAP64[((ptr)>>3)]; - case 'float': return HEAPF32[((ptr)>>2)]; - case 'double': return HEAPF64[((ptr)>>3)]; - case '*': return HEAPU32[((ptr)>>2)]; - default: abort(`invalid type for getValue: ${type}`); - } - } - - var noExitRuntime = true; - - - /** - * @param {number} ptr - * @param {number} value - * @param {string} type - */ - function setValue(ptr, value, type = 'i8') { - if (type.endsWith('*')) type = '*'; - switch (type) { - case 'i1': HEAP8[ptr] = value; break; - case 'i8': HEAP8[ptr] = value; break; - case 'i16': HEAP16[((ptr)>>1)] = value; break; - case 'i32': HEAP32[((ptr)>>2)] = value; break; - case 'i64': HEAP64[((ptr)>>3)] = BigInt(value); break; - case 'float': HEAPF32[((ptr)>>2)] = value; break; - case 'double': HEAPF64[((ptr)>>3)] = value; break; - case '*': HEAPU32[((ptr)>>2)] = value; break; - default: abort(`invalid type for setValue: ${type}`); - } - } - - var stackRestore = (val) => __emscripten_stack_restore(val); - - var stackSave = () => _emscripten_stack_get_current(); - - - - class ExceptionInfo { - // excPtr - Thrown object pointer to wrap. Metadata pointer is calculated from it. - constructor(excPtr) { - this.excPtr = excPtr; - this.ptr = excPtr - 24; - } - - set_type(type) { - HEAPU32[(((this.ptr)+(4))>>2)] = type; - } - - get_type() { - return HEAPU32[(((this.ptr)+(4))>>2)]; - } - - set_destructor(destructor) { - HEAPU32[(((this.ptr)+(8))>>2)] = destructor; - } - - get_destructor() { - return HEAPU32[(((this.ptr)+(8))>>2)]; - } - - set_caught(caught) { - caught = caught ? 1 : 0; - HEAP8[(this.ptr)+(12)] = caught; - } - - get_caught() { - return HEAP8[(this.ptr)+(12)] != 0; - } - - set_rethrown(rethrown) { - rethrown = rethrown ? 1 : 0; - HEAP8[(this.ptr)+(13)] = rethrown; - } - - get_rethrown() { - return HEAP8[(this.ptr)+(13)] != 0; - } - - // Initialize native structure fields. Should be called once after allocated. - init(type, destructor) { - this.set_adjusted_ptr(0); - this.set_type(type); - this.set_destructor(destructor); - } - - set_adjusted_ptr(adjustedPtr) { - HEAPU32[(((this.ptr)+(16))>>2)] = adjustedPtr; - } - - get_adjusted_ptr() { - return HEAPU32[(((this.ptr)+(16))>>2)]; - } - } - - var exceptionLast = 0; - - var uncaughtExceptionCount = 0; - var ___cxa_throw = (ptr, type, destructor) => { - var info = new ExceptionInfo(ptr); - // Initialize ExceptionInfo content after it was allocated in __cxa_allocate_exception. - info.init(type, destructor); - exceptionLast = ptr; - uncaughtExceptionCount++; - throw exceptionLast; - }; - - var syscallGetVarargI = () => { - // the `+` prepended here is necessary to convince the JSCompiler that varargs is indeed a number. - var ret = HEAP32[((+SYSCALLS.varargs)>>2)]; - SYSCALLS.varargs += 4; - return ret; - }; - var syscallGetVarargP = syscallGetVarargI; - - - var PATH = { - isAbs:(path) => path.charAt(0) === '/', - splitPath:(filename) => { - var splitPathRe = /^(\/?|)([\s\S]*?)((?:\.{1,2}|[^\/]+?|)(\.[^.\/]*|))(?:[\/]*)$/; - return splitPathRe.exec(filename).slice(1); - }, - normalizeArray:(parts, allowAboveRoot) => { - // if the path tries to go above the root, `up` ends up > 0 - var up = 0; - for (var i = parts.length - 1; i >= 0; i--) { - var last = parts[i]; - if (last === '.') { - parts.splice(i, 1); - } else if (last === '..') { - parts.splice(i, 1); - up++; - } else if (up) { - parts.splice(i, 1); - up--; - } - } - // if the path is allowed to go above the root, restore leading ..s - if (allowAboveRoot) { - for (; up; up--) { - parts.unshift('..'); - } - } - return parts; - }, - normalize:(path) => { - var isAbsolute = PATH.isAbs(path), - trailingSlash = path.slice(-1) === '/'; - // Normalize the path - path = PATH.normalizeArray(path.split('/').filter((p) => !!p), !isAbsolute).join('/'); - if (!path && !isAbsolute) { - path = '.'; - } - if (path && trailingSlash) { - path += '/'; - } - return (isAbsolute ? '/' : '') + path; - }, - dirname:(path) => { - var result = PATH.splitPath(path), - root = result[0], - dir = result[1]; - if (!root && !dir) { - // No dirname whatsoever - return '.'; - } - if (dir) { - // It has a dirname, strip trailing slash - dir = dir.slice(0, -1); - } - return root + dir; - }, - basename:(path) => path && path.match(/([^\/]+|\/)\/*$/)[1], - join:(...paths) => PATH.normalize(paths.join('/')), - join2:(l, r) => PATH.normalize(l + '/' + r), - }; - - var initRandomFill = () => { - // This block is not needed on v19+ since crypto.getRandomValues is builtin - if (ENVIRONMENT_IS_NODE) { - var nodeCrypto = require('crypto'); - return (view) => nodeCrypto.randomFillSync(view); - } - - return (view) => crypto.getRandomValues(view); - }; - var randomFill = (view) => { - // Lazily init on the first invocation. - (randomFill = initRandomFill())(view); - }; - - - - var PATH_FS = { - resolve:(...args) => { - var resolvedPath = '', - resolvedAbsolute = false; - for (var i = args.length - 1; i >= -1 && !resolvedAbsolute; i--) { - var path = (i >= 0) ? args[i] : FS.cwd(); - // Skip empty and invalid entries - if (typeof path != 'string') { - throw new TypeError('Arguments to path.resolve must be strings'); - } else if (!path) { - return ''; // an invalid portion invalidates the whole thing - } - resolvedPath = path + '/' + resolvedPath; - resolvedAbsolute = PATH.isAbs(path); - } - // At this point the path should be resolved to a full absolute path, but - // handle relative paths to be safe (might happen when process.cwd() fails) - resolvedPath = PATH.normalizeArray(resolvedPath.split('/').filter((p) => !!p), !resolvedAbsolute).join('/'); - return ((resolvedAbsolute ? '/' : '') + resolvedPath) || '.'; - }, - relative:(from, to) => { - from = PATH_FS.resolve(from).slice(1); - to = PATH_FS.resolve(to).slice(1); - function trim(arr) { - var start = 0; - for (; start < arr.length; start++) { - if (arr[start] !== '') break; - } - var end = arr.length - 1; - for (; end >= 0; end--) { - if (arr[end] !== '') break; - } - if (start > end) return []; - return arr.slice(start, end - start + 1); - } - var fromParts = trim(from.split('/')); - var toParts = trim(to.split('/')); - var length = Math.min(fromParts.length, toParts.length); - var samePartsLength = length; - for (var i = 0; i < length; i++) { - if (fromParts[i] !== toParts[i]) { - samePartsLength = i; - break; - } - } - var outputParts = []; - for (var i = samePartsLength; i < fromParts.length; i++) { - outputParts.push('..'); - } - outputParts = outputParts.concat(toParts.slice(samePartsLength)); - return outputParts.join('/'); - }, - }; - - - var UTF8Decoder = globalThis.TextDecoder && new TextDecoder(); - - var findStringEnd = (heapOrArray, idx, maxBytesToRead, ignoreNul) => { - var maxIdx = idx + maxBytesToRead; - if (ignoreNul) return maxIdx; - // TextDecoder needs to know the byte length in advance, it doesn't stop on - // null terminator by itself. - // As a tiny code save trick, compare idx against maxIdx using a negation, - // so that maxBytesToRead=undefined/NaN means Infinity. - while (heapOrArray[idx] && !(idx >= maxIdx)) ++idx; - return idx; - }; - - /** - * Given a pointer 'idx' to a null-terminated UTF8-encoded string in the given - * array that contains uint8 values, returns a copy of that string as a - * Javascript String object. - * heapOrArray is either a regular array, or a JavaScript typed array view. - * @param {number=} idx - * @param {number=} maxBytesToRead - * @param {boolean=} ignoreNul - If true, the function will not stop on a NUL character. - * @return {string} - */ - var UTF8ArrayToString = (heapOrArray, idx = 0, maxBytesToRead, ignoreNul) => { - - var endPtr = findStringEnd(heapOrArray, idx, maxBytesToRead, ignoreNul); - - // When using conditional TextDecoder, skip it for short strings as the overhead of the native call is not worth it. - if (endPtr - idx > 16 && heapOrArray.buffer && UTF8Decoder) { - return UTF8Decoder.decode(heapOrArray.subarray(idx, endPtr)); - } - var str = ''; - while (idx < endPtr) { - // For UTF8 byte structure, see: - // http://en.wikipedia.org/wiki/UTF-8#Description - // https://www.ietf.org/rfc/rfc2279.txt - // https://tools.ietf.org/html/rfc3629 - var u0 = heapOrArray[idx++]; - if (!(u0 & 0x80)) { str += String.fromCharCode(u0); continue; } - var u1 = heapOrArray[idx++] & 63; - if ((u0 & 0xE0) == 0xC0) { str += String.fromCharCode(((u0 & 31) << 6) | u1); continue; } - var u2 = heapOrArray[idx++] & 63; - if ((u0 & 0xF0) == 0xE0) { - u0 = ((u0 & 15) << 12) | (u1 << 6) | u2; - } else { - u0 = ((u0 & 7) << 18) | (u1 << 12) | (u2 << 6) | (heapOrArray[idx++] & 63); - } - - if (u0 < 0x10000) { - str += String.fromCharCode(u0); - } else { - var ch = u0 - 0x10000; - str += String.fromCharCode(0xD800 | (ch >> 10), 0xDC00 | (ch & 0x3FF)); - } - } - return str; - }; - - var FS_stdin_getChar_buffer = []; - - var lengthBytesUTF8 = (str) => { - var len = 0; - for (var i = 0; i < str.length; ++i) { - // Gotcha: charCodeAt returns a 16-bit word that is a UTF-16 encoded code - // unit, not a Unicode code point of the character! So decode - // UTF16->UTF32->UTF8. - // See http://unicode.org/faq/utf_bom.html#utf16-3 - var c = str.charCodeAt(i); // possibly a lead surrogate - if (c <= 0x7F) { - len++; - } else if (c <= 0x7FF) { - len += 2; - } else if (c >= 0xD800 && c <= 0xDFFF) { - len += 4; ++i; - } else { - len += 3; - } - } - return len; - }; - - var stringToUTF8Array = (str, heap, outIdx, maxBytesToWrite) => { - // Parameter maxBytesToWrite is not optional. Negative values, 0, null, - // undefined and false each don't write out any bytes. - if (!(maxBytesToWrite > 0)) - return 0; - - var startIdx = outIdx; - var endIdx = outIdx + maxBytesToWrite - 1; // -1 for string null terminator. - for (var i = 0; i < str.length; ++i) { - // For UTF8 byte structure, see http://en.wikipedia.org/wiki/UTF-8#Description - // and https://www.ietf.org/rfc/rfc2279.txt - // and https://tools.ietf.org/html/rfc3629 - var u = str.codePointAt(i); - if (u <= 0x7F) { - if (outIdx >= endIdx) break; - heap[outIdx++] = u; - } else if (u <= 0x7FF) { - if (outIdx + 1 >= endIdx) break; - heap[outIdx++] = 0xC0 | (u >> 6); - heap[outIdx++] = 0x80 | (u & 63); - } else if (u <= 0xFFFF) { - if (outIdx + 2 >= endIdx) break; - heap[outIdx++] = 0xE0 | (u >> 12); - heap[outIdx++] = 0x80 | ((u >> 6) & 63); - heap[outIdx++] = 0x80 | (u & 63); - } else { - if (outIdx + 3 >= endIdx) break; - heap[outIdx++] = 0xF0 | (u >> 18); - heap[outIdx++] = 0x80 | ((u >> 12) & 63); - heap[outIdx++] = 0x80 | ((u >> 6) & 63); - heap[outIdx++] = 0x80 | (u & 63); - // Gotcha: if codePoint is over 0xFFFF, it is represented as a surrogate pair in UTF-16. - // We need to manually skip over the second code unit for correct iteration. - i++; - } - } - // Null-terminate the pointer to the buffer. - heap[outIdx] = 0; - return outIdx - startIdx; - }; - /** @type {function(string, boolean=, number=)} */ - var intArrayFromString = (stringy, dontAddNull, length) => { - var len = length > 0 ? length : lengthBytesUTF8(stringy)+1; - var u8array = new Array(len); - var numBytesWritten = stringToUTF8Array(stringy, u8array, 0, u8array.length); - if (dontAddNull) u8array.length = numBytesWritten; - return u8array; - }; - var FS_stdin_getChar = () => { - if (!FS_stdin_getChar_buffer.length) { - var result = null; - if (ENVIRONMENT_IS_NODE) { - // we will read data by chunks of BUFSIZE - var BUFSIZE = 256; - var buf = Buffer.alloc(BUFSIZE); - var bytesRead = 0; - - // For some reason we must suppress a closure warning here, even though - // fd definitely exists on process.stdin, and is even the proper way to - // get the fd of stdin, - // https://github.com/nodejs/help/issues/2136#issuecomment-523649904 - // This started to happen after moving this logic out of library_tty.js, - // so it is related to the surrounding code in some unclear manner. - /** @suppress {missingProperties} */ - var fd = process.stdin.fd; - - try { - bytesRead = fs.readSync(fd, buf, 0, BUFSIZE); - } catch(e) { - // Cross-platform differences: on Windows, reading EOF throws an - // exception, but on other OSes, reading EOF returns 0. Uniformize - // behavior by treating the EOF exception to return 0. - if (e.toString().includes('EOF')) bytesRead = 0; - else throw e; - } - - if (bytesRead > 0) { - result = buf.slice(0, bytesRead).toString('utf-8'); - } - } else - if (globalThis.window?.prompt) { - // Browser. - result = window.prompt('Input: '); // returns null on cancel - if (result !== null) { - result += '\n'; - } - } else - {} - if (!result) { - return null; - } - FS_stdin_getChar_buffer = intArrayFromString(result, true); - } - return FS_stdin_getChar_buffer.shift(); - }; - var TTY = { - ttys:[], - init() { - // https://github.com/emscripten-core/emscripten/pull/1555 - // if (ENVIRONMENT_IS_NODE) { - // // currently, FS.init does not distinguish if process.stdin is a file or TTY - // // device, it always assumes it's a TTY device. because of this, we're forcing - // // process.stdin to UTF8 encoding to at least make stdin reading compatible - // // with text files until FS.init can be refactored. - // process.stdin.setEncoding('utf8'); - // } - }, - shutdown() { - // https://github.com/emscripten-core/emscripten/pull/1555 - // if (ENVIRONMENT_IS_NODE) { - // // inolen: any idea as to why node -e 'process.stdin.read()' wouldn't exit immediately (with process.stdin being a tty)? - // // isaacs: because now it's reading from the stream, you've expressed interest in it, so that read() kicks off a _read() which creates a ReadReq operation - // // inolen: I thought read() in that case was a synchronous operation that just grabbed some amount of buffered data if it exists? - // // isaacs: it is. but it also triggers a _read() call, which calls readStart() on the handle - // // isaacs: do process.stdin.pause() and i'd think it'd probably close the pending call - // process.stdin.pause(); - // } - }, - register(dev, ops) { - TTY.ttys[dev] = { input: [], output: [], ops: ops }; - FS.registerDevice(dev, TTY.stream_ops); - }, - stream_ops:{ - open(stream) { - var tty = TTY.ttys[stream.node.rdev]; - if (!tty) { - throw new FS.ErrnoError(43); - } - stream.tty = tty; - stream.seekable = false; - }, - close(stream) { - // flush any pending line data - stream.tty.ops.fsync(stream.tty); - }, - fsync(stream) { - stream.tty.ops.fsync(stream.tty); - }, - read(stream, buffer, offset, length, pos /* ignored */) { - if (!stream.tty || !stream.tty.ops.get_char) { - throw new FS.ErrnoError(60); - } - var bytesRead = 0; - for (var i = 0; i < length; i++) { - var result; - try { - result = stream.tty.ops.get_char(stream.tty); - } catch (e) { - throw new FS.ErrnoError(29); - } - if (result === undefined && bytesRead === 0) { - throw new FS.ErrnoError(6); - } - if (result === null || result === undefined) break; - bytesRead++; - buffer[offset+i] = result; - } - if (bytesRead) { - stream.node.atime = Date.now(); - } - return bytesRead; - }, - write(stream, buffer, offset, length, pos) { - if (!stream.tty || !stream.tty.ops.put_char) { - throw new FS.ErrnoError(60); - } - try { - for (var i = 0; i < length; i++) { - stream.tty.ops.put_char(stream.tty, buffer[offset+i]); - } - } catch (e) { - throw new FS.ErrnoError(29); - } - if (length) { - stream.node.mtime = stream.node.ctime = Date.now(); - } - return i; - }, - }, - default_tty_ops:{ - get_char(tty) { - return FS_stdin_getChar(); - }, - put_char(tty, val) { - if (val === null || val === 10) { - out(UTF8ArrayToString(tty.output)); - tty.output = []; - } else { - if (val != 0) tty.output.push(val); // val == 0 would cut text output off in the middle. - } - }, - fsync(tty) { - if (tty.output?.length > 0) { - out(UTF8ArrayToString(tty.output)); - tty.output = []; - } - }, - ioctl_tcgets(tty) { - // typical setting - return { - c_iflag: 25856, - c_oflag: 5, - c_cflag: 191, - c_lflag: 35387, - c_cc: [ - 0x03, 0x1c, 0x7f, 0x15, 0x04, 0x00, 0x01, 0x00, 0x11, 0x13, 0x1a, 0x00, - 0x12, 0x0f, 0x17, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - ] - }; - }, - ioctl_tcsets(tty, optional_actions, data) { - // currently just ignore - return 0; - }, - ioctl_tiocgwinsz(tty) { - return [24, 80]; - }, - }, - default_tty1_ops:{ - put_char(tty, val) { - if (val === null || val === 10) { - err(UTF8ArrayToString(tty.output)); - tty.output = []; - } else { - if (val != 0) tty.output.push(val); - } - }, - fsync(tty) { - if (tty.output?.length > 0) { - err(UTF8ArrayToString(tty.output)); - tty.output = []; - } - }, - }, - }; - - - var zeroMemory = (ptr, size) => HEAPU8.fill(0, ptr, ptr + size); - - var alignMemory = (size, alignment) => { - return Math.ceil(size / alignment) * alignment; - }; - var mmapAlloc = (size) => { - size = alignMemory(size, 65536); - var ptr = _emscripten_builtin_memalign(65536, size); - if (ptr) zeroMemory(ptr, size); - return ptr; - }; - var MEMFS = { - ops_table:null, - mount(mount) { - return MEMFS.createNode(null, '/', 16895, 0); - }, - createNode(parent, name, mode, dev) { - if (FS.isBlkdev(mode) || FS.isFIFO(mode)) { - // no supported - throw new FS.ErrnoError(63); - } - MEMFS.ops_table ||= { - dir: { - node: { - getattr: MEMFS.node_ops.getattr, - setattr: MEMFS.node_ops.setattr, - lookup: MEMFS.node_ops.lookup, - mknod: MEMFS.node_ops.mknod, - rename: MEMFS.node_ops.rename, - unlink: MEMFS.node_ops.unlink, - rmdir: MEMFS.node_ops.rmdir, - readdir: MEMFS.node_ops.readdir, - symlink: MEMFS.node_ops.symlink - }, - stream: { - llseek: MEMFS.stream_ops.llseek - } - }, - file: { - node: { - getattr: MEMFS.node_ops.getattr, - setattr: MEMFS.node_ops.setattr - }, - stream: { - llseek: MEMFS.stream_ops.llseek, - read: MEMFS.stream_ops.read, - write: MEMFS.stream_ops.write, - mmap: MEMFS.stream_ops.mmap, - msync: MEMFS.stream_ops.msync - } - }, - link: { - node: { - getattr: MEMFS.node_ops.getattr, - setattr: MEMFS.node_ops.setattr, - readlink: MEMFS.node_ops.readlink - }, - stream: {} - }, - chrdev: { - node: { - getattr: MEMFS.node_ops.getattr, - setattr: MEMFS.node_ops.setattr - }, - stream: FS.chrdev_stream_ops - } - }; - var node = FS.createNode(parent, name, mode, dev); - if (FS.isDir(node.mode)) { - node.node_ops = MEMFS.ops_table.dir.node; - node.stream_ops = MEMFS.ops_table.dir.stream; - node.contents = {}; - } else if (FS.isFile(node.mode)) { - node.node_ops = MEMFS.ops_table.file.node; - node.stream_ops = MEMFS.ops_table.file.stream; - node.usedBytes = 0; // The actual number of bytes used in the typed array, as opposed to contents.length which gives the whole capacity. - // When the byte data of the file is populated, this will point to either a typed array, or a normal JS array. Typed arrays are preferred - // for performance, and used by default. However, typed arrays are not resizable like normal JS arrays are, so there is a small disk size - // penalty involved for appending file writes that continuously grow a file similar to std::vector capacity vs used -scheme. - node.contents = null; - } else if (FS.isLink(node.mode)) { - node.node_ops = MEMFS.ops_table.link.node; - node.stream_ops = MEMFS.ops_table.link.stream; - } else if (FS.isChrdev(node.mode)) { - node.node_ops = MEMFS.ops_table.chrdev.node; - node.stream_ops = MEMFS.ops_table.chrdev.stream; - } - node.atime = node.mtime = node.ctime = Date.now(); - // add the new node to the parent - if (parent) { - parent.contents[name] = node; - parent.atime = parent.mtime = parent.ctime = node.atime; - } - return node; - }, - getFileDataAsTypedArray(node) { - if (!node.contents) return new Uint8Array(0); - if (node.contents.subarray) return node.contents.subarray(0, node.usedBytes); // Make sure to not return excess unused bytes. - return new Uint8Array(node.contents); - }, - expandFileStorage(node, newCapacity) { - var prevCapacity = node.contents ? node.contents.length : 0; - if (prevCapacity >= newCapacity) return; // No need to expand, the storage was already large enough. - // Don't expand strictly to the given requested limit if it's only a very small increase, but instead geometrically grow capacity. - // For small filesizes (<1MB), perform size*2 geometric increase, but for large sizes, do a much more conservative size*1.125 increase to - // avoid overshooting the allocation cap by a very large margin. - var CAPACITY_DOUBLING_MAX = 1024 * 1024; - newCapacity = Math.max(newCapacity, (prevCapacity * (prevCapacity < CAPACITY_DOUBLING_MAX ? 2.0 : 1.125)) >>> 0); - if (prevCapacity != 0) newCapacity = Math.max(newCapacity, 256); // At minimum allocate 256b for each file when expanding. - var oldContents = node.contents; - node.contents = new Uint8Array(newCapacity); // Allocate new storage. - if (node.usedBytes > 0) node.contents.set(oldContents.subarray(0, node.usedBytes), 0); // Copy old data over to the new storage. - }, - resizeFileStorage(node, newSize) { - if (node.usedBytes == newSize) return; - if (newSize == 0) { - node.contents = null; // Fully decommit when requesting a resize to zero. - node.usedBytes = 0; - } else { - var oldContents = node.contents; - node.contents = new Uint8Array(newSize); // Allocate new storage. - if (oldContents) { - node.contents.set(oldContents.subarray(0, Math.min(newSize, node.usedBytes))); // Copy old data over to the new storage. - } - node.usedBytes = newSize; - } - }, - node_ops:{ - getattr(node) { - var attr = {}; - // device numbers reuse inode numbers. - attr.dev = FS.isChrdev(node.mode) ? node.id : 1; - attr.ino = node.id; - attr.mode = node.mode; - attr.nlink = 1; - attr.uid = 0; - attr.gid = 0; - attr.rdev = node.rdev; - if (FS.isDir(node.mode)) { - attr.size = 4096; - } else if (FS.isFile(node.mode)) { - attr.size = node.usedBytes; - } else if (FS.isLink(node.mode)) { - attr.size = node.link.length; - } else { - attr.size = 0; - } - attr.atime = new Date(node.atime); - attr.mtime = new Date(node.mtime); - attr.ctime = new Date(node.ctime); - // NOTE: In our implementation, st_blocks = Math.ceil(st_size/st_blksize), - // but this is not required by the standard. - attr.blksize = 4096; - attr.blocks = Math.ceil(attr.size / attr.blksize); - return attr; - }, - setattr(node, attr) { - for (const key of ["mode", "atime", "mtime", "ctime"]) { - if (attr[key] != null) { - node[key] = attr[key]; - } - } - if (attr.size !== undefined) { - MEMFS.resizeFileStorage(node, attr.size); - } - }, - lookup(parent, name) { - // This error may happen quite a bit. To avoid overhead we reuse it (and - // suffer a lack of stack info). - if (!MEMFS.doesNotExistError) { - MEMFS.doesNotExistError = new FS.ErrnoError(44); - /** @suppress {checkTypes} */ - MEMFS.doesNotExistError.stack = ''; - } - throw MEMFS.doesNotExistError; - }, - mknod(parent, name, mode, dev) { - return MEMFS.createNode(parent, name, mode, dev); - }, - rename(old_node, new_dir, new_name) { - var new_node; - try { - new_node = FS.lookupNode(new_dir, new_name); - } catch (e) {} - if (new_node) { - if (FS.isDir(old_node.mode)) { - // if we're overwriting a directory at new_name, make sure it's empty. - for (var i in new_node.contents) { - throw new FS.ErrnoError(55); - } - } - FS.hashRemoveNode(new_node); - } - // do the internal rewiring - delete old_node.parent.contents[old_node.name]; - new_dir.contents[new_name] = old_node; - old_node.name = new_name; - new_dir.ctime = new_dir.mtime = old_node.parent.ctime = old_node.parent.mtime = Date.now(); - }, - unlink(parent, name) { - delete parent.contents[name]; - parent.ctime = parent.mtime = Date.now(); - }, - rmdir(parent, name) { - var node = FS.lookupNode(parent, name); - for (var i in node.contents) { - throw new FS.ErrnoError(55); - } - delete parent.contents[name]; - parent.ctime = parent.mtime = Date.now(); - }, - readdir(node) { - return ['.', '..', ...Object.keys(node.contents)]; - }, - symlink(parent, newname, oldpath) { - var node = MEMFS.createNode(parent, newname, 0o777 | 40960, 0); - node.link = oldpath; - return node; - }, - readlink(node) { - if (!FS.isLink(node.mode)) { - throw new FS.ErrnoError(28); - } - return node.link; - }, - }, - stream_ops:{ - read(stream, buffer, offset, length, position) { - var contents = stream.node.contents; - if (position >= stream.node.usedBytes) return 0; - var size = Math.min(stream.node.usedBytes - position, length); - if (size > 8 && contents.subarray) { // non-trivial, and typed array - buffer.set(contents.subarray(position, position + size), offset); - } else { - for (var i = 0; i < size; i++) buffer[offset + i] = contents[position + i]; - } - return size; - }, - write(stream, buffer, offset, length, position, canOwn) { - // If the buffer is located in main memory (HEAP), and if - // memory can grow, we can't hold on to references of the - // memory buffer, as they may get invalidated. That means we - // need to do copy its contents. - if (buffer.buffer === HEAP8.buffer) { - canOwn = false; - } - - if (!length) return 0; - var node = stream.node; - node.mtime = node.ctime = Date.now(); - - if (buffer.subarray && (!node.contents || node.contents.subarray)) { // This write is from a typed array to a typed array? - if (canOwn) { - node.contents = buffer.subarray(offset, offset + length); - node.usedBytes = length; - return length; - } else if (node.usedBytes === 0 && position === 0) { // If this is a simple first write to an empty file, do a fast set since we don't need to care about old data. - node.contents = buffer.slice(offset, offset + length); - node.usedBytes = length; - return length; - } else if (position + length <= node.usedBytes) { // Writing to an already allocated and used subrange of the file? - node.contents.set(buffer.subarray(offset, offset + length), position); - return length; - } - } - - // Appending to an existing file and we need to reallocate, or source data did not come as a typed array. - MEMFS.expandFileStorage(node, position+length); - if (node.contents.subarray && buffer.subarray) { - // Use typed array write which is available. - node.contents.set(buffer.subarray(offset, offset + length), position); - } else { - for (var i = 0; i < length; i++) { - node.contents[position + i] = buffer[offset + i]; // Or fall back to manual write if not. - } - } - node.usedBytes = Math.max(node.usedBytes, position + length); - return length; - }, - llseek(stream, offset, whence) { - var position = offset; - if (whence === 1) { - position += stream.position; - } else if (whence === 2) { - if (FS.isFile(stream.node.mode)) { - position += stream.node.usedBytes; - } - } - if (position < 0) { - throw new FS.ErrnoError(28); - } - return position; - }, - mmap(stream, length, position, prot, flags) { - if (!FS.isFile(stream.node.mode)) { - throw new FS.ErrnoError(43); - } - var ptr; - var allocated; - var contents = stream.node.contents; - // Only make a new copy when MAP_PRIVATE is specified. - if (!(flags & 2) && contents && contents.buffer === HEAP8.buffer) { - // We can't emulate MAP_SHARED when the file is not backed by the - // buffer we're mapping to (e.g. the HEAP buffer). - allocated = false; - ptr = contents.byteOffset; - } else { - allocated = true; - ptr = mmapAlloc(length); - if (!ptr) { - throw new FS.ErrnoError(48); - } - if (contents) { - // Try to avoid unnecessary slices. - if (position > 0 || position + length < contents.length) { - if (contents.subarray) { - contents = contents.subarray(position, position + length); - } else { - contents = Array.prototype.slice.call(contents, position, position + length); - } - } - HEAP8.set(contents, ptr); - } - } - return { ptr, allocated }; - }, - msync(stream, buffer, offset, length, mmapFlags) { - MEMFS.stream_ops.write(stream, buffer, 0, length, offset, false); - // should we check if bytesWritten and length are the same? - return 0; - }, - }, - }; - - var FS_modeStringToFlags = (str) => { - var flagModes = { - 'r': 0, - 'r+': 2, - 'w': 512 | 64 | 1, - 'w+': 512 | 64 | 2, - 'a': 1024 | 64 | 1, - 'a+': 1024 | 64 | 2, - }; - var flags = flagModes[str]; - if (typeof flags == 'undefined') { - throw new Error(`Unknown file open mode: ${str}`); - } - return flags; - }; - - var FS_getMode = (canRead, canWrite) => { - var mode = 0; - if (canRead) mode |= 292 | 73; - if (canWrite) mode |= 146; - return mode; - }; - - - var asyncLoad = async (url) => { - var arrayBuffer = await readAsync(url); - return new Uint8Array(arrayBuffer); - }; - - - var FS_createDataFile = (...args) => FS.createDataFile(...args); - - var getUniqueRunDependency = (id) => { - return id; - }; - - var runDependencies = 0; - - - var dependenciesFulfilled = null; - var removeRunDependency = (id) => { - runDependencies--; - - Module['monitorRunDependencies']?.(runDependencies); - - if (runDependencies == 0) { - if (dependenciesFulfilled) { - var callback = dependenciesFulfilled; - dependenciesFulfilled = null; - callback(); // can add another dependenciesFulfilled - } - } - }; - var addRunDependency = (id) => { - runDependencies++; - - Module['monitorRunDependencies']?.(runDependencies); - - }; - - - var preloadPlugins = []; - var FS_handledByPreloadPlugin = async (byteArray, fullname) => { - // Ensure plugins are ready. - if (typeof Browser != 'undefined') Browser.init(); - - for (var plugin of preloadPlugins) { - if (plugin['canHandle'](fullname)) { - return plugin['handle'](byteArray, fullname); - } - } - // In no plugin handled this file then return the original/unmodified - // byteArray. - return byteArray; - }; - var FS_preloadFile = async (parent, name, url, canRead, canWrite, dontCreateFile, canOwn, preFinish) => { - // TODO we should allow people to just pass in a complete filename instead - // of parent and name being that we just join them anyways - var fullname = name ? PATH_FS.resolve(PATH.join2(parent, name)) : parent; - var dep = getUniqueRunDependency(`cp ${fullname}`); // might have several active requests for the same fullname - addRunDependency(dep); - - try { - var byteArray = url; - if (typeof url == 'string') { - byteArray = await asyncLoad(url); - } - - byteArray = await FS_handledByPreloadPlugin(byteArray, fullname); - preFinish?.(); - if (!dontCreateFile) { - FS_createDataFile(parent, name, byteArray, canRead, canWrite, canOwn); - } - } finally { - removeRunDependency(dep); - } - }; - var FS_createPreloadedFile = (parent, name, url, canRead, canWrite, onload, onerror, dontCreateFile, canOwn, preFinish) => { - FS_preloadFile(parent, name, url, canRead, canWrite, dontCreateFile, canOwn, preFinish).then(onload).catch(onerror); - }; - var FS = { - root:null, - mounts:[], - devices:{ - }, - streams:[], - nextInode:1, - nameTable:null, - currentPath:"/", - initialized:false, - ignorePermissions:true, - filesystems:null, - syncFSRequests:0, - readFiles:{ - }, - ErrnoError:class { - name = 'ErrnoError'; - // We set the `name` property to be able to identify `FS.ErrnoError` - // - the `name` is a standard ECMA-262 property of error objects. Kind of good to have it anyway. - // - when using PROXYFS, an error can come from an underlying FS - // as different FS objects have their own FS.ErrnoError each, - // the test `err instanceof FS.ErrnoError` won't detect an error coming from another filesystem, causing bugs. - // we'll use the reliable test `err.name == "ErrnoError"` instead - constructor(errno) { - this.errno = errno; - } - }, - FSStream:class { - shared = {}; - get object() { - return this.node; - } - set object(val) { - this.node = val; - } - get isRead() { - return (this.flags & 2097155) !== 1; - } - get isWrite() { - return (this.flags & 2097155) !== 0; - } - get isAppend() { - return (this.flags & 1024); - } - get flags() { - return this.shared.flags; - } - set flags(val) { - this.shared.flags = val; - } - get position() { - return this.shared.position; - } - set position(val) { - this.shared.position = val; - } - }, - FSNode:class { - node_ops = {}; - stream_ops = {}; - readMode = 292 | 73; - writeMode = 146; - mounted = null; - constructor(parent, name, mode, rdev) { - if (!parent) { - parent = this; // root node sets parent to itself - } - this.parent = parent; - this.mount = parent.mount; - this.id = FS.nextInode++; - this.name = name; - this.mode = mode; - this.rdev = rdev; - this.atime = this.mtime = this.ctime = Date.now(); - } - get read() { - return (this.mode & this.readMode) === this.readMode; - } - set read(val) { - val ? this.mode |= this.readMode : this.mode &= ~this.readMode; - } - get write() { - return (this.mode & this.writeMode) === this.writeMode; - } - set write(val) { - val ? this.mode |= this.writeMode : this.mode &= ~this.writeMode; - } - get isFolder() { - return FS.isDir(this.mode); - } - get isDevice() { - return FS.isChrdev(this.mode); - } - }, - lookupPath(path, opts = {}) { - if (!path) { - throw new FS.ErrnoError(44); - } - opts.follow_mount ??= true - - if (!PATH.isAbs(path)) { - path = FS.cwd() + '/' + path; - } - - // limit max consecutive symlinks to 40 (SYMLOOP_MAX). - linkloop: for (var nlinks = 0; nlinks < 40; nlinks++) { - // split the absolute path - var parts = path.split('/').filter((p) => !!p); - - // start at the root - var current = FS.root; - var current_path = '/'; - - for (var i = 0; i < parts.length; i++) { - var islast = (i === parts.length-1); - if (islast && opts.parent) { - // stop resolving - break; - } - - if (parts[i] === '.') { - continue; - } - - if (parts[i] === '..') { - current_path = PATH.dirname(current_path); - if (FS.isRoot(current)) { - path = current_path + '/' + parts.slice(i + 1).join('/'); - // We're making progress here, don't let many consecutive ..'s - // lead to ELOOP - nlinks--; - continue linkloop; - } else { - current = current.parent; - } - continue; - } - - current_path = PATH.join2(current_path, parts[i]); - try { - current = FS.lookupNode(current, parts[i]); - } catch (e) { - // if noent_okay is true, suppress a ENOENT in the last component - // and return an object with an undefined node. This is needed for - // resolving symlinks in the path when creating a file. - if ((e?.errno === 44) && islast && opts.noent_okay) { - return { path: current_path }; - } - throw e; - } - - // jump to the mount's root node if this is a mountpoint - if (FS.isMountpoint(current) && (!islast || opts.follow_mount)) { - current = current.mounted.root; - } - - // by default, lookupPath will not follow a symlink if it is the final path component. - // setting opts.follow = true will override this behavior. - if (FS.isLink(current.mode) && (!islast || opts.follow)) { - if (!current.node_ops.readlink) { - throw new FS.ErrnoError(52); - } - var link = current.node_ops.readlink(current); - if (!PATH.isAbs(link)) { - link = PATH.dirname(current_path) + '/' + link; - } - path = link + '/' + parts.slice(i + 1).join('/'); - continue linkloop; - } - } - return { path: current_path, node: current }; - } - throw new FS.ErrnoError(32); - }, - getPath(node) { - var path; - while (true) { - if (FS.isRoot(node)) { - var mount = node.mount.mountpoint; - if (!path) return mount; - return mount[mount.length-1] !== '/' ? `${mount}/${path}` : mount + path; - } - path = path ? `${node.name}/${path}` : node.name; - node = node.parent; - } - }, - hashName(parentid, name) { - var hash = 0; - - for (var i = 0; i < name.length; i++) { - hash = ((hash << 5) - hash + name.charCodeAt(i)) | 0; - } - return ((parentid + hash) >>> 0) % FS.nameTable.length; - }, - hashAddNode(node) { - var hash = FS.hashName(node.parent.id, node.name); - node.name_next = FS.nameTable[hash]; - FS.nameTable[hash] = node; - }, - hashRemoveNode(node) { - var hash = FS.hashName(node.parent.id, node.name); - if (FS.nameTable[hash] === node) { - FS.nameTable[hash] = node.name_next; - } else { - var current = FS.nameTable[hash]; - while (current) { - if (current.name_next === node) { - current.name_next = node.name_next; - break; - } - current = current.name_next; - } - } - }, - lookupNode(parent, name) { - var errCode = FS.mayLookup(parent); - if (errCode) { - throw new FS.ErrnoError(errCode); - } - var hash = FS.hashName(parent.id, name); - for (var node = FS.nameTable[hash]; node; node = node.name_next) { - var nodeName = node.name; - if (node.parent.id === parent.id && nodeName === name) { - return node; - } - } - // if we failed to find it in the cache, call into the VFS - return FS.lookup(parent, name); - }, - createNode(parent, name, mode, rdev) { - var node = new FS.FSNode(parent, name, mode, rdev); - - FS.hashAddNode(node); - - return node; - }, - destroyNode(node) { - FS.hashRemoveNode(node); - }, - isRoot(node) { - return node === node.parent; - }, - isMountpoint(node) { - return !!node.mounted; - }, - isFile(mode) { - return (mode & 61440) === 32768; - }, - isDir(mode) { - return (mode & 61440) === 16384; - }, - isLink(mode) { - return (mode & 61440) === 40960; - }, - isChrdev(mode) { - return (mode & 61440) === 8192; - }, - isBlkdev(mode) { - return (mode & 61440) === 24576; - }, - isFIFO(mode) { - return (mode & 61440) === 4096; - }, - isSocket(mode) { - return (mode & 49152) === 49152; - }, - flagsToPermissionString(flag) { - var perms = ['r', 'w', 'rw'][flag & 3]; - if ((flag & 512)) { - perms += 'w'; - } - return perms; - }, - nodePermissions(node, perms) { - if (FS.ignorePermissions) { - return 0; - } - // return 0 if any user, group or owner bits are set. - if (perms.includes('r') && !(node.mode & 292)) { - return 2; - } else if (perms.includes('w') && !(node.mode & 146)) { - return 2; - } else if (perms.includes('x') && !(node.mode & 73)) { - return 2; - } - return 0; - }, - mayLookup(dir) { - if (!FS.isDir(dir.mode)) return 54; - var errCode = FS.nodePermissions(dir, 'x'); - if (errCode) return errCode; - if (!dir.node_ops.lookup) return 2; - return 0; - }, - mayCreate(dir, name) { - if (!FS.isDir(dir.mode)) { - return 54; - } - try { - var node = FS.lookupNode(dir, name); - return 20; - } catch (e) { - } - return FS.nodePermissions(dir, 'wx'); - }, - mayDelete(dir, name, isdir) { - var node; - try { - node = FS.lookupNode(dir, name); - } catch (e) { - return e.errno; - } - var errCode = FS.nodePermissions(dir, 'wx'); - if (errCode) { - return errCode; - } - if (isdir) { - if (!FS.isDir(node.mode)) { - return 54; - } - if (FS.isRoot(node) || FS.getPath(node) === FS.cwd()) { - return 10; - } - } else { - if (FS.isDir(node.mode)) { - return 31; - } - } - return 0; - }, - mayOpen(node, flags) { - if (!node) { - return 44; - } - if (FS.isLink(node.mode)) { - return 32; - } else if (FS.isDir(node.mode)) { - if (FS.flagsToPermissionString(flags) !== 'r' // opening for write - || (flags & (512 | 64))) { // TODO: check for O_SEARCH? (== search for dir only) - return 31; - } - } - return FS.nodePermissions(node, FS.flagsToPermissionString(flags)); - }, - checkOpExists(op, err) { - if (!op) { - throw new FS.ErrnoError(err); - } - return op; - }, - MAX_OPEN_FDS:4096, - nextfd() { - for (var fd = 0; fd <= FS.MAX_OPEN_FDS; fd++) { - if (!FS.streams[fd]) { - return fd; - } - } - throw new FS.ErrnoError(33); - }, - getStreamChecked(fd) { - var stream = FS.getStream(fd); - if (!stream) { - throw new FS.ErrnoError(8); - } - return stream; - }, - getStream:(fd) => FS.streams[fd], - createStream(stream, fd = -1) { - - // clone it, so we can return an instance of FSStream - stream = Object.assign(new FS.FSStream(), stream); - if (fd == -1) { - fd = FS.nextfd(); - } - stream.fd = fd; - FS.streams[fd] = stream; - return stream; - }, - closeStream(fd) { - FS.streams[fd] = null; - }, - dupStream(origStream, fd = -1) { - var stream = FS.createStream(origStream, fd); - stream.stream_ops?.dup?.(stream); - return stream; - }, - doSetAttr(stream, node, attr) { - var setattr = stream?.stream_ops.setattr; - var arg = setattr ? stream : node; - setattr ??= node.node_ops.setattr; - FS.checkOpExists(setattr, 63) - setattr(arg, attr); - }, - chrdev_stream_ops:{ - open(stream) { - var device = FS.getDevice(stream.node.rdev); - // override node's stream ops with the device's - stream.stream_ops = device.stream_ops; - // forward the open call - stream.stream_ops.open?.(stream); - }, - llseek() { - throw new FS.ErrnoError(70); - }, - }, - major:(dev) => ((dev) >> 8), - minor:(dev) => ((dev) & 0xff), - makedev:(ma, mi) => ((ma) << 8 | (mi)), - registerDevice(dev, ops) { - FS.devices[dev] = { stream_ops: ops }; - }, - getDevice:(dev) => FS.devices[dev], - getMounts(mount) { - var mounts = []; - var check = [mount]; - - while (check.length) { - var m = check.pop(); - - mounts.push(m); - - check.push(...m.mounts); - } - - return mounts; - }, - syncfs(populate, callback) { - if (typeof populate == 'function') { - callback = populate; - populate = false; - } - - FS.syncFSRequests++; - - if (FS.syncFSRequests > 1) { - err(`warning: ${FS.syncFSRequests} FS.syncfs operations in flight at once, probably just doing extra work`); - } - - var mounts = FS.getMounts(FS.root.mount); - var completed = 0; - - function doCallback(errCode) { - FS.syncFSRequests--; - return callback(errCode); - } - - function done(errCode) { - if (errCode) { - if (!done.errored) { - done.errored = true; - return doCallback(errCode); - } - return; - } - if (++completed >= mounts.length) { - doCallback(null); - } - }; - - // sync all mounts - for (var mount of mounts) { - if (mount.type.syncfs) { - mount.type.syncfs(mount, populate, done); - } else { - done(null); - } - } - }, - mount(type, opts, mountpoint) { - var root = mountpoint === '/'; - var pseudo = !mountpoint; - var node; - - if (root && FS.root) { - throw new FS.ErrnoError(10); - } else if (!root && !pseudo) { - var lookup = FS.lookupPath(mountpoint, { follow_mount: false }); - - mountpoint = lookup.path; // use the absolute path - node = lookup.node; - - if (FS.isMountpoint(node)) { - throw new FS.ErrnoError(10); - } - - if (!FS.isDir(node.mode)) { - throw new FS.ErrnoError(54); - } - } - - var mount = { - type, - opts, - mountpoint, - mounts: [] - }; - - // create a root node for the fs - var mountRoot = type.mount(mount); - mountRoot.mount = mount; - mount.root = mountRoot; - - if (root) { - FS.root = mountRoot; - } else if (node) { - // set as a mountpoint - node.mounted = mount; - - // add the new mount to the current mount's children - if (node.mount) { - node.mount.mounts.push(mount); - } - } - - return mountRoot; - }, - unmount(mountpoint) { - var lookup = FS.lookupPath(mountpoint, { follow_mount: false }); - - if (!FS.isMountpoint(lookup.node)) { - throw new FS.ErrnoError(28); - } - - // destroy the nodes for this mount, and all its child mounts - var node = lookup.node; - var mount = node.mounted; - var mounts = FS.getMounts(mount); - - for (var [hash, current] of Object.entries(FS.nameTable)) { - while (current) { - var next = current.name_next; - - if (mounts.includes(current.mount)) { - FS.destroyNode(current); - } - - current = next; - } - } - - // no longer a mountpoint - node.mounted = null; - - // remove this mount from the child mounts - var idx = node.mount.mounts.indexOf(mount); - node.mount.mounts.splice(idx, 1); - }, - lookup(parent, name) { - return parent.node_ops.lookup(parent, name); - }, - mknod(path, mode, dev) { - var lookup = FS.lookupPath(path, { parent: true }); - var parent = lookup.node; - var name = PATH.basename(path); - if (!name) { - throw new FS.ErrnoError(28); - } - if (name === '.' || name === '..') { - throw new FS.ErrnoError(20); - } - var errCode = FS.mayCreate(parent, name); - if (errCode) { - throw new FS.ErrnoError(errCode); - } - if (!parent.node_ops.mknod) { - throw new FS.ErrnoError(63); - } - return parent.node_ops.mknod(parent, name, mode, dev); - }, - statfs(path) { - return FS.statfsNode(FS.lookupPath(path, {follow: true}).node); - }, - statfsStream(stream) { - // We keep a separate statfsStream function because noderawfs overrides - // it. In noderawfs, stream.node is sometimes null. Instead, we need to - // look at stream.path. - return FS.statfsNode(stream.node); - }, - statfsNode(node) { - // NOTE: None of the defaults here are true. We're just returning safe and - // sane values. Currently nodefs and rawfs replace these defaults, - // other file systems leave them alone. - var rtn = { - bsize: 4096, - frsize: 4096, - blocks: 1e6, - bfree: 5e5, - bavail: 5e5, - files: FS.nextInode, - ffree: FS.nextInode - 1, - fsid: 42, - flags: 2, - namelen: 255, - }; - - if (node.node_ops.statfs) { - Object.assign(rtn, node.node_ops.statfs(node.mount.opts.root)); - } - return rtn; - }, - create(path, mode = 0o666) { - mode &= 4095; - mode |= 32768; - return FS.mknod(path, mode, 0); - }, - mkdir(path, mode = 0o777) { - mode &= 511 | 512; - mode |= 16384; - return FS.mknod(path, mode, 0); - }, - mkdirTree(path, mode) { - var dirs = path.split('/'); - var d = ''; - for (var dir of dirs) { - if (!dir) continue; - if (d || PATH.isAbs(path)) d += '/'; - d += dir; - try { - FS.mkdir(d, mode); - } catch(e) { - if (e.errno != 20) throw e; - } - } - }, - mkdev(path, mode, dev) { - if (typeof dev == 'undefined') { - dev = mode; - mode = 0o666; - } - mode |= 8192; - return FS.mknod(path, mode, dev); - }, - symlink(oldpath, newpath) { - if (!PATH_FS.resolve(oldpath)) { - throw new FS.ErrnoError(44); - } - var lookup = FS.lookupPath(newpath, { parent: true }); - var parent = lookup.node; - if (!parent) { - throw new FS.ErrnoError(44); - } - var newname = PATH.basename(newpath); - var errCode = FS.mayCreate(parent, newname); - if (errCode) { - throw new FS.ErrnoError(errCode); - } - if (!parent.node_ops.symlink) { - throw new FS.ErrnoError(63); - } - return parent.node_ops.symlink(parent, newname, oldpath); - }, - rename(old_path, new_path) { - var old_dirname = PATH.dirname(old_path); - var new_dirname = PATH.dirname(new_path); - var old_name = PATH.basename(old_path); - var new_name = PATH.basename(new_path); - // parents must exist - var lookup, old_dir, new_dir; - - // let the errors from non existent directories percolate up - lookup = FS.lookupPath(old_path, { parent: true }); - old_dir = lookup.node; - lookup = FS.lookupPath(new_path, { parent: true }); - new_dir = lookup.node; - - if (!old_dir || !new_dir) throw new FS.ErrnoError(44); - // need to be part of the same mount - if (old_dir.mount !== new_dir.mount) { - throw new FS.ErrnoError(75); - } - // source must exist - var old_node = FS.lookupNode(old_dir, old_name); - // old path should not be an ancestor of the new path - var relative = PATH_FS.relative(old_path, new_dirname); - if (relative.charAt(0) !== '.') { - throw new FS.ErrnoError(28); - } - // new path should not be an ancestor of the old path - relative = PATH_FS.relative(new_path, old_dirname); - if (relative.charAt(0) !== '.') { - throw new FS.ErrnoError(55); - } - // see if the new path already exists - var new_node; - try { - new_node = FS.lookupNode(new_dir, new_name); - } catch (e) { - // not fatal - } - // early out if nothing needs to change - if (old_node === new_node) { - return; - } - // we'll need to delete the old entry - var isdir = FS.isDir(old_node.mode); - var errCode = FS.mayDelete(old_dir, old_name, isdir); - if (errCode) { - throw new FS.ErrnoError(errCode); - } - // need delete permissions if we'll be overwriting. - // need create permissions if new doesn't already exist. - errCode = new_node ? - FS.mayDelete(new_dir, new_name, isdir) : - FS.mayCreate(new_dir, new_name); - if (errCode) { - throw new FS.ErrnoError(errCode); - } - if (!old_dir.node_ops.rename) { - throw new FS.ErrnoError(63); - } - if (FS.isMountpoint(old_node) || (new_node && FS.isMountpoint(new_node))) { - throw new FS.ErrnoError(10); - } - // if we are going to change the parent, check write permissions - if (new_dir !== old_dir) { - errCode = FS.nodePermissions(old_dir, 'w'); - if (errCode) { - throw new FS.ErrnoError(errCode); - } - } - // remove the node from the lookup hash - FS.hashRemoveNode(old_node); - // do the underlying fs rename - try { - old_dir.node_ops.rename(old_node, new_dir, new_name); - // update old node (we do this here to avoid each backend - // needing to) - old_node.parent = new_dir; - } catch (e) { - throw e; - } finally { - // add the node back to the hash (in case node_ops.rename - // changed its name) - FS.hashAddNode(old_node); - } - }, - rmdir(path) { - var lookup = FS.lookupPath(path, { parent: true }); - var parent = lookup.node; - var name = PATH.basename(path); - var node = FS.lookupNode(parent, name); - var errCode = FS.mayDelete(parent, name, true); - if (errCode) { - throw new FS.ErrnoError(errCode); - } - if (!parent.node_ops.rmdir) { - throw new FS.ErrnoError(63); - } - if (FS.isMountpoint(node)) { - throw new FS.ErrnoError(10); - } - parent.node_ops.rmdir(parent, name); - FS.destroyNode(node); - }, - readdir(path) { - var lookup = FS.lookupPath(path, { follow: true }); - var node = lookup.node; - var readdir = FS.checkOpExists(node.node_ops.readdir, 54); - return readdir(node); - }, - unlink(path) { - var lookup = FS.lookupPath(path, { parent: true }); - var parent = lookup.node; - if (!parent) { - throw new FS.ErrnoError(44); - } - var name = PATH.basename(path); - var node = FS.lookupNode(parent, name); - var errCode = FS.mayDelete(parent, name, false); - if (errCode) { - // According to POSIX, we should map EISDIR to EPERM, but - // we instead do what Linux does (and we must, as we use - // the musl linux libc). - throw new FS.ErrnoError(errCode); - } - if (!parent.node_ops.unlink) { - throw new FS.ErrnoError(63); - } - if (FS.isMountpoint(node)) { - throw new FS.ErrnoError(10); - } - parent.node_ops.unlink(parent, name); - FS.destroyNode(node); - }, - readlink(path) { - var lookup = FS.lookupPath(path); - var link = lookup.node; - if (!link) { - throw new FS.ErrnoError(44); - } - if (!link.node_ops.readlink) { - throw new FS.ErrnoError(28); - } - return link.node_ops.readlink(link); - }, - stat(path, dontFollow) { - var lookup = FS.lookupPath(path, { follow: !dontFollow }); - var node = lookup.node; - var getattr = FS.checkOpExists(node.node_ops.getattr, 63); - return getattr(node); - }, - fstat(fd) { - var stream = FS.getStreamChecked(fd); - var node = stream.node; - var getattr = stream.stream_ops.getattr; - var arg = getattr ? stream : node; - getattr ??= node.node_ops.getattr; - FS.checkOpExists(getattr, 63) - return getattr(arg); - }, - lstat(path) { - return FS.stat(path, true); - }, - doChmod(stream, node, mode, dontFollow) { - FS.doSetAttr(stream, node, { - mode: (mode & 4095) | (node.mode & ~4095), - ctime: Date.now(), - dontFollow - }); - }, - chmod(path, mode, dontFollow) { - var node; - if (typeof path == 'string') { - var lookup = FS.lookupPath(path, { follow: !dontFollow }); - node = lookup.node; - } else { - node = path; - } - FS.doChmod(null, node, mode, dontFollow); - }, - lchmod(path, mode) { - FS.chmod(path, mode, true); - }, - fchmod(fd, mode) { - var stream = FS.getStreamChecked(fd); - FS.doChmod(stream, stream.node, mode, false); - }, - doChown(stream, node, dontFollow) { - FS.doSetAttr(stream, node, { - timestamp: Date.now(), - dontFollow - // we ignore the uid / gid for now - }); - }, - chown(path, uid, gid, dontFollow) { - var node; - if (typeof path == 'string') { - var lookup = FS.lookupPath(path, { follow: !dontFollow }); - node = lookup.node; - } else { - node = path; - } - FS.doChown(null, node, dontFollow); - }, - lchown(path, uid, gid) { - FS.chown(path, uid, gid, true); - }, - fchown(fd, uid, gid) { - var stream = FS.getStreamChecked(fd); - FS.doChown(stream, stream.node, false); - }, - doTruncate(stream, node, len) { - if (FS.isDir(node.mode)) { - throw new FS.ErrnoError(31); - } - if (!FS.isFile(node.mode)) { - throw new FS.ErrnoError(28); - } - var errCode = FS.nodePermissions(node, 'w'); - if (errCode) { - throw new FS.ErrnoError(errCode); - } - FS.doSetAttr(stream, node, { - size: len, - timestamp: Date.now() - }); - }, - truncate(path, len) { - if (len < 0) { - throw new FS.ErrnoError(28); - } - var node; - if (typeof path == 'string') { - var lookup = FS.lookupPath(path, { follow: true }); - node = lookup.node; - } else { - node = path; - } - FS.doTruncate(null, node, len); - }, - ftruncate(fd, len) { - var stream = FS.getStreamChecked(fd); - if (len < 0 || (stream.flags & 2097155) === 0) { - throw new FS.ErrnoError(28); - } - FS.doTruncate(stream, stream.node, len); - }, - utime(path, atime, mtime) { - var lookup = FS.lookupPath(path, { follow: true }); - var node = lookup.node; - var setattr = FS.checkOpExists(node.node_ops.setattr, 63); - setattr(node, { - atime: atime, - mtime: mtime - }); - }, - open(path, flags, mode = 0o666) { - if (path === "") { - throw new FS.ErrnoError(44); - } - flags = typeof flags == 'string' ? FS_modeStringToFlags(flags) : flags; - if ((flags & 64)) { - mode = (mode & 4095) | 32768; - } else { - mode = 0; - } - var node; - var isDirPath; - if (typeof path == 'object') { - node = path; - } else { - isDirPath = path.endsWith("/"); - // noent_okay makes it so that if the final component of the path - // doesn't exist, lookupPath returns `node: undefined`. `path` will be - // updated to point to the target of all symlinks. - var lookup = FS.lookupPath(path, { - follow: !(flags & 131072), - noent_okay: true - }); - node = lookup.node; - path = lookup.path; - } - // perhaps we need to create the node - var created = false; - if ((flags & 64)) { - if (node) { - // if O_CREAT and O_EXCL are set, error out if the node already exists - if ((flags & 128)) { - throw new FS.ErrnoError(20); - } - } else if (isDirPath) { - throw new FS.ErrnoError(31); - } else { - // node doesn't exist, try to create it - // Ignore the permission bits here to ensure we can `open` this new - // file below. We use chmod below the apply the permissions once the - // file is open. - node = FS.mknod(path, mode | 0o777, 0); - created = true; - } - } - if (!node) { - throw new FS.ErrnoError(44); - } - // can't truncate a device - if (FS.isChrdev(node.mode)) { - flags &= ~512; - } - // if asked only for a directory, then this must be one - if ((flags & 65536) && !FS.isDir(node.mode)) { - throw new FS.ErrnoError(54); - } - // check permissions, if this is not a file we just created now (it is ok to - // create and write to a file with read-only permissions; it is read-only - // for later use) - if (!created) { - var errCode = FS.mayOpen(node, flags); - if (errCode) { - throw new FS.ErrnoError(errCode); - } - } - // do truncation if necessary - if ((flags & 512) && !created) { - FS.truncate(node, 0); - } - // we've already handled these, don't pass down to the underlying vfs - flags &= ~(128 | 512 | 131072); - - // register the stream with the filesystem - var stream = FS.createStream({ - node, - path: FS.getPath(node), // we want the absolute path to the node - flags, - seekable: true, - position: 0, - stream_ops: node.stream_ops, - // used by the file family libc calls (fopen, fwrite, ferror, etc.) - ungotten: [], - error: false - }); - // call the new stream's open function - if (stream.stream_ops.open) { - stream.stream_ops.open(stream); - } - if (created) { - FS.chmod(node, mode & 0o777); - } - if (Module['logReadFiles'] && !(flags & 1)) { - if (!(path in FS.readFiles)) { - FS.readFiles[path] = 1; - } - } - return stream; - }, - close(stream) { - if (FS.isClosed(stream)) { - throw new FS.ErrnoError(8); - } - if (stream.getdents) stream.getdents = null; // free readdir state - try { - if (stream.stream_ops.close) { - stream.stream_ops.close(stream); - } - } catch (e) { - throw e; - } finally { - FS.closeStream(stream.fd); - } - stream.fd = null; - }, - isClosed(stream) { - return stream.fd === null; - }, - llseek(stream, offset, whence) { - if (FS.isClosed(stream)) { - throw new FS.ErrnoError(8); - } - if (!stream.seekable || !stream.stream_ops.llseek) { - throw new FS.ErrnoError(70); - } - if (whence != 0 && whence != 1 && whence != 2) { - throw new FS.ErrnoError(28); - } - stream.position = stream.stream_ops.llseek(stream, offset, whence); - stream.ungotten = []; - return stream.position; - }, - read(stream, buffer, offset, length, position) { - if (length < 0 || position < 0) { - throw new FS.ErrnoError(28); - } - if (FS.isClosed(stream)) { - throw new FS.ErrnoError(8); - } - if ((stream.flags & 2097155) === 1) { - throw new FS.ErrnoError(8); - } - if (FS.isDir(stream.node.mode)) { - throw new FS.ErrnoError(31); - } - if (!stream.stream_ops.read) { - throw new FS.ErrnoError(28); - } - var seeking = typeof position != 'undefined'; - if (!seeking) { - position = stream.position; - } else if (!stream.seekable) { - throw new FS.ErrnoError(70); - } - var bytesRead = stream.stream_ops.read(stream, buffer, offset, length, position); - if (!seeking) stream.position += bytesRead; - return bytesRead; - }, - write(stream, buffer, offset, length, position, canOwn) { - if (length < 0 || position < 0) { - throw new FS.ErrnoError(28); - } - if (FS.isClosed(stream)) { - throw new FS.ErrnoError(8); - } - if ((stream.flags & 2097155) === 0) { - throw new FS.ErrnoError(8); - } - if (FS.isDir(stream.node.mode)) { - throw new FS.ErrnoError(31); - } - if (!stream.stream_ops.write) { - throw new FS.ErrnoError(28); - } - if (stream.seekable && stream.flags & 1024) { - // seek to the end before writing in append mode - FS.llseek(stream, 0, 2); - } - var seeking = typeof position != 'undefined'; - if (!seeking) { - position = stream.position; - } else if (!stream.seekable) { - throw new FS.ErrnoError(70); - } - var bytesWritten = stream.stream_ops.write(stream, buffer, offset, length, position, canOwn); - if (!seeking) stream.position += bytesWritten; - return bytesWritten; - }, - mmap(stream, length, position, prot, flags) { - // User requests writing to file (prot & PROT_WRITE != 0). - // Checking if we have permissions to write to the file unless - // MAP_PRIVATE flag is set. According to POSIX spec it is possible - // to write to file opened in read-only mode with MAP_PRIVATE flag, - // as all modifications will be visible only in the memory of - // the current process. - if ((prot & 2) !== 0 - && (flags & 2) === 0 - && (stream.flags & 2097155) !== 2) { - throw new FS.ErrnoError(2); - } - if ((stream.flags & 2097155) === 1) { - throw new FS.ErrnoError(2); - } - if (!stream.stream_ops.mmap) { - throw new FS.ErrnoError(43); - } - if (!length) { - throw new FS.ErrnoError(28); - } - return stream.stream_ops.mmap(stream, length, position, prot, flags); - }, - msync(stream, buffer, offset, length, mmapFlags) { - if (!stream.stream_ops.msync) { - return 0; - } - return stream.stream_ops.msync(stream, buffer, offset, length, mmapFlags); - }, - ioctl(stream, cmd, arg) { - if (!stream.stream_ops.ioctl) { - throw new FS.ErrnoError(59); - } - return stream.stream_ops.ioctl(stream, cmd, arg); - }, - readFile(path, opts = {}) { - opts.flags = opts.flags || 0; - opts.encoding = opts.encoding || 'binary'; - if (opts.encoding !== 'utf8' && opts.encoding !== 'binary') { - abort(`Invalid encoding type "${opts.encoding}"`); - } - var stream = FS.open(path, opts.flags); - var stat = FS.stat(path); - var length = stat.size; - var buf = new Uint8Array(length); - FS.read(stream, buf, 0, length, 0); - if (opts.encoding === 'utf8') { - buf = UTF8ArrayToString(buf); - } - FS.close(stream); - return buf; - }, - writeFile(path, data, opts = {}) { - opts.flags = opts.flags || 577; - var stream = FS.open(path, opts.flags, opts.mode); - if (typeof data == 'string') { - data = new Uint8Array(intArrayFromString(data, true)); - } - if (ArrayBuffer.isView(data)) { - FS.write(stream, data, 0, data.byteLength, undefined, opts.canOwn); - } else { - abort('Unsupported data type'); - } - FS.close(stream); - }, - cwd:() => FS.currentPath, - chdir(path) { - var lookup = FS.lookupPath(path, { follow: true }); - if (lookup.node === null) { - throw new FS.ErrnoError(44); - } - if (!FS.isDir(lookup.node.mode)) { - throw new FS.ErrnoError(54); - } - var errCode = FS.nodePermissions(lookup.node, 'x'); - if (errCode) { - throw new FS.ErrnoError(errCode); - } - FS.currentPath = lookup.path; - }, - createDefaultDirectories() { - FS.mkdir('/tmp'); - FS.mkdir('/home'); - FS.mkdir('/home/web_user'); - }, - createDefaultDevices() { - // create /dev - FS.mkdir('/dev'); - // setup /dev/null - FS.registerDevice(FS.makedev(1, 3), { - read: () => 0, - write: (stream, buffer, offset, length, pos) => length, - llseek: () => 0, - }); - FS.mkdev('/dev/null', FS.makedev(1, 3)); - // setup /dev/tty and /dev/tty1 - // stderr needs to print output using err() rather than out() - // so we register a second tty just for it. - TTY.register(FS.makedev(5, 0), TTY.default_tty_ops); - TTY.register(FS.makedev(6, 0), TTY.default_tty1_ops); - FS.mkdev('/dev/tty', FS.makedev(5, 0)); - FS.mkdev('/dev/tty1', FS.makedev(6, 0)); - // setup /dev/[u]random - // use a buffer to avoid overhead of individual crypto calls per byte - var randomBuffer = new Uint8Array(1024), randomLeft = 0; - var randomByte = () => { - if (randomLeft === 0) { - randomFill(randomBuffer); - randomLeft = randomBuffer.byteLength; - } - return randomBuffer[--randomLeft]; - }; - FS.createDevice('/dev', 'random', randomByte); - FS.createDevice('/dev', 'urandom', randomByte); - // we're not going to emulate the actual shm device, - // just create the tmp dirs that reside in it commonly - FS.mkdir('/dev/shm'); - FS.mkdir('/dev/shm/tmp'); - }, - createSpecialDirectories() { - // create /proc/self/fd which allows /proc/self/fd/6 => readlink gives the - // name of the stream for fd 6 (see test_unistd_ttyname) - FS.mkdir('/proc'); - var proc_self = FS.mkdir('/proc/self'); - FS.mkdir('/proc/self/fd'); - FS.mount({ - mount() { - var node = FS.createNode(proc_self, 'fd', 16895, 73); - node.stream_ops = { - llseek: MEMFS.stream_ops.llseek, - }; - node.node_ops = { - lookup(parent, name) { - var fd = +name; - var stream = FS.getStreamChecked(fd); - var ret = { - parent: null, - mount: { mountpoint: 'fake' }, - node_ops: { readlink: () => stream.path }, - id: fd + 1, - }; - ret.parent = ret; // make it look like a simple root node - return ret; - }, - readdir() { - return Array.from(FS.streams.entries()) - .filter(([k, v]) => v) - .map(([k, v]) => k.toString()); - } - }; - return node; - } - }, {}, '/proc/self/fd'); - }, - createStandardStreams(input, output, error) { - // TODO deprecate the old functionality of a single - // input / output callback and that utilizes FS.createDevice - // and instead require a unique set of stream ops - - // by default, we symlink the standard streams to the - // default tty devices. however, if the standard streams - // have been overwritten we create a unique device for - // them instead. - if (input) { - FS.createDevice('/dev', 'stdin', input); - } else { - FS.symlink('/dev/tty', '/dev/stdin'); - } - if (output) { - FS.createDevice('/dev', 'stdout', null, output); - } else { - FS.symlink('/dev/tty', '/dev/stdout'); - } - if (error) { - FS.createDevice('/dev', 'stderr', null, error); - } else { - FS.symlink('/dev/tty1', '/dev/stderr'); - } - - // open default streams for the stdin, stdout and stderr devices - var stdin = FS.open('/dev/stdin', 0); - var stdout = FS.open('/dev/stdout', 1); - var stderr = FS.open('/dev/stderr', 1); - }, - staticInit() { - FS.nameTable = new Array(4096); - - FS.mount(MEMFS, {}, '/'); - - FS.createDefaultDirectories(); - FS.createDefaultDevices(); - FS.createSpecialDirectories(); - - FS.filesystems = { - 'MEMFS': MEMFS, - }; - }, - init(input, output, error) { - FS.initialized = true; - - // Allow Module.stdin etc. to provide defaults, if none explicitly passed to us here - input ??= Module['stdin']; - output ??= Module['stdout']; - error ??= Module['stderr']; - - FS.createStandardStreams(input, output, error); - }, - quit() { - FS.initialized = false; - // force-flush all streams, so we get musl std streams printed out - // close all of our streams - for (var stream of FS.streams) { - if (stream) { - FS.close(stream); - } - } - }, - findObject(path, dontResolveLastLink) { - var ret = FS.analyzePath(path, dontResolveLastLink); - if (!ret.exists) { - return null; - } - return ret.object; - }, - analyzePath(path, dontResolveLastLink) { - // operate from within the context of the symlink's target - try { - var lookup = FS.lookupPath(path, { follow: !dontResolveLastLink }); - path = lookup.path; - } catch (e) { - } - var ret = { - isRoot: false, exists: false, error: 0, name: null, path: null, object: null, - parentExists: false, parentPath: null, parentObject: null - }; - try { - var lookup = FS.lookupPath(path, { parent: true }); - ret.parentExists = true; - ret.parentPath = lookup.path; - ret.parentObject = lookup.node; - ret.name = PATH.basename(path); - lookup = FS.lookupPath(path, { follow: !dontResolveLastLink }); - ret.exists = true; - ret.path = lookup.path; - ret.object = lookup.node; - ret.name = lookup.node.name; - ret.isRoot = lookup.path === '/'; - } catch (e) { - ret.error = e.errno; - }; - return ret; - }, - createPath(parent, path, canRead, canWrite) { - parent = typeof parent == 'string' ? parent : FS.getPath(parent); - var parts = path.split('/').reverse(); - while (parts.length) { - var part = parts.pop(); - if (!part) continue; - var current = PATH.join2(parent, part); - try { - FS.mkdir(current); - } catch (e) { - if (e.errno != 20) throw e; - } - parent = current; - } - return current; - }, - createFile(parent, name, properties, canRead, canWrite) { - var path = PATH.join2(typeof parent == 'string' ? parent : FS.getPath(parent), name); - var mode = FS_getMode(canRead, canWrite); - return FS.create(path, mode); - }, - createDataFile(parent, name, data, canRead, canWrite, canOwn) { - var path = name; - if (parent) { - parent = typeof parent == 'string' ? parent : FS.getPath(parent); - path = name ? PATH.join2(parent, name) : parent; - } - var mode = FS_getMode(canRead, canWrite); - var node = FS.create(path, mode); - if (data) { - if (typeof data == 'string') { - var arr = new Array(data.length); - for (var i = 0, len = data.length; i < len; ++i) arr[i] = data.charCodeAt(i); - data = arr; - } - // make sure we can write to the file - FS.chmod(node, mode | 146); - var stream = FS.open(node, 577); - FS.write(stream, data, 0, data.length, 0, canOwn); - FS.close(stream); - FS.chmod(node, mode); - } - }, - createDevice(parent, name, input, output) { - var path = PATH.join2(typeof parent == 'string' ? parent : FS.getPath(parent), name); - var mode = FS_getMode(!!input, !!output); - FS.createDevice.major ??= 64; - var dev = FS.makedev(FS.createDevice.major++, 0); - // Create a fake device that a set of stream ops to emulate - // the old behavior. - FS.registerDevice(dev, { - open(stream) { - stream.seekable = false; - }, - close(stream) { - // flush any pending line data - if (output?.buffer?.length) { - output(10); - } - }, - read(stream, buffer, offset, length, pos /* ignored */) { - var bytesRead = 0; - for (var i = 0; i < length; i++) { - var result; - try { - result = input(); - } catch (e) { - throw new FS.ErrnoError(29); - } - if (result === undefined && bytesRead === 0) { - throw new FS.ErrnoError(6); - } - if (result === null || result === undefined) break; - bytesRead++; - buffer[offset+i] = result; - } - if (bytesRead) { - stream.node.atime = Date.now(); - } - return bytesRead; - }, - write(stream, buffer, offset, length, pos) { - for (var i = 0; i < length; i++) { - try { - output(buffer[offset+i]); - } catch (e) { - throw new FS.ErrnoError(29); - } - } - if (length) { - stream.node.mtime = stream.node.ctime = Date.now(); - } - return i; - } - }); - return FS.mkdev(path, mode, dev); - }, - forceLoadFile(obj) { - if (obj.isDevice || obj.isFolder || obj.link || obj.contents) return true; - if (globalThis.XMLHttpRequest) { - abort("Lazy loading should have been performed (contents set) in createLazyFile, but it was not. Lazy loading only works in web workers. Use --embed-file or --preload-file in emcc on the main thread."); - } else { // Command-line. - try { - obj.contents = readBinary(obj.url); - } catch (e) { - throw new FS.ErrnoError(29); - } - } - }, - createLazyFile(parent, name, url, canRead, canWrite) { - // Lazy chunked Uint8Array (implements get and length from Uint8Array). - // Actual getting is abstracted away for eventual reuse. - class LazyUint8Array { - lengthKnown = false; - chunks = []; // Loaded chunks. Index is the chunk number - get(idx) { - if (idx > this.length-1 || idx < 0) { - return undefined; - } - var chunkOffset = idx % this.chunkSize; - var chunkNum = (idx / this.chunkSize)|0; - return this.getter(chunkNum)[chunkOffset]; - } - setDataGetter(getter) { - this.getter = getter; - } - cacheLength() { - // Find length - var xhr = new XMLHttpRequest(); - xhr.open('HEAD', url, false); - xhr.send(null); - if (!(xhr.status >= 200 && xhr.status < 300 || xhr.status === 304)) abort("Couldn't load " + url + ". Status: " + xhr.status); - var datalength = Number(xhr.getResponseHeader("Content-length")); - var header; - var hasByteServing = (header = xhr.getResponseHeader("Accept-Ranges")) && header === "bytes"; - var usesGzip = (header = xhr.getResponseHeader("Content-Encoding")) && header === "gzip"; - - var chunkSize = 1024*1024; // Chunk size in bytes - - if (!hasByteServing) chunkSize = datalength; - - // Function to get a range from the remote URL. - var doXHR = (from, to) => { - if (from > to) abort("invalid range (" + from + ", " + to + ") or no bytes requested!"); - if (to > datalength-1) abort("only " + datalength + " bytes available! programmer error!"); - - // TODO: Use mozResponseArrayBuffer, responseStream, etc. if available. - var xhr = new XMLHttpRequest(); - xhr.open('GET', url, false); - if (datalength !== chunkSize) xhr.setRequestHeader("Range", "bytes=" + from + "-" + to); - - // Some hints to the browser that we want binary data. - xhr.responseType = 'arraybuffer'; - if (xhr.overrideMimeType) { - xhr.overrideMimeType('text/plain; charset=x-user-defined'); - } - - xhr.send(null); - if (!(xhr.status >= 200 && xhr.status < 300 || xhr.status === 304)) abort("Couldn't load " + url + ". Status: " + xhr.status); - if (xhr.response !== undefined) { - return new Uint8Array(/** @type{Array} */(xhr.response || [])); - } - return intArrayFromString(xhr.responseText || '', true); - }; - var lazyArray = this; - lazyArray.setDataGetter((chunkNum) => { - var start = chunkNum * chunkSize; - var end = (chunkNum+1) * chunkSize - 1; // including this byte - end = Math.min(end, datalength-1); // if datalength-1 is selected, this is the last block - if (typeof lazyArray.chunks[chunkNum] == 'undefined') { - lazyArray.chunks[chunkNum] = doXHR(start, end); - } - if (typeof lazyArray.chunks[chunkNum] == 'undefined') abort('doXHR failed!'); - return lazyArray.chunks[chunkNum]; - }); - - if (usesGzip || !datalength) { - // if the server uses gzip or doesn't supply the length, we have to download the whole file to get the (uncompressed) length - chunkSize = datalength = 1; // this will force getter(0)/doXHR do download the whole file - datalength = this.getter(0).length; - chunkSize = datalength; - out("LazyFiles on gzip forces download of the whole file when length is accessed"); - } - - this._length = datalength; - this._chunkSize = chunkSize; - this.lengthKnown = true; - } - get length() { - if (!this.lengthKnown) { - this.cacheLength(); - } - return this._length; - } - get chunkSize() { - if (!this.lengthKnown) { - this.cacheLength(); - } - return this._chunkSize; - } - } - - if (globalThis.XMLHttpRequest) { - if (!ENVIRONMENT_IS_WORKER) abort('Cannot do synchronous binary XHRs outside webworkers in modern browsers. Use --embed-file or --preload-file in emcc'); - var lazyArray = new LazyUint8Array(); - var properties = { isDevice: false, contents: lazyArray }; - } else { - var properties = { isDevice: false, url: url }; - } - - var node = FS.createFile(parent, name, properties, canRead, canWrite); - // This is a total hack, but I want to get this lazy file code out of the - // core of MEMFS. If we want to keep this lazy file concept I feel it should - // be its own thin LAZYFS proxying calls to MEMFS. - if (properties.contents) { - node.contents = properties.contents; - } else if (properties.url) { - node.contents = null; - node.url = properties.url; - } - // Add a function that defers querying the file size until it is asked the first time. - Object.defineProperties(node, { - usedBytes: { - get: function() { return this.contents.length; } - } - }); - // override each stream op with one that tries to force load the lazy file first - var stream_ops = {}; - for (const [key, fn] of Object.entries(node.stream_ops)) { - stream_ops[key] = (...args) => { - FS.forceLoadFile(node); - return fn(...args); - }; - } - function writeChunks(stream, buffer, offset, length, position) { - var contents = stream.node.contents; - if (position >= contents.length) - return 0; - var size = Math.min(contents.length - position, length); - if (contents.slice) { // normal array - for (var i = 0; i < size; i++) { - buffer[offset + i] = contents[position + i]; - } - } else { - for (var i = 0; i < size; i++) { // LazyUint8Array from sync binary XHR - buffer[offset + i] = contents.get(position + i); - } - } - return size; - } - // use a custom read function - stream_ops.read = (stream, buffer, offset, length, position) => { - FS.forceLoadFile(node); - return writeChunks(stream, buffer, offset, length, position) - }; - // use a custom mmap function - stream_ops.mmap = (stream, length, position, prot, flags) => { - FS.forceLoadFile(node); - var ptr = mmapAlloc(length); - if (!ptr) { - throw new FS.ErrnoError(48); - } - writeChunks(stream, HEAP8, ptr, length, position); - return { ptr, allocated: true }; - }; - node.stream_ops = stream_ops; - return node; - }, - }; - - - /** - * Given a pointer 'ptr' to a null-terminated UTF8-encoded string in the - * emscripten HEAP, returns a copy of that string as a Javascript String object. - * - * @param {number} ptr - * @param {number=} maxBytesToRead - An optional length that specifies the - * maximum number of bytes to read. You can omit this parameter to scan the - * string until the first 0 byte. If maxBytesToRead is passed, and the string - * at [ptr, ptr+maxBytesToReadr[ contains a null byte in the middle, then the - * string will cut short at that byte index. - * @param {boolean=} ignoreNul - If true, the function will not stop on a NUL character. - * @return {string} - */ - var UTF8ToString = (ptr, maxBytesToRead, ignoreNul) => { - return ptr ? UTF8ArrayToString(HEAPU8, ptr, maxBytesToRead, ignoreNul) : ''; - }; - var SYSCALLS = { - DEFAULT_POLLMASK:5, - calculateAt(dirfd, path, allowEmpty) { - if (PATH.isAbs(path)) { - return path; - } - // relative path - var dir; - if (dirfd === -100) { - dir = FS.cwd(); - } else { - var dirstream = SYSCALLS.getStreamFromFD(dirfd); - dir = dirstream.path; - } - if (path.length == 0) { - if (!allowEmpty) { - throw new FS.ErrnoError(44);; - } - return dir; - } - return dir + '/' + path; - }, - writeStat(buf, stat) { - HEAPU32[((buf)>>2)] = stat.dev; - HEAPU32[(((buf)+(4))>>2)] = stat.mode; - HEAPU32[(((buf)+(8))>>2)] = stat.nlink; - HEAPU32[(((buf)+(12))>>2)] = stat.uid; - HEAPU32[(((buf)+(16))>>2)] = stat.gid; - HEAPU32[(((buf)+(20))>>2)] = stat.rdev; - HEAP64[(((buf)+(24))>>3)] = BigInt(stat.size); - HEAP32[(((buf)+(32))>>2)] = 4096; - HEAP32[(((buf)+(36))>>2)] = stat.blocks; - var atime = stat.atime.getTime(); - var mtime = stat.mtime.getTime(); - var ctime = stat.ctime.getTime(); - HEAP64[(((buf)+(40))>>3)] = BigInt(Math.floor(atime / 1000)); - HEAPU32[(((buf)+(48))>>2)] = (atime % 1000) * 1000 * 1000; - HEAP64[(((buf)+(56))>>3)] = BigInt(Math.floor(mtime / 1000)); - HEAPU32[(((buf)+(64))>>2)] = (mtime % 1000) * 1000 * 1000; - HEAP64[(((buf)+(72))>>3)] = BigInt(Math.floor(ctime / 1000)); - HEAPU32[(((buf)+(80))>>2)] = (ctime % 1000) * 1000 * 1000; - HEAP64[(((buf)+(88))>>3)] = BigInt(stat.ino); - return 0; - }, - writeStatFs(buf, stats) { - HEAPU32[(((buf)+(4))>>2)] = stats.bsize; - HEAPU32[(((buf)+(60))>>2)] = stats.bsize; - HEAP64[(((buf)+(8))>>3)] = BigInt(stats.blocks); - HEAP64[(((buf)+(16))>>3)] = BigInt(stats.bfree); - HEAP64[(((buf)+(24))>>3)] = BigInt(stats.bavail); - HEAP64[(((buf)+(32))>>3)] = BigInt(stats.files); - HEAP64[(((buf)+(40))>>3)] = BigInt(stats.ffree); - HEAPU32[(((buf)+(48))>>2)] = stats.fsid; - HEAPU32[(((buf)+(64))>>2)] = stats.flags; // ST_NOSUID - HEAPU32[(((buf)+(56))>>2)] = stats.namelen; - }, - doMsync(addr, stream, len, flags, offset) { - if (!FS.isFile(stream.node.mode)) { - throw new FS.ErrnoError(43); - } - if (flags & 2) { - // MAP_PRIVATE calls need not to be synced back to underlying fs - return 0; - } - var buffer = HEAPU8.slice(addr, addr + len); - FS.msync(stream, buffer, offset, len, flags); - }, - getStreamFromFD(fd) { - var stream = FS.getStreamChecked(fd); - return stream; - }, - varargs:undefined, - getStr(ptr) { - var ret = UTF8ToString(ptr); - return ret; - }, - }; - function ___syscall_fcntl64(fd, cmd, varargs) { - SYSCALLS.varargs = varargs; - try { - - var stream = SYSCALLS.getStreamFromFD(fd); - switch (cmd) { - case 0: { - var arg = syscallGetVarargI(); - if (arg < 0) { - return -28; - } - while (FS.streams[arg]) { - arg++; - } - var newStream; - newStream = FS.dupStream(stream, arg); - return newStream.fd; - } - case 1: - case 2: - return 0; // FD_CLOEXEC makes no sense for a single process. - case 3: - return stream.flags; - case 4: { - var arg = syscallGetVarargI(); - stream.flags |= arg; - return 0; - } - case 12: { - var arg = syscallGetVarargP(); - var offset = 0; - // We're always unlocked. - HEAP16[(((arg)+(offset))>>1)] = 2; - return 0; - } - case 13: - case 14: - // Pretend that the locking is successful. These are process-level locks, - // and Emscripten programs are a single process. If we supported linking a - // filesystem between programs, we'd need to do more here. - // See https://github.com/emscripten-core/emscripten/issues/23697 - return 0; - } - return -28; - } catch (e) { - if (typeof FS == 'undefined' || !(e.name === 'ErrnoError')) throw e; - return -e.errno; - } - } - - function ___syscall_fstat64(fd, buf) { - try { - - return SYSCALLS.writeStat(buf, FS.fstat(fd)); - } catch (e) { - if (typeof FS == 'undefined' || !(e.name === 'ErrnoError')) throw e; - return -e.errno; - } - } - - - function ___syscall_ioctl(fd, op, varargs) { - SYSCALLS.varargs = varargs; - try { - - var stream = SYSCALLS.getStreamFromFD(fd); - switch (op) { - case 21509: { - if (!stream.tty) return -59; - return 0; - } - case 21505: { - if (!stream.tty) return -59; - if (stream.tty.ops.ioctl_tcgets) { - var termios = stream.tty.ops.ioctl_tcgets(stream); - var argp = syscallGetVarargP(); - HEAP32[((argp)>>2)] = termios.c_iflag || 0; - HEAP32[(((argp)+(4))>>2)] = termios.c_oflag || 0; - HEAP32[(((argp)+(8))>>2)] = termios.c_cflag || 0; - HEAP32[(((argp)+(12))>>2)] = termios.c_lflag || 0; - for (var i = 0; i < 32; i++) { - HEAP8[(argp + i)+(17)] = termios.c_cc[i] || 0; - } - return 0; - } - return 0; - } - case 21510: - case 21511: - case 21512: { - if (!stream.tty) return -59; - return 0; // no-op, not actually adjusting terminal settings - } - case 21506: - case 21507: - case 21508: { - if (!stream.tty) return -59; - if (stream.tty.ops.ioctl_tcsets) { - var argp = syscallGetVarargP(); - var c_iflag = HEAP32[((argp)>>2)]; - var c_oflag = HEAP32[(((argp)+(4))>>2)]; - var c_cflag = HEAP32[(((argp)+(8))>>2)]; - var c_lflag = HEAP32[(((argp)+(12))>>2)]; - var c_cc = [] - for (var i = 0; i < 32; i++) { - c_cc.push(HEAP8[(argp + i)+(17)]); - } - return stream.tty.ops.ioctl_tcsets(stream.tty, op, { c_iflag, c_oflag, c_cflag, c_lflag, c_cc }); - } - return 0; // no-op, not actually adjusting terminal settings - } - case 21519: { - if (!stream.tty) return -59; - var argp = syscallGetVarargP(); - HEAP32[((argp)>>2)] = 0; - return 0; - } - case 21520: { - if (!stream.tty) return -59; - return -28; // not supported - } - case 21537: - case 21531: { - var argp = syscallGetVarargP(); - return FS.ioctl(stream, op, argp); - } - case 21523: { - // TODO: in theory we should write to the winsize struct that gets - // passed in, but for now musl doesn't read anything on it - if (!stream.tty) return -59; - if (stream.tty.ops.ioctl_tiocgwinsz) { - var winsize = stream.tty.ops.ioctl_tiocgwinsz(stream.tty); - var argp = syscallGetVarargP(); - HEAP16[((argp)>>1)] = winsize[0]; - HEAP16[(((argp)+(2))>>1)] = winsize[1]; - } - return 0; - } - case 21524: { - // TODO: technically, this ioctl call should change the window size. - // but, since emscripten doesn't have any concept of a terminal window - // yet, we'll just silently throw it away as we do TIOCGWINSZ - if (!stream.tty) return -59; - return 0; - } - case 21515: { - if (!stream.tty) return -59; - return 0; - } - default: return -28; // not supported - } - } catch (e) { - if (typeof FS == 'undefined' || !(e.name === 'ErrnoError')) throw e; - return -e.errno; - } - } - - function ___syscall_lstat64(path, buf) { - try { - - path = SYSCALLS.getStr(path); - return SYSCALLS.writeStat(buf, FS.lstat(path)); - } catch (e) { - if (typeof FS == 'undefined' || !(e.name === 'ErrnoError')) throw e; - return -e.errno; - } - } - - function ___syscall_newfstatat(dirfd, path, buf, flags) { - try { - - path = SYSCALLS.getStr(path); - var nofollow = flags & 256; - var allowEmpty = flags & 4096; - flags = flags & (~6400); - path = SYSCALLS.calculateAt(dirfd, path, allowEmpty); - return SYSCALLS.writeStat(buf, nofollow ? FS.lstat(path) : FS.stat(path)); - } catch (e) { - if (typeof FS == 'undefined' || !(e.name === 'ErrnoError')) throw e; - return -e.errno; - } - } - - - function ___syscall_openat(dirfd, path, flags, varargs) { - SYSCALLS.varargs = varargs; - try { - - path = SYSCALLS.getStr(path); - path = SYSCALLS.calculateAt(dirfd, path); - var mode = varargs ? syscallGetVarargI() : 0; - return FS.open(path, flags, mode).fd; - } catch (e) { - if (typeof FS == 'undefined' || !(e.name === 'ErrnoError')) throw e; - return -e.errno; - } - } - - function ___syscall_stat64(path, buf) { - try { - - path = SYSCALLS.getStr(path); - return SYSCALLS.writeStat(buf, FS.stat(path)); - } catch (e) { - if (typeof FS == 'undefined' || !(e.name === 'ErrnoError')) throw e; - return -e.errno; - } - } - - var __abort_js = () => - abort(''); - - var structRegistrations = { - }; - - var runDestructors = (destructors) => { - while (destructors.length) { - var ptr = destructors.pop(); - var del = destructors.pop(); - del(ptr); - } - }; - - /** @suppress {globalThis} */ - function readPointer(pointer) { - return this.fromWireType(HEAPU32[((pointer)>>2)]); - } - - var awaitingDependencies = { - }; - - var registeredTypes = { - }; - - var typeDependencies = { - }; - - var InternalError = class InternalError extends Error { constructor(message) { super(message); this.name = 'InternalError'; }}; - var throwInternalError = (message) => { throw new InternalError(message); }; - var whenDependentTypesAreResolved = (myTypes, dependentTypes, getTypeConverters) => { - myTypes.forEach((type) => typeDependencies[type] = dependentTypes); - - function onComplete(typeConverters) { - var myTypeConverters = getTypeConverters(typeConverters); - if (myTypeConverters.length !== myTypes.length) { - throwInternalError('Mismatched type converter count'); - } - for (var i = 0; i < myTypes.length; ++i) { - registerType(myTypes[i], myTypeConverters[i]); - } - } - - var typeConverters = new Array(dependentTypes.length); - var unregisteredTypes = []; - var registered = 0; - for (let [i, dt] of dependentTypes.entries()) { - if (registeredTypes.hasOwnProperty(dt)) { - typeConverters[i] = registeredTypes[dt]; - } else { - unregisteredTypes.push(dt); - if (!awaitingDependencies.hasOwnProperty(dt)) { - awaitingDependencies[dt] = []; - } - awaitingDependencies[dt].push(() => { - typeConverters[i] = registeredTypes[dt]; - ++registered; - if (registered === unregisteredTypes.length) { - onComplete(typeConverters); - } - }); - } - } - if (0 === unregisteredTypes.length) { - onComplete(typeConverters); - } - }; - var __embind_finalize_value_object = (structType) => { - var reg = structRegistrations[structType]; - delete structRegistrations[structType]; - - var rawConstructor = reg.rawConstructor; - var rawDestructor = reg.rawDestructor; - var fieldRecords = reg.fields; - var fieldTypes = fieldRecords.map((field) => field.getterReturnType). - concat(fieldRecords.map((field) => field.setterArgumentType)); - whenDependentTypesAreResolved([structType], fieldTypes, (fieldTypes) => { - var fields = {}; - for (var [i, field] of fieldRecords.entries()) { - const getterReturnType = fieldTypes[i]; - const getter = field.getter; - const getterContext = field.getterContext; - const setterArgumentType = fieldTypes[i + fieldRecords.length]; - const setter = field.setter; - const setterContext = field.setterContext; - fields[field.fieldName] = { - read: (ptr) => getterReturnType.fromWireType(getter(getterContext, ptr)), - write: (ptr, o) => { - var destructors = []; - setter(setterContext, ptr, setterArgumentType.toWireType(destructors, o)); - runDestructors(destructors); - }, - optional: getterReturnType.optional, - }; - } - - return [{ - name: reg.name, - fromWireType: (ptr) => { - var rv = {}; - for (var i in fields) { - rv[i] = fields[i].read(ptr); - } - rawDestructor(ptr); - return rv; - }, - toWireType: (destructors, o) => { - // todo: Here we have an opportunity for -O3 level "unsafe" optimizations: - // assume all fields are present without checking. - for (var fieldName in fields) { - if (!(fieldName in o) && !fields[fieldName].optional) { - throw new TypeError(`Missing field: "${fieldName}"`); - } - } - var ptr = rawConstructor(); - for (fieldName in fields) { - fields[fieldName].write(ptr, o[fieldName]); - } - if (destructors !== null) { - destructors.push(rawDestructor, ptr); - } - return ptr; - }, - readValueFromPointer: readPointer, - destructorFunction: rawDestructor, - }]; - }); - }; - - var AsciiToString = (ptr) => { - var str = ''; - while (1) { - var ch = HEAPU8[ptr++]; - if (!ch) return str; - str += String.fromCharCode(ch); - } - }; - - - - - var BindingError = class BindingError extends Error { constructor(message) { super(message); this.name = 'BindingError'; }}; - var throwBindingError = (message) => { throw new BindingError(message); }; - /** @param {Object=} options */ - function sharedRegisterType(rawType, registeredInstance, options = {}) { - var name = registeredInstance.name; - if (!rawType) { - throwBindingError(`type "${name}" must have a positive integer typeid pointer`); - } - if (registeredTypes.hasOwnProperty(rawType)) { - if (options.ignoreDuplicateRegistrations) { - return; - } else { - throwBindingError(`Cannot register type '${name}' twice`); - } - } - - registeredTypes[rawType] = registeredInstance; - delete typeDependencies[rawType]; - - if (awaitingDependencies.hasOwnProperty(rawType)) { - var callbacks = awaitingDependencies[rawType]; - delete awaitingDependencies[rawType]; - callbacks.forEach((cb) => cb()); - } - } - /** @param {Object=} options */ - function registerType(rawType, registeredInstance, options = {}) { - return sharedRegisterType(rawType, registeredInstance, options); - } - - var integerReadValueFromPointer = (name, width, signed) => { - // integers are quite common, so generate very specialized functions - switch (width) { - case 1: return signed ? - (pointer) => HEAP8[pointer] : - (pointer) => HEAPU8[pointer]; - case 2: return signed ? - (pointer) => HEAP16[((pointer)>>1)] : - (pointer) => HEAPU16[((pointer)>>1)] - case 4: return signed ? - (pointer) => HEAP32[((pointer)>>2)] : - (pointer) => HEAPU32[((pointer)>>2)] - case 8: return signed ? - (pointer) => HEAP64[((pointer)>>3)] : - (pointer) => HEAPU64[((pointer)>>3)] - default: - throw new TypeError(`invalid integer width (${width}): ${name}`); - } - }; - /** @suppress {globalThis} */ - var __embind_register_bigint = (primitiveType, name, size, minRange, maxRange) => { - name = AsciiToString(name); - - const isUnsignedType = minRange === 0n; - - let fromWireType = (value) => value; - if (isUnsignedType) { - // uint64 get converted to int64 in ABI, fix them up like we do for 32-bit integers. - const bitSize = size * 8; - fromWireType = (value) => { - return BigInt.asUintN(bitSize, value); - } - maxRange = fromWireType(maxRange); - } - - registerType(primitiveType, { - name, - fromWireType: fromWireType, - toWireType: (destructors, value) => { - if (typeof value == "number") { - value = BigInt(value); - } - return value; - }, - readValueFromPointer: integerReadValueFromPointer(name, size, !isUnsignedType), - destructorFunction: null, // This type does not need a destructor - }); - }; - - - /** @suppress {globalThis} */ - var __embind_register_bool = (rawType, name, trueValue, falseValue) => { - name = AsciiToString(name); - registerType(rawType, { - name, - fromWireType: function(wt) { - // ambiguous emscripten ABI: sometimes return values are - // true or false, and sometimes integers (0 or 1) - return !!wt; - }, - toWireType: function(destructors, o) { - return o ? trueValue : falseValue; - }, - readValueFromPointer: function(pointer) { - return this.fromWireType(HEAPU8[pointer]); - }, - destructorFunction: null, // This type does not need a destructor - }); - }; - - - - var shallowCopyInternalPointer = (o) => { - return { - count: o.count, - deleteScheduled: o.deleteScheduled, - preservePointerOnDelete: o.preservePointerOnDelete, - ptr: o.ptr, - ptrType: o.ptrType, - smartPtr: o.smartPtr, - smartPtrType: o.smartPtrType, - }; - }; - - var throwInstanceAlreadyDeleted = (obj) => { - function getInstanceTypeName(handle) { - return handle.$$.ptrType.registeredClass.name; - } - throwBindingError(getInstanceTypeName(obj) + ' instance already deleted'); - }; - - var finalizationRegistry = false; - - var detachFinalizer = (handle) => {}; - - var runDestructor = ($$) => { - if ($$.smartPtr) { - $$.smartPtrType.rawDestructor($$.smartPtr); - } else { - $$.ptrType.registeredClass.rawDestructor($$.ptr); - } - }; - var releaseClassHandle = ($$) => { - $$.count.value -= 1; - var toDelete = 0 === $$.count.value; - if (toDelete) { - runDestructor($$); - } - }; - var attachFinalizer = (handle) => { - if (!globalThis.FinalizationRegistry) { - attachFinalizer = (handle) => handle; - return handle; - } - // If the running environment has a FinalizationRegistry (see - // https://github.com/tc39/proposal-weakrefs), then attach finalizers - // for class handles. We check for the presence of FinalizationRegistry - // at run-time, not build-time. - finalizationRegistry = new FinalizationRegistry((info) => { - releaseClassHandle(info.$$); - }); - attachFinalizer = (handle) => { - var $$ = handle.$$; - var hasSmartPtr = !!$$.smartPtr; - if (hasSmartPtr) { - // We should not call the destructor on raw pointers in case other code expects the pointee to live - var info = { $$: $$ }; - finalizationRegistry.register(handle, info, handle); - } - return handle; - }; - detachFinalizer = (handle) => finalizationRegistry.unregister(handle); - return attachFinalizer(handle); - }; - - - - - var deletionQueue = []; - var flushPendingDeletes = () => { - while (deletionQueue.length) { - var obj = deletionQueue.pop(); - obj.$$.deleteScheduled = false; - obj['delete'](); - } - }; - - var delayFunction; - var init_ClassHandle = () => { - let proto = ClassHandle.prototype; - - Object.assign(proto, { - "isAliasOf"(other) { - if (!(this instanceof ClassHandle)) { - return false; - } - if (!(other instanceof ClassHandle)) { - return false; - } - - var leftClass = this.$$.ptrType.registeredClass; - var left = this.$$.ptr; - other.$$ = /** @type {Object} */ (other.$$); - var rightClass = other.$$.ptrType.registeredClass; - var right = other.$$.ptr; - - while (leftClass.baseClass) { - left = leftClass.upcast(left); - leftClass = leftClass.baseClass; - } - - while (rightClass.baseClass) { - right = rightClass.upcast(right); - rightClass = rightClass.baseClass; - } - - return leftClass === rightClass && left === right; - }, - - "clone"() { - if (!this.$$.ptr) { - throwInstanceAlreadyDeleted(this); - } - - if (this.$$.preservePointerOnDelete) { - this.$$.count.value += 1; - return this; - } else { - var clone = attachFinalizer(Object.create(Object.getPrototypeOf(this), { - $$: { - value: shallowCopyInternalPointer(this.$$), - } - })); - - clone.$$.count.value += 1; - clone.$$.deleteScheduled = false; - return clone; - } - }, - - "delete"() { - if (!this.$$.ptr) { - throwInstanceAlreadyDeleted(this); - } - - if (this.$$.deleteScheduled && !this.$$.preservePointerOnDelete) { - throwBindingError('Object already scheduled for deletion'); - } - - detachFinalizer(this); - releaseClassHandle(this.$$); - - if (!this.$$.preservePointerOnDelete) { - this.$$.smartPtr = undefined; - this.$$.ptr = undefined; - } - }, - - "isDeleted"() { - return !this.$$.ptr; - }, - - "deleteLater"() { - if (!this.$$.ptr) { - throwInstanceAlreadyDeleted(this); - } - if (this.$$.deleteScheduled && !this.$$.preservePointerOnDelete) { - throwBindingError('Object already scheduled for deletion'); - } - deletionQueue.push(this); - if (deletionQueue.length === 1 && delayFunction) { - delayFunction(flushPendingDeletes); - } - this.$$.deleteScheduled = true; - return this; - }, - }); - - // Support `using ...` from https://github.com/tc39/proposal-explicit-resource-management. - const symbolDispose = Symbol.dispose; - if (symbolDispose) { - proto[symbolDispose] = proto['delete']; - } - }; - /** @constructor */ - function ClassHandle() { - } - - var createNamedFunction = (name, func) => Object.defineProperty(func, 'name', { value: name }); - - var registeredPointers = { - }; - - var ensureOverloadTable = (proto, methodName, humanName) => { - if (undefined === proto[methodName].overloadTable) { - var prevFunc = proto[methodName]; - // Inject an overload resolver function that routes to the appropriate overload based on the number of arguments. - proto[methodName] = function(...args) { - // TODO This check can be removed in -O3 level "unsafe" optimizations. - if (!proto[methodName].overloadTable.hasOwnProperty(args.length)) { - throwBindingError(`Function '${humanName}' called with an invalid number of arguments (${args.length}) - expects one of (${proto[methodName].overloadTable})!`); - } - return proto[methodName].overloadTable[args.length].apply(this, args); - }; - // Move the previous function into the overload table. - proto[methodName].overloadTable = []; - proto[methodName].overloadTable[prevFunc.argCount] = prevFunc; - } - }; - - /** @param {number=} numArguments */ - var exposePublicSymbol = (name, value, numArguments) => { - if (Module.hasOwnProperty(name)) { - if (undefined === numArguments || (undefined !== Module[name].overloadTable && undefined !== Module[name].overloadTable[numArguments])) { - throwBindingError(`Cannot register public name '${name}' twice`); - } - - // We are exposing a function with the same name as an existing function. Create an overload table and a function selector - // that routes between the two. - ensureOverloadTable(Module, name, name); - if (Module[name].overloadTable.hasOwnProperty(numArguments)) { - throwBindingError(`Cannot register multiple overloads of a function with the same number of arguments (${numArguments})!`); - } - // Add the new function into the overload table. - Module[name].overloadTable[numArguments] = value; - } else { - Module[name] = value; - Module[name].argCount = numArguments; - } - }; - - var char_0 = 48; - - var char_9 = 57; - var makeLegalFunctionName = (name) => { - name = name.replace(/[^a-zA-Z0-9_]/g, '$'); - var f = name.charCodeAt(0); - if (f >= char_0 && f <= char_9) { - return `_${name}`; - } - return name; - }; - - - /** @constructor */ - function RegisteredClass(name, - constructor, - instancePrototype, - rawDestructor, - baseClass, - getActualType, - upcast, - downcast) { - this.name = name; - this.constructor = constructor; - this.instancePrototype = instancePrototype; - this.rawDestructor = rawDestructor; - this.baseClass = baseClass; - this.getActualType = getActualType; - this.upcast = upcast; - this.downcast = downcast; - this.pureVirtualFunctions = []; - } - - - var upcastPointer = (ptr, ptrClass, desiredClass) => { - while (ptrClass !== desiredClass) { - if (!ptrClass.upcast) { - throwBindingError(`Expected null or instance of ${desiredClass.name}, got an instance of ${ptrClass.name}`); - } - ptr = ptrClass.upcast(ptr); - ptrClass = ptrClass.baseClass; - } - return ptr; - }; - - var embindRepr = (v) => { - if (v === null) { - return 'null'; - } - var t = typeof v; - if (t === 'object' || t === 'array' || t === 'function') { - return v.toString(); - } else { - return '' + v; - } - }; - /** @suppress {globalThis} */ - function constNoSmartPtrRawPointerToWireType(destructors, handle) { - if (handle === null) { - if (this.isReference) { - throwBindingError(`null is not a valid ${this.name}`); - } - return 0; - } - - if (!handle.$$) { - throwBindingError(`Cannot pass "${embindRepr(handle)}" as a ${this.name}`); - } - if (!handle.$$.ptr) { - throwBindingError(`Cannot pass deleted object as a pointer of type ${this.name}`); - } - var handleClass = handle.$$.ptrType.registeredClass; - var ptr = upcastPointer(handle.$$.ptr, handleClass, this.registeredClass); - return ptr; - } - - - /** @suppress {globalThis} */ - function genericPointerToWireType(destructors, handle) { - var ptr; - if (handle === null) { - if (this.isReference) { - throwBindingError(`null is not a valid ${this.name}`); - } - - if (this.isSmartPointer) { - ptr = this.rawConstructor(); - if (destructors !== null) { - destructors.push(this.rawDestructor, ptr); - } - return ptr; - } else { - return 0; - } - } - - if (!handle || !handle.$$) { - throwBindingError(`Cannot pass "${embindRepr(handle)}" as a ${this.name}`); - } - if (!handle.$$.ptr) { - throwBindingError(`Cannot pass deleted object as a pointer of type ${this.name}`); - } - if (!this.isConst && handle.$$.ptrType.isConst) { - throwBindingError(`Cannot convert argument of type ${(handle.$$.smartPtrType ? handle.$$.smartPtrType.name : handle.$$.ptrType.name)} to parameter type ${this.name}`); - } - var handleClass = handle.$$.ptrType.registeredClass; - ptr = upcastPointer(handle.$$.ptr, handleClass, this.registeredClass); - - if (this.isSmartPointer) { - // TODO: this is not strictly true - // We could support BY_EMVAL conversions from raw pointers to smart pointers - // because the smart pointer can hold a reference to the handle - if (undefined === handle.$$.smartPtr) { - throwBindingError('Passing raw pointer to smart pointer is illegal'); - } - - switch (this.sharingPolicy) { - case 0: // NONE - // no upcasting - if (handle.$$.smartPtrType === this) { - ptr = handle.$$.smartPtr; - } else { - throwBindingError(`Cannot convert argument of type ${(handle.$$.smartPtrType ? handle.$$.smartPtrType.name : handle.$$.ptrType.name)} to parameter type ${this.name}`); - } - break; - - case 1: // INTRUSIVE - ptr = handle.$$.smartPtr; - break; - - case 2: // BY_EMVAL - if (handle.$$.smartPtrType === this) { - ptr = handle.$$.smartPtr; - } else { - var clonedHandle = handle['clone'](); - ptr = this.rawShare( - ptr, - Emval.toHandle(() => clonedHandle['delete']()) - ); - if (destructors !== null) { - destructors.push(this.rawDestructor, ptr); - } - } - break; - - default: - throwBindingError('Unsupporting sharing policy'); - } - } - return ptr; - } - - - - /** @suppress {globalThis} */ - function nonConstNoSmartPtrRawPointerToWireType(destructors, handle) { - if (handle === null) { - if (this.isReference) { - throwBindingError(`null is not a valid ${this.name}`); - } - return 0; - } - - if (!handle.$$) { - throwBindingError(`Cannot pass "${embindRepr(handle)}" as a ${this.name}`); - } - if (!handle.$$.ptr) { - throwBindingError(`Cannot pass deleted object as a pointer of type ${this.name}`); - } - if (handle.$$.ptrType.isConst) { - throwBindingError(`Cannot convert argument of type ${handle.$$.ptrType.name} to parameter type ${this.name}`); - } - var handleClass = handle.$$.ptrType.registeredClass; - var ptr = upcastPointer(handle.$$.ptr, handleClass, this.registeredClass); - return ptr; - } - - - - var downcastPointer = (ptr, ptrClass, desiredClass) => { - if (ptrClass === desiredClass) { - return ptr; - } - if (undefined === desiredClass.baseClass) { - return null; // no conversion - } - - var rv = downcastPointer(ptr, ptrClass, desiredClass.baseClass); - if (rv === null) { - return null; - } - return desiredClass.downcast(rv); - }; - - - var registeredInstances = { - }; - - var getBasestPointer = (class_, ptr) => { - if (ptr === undefined) { - throwBindingError('ptr should not be undefined'); - } - while (class_.baseClass) { - ptr = class_.upcast(ptr); - class_ = class_.baseClass; - } - return ptr; - }; - var getInheritedInstance = (class_, ptr) => { - ptr = getBasestPointer(class_, ptr); - return registeredInstances[ptr]; - }; - - - var makeClassHandle = (prototype, record) => { - if (!record.ptrType || !record.ptr) { - throwInternalError('makeClassHandle requires ptr and ptrType'); - } - var hasSmartPtrType = !!record.smartPtrType; - var hasSmartPtr = !!record.smartPtr; - if (hasSmartPtrType !== hasSmartPtr) { - throwInternalError('Both smartPtrType and smartPtr must be specified'); - } - record.count = { value: 1 }; - return attachFinalizer(Object.create(prototype, { - $$: { - value: record, - writable: true, - }, - })); - }; - /** @suppress {globalThis} */ - function RegisteredPointer_fromWireType(ptr) { - // ptr is a raw pointer (or a raw smartpointer) - - // rawPointer is a maybe-null raw pointer - var rawPointer = this.getPointee(ptr); - if (!rawPointer) { - this.destructor(ptr); - return null; - } - - var registeredInstance = getInheritedInstance(this.registeredClass, rawPointer); - if (undefined !== registeredInstance) { - // JS object has been neutered, time to repopulate it - if (0 === registeredInstance.$$.count.value) { - registeredInstance.$$.ptr = rawPointer; - registeredInstance.$$.smartPtr = ptr; - return registeredInstance['clone'](); - } else { - // else, just increment reference count on existing object - // it already has a reference to the smart pointer - var rv = registeredInstance['clone'](); - this.destructor(ptr); - return rv; - } - } - - function makeDefaultHandle() { - if (this.isSmartPointer) { - return makeClassHandle(this.registeredClass.instancePrototype, { - ptrType: this.pointeeType, - ptr: rawPointer, - smartPtrType: this, - smartPtr: ptr, - }); - } else { - return makeClassHandle(this.registeredClass.instancePrototype, { - ptrType: this, - ptr, - }); - } - } - - var actualType = this.registeredClass.getActualType(rawPointer); - var registeredPointerRecord = registeredPointers[actualType]; - if (!registeredPointerRecord) { - return makeDefaultHandle.call(this); - } - - var toType; - if (this.isConst) { - toType = registeredPointerRecord.constPointerType; - } else { - toType = registeredPointerRecord.pointerType; - } - var dp = downcastPointer( - rawPointer, - this.registeredClass, - toType.registeredClass); - if (dp === null) { - return makeDefaultHandle.call(this); - } - if (this.isSmartPointer) { - return makeClassHandle(toType.registeredClass.instancePrototype, { - ptrType: toType, - ptr: dp, - smartPtrType: this, - smartPtr: ptr, - }); - } else { - return makeClassHandle(toType.registeredClass.instancePrototype, { - ptrType: toType, - ptr: dp, - }); - } - } - var init_RegisteredPointer = () => { - Object.assign(RegisteredPointer.prototype, { - getPointee(ptr) { - if (this.rawGetPointee) { - ptr = this.rawGetPointee(ptr); - } - return ptr; - }, - destructor(ptr) { - this.rawDestructor?.(ptr); - }, - readValueFromPointer: readPointer, - fromWireType: RegisteredPointer_fromWireType, - }); - }; - /** @constructor - @param {*=} pointeeType, - @param {*=} sharingPolicy, - @param {*=} rawGetPointee, - @param {*=} rawConstructor, - @param {*=} rawShare, - @param {*=} rawDestructor, - */ - function RegisteredPointer( - name, - registeredClass, - isReference, - isConst, - - // smart pointer properties - isSmartPointer, - pointeeType, - sharingPolicy, - rawGetPointee, - rawConstructor, - rawShare, - rawDestructor - ) { - this.name = name; - this.registeredClass = registeredClass; - this.isReference = isReference; - this.isConst = isConst; - - // smart pointer properties - this.isSmartPointer = isSmartPointer; - this.pointeeType = pointeeType; - this.sharingPolicy = sharingPolicy; - this.rawGetPointee = rawGetPointee; - this.rawConstructor = rawConstructor; - this.rawShare = rawShare; - this.rawDestructor = rawDestructor; - - if (!isSmartPointer && registeredClass.baseClass === undefined) { - if (isConst) { - this.toWireType = constNoSmartPtrRawPointerToWireType; - this.destructorFunction = null; - } else { - this.toWireType = nonConstNoSmartPtrRawPointerToWireType; - this.destructorFunction = null; - } - } else { - this.toWireType = genericPointerToWireType; - // Here we must leave this.destructorFunction undefined, since whether genericPointerToWireType returns - // a pointer that needs to be freed up is runtime-dependent, and cannot be evaluated at registration time. - // TODO: Create an alternative mechanism that allows removing the use of var destructors = []; array in - // craftInvokerFunction altogether. - } - } - - /** @param {number=} numArguments */ - var replacePublicSymbol = (name, value, numArguments) => { - if (!Module.hasOwnProperty(name)) { - throwInternalError('Replacing nonexistent public symbol'); - } - // If there's an overload table for this symbol, replace the symbol in the overload table instead. - if (undefined !== Module[name].overloadTable && undefined !== numArguments) { - Module[name].overloadTable[numArguments] = value; - } else { - Module[name] = value; - Module[name].argCount = numArguments; - } - }; - - - - var wasmTableMirror = []; - - - var getWasmTableEntry = (funcPtr) => { - var func = wasmTableMirror[funcPtr]; - if (!func) { - /** @suppress {checkTypes} */ - wasmTableMirror[funcPtr] = func = wasmTable.get(funcPtr); - } - return func; - }; - var embind__requireFunction = (signature, rawFunction, isAsync = false) => { - - signature = AsciiToString(signature); - - function makeDynCaller() { - var rtn = getWasmTableEntry(rawFunction); - return rtn; - } - - var fp = makeDynCaller(); - if (typeof fp != 'function') { - throwBindingError(`unknown function pointer with signature ${signature}: ${rawFunction}`); - } - return fp; - }; - - - - class UnboundTypeError extends Error {} - - - - var getTypeName = (type) => { - var ptr = ___getTypeName(type); - var rv = AsciiToString(ptr); - _free(ptr); - return rv; - }; - var throwUnboundTypeError = (message, types) => { - var unboundTypes = []; - var seen = {}; - function visit(type) { - if (seen[type]) { - return; - } - if (registeredTypes[type]) { - return; - } - if (typeDependencies[type]) { - typeDependencies[type].forEach(visit); - return; - } - unboundTypes.push(type); - seen[type] = true; - } - types.forEach(visit); - - throw new UnboundTypeError(`${message}: ` + unboundTypes.map(getTypeName).join([', '])); - }; - - var __embind_register_class = (rawType, - rawPointerType, - rawConstPointerType, - baseClassRawType, - getActualTypeSignature, - getActualType, - upcastSignature, - upcast, - downcastSignature, - downcast, - name, - destructorSignature, - rawDestructor) => { - name = AsciiToString(name); - getActualType = embind__requireFunction(getActualTypeSignature, getActualType); - upcast &&= embind__requireFunction(upcastSignature, upcast); - downcast &&= embind__requireFunction(downcastSignature, downcast); - rawDestructor = embind__requireFunction(destructorSignature, rawDestructor); - var legalFunctionName = makeLegalFunctionName(name); - - exposePublicSymbol(legalFunctionName, function() { - // this code cannot run if baseClassRawType is zero - throwUnboundTypeError(`Cannot construct ${name} due to unbound types`, [baseClassRawType]); - }); - - whenDependentTypesAreResolved( - [rawType, rawPointerType, rawConstPointerType], - baseClassRawType ? [baseClassRawType] : [], - (base) => { - base = base[0]; - - var baseClass; - var basePrototype; - if (baseClassRawType) { - baseClass = base.registeredClass; - basePrototype = baseClass.instancePrototype; - } else { - basePrototype = ClassHandle.prototype; - } - - var constructor = createNamedFunction(name, function(...args) { - if (Object.getPrototypeOf(this) !== instancePrototype) { - throw new BindingError(`Use 'new' to construct ${name}`); - } - if (undefined === registeredClass.constructor_body) { - throw new BindingError(`${name} has no accessible constructor`); - } - var body = registeredClass.constructor_body[args.length]; - if (undefined === body) { - throw new BindingError(`Tried to invoke ctor of ${name} with invalid number of parameters (${args.length}) - expected (${Object.keys(registeredClass.constructor_body).toString()}) parameters instead!`); - } - return body.apply(this, args); - }); - - var instancePrototype = Object.create(basePrototype, { - constructor: { value: constructor }, - }); - - constructor.prototype = instancePrototype; - - var registeredClass = new RegisteredClass(name, - constructor, - instancePrototype, - rawDestructor, - baseClass, - getActualType, - upcast, - downcast); - - if (registeredClass.baseClass) { - // Keep track of class hierarchy. Used to allow sub-classes to inherit class functions. - registeredClass.baseClass.__derivedClasses ??= []; - - registeredClass.baseClass.__derivedClasses.push(registeredClass); - } - - var referenceConverter = new RegisteredPointer(name, - registeredClass, - true, - false, - false); - - var pointerConverter = new RegisteredPointer(name + '*', - registeredClass, - false, - false, - false); - - var constPointerConverter = new RegisteredPointer(name + ' const*', - registeredClass, - false, - true, - false); - - registeredPointers[rawType] = { - pointerType: pointerConverter, - constPointerType: constPointerConverter - }; - - replacePublicSymbol(legalFunctionName, constructor); - - return [referenceConverter, pointerConverter, constPointerConverter]; - } - ); - }; - - var heap32VectorToArray = (count, firstElement) => { - var array = []; - for (var i = 0; i < count; i++) { - // TODO(https://github.com/emscripten-core/emscripten/issues/17310): - // Find a way to hoist the `>> 2` or `>> 3` out of this loop. - array.push(HEAPU32[(((firstElement)+(i * 4))>>2)]); - } - return array; - }; - - - - - - - function usesDestructorStack(argTypes) { - // Skip return value at index 0 - it's not deleted here. - for (var i = 1; i < argTypes.length; ++i) { - // The type does not define a destructor function - must use dynamic stack - if (argTypes[i] !== null && argTypes[i].destructorFunction === undefined) { - return true; - } - } - return false; - } - - function createJsInvoker(argTypes, isClassMethodFunc, returns, isAsync) { - var needsDestructorStack = usesDestructorStack(argTypes); - var argCount = argTypes.length - 2; - var argsList = []; - var argsListWired = ['fn']; - if (isClassMethodFunc) { - argsListWired.push('thisWired'); - } - for (var i = 0; i < argCount; ++i) { - argsList.push(`arg${i}`) - argsListWired.push(`arg${i}Wired`) - } - argsList = argsList.join(',') - argsListWired = argsListWired.join(',') - - var invokerFnBody = `return function (${argsList}) {\n`; - - if (needsDestructorStack) { - invokerFnBody += "var destructors = [];\n"; - } - - var dtorStack = needsDestructorStack ? "destructors" : "null"; - var args1 = ["humanName", "throwBindingError", "invoker", "fn", "runDestructors", "fromRetWire", "toClassParamWire"]; - - if (isClassMethodFunc) { - invokerFnBody += `var thisWired = toClassParamWire(${dtorStack}, this);\n`; - } - - for (var i = 0; i < argCount; ++i) { - var argName = `toArg${i}Wire`; - invokerFnBody += `var arg${i}Wired = ${argName}(${dtorStack}, arg${i});\n`; - args1.push(argName); - } - - invokerFnBody += (returns || isAsync ? "var rv = ":"") + `invoker(${argsListWired});\n`; - - var returnVal = returns ? "rv" : ""; - - if (needsDestructorStack) { - invokerFnBody += "runDestructors(destructors);\n"; - } else { - for (var i = isClassMethodFunc?1:2; i < argTypes.length; ++i) { // Skip return value at index 0 - it's not deleted here. Also skip class type if not a method. - var paramName = (i === 1 ? "thisWired" : ("arg"+(i - 2)+"Wired")); - if (argTypes[i].destructorFunction !== null) { - invokerFnBody += `${paramName}_dtor(${paramName});\n`; - args1.push(`${paramName}_dtor`); - } - } - } - - if (returns) { - invokerFnBody += "var ret = fromRetWire(rv);\n" + - "return ret;\n"; - } else { - } - - invokerFnBody += "}\n"; - - return new Function(args1, invokerFnBody); - } - function craftInvokerFunction(humanName, argTypes, classType, cppInvokerFunc, cppTargetFunc, /** boolean= */ isAsync) { - // humanName: a human-readable string name for the function to be generated. - // argTypes: An array that contains the embind type objects for all types in the function signature. - // argTypes[0] is the type object for the function return value. - // argTypes[1] is the type object for function this object/class type, or null if not crafting an invoker for a class method. - // argTypes[2...] are the actual function parameters. - // classType: The embind type object for the class to be bound, or null if this is not a method of a class. - // cppInvokerFunc: JS Function object to the C++-side function that interops into C++ code. - // cppTargetFunc: Function pointer (an integer to FUNCTION_TABLE) to the target C++ function the cppInvokerFunc will end up calling. - // isAsync: Optional. If true, returns an async function. Async bindings are only supported with JSPI. - var argCount = argTypes.length; - - if (argCount < 2) { - throwBindingError("argTypes array size mismatch! Must at least get return value and 'this' types!"); - } - - var isClassMethodFunc = (argTypes[1] !== null && classType !== null); - - // Free functions with signature "void function()" do not need an invoker that marshalls between wire types. - // TODO: This omits argument count check - enable only at -O3 or similar. - // if (ENABLE_UNSAFE_OPTS && argCount == 2 && argTypes[0].name == "void" && !isClassMethodFunc) { - // return FUNCTION_TABLE[fn]; - // } - - // Determine if we need to use a dynamic stack to store the destructors for the function parameters. - // TODO: Remove this completely once all function invokers are being dynamically generated. - var needsDestructorStack = usesDestructorStack(argTypes); - - var returns = !argTypes[0].isVoid; - - var expectedArgCount = argCount - 2; - // Builld the arguments that will be passed into the closure around the invoker - // function. - var retType = argTypes[0]; - var instType = argTypes[1]; - var closureArgs = [humanName, throwBindingError, cppInvokerFunc, cppTargetFunc, runDestructors, retType.fromWireType.bind(retType), instType?.toWireType.bind(instType)]; - for (var i = 2; i < argCount; ++i) { - var argType = argTypes[i]; - closureArgs.push(argType.toWireType.bind(argType)); - } - if (!needsDestructorStack) { - // Skip return value at index 0 - it's not deleted here. Also skip class type if not a method. - for (var i = isClassMethodFunc?1:2; i < argTypes.length; ++i) { - if (argTypes[i].destructorFunction !== null) { - closureArgs.push(argTypes[i].destructorFunction); - } - } - } - - let invokerFactory = createJsInvoker(argTypes, isClassMethodFunc, returns, isAsync); - var invokerFn = invokerFactory(...closureArgs); - return createNamedFunction(humanName, invokerFn); - } - var __embind_register_class_constructor = ( - rawClassType, - argCount, - rawArgTypesAddr, - invokerSignature, - invoker, - rawConstructor - ) => { - var rawArgTypes = heap32VectorToArray(argCount, rawArgTypesAddr); - invoker = embind__requireFunction(invokerSignature, invoker); - var args = [rawConstructor]; - var destructors = []; - - whenDependentTypesAreResolved([], [rawClassType], (classType) => { - classType = classType[0]; - var humanName = `constructor ${classType.name}`; - - if (undefined === classType.registeredClass.constructor_body) { - classType.registeredClass.constructor_body = []; - } - if (undefined !== classType.registeredClass.constructor_body[argCount - 1]) { - throw new BindingError(`Cannot register multiple constructors with identical number of parameters (${argCount-1}) for class '${classType.name}'! Overload resolution is currently only performed using the parameter count, not actual type info!`); - } - classType.registeredClass.constructor_body[argCount - 1] = () => { - throwUnboundTypeError(`Cannot construct ${classType.name} due to unbound types`, rawArgTypes); - }; - - whenDependentTypesAreResolved([], rawArgTypes, (argTypes) => { - // Insert empty slot for context type (argTypes[1]). - argTypes.splice(1, 0, null); - classType.registeredClass.constructor_body[argCount - 1] = craftInvokerFunction(humanName, argTypes, null, invoker, rawConstructor); - return []; - }); - return []; - }); - }; - - - - - - - - var getFunctionName = (signature) => { - signature = signature.trim(); - const argsIndex = signature.indexOf("("); - if (argsIndex === -1) return signature; - return signature.slice(0, argsIndex); - }; - var __embind_register_class_function = (rawClassType, - methodName, - argCount, - rawArgTypesAddr, // [ReturnType, ThisType, Args...] - invokerSignature, - rawInvoker, - context, - isPureVirtual, - isAsync, - isNonnullReturn) => { - var rawArgTypes = heap32VectorToArray(argCount, rawArgTypesAddr); - methodName = AsciiToString(methodName); - methodName = getFunctionName(methodName); - rawInvoker = embind__requireFunction(invokerSignature, rawInvoker, isAsync); - - whenDependentTypesAreResolved([], [rawClassType], (classType) => { - classType = classType[0]; - var humanName = `${classType.name}.${methodName}`; - - if (methodName.startsWith("@@")) { - methodName = Symbol[methodName.substring(2)]; - } - - if (isPureVirtual) { - classType.registeredClass.pureVirtualFunctions.push(methodName); - } - - function unboundTypesHandler() { - throwUnboundTypeError(`Cannot call ${humanName} due to unbound types`, rawArgTypes); - } - - var proto = classType.registeredClass.instancePrototype; - var method = proto[methodName]; - if (undefined === method || (undefined === method.overloadTable && method.className !== classType.name && method.argCount === argCount - 2)) { - // This is the first overload to be registered, OR we are replacing a - // function in the base class with a function in the derived class. - unboundTypesHandler.argCount = argCount - 2; - unboundTypesHandler.className = classType.name; - proto[methodName] = unboundTypesHandler; - } else { - // There was an existing function with the same name registered. Set up - // a function overload routing table. - ensureOverloadTable(proto, methodName, humanName); - proto[methodName].overloadTable[argCount - 2] = unboundTypesHandler; - } - - whenDependentTypesAreResolved([], rawArgTypes, (argTypes) => { - var memberFunction = craftInvokerFunction(humanName, argTypes, classType, rawInvoker, context, isAsync); - - // Replace the initial unbound-handler-stub function with the - // appropriate member function, now that all types are resolved. If - // multiple overloads are registered for this function, the function - // goes into an overload table. - if (undefined === proto[methodName].overloadTable) { - // Set argCount in case an overload is registered later - memberFunction.argCount = argCount - 2; - proto[methodName] = memberFunction; - } else { - proto[methodName].overloadTable[argCount - 2] = memberFunction; - } - - return []; - }); - return []; - }); - }; - - - var __embind_register_constant = (name, type, value) => { - name = AsciiToString(name); - whenDependentTypesAreResolved([], [type], (type) => { - type = type[0]; - Module[name] = type.fromWireType(value); - return []; - }); - }; - - - var emval_freelist = []; - - var emval_handles = [0,1,,1,null,1,true,1,false,1]; - var __emval_decref = (handle) => { - if (handle > 9 && 0 === --emval_handles[handle + 1]) { - emval_handles[handle] = undefined; - emval_freelist.push(handle); - } - }; - - - - var Emval = { - toValue:(handle) => { - if (!handle) { - throwBindingError(`Cannot use deleted val. handle = ${handle}`); - } - return emval_handles[handle]; - }, - toHandle:(value) => { - switch (value) { - case undefined: return 2; - case null: return 4; - case true: return 6; - case false: return 8; - default:{ - const handle = emval_freelist.pop() || emval_handles.length; - emval_handles[handle] = value; - emval_handles[handle + 1] = 1; - return handle; - } - } - }, - }; - - var EmValType = { - name: 'emscripten::val', - fromWireType: (handle) => { - var rv = Emval.toValue(handle); - __emval_decref(handle); - return rv; - }, - toWireType: (destructors, value) => Emval.toHandle(value), - readValueFromPointer: readPointer, - destructorFunction: null, // This type does not need a destructor - - // TODO: do we need a deleteObject here? write a test where - // emval is passed into JS via an interface - }; - var __embind_register_emval = (rawType) => registerType(rawType, EmValType); - - - var enumReadValueFromPointer = (name, width, signed) => { - switch (width) { - case 1: return signed ? - function(pointer) { return this.fromWireType(HEAP8[pointer]) } : - function(pointer) { return this.fromWireType(HEAPU8[pointer]) }; - case 2: return signed ? - function(pointer) { return this.fromWireType(HEAP16[((pointer)>>1)]) } : - function(pointer) { return this.fromWireType(HEAPU16[((pointer)>>1)]) }; - case 4: return signed ? - function(pointer) { return this.fromWireType(HEAP32[((pointer)>>2)]) } : - function(pointer) { return this.fromWireType(HEAPU32[((pointer)>>2)]) }; - default: - throw new TypeError(`invalid integer width (${width}): ${name}`); - } - }; - - - /** @suppress {globalThis} */ - var __embind_register_enum = (rawType, name, size, isSigned) => { - name = AsciiToString(name); - - function ctor() {} - ctor.values = {}; - - registerType(rawType, { - name, - constructor: ctor, - fromWireType: function(c) { - return this.constructor.values[c]; - }, - toWireType: (destructors, c) => c.value, - readValueFromPointer: enumReadValueFromPointer(name, size, isSigned), - destructorFunction: null, - }); - exposePublicSymbol(name, ctor); - }; - - - - - - var requireRegisteredType = (rawType, humanName) => { - var impl = registeredTypes[rawType]; - if (undefined === impl) { - throwBindingError(`${humanName} has unknown type ${getTypeName(rawType)}`); - } - return impl; - }; - var __embind_register_enum_value = (rawEnumType, name, enumValue) => { - var enumType = requireRegisteredType(rawEnumType, 'enum'); - name = AsciiToString(name); - - var Enum = enumType.constructor; - - var Value = Object.create(enumType.constructor.prototype, { - value: {value: enumValue}, - constructor: {value: createNamedFunction(`${enumType.name}_${name}`, function() {})}, - }); - Enum.values[enumValue] = Value; - Enum[name] = Value; - }; - - var floatReadValueFromPointer = (name, width) => { - switch (width) { - case 4: return function(pointer) { - return this.fromWireType(HEAPF32[((pointer)>>2)]); - }; - case 8: return function(pointer) { - return this.fromWireType(HEAPF64[((pointer)>>3)]); - }; - default: - throw new TypeError(`invalid float width (${width}): ${name}`); - } - }; - - - var __embind_register_float = (rawType, name, size) => { - name = AsciiToString(name); - registerType(rawType, { - name, - fromWireType: (value) => value, - toWireType: (destructors, value) => { - // The VM will perform JS to Wasm value conversion, according to the spec: - // https://www.w3.org/TR/wasm-js-api-1/#towebassemblyvalue - return value; - }, - readValueFromPointer: floatReadValueFromPointer(name, size), - destructorFunction: null, // This type does not need a destructor - }); - }; - - - - - - - - - - var __embind_register_function = (name, argCount, rawArgTypesAddr, signature, rawInvoker, fn, isAsync, isNonnullReturn) => { - var argTypes = heap32VectorToArray(argCount, rawArgTypesAddr); - name = AsciiToString(name); - name = getFunctionName(name); - - rawInvoker = embind__requireFunction(signature, rawInvoker, isAsync); - - exposePublicSymbol(name, function() { - throwUnboundTypeError(`Cannot call ${name} due to unbound types`, argTypes); - }, argCount - 1); - - whenDependentTypesAreResolved([], argTypes, (argTypes) => { - var invokerArgsArray = [argTypes[0] /* return value */, null /* no class 'this'*/].concat(argTypes.slice(1) /* actual params */); - replacePublicSymbol(name, craftInvokerFunction(name, invokerArgsArray, null /* no class 'this'*/, rawInvoker, fn, isAsync), argCount - 1); - return []; - }); - }; - - - - /** @suppress {globalThis} */ - var __embind_register_integer = (primitiveType, name, size, minRange, maxRange) => { - name = AsciiToString(name); - - const isUnsignedType = minRange === 0; - - let fromWireType = (value) => value; - if (isUnsignedType) { - var bitshift = 32 - 8*size; - fromWireType = (value) => (value << bitshift) >>> bitshift; - maxRange = fromWireType(maxRange); - } - - registerType(primitiveType, { - name, - fromWireType: fromWireType, - toWireType: (destructors, value) => { - // The VM will perform JS to Wasm value conversion, according to the spec: - // https://www.w3.org/TR/wasm-js-api-1/#towebassemblyvalue - return value; - }, - readValueFromPointer: integerReadValueFromPointer(name, size, minRange !== 0), - destructorFunction: null, // This type does not need a destructor - }); - }; - - - var __embind_register_memory_view = (rawType, dataTypeIndex, name) => { - var typeMapping = [ - Int8Array, - Uint8Array, - Int16Array, - Uint16Array, - Int32Array, - Uint32Array, - Float32Array, - Float64Array, - BigInt64Array, - BigUint64Array, - ]; - - var TA = typeMapping[dataTypeIndex]; - - function decodeMemoryView(handle) { - var size = HEAPU32[((handle)>>2)]; - var data = HEAPU32[(((handle)+(4))>>2)]; - return new TA(HEAP8.buffer, data, size); - } - - name = AsciiToString(name); - registerType(rawType, { - name, - fromWireType: decodeMemoryView, - readValueFromPointer: decodeMemoryView, - }, { - ignoreDuplicateRegistrations: true, - }); - }; - - - - - - var stringToUTF8 = (str, outPtr, maxBytesToWrite) => { - return stringToUTF8Array(str, HEAPU8, outPtr, maxBytesToWrite); - }; - - - - - var __embind_register_std_string = (rawType, name) => { - name = AsciiToString(name); - var stdStringIsUTF8 = true; - - registerType(rawType, { - name, - // For some method names we use string keys here since they are part of - // the public/external API and/or used by the runtime-generated code. - fromWireType(value) { - var length = HEAPU32[((value)>>2)]; - var payload = value + 4; - - var str; - if (stdStringIsUTF8) { - str = UTF8ToString(payload, length, true); - } else { - str = ''; - for (var i = 0; i < length; ++i) { - str += String.fromCharCode(HEAPU8[payload + i]); - } - } - - _free(value); - - return str; - }, - toWireType(destructors, value) { - if (value instanceof ArrayBuffer) { - value = new Uint8Array(value); - } - - var length; - var valueIsOfTypeString = (typeof value == 'string'); - - // We accept `string` or array views with single byte elements - if (!(valueIsOfTypeString || (ArrayBuffer.isView(value) && value.BYTES_PER_ELEMENT == 1))) { - throwBindingError('Cannot pass non-string to std::string'); - } - if (stdStringIsUTF8 && valueIsOfTypeString) { - length = lengthBytesUTF8(value); - } else { - length = value.length; - } - - // assumes POINTER_SIZE alignment - var base = _malloc(4 + length + 1); - var ptr = base + 4; - HEAPU32[((base)>>2)] = length; - if (valueIsOfTypeString) { - if (stdStringIsUTF8) { - stringToUTF8(value, ptr, length + 1); - } else { - for (var i = 0; i < length; ++i) { - var charCode = value.charCodeAt(i); - if (charCode > 255) { - _free(base); - throwBindingError('String has UTF-16 code units that do not fit in 8 bits'); - } - HEAPU8[ptr + i] = charCode; - } - } - } else { - HEAPU8.set(value, ptr); - } - - if (destructors !== null) { - destructors.push(_free, base); - } - return base; - }, - readValueFromPointer: readPointer, - destructorFunction(ptr) { - _free(ptr); - }, - }); - }; - - - - - var UTF16Decoder = globalThis.TextDecoder ? new TextDecoder('utf-16le') : undefined;; - - var UTF16ToString = (ptr, maxBytesToRead, ignoreNul) => { - var idx = ((ptr)>>1); - var endIdx = findStringEnd(HEAPU16, idx, maxBytesToRead / 2, ignoreNul); - - // When using conditional TextDecoder, skip it for short strings as the overhead of the native call is not worth it. - if (endIdx - idx > 16 && UTF16Decoder) - return UTF16Decoder.decode(HEAPU16.subarray(idx, endIdx)); - - // Fallback: decode without UTF16Decoder - var str = ''; - - // If maxBytesToRead is not passed explicitly, it will be undefined, and the - // for-loop's condition will always evaluate to true. The loop is then - // terminated on the first null char. - for (var i = idx; i < endIdx; ++i) { - var codeUnit = HEAPU16[i]; - // fromCharCode constructs a character from a UTF-16 code unit, so we can - // pass the UTF16 string right through. - str += String.fromCharCode(codeUnit); - } - - return str; - }; - - var stringToUTF16 = (str, outPtr, maxBytesToWrite) => { - // Backwards compatibility: if max bytes is not specified, assume unsafe unbounded write is allowed. - maxBytesToWrite ??= 0x7FFFFFFF; - if (maxBytesToWrite < 2) return 0; - maxBytesToWrite -= 2; // Null terminator. - var startPtr = outPtr; - var numCharsToWrite = (maxBytesToWrite < str.length*2) ? (maxBytesToWrite / 2) : str.length; - for (var i = 0; i < numCharsToWrite; ++i) { - // charCodeAt returns a UTF-16 encoded code unit, so it can be directly written to the HEAP. - var codeUnit = str.charCodeAt(i); // possibly a lead surrogate - HEAP16[((outPtr)>>1)] = codeUnit; - outPtr += 2; - } - // Null-terminate the pointer to the HEAP. - HEAP16[((outPtr)>>1)] = 0; - return outPtr - startPtr; - }; - - var lengthBytesUTF16 = (str) => str.length*2; - - var UTF32ToString = (ptr, maxBytesToRead, ignoreNul) => { - var str = ''; - var startIdx = ((ptr)>>2); - // If maxBytesToRead is not passed explicitly, it will be undefined, and this - // will always evaluate to true. This saves on code size. - for (var i = 0; !(i >= maxBytesToRead / 4); i++) { - var utf32 = HEAPU32[startIdx + i]; - if (!utf32 && !ignoreNul) break; - str += String.fromCodePoint(utf32); - } - return str; - }; - - var stringToUTF32 = (str, outPtr, maxBytesToWrite) => { - // Backwards compatibility: if max bytes is not specified, assume unsafe unbounded write is allowed. - maxBytesToWrite ??= 0x7FFFFFFF; - if (maxBytesToWrite < 4) return 0; - var startPtr = outPtr; - var endPtr = startPtr + maxBytesToWrite - 4; - for (var i = 0; i < str.length; ++i) { - var codePoint = str.codePointAt(i); - // Gotcha: if codePoint is over 0xFFFF, it is represented as a surrogate pair in UTF-16. - // We need to manually skip over the second code unit for correct iteration. - if (codePoint > 0xFFFF) { - i++; - } - HEAP32[((outPtr)>>2)] = codePoint; - outPtr += 4; - if (outPtr + 4 > endPtr) break; - } - // Null-terminate the pointer to the HEAP. - HEAP32[((outPtr)>>2)] = 0; - return outPtr - startPtr; - }; - - var lengthBytesUTF32 = (str) => { - var len = 0; - for (var i = 0; i < str.length; ++i) { - var codePoint = str.codePointAt(i); - // Gotcha: if codePoint is over 0xFFFF, it is represented as a surrogate pair in UTF-16. - // We need to manually skip over the second code unit for correct iteration. - if (codePoint > 0xFFFF) { - i++; - } - len += 4; - } - - return len; - }; - var __embind_register_std_wstring = (rawType, charSize, name) => { - name = AsciiToString(name); - var decodeString, encodeString, lengthBytesUTF; - if (charSize === 2) { - decodeString = UTF16ToString; - encodeString = stringToUTF16; - lengthBytesUTF = lengthBytesUTF16; - } else { - decodeString = UTF32ToString; - encodeString = stringToUTF32; - lengthBytesUTF = lengthBytesUTF32; - } - registerType(rawType, { - name, - fromWireType: (value) => { - // Code mostly taken from _embind_register_std_string fromWireType - var length = HEAPU32[((value)>>2)]; - var str = decodeString(value + 4, length * charSize, true); - - _free(value); - - return str; - }, - toWireType: (destructors, value) => { - if (!(typeof value == 'string')) { - throwBindingError(`Cannot pass non-string to C++ string type ${name}`); - } - - // assumes POINTER_SIZE alignment - var length = lengthBytesUTF(value); - var ptr = _malloc(4 + length + charSize); - HEAPU32[((ptr)>>2)] = length / charSize; - - encodeString(value, ptr + 4, length + charSize); - - if (destructors !== null) { - destructors.push(_free, ptr); - } - return ptr; - }, - readValueFromPointer: readPointer, - destructorFunction(ptr) { - _free(ptr); - } - }); - }; - - - - var __embind_register_value_object = ( - rawType, - name, - constructorSignature, - rawConstructor, - destructorSignature, - rawDestructor - ) => { - structRegistrations[rawType] = { - name: AsciiToString(name), - rawConstructor: embind__requireFunction(constructorSignature, rawConstructor), - rawDestructor: embind__requireFunction(destructorSignature, rawDestructor), - fields: [], - }; - }; - - - - var __embind_register_value_object_field = ( - structType, - fieldName, - getterReturnType, - getterSignature, - getter, - getterContext, - setterArgumentType, - setterSignature, - setter, - setterContext - ) => { - structRegistrations[structType].fields.push({ - fieldName: AsciiToString(fieldName), - getterReturnType, - getter: embind__requireFunction(getterSignature, getter), - getterContext, - setterArgumentType, - setter: embind__requireFunction(setterSignature, setter), - setterContext, - }); - }; - - - var __embind_register_void = (rawType, name) => { - name = AsciiToString(name); - registerType(rawType, { - isVoid: true, // void return values can be optimized out sometimes - name, - fromWireType: () => undefined, - // TODO: assert if anything else is given? - toWireType: (destructors, o) => undefined, - }); - }; - - var __emscripten_throw_longjmp = () => { - throw Infinity; - }; - - var emval_methodCallers = []; - var emval_addMethodCaller = (caller) => { - var id = emval_methodCallers.length; - emval_methodCallers.push(caller); - return id; - }; - - var emval_lookupTypes = (argCount, argTypes) => { - var a = new Array(argCount); - for (var i = 0; i < argCount; ++i) { - a[i] = requireRegisteredType(HEAPU32[(((argTypes)+(i*4))>>2)], - `parameter ${i}`); - } - return a; - }; - - - var emval_returnValue = (toReturnWire, destructorsRef, handle) => { - var destructors = []; - var result = toReturnWire(destructors, handle); - if (destructors.length) { - // void, primitives and any other types w/o destructors don't need to allocate a handle - HEAPU32[((destructorsRef)>>2)] = Emval.toHandle(destructors); - } - return result; - }; - - - var emval_symbols = { - }; - - var getStringOrSymbol = (address) => { - var symbol = emval_symbols[address]; - if (symbol === undefined) { - return AsciiToString(address); - } - return symbol; - }; - var __emval_create_invoker = (argCount, argTypesPtr, kind) => { - var GenericWireTypeSize = 8; - - var [retType, ...argTypes] = emval_lookupTypes(argCount, argTypesPtr); - var toReturnWire = retType.toWireType.bind(retType); - var argFromPtr = argTypes.map(type => type.readValueFromPointer.bind(type)); - argCount--; // remove the extracted return type - - var captures = {'toValue': Emval.toValue}; - var args = argFromPtr.map((argFromPtr, i) => { - var captureName = `argFromPtr${i}`; - captures[captureName] = argFromPtr; - return `${captureName}(args${i ? '+' + i * GenericWireTypeSize : ''})`; - }); - var functionBody; - switch (kind){ - case 0: - functionBody = 'toValue(handle)'; - break; - case 2: - functionBody = 'new (toValue(handle))'; - break; - case 3: - functionBody = ''; - break; - case 1: - captures['getStringOrSymbol'] = getStringOrSymbol; - functionBody = 'toValue(handle)[getStringOrSymbol(methodName)]'; - break; - } - functionBody += `(${args})`; - if (!retType.isVoid) { - captures['toReturnWire'] = toReturnWire; - captures['emval_returnValue'] = emval_returnValue; - functionBody = `return emval_returnValue(toReturnWire, destructorsRef, ${functionBody})`; - } - functionBody = `return function (handle, methodName, destructorsRef, args) { - ${functionBody} - }`; - - var invokerFunction = new Function(Object.keys(captures), functionBody)(...Object.values(captures)); - var functionName = `methodCaller<(${argTypes.map(t => t.name)}) => ${retType.name}>`; - return emval_addMethodCaller(createNamedFunction(functionName, invokerFunction)); - }; - - - - var __emval_get_global = (name) => { - if (!name) { - return Emval.toHandle(globalThis); - } - name = getStringOrSymbol(name); - return Emval.toHandle(globalThis[name]); - }; - - - var __emval_get_module_property = (name) => { - name = getStringOrSymbol(name); - return Emval.toHandle(Module[name]); - }; - - var __emval_get_property = (handle, key) => { - handle = Emval.toValue(handle); - key = Emval.toValue(key); - return Emval.toHandle(handle[key]); - }; - - var __emval_incref = (handle) => { - if (handle > 9) { - emval_handles[handle + 1] += 1; - } - }; - - - - var __emval_invoke = (caller, handle, methodName, destructorsRef, args) => { - return emval_methodCallers[caller](handle, methodName, destructorsRef, args); - }; - - - var __emval_new_cstring = (v) => Emval.toHandle(getStringOrSymbol(v)); - - - - var __emval_run_destructors = (handle) => { - var destructors = Emval.toValue(handle); - runDestructors(destructors); - __emval_decref(handle); - }; - - - - - - - var INT53_MAX = 9007199254740992; - - var INT53_MIN = -9007199254740992; - var bigintToI53Checked = (num) => (num < INT53_MIN || num > INT53_MAX) ? NaN : Number(num); - function __mmap_js(len, prot, flags, fd, offset, allocated, addr) { - offset = bigintToI53Checked(offset); - - - try { - - var stream = SYSCALLS.getStreamFromFD(fd); - var res = FS.mmap(stream, len, offset, prot, flags); - var ptr = res.ptr; - HEAP32[((allocated)>>2)] = res.allocated; - HEAPU32[((addr)>>2)] = ptr; - return 0; - } catch (e) { - if (typeof FS == 'undefined' || !(e.name === 'ErrnoError')) throw e; - return -e.errno; - } - ; - } - - - function __munmap_js(addr, len, prot, flags, fd, offset) { - offset = bigintToI53Checked(offset); - - - try { - - var stream = SYSCALLS.getStreamFromFD(fd); - if (prot & 2) { - SYSCALLS.doMsync(addr, stream, len, flags, offset); - } - } catch (e) { - if (typeof FS == 'undefined' || !(e.name === 'ErrnoError')) throw e; - return -e.errno; - } - ; - } - - var __tzset_js = (timezone, daylight, std_name, dst_name) => { - // TODO: Use (malleable) environment variables instead of system settings. - var currentYear = new Date().getFullYear(); - var winter = new Date(currentYear, 0, 1); - var summer = new Date(currentYear, 6, 1); - var winterOffset = winter.getTimezoneOffset(); - var summerOffset = summer.getTimezoneOffset(); - - // Local standard timezone offset. Local standard time is not adjusted for - // daylight savings. This code uses the fact that getTimezoneOffset returns - // a greater value during Standard Time versus Daylight Saving Time (DST). - // Thus it determines the expected output during Standard Time, and it - // compares whether the output of the given date the same (Standard) or less - // (DST). - var stdTimezoneOffset = Math.max(winterOffset, summerOffset); - - // timezone is specified as seconds west of UTC ("The external variable - // `timezone` shall be set to the difference, in seconds, between - // Coordinated Universal Time (UTC) and local standard time."), the same - // as returned by stdTimezoneOffset. - // See http://pubs.opengroup.org/onlinepubs/009695399/functions/tzset.html - HEAPU32[((timezone)>>2)] = stdTimezoneOffset * 60; - - HEAP32[((daylight)>>2)] = Number(winterOffset != summerOffset); - - var extractZone = (timezoneOffset) => { - // Why inverse sign? - // Read here https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/getTimezoneOffset - var sign = timezoneOffset >= 0 ? "-" : "+"; - - var absOffset = Math.abs(timezoneOffset) - var hours = String(Math.floor(absOffset / 60)).padStart(2, "0"); - var minutes = String(absOffset % 60).padStart(2, "0"); - - return `UTC${sign}${hours}${minutes}`; - } - - var winterName = extractZone(winterOffset); - var summerName = extractZone(summerOffset); - if (summerOffset < winterOffset) { - // Northern hemisphere - stringToUTF8(winterName, std_name, 17); - stringToUTF8(summerName, dst_name, 17); - } else { - stringToUTF8(winterName, dst_name, 17); - stringToUTF8(summerName, std_name, 17); - } - }; - - var _emscripten_get_now = () => performance.now(); - - var _emscripten_date_now = () => Date.now(); - - var nowIsMonotonic = 1; - - var checkWasiClock = (clock_id) => clock_id >= 0 && clock_id <= 3; - - function _clock_time_get(clk_id, ignored_precision, ptime) { - ignored_precision = bigintToI53Checked(ignored_precision); - - - if (!checkWasiClock(clk_id)) { - return 28; - } - var now; - // all wasi clocks but realtime are monotonic - if (clk_id === 0) { - now = _emscripten_date_now(); - } else if (nowIsMonotonic) { - now = _emscripten_get_now(); - } else { - return 52; - } - // "now" is in ms, and wasi times are in ns. - var nsec = Math.round(now * 1000 * 1000); - HEAP64[((ptime)>>3)] = BigInt(nsec); - return 0; - ; - } - - - var getHeapMax = () => - // Stay one Wasm page short of 4GB: while e.g. Chrome is able to allocate - // full 4GB Wasm memories, the size will wrap back to 0 bytes in Wasm side - // for any code that deals with heap sizes, which would require special - // casing all heap size related code to treat 0 specially. - 2147483648; - var _emscripten_get_heap_max = () => getHeapMax(); - - - - - var growMemory = (size) => { - var oldHeapSize = wasmMemory.buffer.byteLength; - var pages = ((size - oldHeapSize + 65535) / 65536) | 0; - try { - // round size grow request up to wasm page size (fixed 64KB per spec) - wasmMemory.grow(pages); // .grow() takes a delta compared to the previous size - updateMemoryViews(); - return 1 /*success*/; - } catch(e) { - } - // implicit 0 return to save code size (caller will cast "undefined" into 0 - // anyhow) - }; - var _emscripten_resize_heap = (requestedSize) => { - var oldSize = HEAPU8.length; - // With CAN_ADDRESS_2GB or MEMORY64, pointers are already unsigned. - requestedSize >>>= 0; - // With multithreaded builds, races can happen (another thread might increase the size - // in between), so return a failure, and let the caller retry. - - // Memory resize rules: - // 1. Always increase heap size to at least the requested size, rounded up - // to next page multiple. - // 2a. If MEMORY_GROWTH_LINEAR_STEP == -1, excessively resize the heap - // geometrically: increase the heap size according to - // MEMORY_GROWTH_GEOMETRIC_STEP factor (default +20%), At most - // overreserve by MEMORY_GROWTH_GEOMETRIC_CAP bytes (default 96MB). - // 2b. If MEMORY_GROWTH_LINEAR_STEP != -1, excessively resize the heap - // linearly: increase the heap size by at least - // MEMORY_GROWTH_LINEAR_STEP bytes. - // 3. Max size for the heap is capped at 2048MB-WASM_PAGE_SIZE, or by - // MAXIMUM_MEMORY, or by ASAN limit, depending on which is smallest - // 4. If we were unable to allocate as much memory, it may be due to - // over-eager decision to excessively reserve due to (3) above. - // Hence if an allocation fails, cut down on the amount of excess - // growth, in an attempt to succeed to perform a smaller allocation. - - // A limit is set for how much we can grow. We should not exceed that - // (the wasm binary specifies it, so if we tried, we'd fail anyhow). - var maxHeapSize = getHeapMax(); - if (requestedSize > maxHeapSize) { - return false; - } - - // Loop through potential heap size increases. If we attempt a too eager - // reservation that fails, cut down on the attempted size and reserve a - // smaller bump instead. (max 3 times, chosen somewhat arbitrarily) - for (var cutDown = 1; cutDown <= 4; cutDown *= 2) { - var overGrownHeapSize = oldSize * (1 + 0.2 / cutDown); // ensure geometric growth - // but limit overreserving (default to capping at +96MB overgrowth at most) - overGrownHeapSize = Math.min(overGrownHeapSize, requestedSize + 100663296 ); - - var newSize = Math.min(maxHeapSize, alignMemory(Math.max(requestedSize, overGrownHeapSize), 65536)); - - var replacement = growMemory(newSize); - if (replacement) { - - return true; - } - } - return false; - }; - - var ENV = { - }; - - var getExecutableName = () => thisProgram || './this.program'; - var getEnvStrings = () => { - if (!getEnvStrings.strings) { - // Default values. - // Browser language detection #8751 - var lang = ((typeof navigator == 'object' && navigator.language) || 'C').replace('-', '_') + '.UTF-8'; - var env = { - 'USER': 'web_user', - 'LOGNAME': 'web_user', - 'PATH': '/', - 'PWD': '/', - 'HOME': '/home/web_user', - 'LANG': lang, - '_': getExecutableName() - }; - // Apply the user-provided values, if any. - for (var x in ENV) { - // x is a key in ENV; if ENV[x] is undefined, that means it was - // explicitly set to be so. We allow user code to do that to - // force variables with default values to remain unset. - if (ENV[x] === undefined) delete env[x]; - else env[x] = ENV[x]; - } - var strings = []; - for (var x in env) { - strings.push(`${x}=${env[x]}`); - } - getEnvStrings.strings = strings; - } - return getEnvStrings.strings; - }; - - var _environ_get = (__environ, environ_buf) => { - var bufSize = 0; - var envp = 0; - for (var string of getEnvStrings()) { - var ptr = environ_buf + bufSize; - HEAPU32[(((__environ)+(envp))>>2)] = ptr; - bufSize += stringToUTF8(string, ptr, Infinity) + 1; - envp += 4; - } - return 0; - }; - - - var _environ_sizes_get = (penviron_count, penviron_buf_size) => { - var strings = getEnvStrings(); - HEAPU32[((penviron_count)>>2)] = strings.length; - var bufSize = 0; - for (var string of strings) { - bufSize += lengthBytesUTF8(string) + 1; - } - HEAPU32[((penviron_buf_size)>>2)] = bufSize; - return 0; - }; - - function _fd_close(fd) { - try { - - var stream = SYSCALLS.getStreamFromFD(fd); - FS.close(stream); - return 0; - } catch (e) { - if (typeof FS == 'undefined' || !(e.name === 'ErrnoError')) throw e; - return e.errno; - } - } - - /** @param {number=} offset */ - var doReadv = (stream, iov, iovcnt, offset) => { - var ret = 0; - for (var i = 0; i < iovcnt; i++) { - var ptr = HEAPU32[((iov)>>2)]; - var len = HEAPU32[(((iov)+(4))>>2)]; - iov += 8; - var curr = FS.read(stream, HEAP8, ptr, len, offset); - if (curr < 0) return -1; - ret += curr; - if (curr < len) break; // nothing more to read - if (typeof offset != 'undefined') { - offset += curr; - } - } - return ret; - }; - - function _fd_read(fd, iov, iovcnt, pnum) { - try { - - var stream = SYSCALLS.getStreamFromFD(fd); - var num = doReadv(stream, iov, iovcnt); - HEAPU32[((pnum)>>2)] = num; - return 0; - } catch (e) { - if (typeof FS == 'undefined' || !(e.name === 'ErrnoError')) throw e; - return e.errno; - } - } - - - function _fd_seek(fd, offset, whence, newOffset) { - offset = bigintToI53Checked(offset); - - - try { - - if (isNaN(offset)) return 61; - var stream = SYSCALLS.getStreamFromFD(fd); - FS.llseek(stream, offset, whence); - HEAP64[((newOffset)>>3)] = BigInt(stream.position); - if (stream.getdents && offset === 0 && whence === 0) stream.getdents = null; // reset readdir state - return 0; - } catch (e) { - if (typeof FS == 'undefined' || !(e.name === 'ErrnoError')) throw e; - return e.errno; - } - ; - } - - /** @param {number=} offset */ - var doWritev = (stream, iov, iovcnt, offset) => { - var ret = 0; - for (var i = 0; i < iovcnt; i++) { - var ptr = HEAPU32[((iov)>>2)]; - var len = HEAPU32[(((iov)+(4))>>2)]; - iov += 8; - var curr = FS.write(stream, HEAP8, ptr, len, offset); - if (curr < 0) return -1; - ret += curr; - if (curr < len) { - // No more space to write. - break; - } - if (typeof offset != 'undefined') { - offset += curr; - } - } - return ret; - }; - - function _fd_write(fd, iov, iovcnt, pnum) { - try { - - var stream = SYSCALLS.getStreamFromFD(fd); - var num = doWritev(stream, iov, iovcnt); - HEAPU32[((pnum)>>2)] = num; - return 0; - } catch (e) { - if (typeof FS == 'undefined' || !(e.name === 'ErrnoError')) throw e; - return e.errno; - } - } - - - FS.createPreloadedFile = FS_createPreloadedFile; - FS.preloadFile = FS_preloadFile; - FS.staticInit();; -init_ClassHandle(); -init_RegisteredPointer(); -// End JS library code - -// include: postlibrary.js -// This file is included after the automatically-generated JS library code -// but before the wasm module is created. - -{ - - // Begin ATMODULES hooks - if (Module['noExitRuntime']) noExitRuntime = Module['noExitRuntime']; -if (Module['preloadPlugins']) preloadPlugins = Module['preloadPlugins']; -if (Module['print']) out = Module['print']; -if (Module['printErr']) err = Module['printErr']; -if (Module['wasmBinary']) wasmBinary = Module['wasmBinary']; - // End ATMODULES hooks - - if (Module['arguments']) arguments_ = Module['arguments']; - if (Module['thisProgram']) thisProgram = Module['thisProgram']; - - if (Module['preInit']) { - if (typeof Module['preInit'] == 'function') Module['preInit'] = [Module['preInit']]; - while (Module['preInit'].length > 0) { - Module['preInit'].shift()(); - } - } -} - -// Begin runtime exports - // End runtime exports - // Begin JS library exports - // End JS library exports - -// end include: postlibrary.js - - -// Imports from the Wasm binary. -var ___getTypeName, - _free, - _malloc, - _emscripten_builtin_memalign, - _setThrew, - __emscripten_stack_restore, - __emscripten_stack_alloc, - _emscripten_stack_get_current, - memory, - __indirect_function_table, - wasmMemory, - wasmTable; - - -function assignWasmExports(wasmExports) { - ___getTypeName = wasmExports['__getTypeName']; - _free = wasmExports['free']; - _malloc = wasmExports['malloc']; - _emscripten_builtin_memalign = wasmExports['emscripten_builtin_memalign']; - _setThrew = wasmExports['setThrew']; - __emscripten_stack_restore = wasmExports['_emscripten_stack_restore']; - __emscripten_stack_alloc = wasmExports['_emscripten_stack_alloc']; - _emscripten_stack_get_current = wasmExports['emscripten_stack_get_current']; - memory = wasmMemory = wasmExports['memory']; - __indirect_function_table = wasmTable = wasmExports['__indirect_function_table']; -} - -var wasmImports = { - /** @export */ - __cxa_throw: ___cxa_throw, - /** @export */ - __syscall_fcntl64: ___syscall_fcntl64, - /** @export */ - __syscall_fstat64: ___syscall_fstat64, - /** @export */ - __syscall_ioctl: ___syscall_ioctl, - /** @export */ - __syscall_lstat64: ___syscall_lstat64, - /** @export */ - __syscall_newfstatat: ___syscall_newfstatat, - /** @export */ - __syscall_openat: ___syscall_openat, - /** @export */ - __syscall_stat64: ___syscall_stat64, - /** @export */ - _abort_js: __abort_js, - /** @export */ - _embind_finalize_value_object: __embind_finalize_value_object, - /** @export */ - _embind_register_bigint: __embind_register_bigint, - /** @export */ - _embind_register_bool: __embind_register_bool, - /** @export */ - _embind_register_class: __embind_register_class, - /** @export */ - _embind_register_class_constructor: __embind_register_class_constructor, - /** @export */ - _embind_register_class_function: __embind_register_class_function, - /** @export */ - _embind_register_constant: __embind_register_constant, - /** @export */ - _embind_register_emval: __embind_register_emval, - /** @export */ - _embind_register_enum: __embind_register_enum, - /** @export */ - _embind_register_enum_value: __embind_register_enum_value, - /** @export */ - _embind_register_float: __embind_register_float, - /** @export */ - _embind_register_function: __embind_register_function, - /** @export */ - _embind_register_integer: __embind_register_integer, - /** @export */ - _embind_register_memory_view: __embind_register_memory_view, - /** @export */ - _embind_register_std_string: __embind_register_std_string, - /** @export */ - _embind_register_std_wstring: __embind_register_std_wstring, - /** @export */ - _embind_register_value_object: __embind_register_value_object, - /** @export */ - _embind_register_value_object_field: __embind_register_value_object_field, - /** @export */ - _embind_register_void: __embind_register_void, - /** @export */ - _emscripten_throw_longjmp: __emscripten_throw_longjmp, - /** @export */ - _emval_create_invoker: __emval_create_invoker, - /** @export */ - _emval_decref: __emval_decref, - /** @export */ - _emval_get_global: __emval_get_global, - /** @export */ - _emval_get_module_property: __emval_get_module_property, - /** @export */ - _emval_get_property: __emval_get_property, - /** @export */ - _emval_incref: __emval_incref, - /** @export */ - _emval_invoke: __emval_invoke, - /** @export */ - _emval_new_cstring: __emval_new_cstring, - /** @export */ - _emval_run_destructors: __emval_run_destructors, - /** @export */ - _mmap_js: __mmap_js, - /** @export */ - _munmap_js: __munmap_js, - /** @export */ - _tzset_js: __tzset_js, - /** @export */ - clock_time_get: _clock_time_get, - /** @export */ - emscripten_date_now: _emscripten_date_now, - /** @export */ - emscripten_get_heap_max: _emscripten_get_heap_max, - /** @export */ - emscripten_get_now: _emscripten_get_now, - /** @export */ - emscripten_resize_heap: _emscripten_resize_heap, - /** @export */ - environ_get: _environ_get, - /** @export */ - environ_sizes_get: _environ_sizes_get, - /** @export */ - fd_close: _fd_close, - /** @export */ - fd_read: _fd_read, - /** @export */ - fd_seek: _fd_seek, - /** @export */ - fd_write: _fd_write, - /** @export */ - invoke_ii, - /** @export */ - invoke_vi, - /** @export */ - invoke_vii, - /** @export */ - invoke_viii -}; - -function invoke_vi(index,a1) { - var sp = stackSave(); - try { - getWasmTableEntry(index)(a1); - } catch(e) { - stackRestore(sp); - if (e !== e+0) throw e; - _setThrew(1, 0); - } -} - -function invoke_viii(index,a1,a2,a3) { - var sp = stackSave(); - try { - getWasmTableEntry(index)(a1,a2,a3); - } catch(e) { - stackRestore(sp); - if (e !== e+0) throw e; - _setThrew(1, 0); - } -} - -function invoke_ii(index,a1) { - var sp = stackSave(); - try { - return getWasmTableEntry(index)(a1); - } catch(e) { - stackRestore(sp); - if (e !== e+0) throw e; - _setThrew(1, 0); - } -} - -function invoke_vii(index,a1,a2) { - var sp = stackSave(); - try { - getWasmTableEntry(index)(a1,a2); - } catch(e) { - stackRestore(sp); - if (e !== e+0) throw e; - _setThrew(1, 0); - } -} - - -// include: postamble.js -// === Auto-generated postamble setup entry stuff === - -function run() { - - if (runDependencies > 0) { - dependenciesFulfilled = run; - return; - } - - preRun(); - - // a preRun added a dependency, run will be called later - if (runDependencies > 0) { - dependenciesFulfilled = run; - return; - } - - function doRun() { - // run may have just been called through dependencies being fulfilled just in this very frame, - // or while the async setStatus time below was happening - Module['calledRun'] = true; - - if (ABORT) return; - - initRuntime(); - - readyPromiseResolve?.(Module); - Module['onRuntimeInitialized']?.(); - - postRun(); - } - - if (Module['setStatus']) { - Module['setStatus']('Running...'); - setTimeout(() => { - setTimeout(() => Module['setStatus'](''), 1); - doRun(); - }, 1); - } else - { - doRun(); - } -} - -var wasmExports; - -// In modularize mode the generated code is within a factory function so we -// can use await here (since it's not top-level-await). -wasmExports = await (createWasm()); - -run(); - -// end include: postamble.js - -// include: postamble_modularize.js -// In MODULARIZE mode we wrap the generated code in a factory function -// and return either the Module itself, or a promise of the module. -// -// We assign to the `moduleRtn` global here and configure closure to see -// this as and extern so it won't get minified. - -if (runtimeInitialized) { - moduleRtn = Module; -} else { - // Set up the promise that indicates the Module is initialized - moduleRtn = new Promise((resolve, reject) => { - readyPromiseResolve = resolve; - readyPromiseReject = reject; - }); -} - -// end include: postamble_modularize.js - - - - return moduleRtn; - }; -})(); - -// Export using a UMD style export, or ES6 exports if selected -if (typeof exports === 'object' && typeof module === 'object') { - module.exports = BASIS; - // This default export looks redundant, but it allows TS to import this - // commonjs style module. - module.exports.default = BASIS; -} else if (typeof define === 'function' && define['amd']) - define([], () => BASIS); +var BASIS=(()=>{var _scriptName=globalThis.document?.currentScript?.src;return async function(moduleArg={}){var moduleRtn;var Module=moduleArg;var ENVIRONMENT_IS_WEB=!!globalThis.window;var ENVIRONMENT_IS_WORKER=!!globalThis.WorkerGlobalScope;var ENVIRONMENT_IS_NODE=globalThis.process?.versions?.node&&globalThis.process?.type!="renderer";var arguments_=[];var thisProgram="./this.program";var quit_=(status,toThrow)=>{throw toThrow};if(typeof __filename!="undefined"){_scriptName=__filename}else if(ENVIRONMENT_IS_WORKER){_scriptName=self.location.href}var scriptDirectory="";function locateFile(path){if(Module["locateFile"]){return Module["locateFile"](path,scriptDirectory)}return scriptDirectory+path}var readAsync,readBinary;if(ENVIRONMENT_IS_NODE){var fs=require("fs");scriptDirectory=__dirname+"/";readBinary=filename=>{filename=isFileURI(filename)?new URL(filename):filename;var ret=fs.readFileSync(filename);return ret};readAsync=async(filename,binary=true)=>{filename=isFileURI(filename)?new URL(filename):filename;var ret=fs.readFileSync(filename,binary?undefined:"utf8");return ret};if(process.argv.length>1){thisProgram=process.argv[1].replace(/\\/g,"/")}arguments_=process.argv.slice(2);quit_=(status,toThrow)=>{process.exitCode=status;throw toThrow}}else if(ENVIRONMENT_IS_WEB||ENVIRONMENT_IS_WORKER){try{scriptDirectory=new URL(".",_scriptName).href}catch{}{if(ENVIRONMENT_IS_WORKER){readBinary=url=>{var xhr=new XMLHttpRequest;xhr.open("GET",url,false);xhr.responseType="arraybuffer";xhr.send(null);return new Uint8Array(xhr.response)}}readAsync=async url=>{if(isFileURI(url)){return new Promise((resolve,reject)=>{var xhr=new XMLHttpRequest;xhr.open("GET",url,true);xhr.responseType="arraybuffer";xhr.onload=()=>{if(xhr.status==200||xhr.status==0&&xhr.response){resolve(xhr.response);return}reject(xhr.status)};xhr.onerror=reject;xhr.send(null)})}var response=await fetch(url,{credentials:"same-origin"});if(response.ok){return response.arrayBuffer()}throw new Error(response.status+" : "+response.url)}}}else{}var out=console.log.bind(console);var err=console.error.bind(console);var wasmBinary;var ABORT=false;var isFileURI=filename=>filename.startsWith("file://");var readyPromiseResolve,readyPromiseReject;var HEAP8,HEAPU8,HEAP16,HEAPU16,HEAP32,HEAPU32,HEAPF32,HEAPF64;var HEAP64,HEAPU64;var runtimeInitialized=false;function updateMemoryViews(){var b=wasmMemory.buffer;Module["HEAP8"]=HEAP8=new Int8Array(b);HEAP16=new Int16Array(b);HEAPU8=new Uint8Array(b);HEAPU16=new Uint16Array(b);HEAP32=new Int32Array(b);HEAPU32=new Uint32Array(b);HEAPF32=new Float32Array(b);HEAPF64=new Float64Array(b);HEAP64=new BigInt64Array(b);HEAPU64=new BigUint64Array(b)}function preRun(){if(Module["preRun"]){if(typeof Module["preRun"]=="function")Module["preRun"]=[Module["preRun"]];while(Module["preRun"].length){addOnPreRun(Module["preRun"].shift())}}callRuntimeCallbacks(onPreRuns)}function initRuntime(){runtimeInitialized=true;if(!Module["noFSInit"]&&!FS.initialized)FS.init();TTY.init();wasmExports["$"]();FS.ignorePermissions=false}function postRun(){if(Module["postRun"]){if(typeof Module["postRun"]=="function")Module["postRun"]=[Module["postRun"]];while(Module["postRun"].length){addOnPostRun(Module["postRun"].shift())}}callRuntimeCallbacks(onPostRuns)}function abort(what){Module["onAbort"]?.(what);what="Aborted("+what+")";err(what);ABORT=true;what+=". Build with -sASSERTIONS for more info.";var e=new WebAssembly.RuntimeError(what);readyPromiseReject?.(e);throw e}var wasmBinaryFile;function findWasmBinary(){return locateFile("basis_encoder.wasm")}function getBinarySync(file){if(file==wasmBinaryFile&&wasmBinary){return new Uint8Array(wasmBinary)}if(readBinary){return readBinary(file)}throw"both async and sync fetching of the wasm failed"}async function getWasmBinary(binaryFile){if(!wasmBinary){try{var response=await readAsync(binaryFile);return new Uint8Array(response)}catch{}}return getBinarySync(binaryFile)}async function instantiateArrayBuffer(binaryFile,imports){try{var binary=await getWasmBinary(binaryFile);var instance=await WebAssembly.instantiate(binary,imports);return instance}catch(reason){err(`failed to asynchronously prepare wasm: ${reason}`);abort(reason)}}async function instantiateAsync(binary,binaryFile,imports){if(!binary&&!isFileURI(binaryFile)&&!ENVIRONMENT_IS_NODE){try{var response=fetch(binaryFile,{credentials:"same-origin"});var instantiationResult=await WebAssembly.instantiateStreaming(response,imports);return instantiationResult}catch(reason){err(`wasm streaming compile failed: ${reason}`);err("falling back to ArrayBuffer instantiation")}}return instantiateArrayBuffer(binaryFile,imports)}function getWasmImports(){var imports={a:wasmImports};return imports}async function createWasm(){function receiveInstance(instance,module){wasmExports=instance.exports;assignWasmExports(wasmExports);updateMemoryViews();return wasmExports}function receiveInstantiationResult(result){return receiveInstance(result["instance"])}var info=getWasmImports();if(Module["instantiateWasm"]){return new Promise((resolve,reject)=>{Module["instantiateWasm"](info,(inst,mod)=>{resolve(receiveInstance(inst,mod))})})}wasmBinaryFile??=findWasmBinary();var result=await instantiateAsync(wasmBinary,wasmBinaryFile,info);var exports=receiveInstantiationResult(result);return exports}class ExitStatus{name="ExitStatus";constructor(status){this.message=`Program terminated with exit(${status})`;this.status=status}}var callRuntimeCallbacks=callbacks=>{while(callbacks.length>0){callbacks.shift()(Module)}};var onPostRuns=[];var addOnPostRun=cb=>onPostRuns.push(cb);var onPreRuns=[];var addOnPreRun=cb=>onPreRuns.push(cb);var noExitRuntime=true;var stackRestore=val=>__emscripten_stack_restore(val);var stackSave=()=>_emscripten_stack_get_current();class ExceptionInfo{constructor(excPtr){this.excPtr=excPtr;this.ptr=excPtr-24}set_type(type){HEAPU32[this.ptr+4>>2]=type}get_type(){return HEAPU32[this.ptr+4>>2]}set_destructor(destructor){HEAPU32[this.ptr+8>>2]=destructor}get_destructor(){return HEAPU32[this.ptr+8>>2]}set_caught(caught){caught=caught?1:0;HEAP8[this.ptr+12]=caught}get_caught(){return HEAP8[this.ptr+12]!=0}set_rethrown(rethrown){rethrown=rethrown?1:0;HEAP8[this.ptr+13]=rethrown}get_rethrown(){return HEAP8[this.ptr+13]!=0}init(type,destructor){this.set_adjusted_ptr(0);this.set_type(type);this.set_destructor(destructor)}set_adjusted_ptr(adjustedPtr){HEAPU32[this.ptr+16>>2]=adjustedPtr}get_adjusted_ptr(){return HEAPU32[this.ptr+16>>2]}}var exceptionLast=0;var uncaughtExceptionCount=0;var ___cxa_throw=(ptr,type,destructor)=>{var info=new ExceptionInfo(ptr);info.init(type,destructor);exceptionLast=ptr;uncaughtExceptionCount++;throw exceptionLast};var syscallGetVarargI=()=>{var ret=HEAP32[+SYSCALLS.varargs>>2];SYSCALLS.varargs+=4;return ret};var syscallGetVarargP=syscallGetVarargI;var PATH={isAbs:path=>path.charAt(0)==="/",splitPath:filename=>{var splitPathRe=/^(\/?|)([\s\S]*?)((?:\.{1,2}|[^\/]+?|)(\.[^.\/]*|))(?:[\/]*)$/;return splitPathRe.exec(filename).slice(1)},normalizeArray:(parts,allowAboveRoot)=>{var up=0;for(var i=parts.length-1;i>=0;i--){var last=parts[i];if(last==="."){parts.splice(i,1)}else if(last===".."){parts.splice(i,1);up++}else if(up){parts.splice(i,1);up--}}if(allowAboveRoot){for(;up;up--){parts.unshift("..")}}return parts},normalize:path=>{var isAbsolute=PATH.isAbs(path),trailingSlash=path.slice(-1)==="/";path=PATH.normalizeArray(path.split("/").filter(p=>!!p),!isAbsolute).join("/");if(!path&&!isAbsolute){path="."}if(path&&trailingSlash){path+="/"}return(isAbsolute?"/":"")+path},dirname:path=>{var result=PATH.splitPath(path),root=result[0],dir=result[1];if(!root&&!dir){return"."}if(dir){dir=dir.slice(0,-1)}return root+dir},basename:path=>path&&path.match(/([^\/]+|\/)\/*$/)[1],join:(...paths)=>PATH.normalize(paths.join("/")),join2:(l,r)=>PATH.normalize(l+"/"+r)};var initRandomFill=()=>{if(ENVIRONMENT_IS_NODE){var nodeCrypto=require("crypto");return view=>nodeCrypto.randomFillSync(view)}return view=>crypto.getRandomValues(view)};var randomFill=view=>{(randomFill=initRandomFill())(view)};var PATH_FS={resolve:(...args)=>{var resolvedPath="",resolvedAbsolute=false;for(var i=args.length-1;i>=-1&&!resolvedAbsolute;i--){var path=i>=0?args[i]:FS.cwd();if(typeof path!="string"){throw new TypeError("Arguments to path.resolve must be strings")}else if(!path){return""}resolvedPath=path+"/"+resolvedPath;resolvedAbsolute=PATH.isAbs(path)}resolvedPath=PATH.normalizeArray(resolvedPath.split("/").filter(p=>!!p),!resolvedAbsolute).join("/");return(resolvedAbsolute?"/":"")+resolvedPath||"."},relative:(from,to)=>{from=PATH_FS.resolve(from).slice(1);to=PATH_FS.resolve(to).slice(1);function trim(arr){var start=0;for(;start=0;end--){if(arr[end]!=="")break}if(start>end)return[];return arr.slice(start,end-start+1)}var fromParts=trim(from.split("/"));var toParts=trim(to.split("/"));var length=Math.min(fromParts.length,toParts.length);var samePartsLength=length;for(var i=0;i{var maxIdx=idx+maxBytesToRead;if(ignoreNul)return maxIdx;while(heapOrArray[idx]&&!(idx>=maxIdx))++idx;return idx};var UTF8ArrayToString=(heapOrArray,idx=0,maxBytesToRead,ignoreNul)=>{var endPtr=findStringEnd(heapOrArray,idx,maxBytesToRead,ignoreNul);if(endPtr-idx>16&&heapOrArray.buffer&&UTF8Decoder){return UTF8Decoder.decode(heapOrArray.subarray(idx,endPtr))}var str="";while(idx>10,56320|ch&1023)}}return str};var FS_stdin_getChar_buffer=[];var lengthBytesUTF8=str=>{var len=0;for(var i=0;i=55296&&c<=57343){len+=4;++i}else{len+=3}}return len};var stringToUTF8Array=(str,heap,outIdx,maxBytesToWrite)=>{if(!(maxBytesToWrite>0))return 0;var startIdx=outIdx;var endIdx=outIdx+maxBytesToWrite-1;for(var i=0;i=endIdx)break;heap[outIdx++]=u}else if(u<=2047){if(outIdx+1>=endIdx)break;heap[outIdx++]=192|u>>6;heap[outIdx++]=128|u&63}else if(u<=65535){if(outIdx+2>=endIdx)break;heap[outIdx++]=224|u>>12;heap[outIdx++]=128|u>>6&63;heap[outIdx++]=128|u&63}else{if(outIdx+3>=endIdx)break;heap[outIdx++]=240|u>>18;heap[outIdx++]=128|u>>12&63;heap[outIdx++]=128|u>>6&63;heap[outIdx++]=128|u&63;i++}}heap[outIdx]=0;return outIdx-startIdx};var intArrayFromString=(stringy,dontAddNull,length)=>{var len=length>0?length:lengthBytesUTF8(stringy)+1;var u8array=new Array(len);var numBytesWritten=stringToUTF8Array(stringy,u8array,0,u8array.length);if(dontAddNull)u8array.length=numBytesWritten;return u8array};var FS_stdin_getChar=()=>{if(!FS_stdin_getChar_buffer.length){var result=null;if(ENVIRONMENT_IS_NODE){var BUFSIZE=256;var buf=Buffer.alloc(BUFSIZE);var bytesRead=0;var fd=process.stdin.fd;try{bytesRead=fs.readSync(fd,buf,0,BUFSIZE)}catch(e){if(e.toString().includes("EOF"))bytesRead=0;else throw e}if(bytesRead>0){result=buf.slice(0,bytesRead).toString("utf-8")}}else if(globalThis.window?.prompt){result=window.prompt("Input: ");if(result!==null){result+="\n"}}else{}if(!result){return null}FS_stdin_getChar_buffer=intArrayFromString(result,true)}return FS_stdin_getChar_buffer.shift()};var TTY={ttys:[],init(){},shutdown(){},register(dev,ops){TTY.ttys[dev]={input:[],output:[],ops};FS.registerDevice(dev,TTY.stream_ops)},stream_ops:{open(stream){var tty=TTY.ttys[stream.node.rdev];if(!tty){throw new FS.ErrnoError(43)}stream.tty=tty;stream.seekable=false},close(stream){stream.tty.ops.fsync(stream.tty)},fsync(stream){stream.tty.ops.fsync(stream.tty)},read(stream,buffer,offset,length,pos){if(!stream.tty||!stream.tty.ops.get_char){throw new FS.ErrnoError(60)}var bytesRead=0;for(var i=0;i0){out(UTF8ArrayToString(tty.output));tty.output=[]}},ioctl_tcgets(tty){return{c_iflag:25856,c_oflag:5,c_cflag:191,c_lflag:35387,c_cc:[3,28,127,21,4,0,1,0,17,19,26,0,18,15,23,22,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]}},ioctl_tcsets(tty,optional_actions,data){return 0},ioctl_tiocgwinsz(tty){return[24,80]}},default_tty1_ops:{put_char(tty,val){if(val===null||val===10){err(UTF8ArrayToString(tty.output));tty.output=[]}else{if(val!=0)tty.output.push(val)}},fsync(tty){if(tty.output?.length>0){err(UTF8ArrayToString(tty.output));tty.output=[]}}}};var zeroMemory=(ptr,size)=>HEAPU8.fill(0,ptr,ptr+size);var alignMemory=(size,alignment)=>Math.ceil(size/alignment)*alignment;var mmapAlloc=size=>{size=alignMemory(size,65536);var ptr=_emscripten_builtin_memalign(65536,size);if(ptr)zeroMemory(ptr,size);return ptr};var MEMFS={ops_table:null,mount(mount){return MEMFS.createNode(null,"/",16895,0)},createNode(parent,name,mode,dev){if(FS.isBlkdev(mode)||FS.isFIFO(mode)){throw new FS.ErrnoError(63)}MEMFS.ops_table||={dir:{node:{getattr:MEMFS.node_ops.getattr,setattr:MEMFS.node_ops.setattr,lookup:MEMFS.node_ops.lookup,mknod:MEMFS.node_ops.mknod,rename:MEMFS.node_ops.rename,unlink:MEMFS.node_ops.unlink,rmdir:MEMFS.node_ops.rmdir,readdir:MEMFS.node_ops.readdir,symlink:MEMFS.node_ops.symlink},stream:{llseek:MEMFS.stream_ops.llseek}},file:{node:{getattr:MEMFS.node_ops.getattr,setattr:MEMFS.node_ops.setattr},stream:{llseek:MEMFS.stream_ops.llseek,read:MEMFS.stream_ops.read,write:MEMFS.stream_ops.write,mmap:MEMFS.stream_ops.mmap,msync:MEMFS.stream_ops.msync}},link:{node:{getattr:MEMFS.node_ops.getattr,setattr:MEMFS.node_ops.setattr,readlink:MEMFS.node_ops.readlink},stream:{}},chrdev:{node:{getattr:MEMFS.node_ops.getattr,setattr:MEMFS.node_ops.setattr},stream:FS.chrdev_stream_ops}};var node=FS.createNode(parent,name,mode,dev);if(FS.isDir(node.mode)){node.node_ops=MEMFS.ops_table.dir.node;node.stream_ops=MEMFS.ops_table.dir.stream;node.contents={}}else if(FS.isFile(node.mode)){node.node_ops=MEMFS.ops_table.file.node;node.stream_ops=MEMFS.ops_table.file.stream;node.usedBytes=0;node.contents=null}else if(FS.isLink(node.mode)){node.node_ops=MEMFS.ops_table.link.node;node.stream_ops=MEMFS.ops_table.link.stream}else if(FS.isChrdev(node.mode)){node.node_ops=MEMFS.ops_table.chrdev.node;node.stream_ops=MEMFS.ops_table.chrdev.stream}node.atime=node.mtime=node.ctime=Date.now();if(parent){parent.contents[name]=node;parent.atime=parent.mtime=parent.ctime=node.atime}return node},getFileDataAsTypedArray(node){if(!node.contents)return new Uint8Array(0);if(node.contents.subarray)return node.contents.subarray(0,node.usedBytes);return new Uint8Array(node.contents)},expandFileStorage(node,newCapacity){var prevCapacity=node.contents?node.contents.length:0;if(prevCapacity>=newCapacity)return;var CAPACITY_DOUBLING_MAX=1024*1024;newCapacity=Math.max(newCapacity,prevCapacity*(prevCapacity>>0);if(prevCapacity!=0)newCapacity=Math.max(newCapacity,256);var oldContents=node.contents;node.contents=new Uint8Array(newCapacity);if(node.usedBytes>0)node.contents.set(oldContents.subarray(0,node.usedBytes),0)},resizeFileStorage(node,newSize){if(node.usedBytes==newSize)return;if(newSize==0){node.contents=null;node.usedBytes=0}else{var oldContents=node.contents;node.contents=new Uint8Array(newSize);if(oldContents){node.contents.set(oldContents.subarray(0,Math.min(newSize,node.usedBytes)))}node.usedBytes=newSize}},node_ops:{getattr(node){var attr={};attr.dev=FS.isChrdev(node.mode)?node.id:1;attr.ino=node.id;attr.mode=node.mode;attr.nlink=1;attr.uid=0;attr.gid=0;attr.rdev=node.rdev;if(FS.isDir(node.mode)){attr.size=4096}else if(FS.isFile(node.mode)){attr.size=node.usedBytes}else if(FS.isLink(node.mode)){attr.size=node.link.length}else{attr.size=0}attr.atime=new Date(node.atime);attr.mtime=new Date(node.mtime);attr.ctime=new Date(node.ctime);attr.blksize=4096;attr.blocks=Math.ceil(attr.size/attr.blksize);return attr},setattr(node,attr){for(const key of["mode","atime","mtime","ctime"]){if(attr[key]!=null){node[key]=attr[key]}}if(attr.size!==undefined){MEMFS.resizeFileStorage(node,attr.size)}},lookup(parent,name){if(!MEMFS.doesNotExistError){MEMFS.doesNotExistError=new FS.ErrnoError(44);MEMFS.doesNotExistError.stack=""}throw MEMFS.doesNotExistError},mknod(parent,name,mode,dev){return MEMFS.createNode(parent,name,mode,dev)},rename(old_node,new_dir,new_name){var new_node;try{new_node=FS.lookupNode(new_dir,new_name)}catch(e){}if(new_node){if(FS.isDir(old_node.mode)){for(var i in new_node.contents){throw new FS.ErrnoError(55)}}FS.hashRemoveNode(new_node)}delete old_node.parent.contents[old_node.name];new_dir.contents[new_name]=old_node;old_node.name=new_name;new_dir.ctime=new_dir.mtime=old_node.parent.ctime=old_node.parent.mtime=Date.now()},unlink(parent,name){delete parent.contents[name];parent.ctime=parent.mtime=Date.now()},rmdir(parent,name){var node=FS.lookupNode(parent,name);for(var i in node.contents){throw new FS.ErrnoError(55)}delete parent.contents[name];parent.ctime=parent.mtime=Date.now()},readdir(node){return[".","..",...Object.keys(node.contents)]},symlink(parent,newname,oldpath){var node=MEMFS.createNode(parent,newname,511|40960,0);node.link=oldpath;return node},readlink(node){if(!FS.isLink(node.mode)){throw new FS.ErrnoError(28)}return node.link}},stream_ops:{read(stream,buffer,offset,length,position){var contents=stream.node.contents;if(position>=stream.node.usedBytes)return 0;var size=Math.min(stream.node.usedBytes-position,length);if(size>8&&contents.subarray){buffer.set(contents.subarray(position,position+size),offset)}else{for(var i=0;i0||position+length{var flagModes={r:0,"r+":2,w:512|64|1,"w+":512|64|2,a:1024|64|1,"a+":1024|64|2};var flags=flagModes[str];if(typeof flags=="undefined"){throw new Error(`Unknown file open mode: ${str}`)}return flags};var FS_getMode=(canRead,canWrite)=>{var mode=0;if(canRead)mode|=292|73;if(canWrite)mode|=146;return mode};var asyncLoad=async url=>{var arrayBuffer=await readAsync(url);return new Uint8Array(arrayBuffer)};var FS_createDataFile=(...args)=>FS.createDataFile(...args);var getUniqueRunDependency=id=>id;var runDependencies=0;var dependenciesFulfilled=null;var removeRunDependency=id=>{runDependencies--;Module["monitorRunDependencies"]?.(runDependencies);if(runDependencies==0){if(dependenciesFulfilled){var callback=dependenciesFulfilled;dependenciesFulfilled=null;callback()}}};var addRunDependency=id=>{runDependencies++;Module["monitorRunDependencies"]?.(runDependencies)};var preloadPlugins=[];var FS_handledByPreloadPlugin=async(byteArray,fullname)=>{if(typeof Browser!="undefined")Browser.init();for(var plugin of preloadPlugins){if(plugin["canHandle"](fullname)){return plugin["handle"](byteArray,fullname)}}return byteArray};var FS_preloadFile=async(parent,name,url,canRead,canWrite,dontCreateFile,canOwn,preFinish)=>{var fullname=name?PATH_FS.resolve(PATH.join2(parent,name)):parent;var dep=getUniqueRunDependency(`cp ${fullname}`);addRunDependency(dep);try{var byteArray=url;if(typeof url=="string"){byteArray=await asyncLoad(url)}byteArray=await FS_handledByPreloadPlugin(byteArray,fullname);preFinish?.();if(!dontCreateFile){FS_createDataFile(parent,name,byteArray,canRead,canWrite,canOwn)}}finally{removeRunDependency(dep)}};var FS_createPreloadedFile=(parent,name,url,canRead,canWrite,onload,onerror,dontCreateFile,canOwn,preFinish)=>{FS_preloadFile(parent,name,url,canRead,canWrite,dontCreateFile,canOwn,preFinish).then(onload).catch(onerror)};var FS={root:null,mounts:[],devices:{},streams:[],nextInode:1,nameTable:null,currentPath:"/",initialized:false,ignorePermissions:true,filesystems:null,syncFSRequests:0,readFiles:{},ErrnoError:class{name="ErrnoError";constructor(errno){this.errno=errno}},FSStream:class{shared={};get object(){return this.node}set object(val){this.node=val}get isRead(){return(this.flags&2097155)!==1}get isWrite(){return(this.flags&2097155)!==0}get isAppend(){return this.flags&1024}get flags(){return this.shared.flags}set flags(val){this.shared.flags=val}get position(){return this.shared.position}set position(val){this.shared.position=val}},FSNode:class{node_ops={};stream_ops={};readMode=292|73;writeMode=146;mounted=null;constructor(parent,name,mode,rdev){if(!parent){parent=this}this.parent=parent;this.mount=parent.mount;this.id=FS.nextInode++;this.name=name;this.mode=mode;this.rdev=rdev;this.atime=this.mtime=this.ctime=Date.now()}get read(){return(this.mode&this.readMode)===this.readMode}set read(val){val?this.mode|=this.readMode:this.mode&=~this.readMode}get write(){return(this.mode&this.writeMode)===this.writeMode}set write(val){val?this.mode|=this.writeMode:this.mode&=~this.writeMode}get isFolder(){return FS.isDir(this.mode)}get isDevice(){return FS.isChrdev(this.mode)}},lookupPath(path,opts={}){if(!path){throw new FS.ErrnoError(44)}opts.follow_mount??=true;if(!PATH.isAbs(path)){path=FS.cwd()+"/"+path}linkloop:for(var nlinks=0;nlinks<40;nlinks++){var parts=path.split("/").filter(p=>!!p);var current=FS.root;var current_path="/";for(var i=0;i>>0)%FS.nameTable.length},hashAddNode(node){var hash=FS.hashName(node.parent.id,node.name);node.name_next=FS.nameTable[hash];FS.nameTable[hash]=node},hashRemoveNode(node){var hash=FS.hashName(node.parent.id,node.name);if(FS.nameTable[hash]===node){FS.nameTable[hash]=node.name_next}else{var current=FS.nameTable[hash];while(current){if(current.name_next===node){current.name_next=node.name_next;break}current=current.name_next}}},lookupNode(parent,name){var errCode=FS.mayLookup(parent);if(errCode){throw new FS.ErrnoError(errCode)}var hash=FS.hashName(parent.id,name);for(var node=FS.nameTable[hash];node;node=node.name_next){var nodeName=node.name;if(node.parent.id===parent.id&&nodeName===name){return node}}return FS.lookup(parent,name)},createNode(parent,name,mode,rdev){var node=new FS.FSNode(parent,name,mode,rdev);FS.hashAddNode(node);return node},destroyNode(node){FS.hashRemoveNode(node)},isRoot(node){return node===node.parent},isMountpoint(node){return!!node.mounted},isFile(mode){return(mode&61440)===32768},isDir(mode){return(mode&61440)===16384},isLink(mode){return(mode&61440)===40960},isChrdev(mode){return(mode&61440)===8192},isBlkdev(mode){return(mode&61440)===24576},isFIFO(mode){return(mode&61440)===4096},isSocket(mode){return(mode&49152)===49152},flagsToPermissionString(flag){var perms=["r","w","rw"][flag&3];if(flag&512){perms+="w"}return perms},nodePermissions(node,perms){if(FS.ignorePermissions){return 0}if(perms.includes("r")&&!(node.mode&292)){return 2}else if(perms.includes("w")&&!(node.mode&146)){return 2}else if(perms.includes("x")&&!(node.mode&73)){return 2}return 0},mayLookup(dir){if(!FS.isDir(dir.mode))return 54;var errCode=FS.nodePermissions(dir,"x");if(errCode)return errCode;if(!dir.node_ops.lookup)return 2;return 0},mayCreate(dir,name){if(!FS.isDir(dir.mode)){return 54}try{var node=FS.lookupNode(dir,name);return 20}catch(e){}return FS.nodePermissions(dir,"wx")},mayDelete(dir,name,isdir){var node;try{node=FS.lookupNode(dir,name)}catch(e){return e.errno}var errCode=FS.nodePermissions(dir,"wx");if(errCode){return errCode}if(isdir){if(!FS.isDir(node.mode)){return 54}if(FS.isRoot(node)||FS.getPath(node)===FS.cwd()){return 10}}else{if(FS.isDir(node.mode)){return 31}}return 0},mayOpen(node,flags){if(!node){return 44}if(FS.isLink(node.mode)){return 32}else if(FS.isDir(node.mode)){if(FS.flagsToPermissionString(flags)!=="r"||flags&(512|64)){return 31}}return FS.nodePermissions(node,FS.flagsToPermissionString(flags))},checkOpExists(op,err){if(!op){throw new FS.ErrnoError(err)}return op},MAX_OPEN_FDS:4096,nextfd(){for(var fd=0;fd<=FS.MAX_OPEN_FDS;fd++){if(!FS.streams[fd]){return fd}}throw new FS.ErrnoError(33)},getStreamChecked(fd){var stream=FS.getStream(fd);if(!stream){throw new FS.ErrnoError(8)}return stream},getStream:fd=>FS.streams[fd],createStream(stream,fd=-1){stream=Object.assign(new FS.FSStream,stream);if(fd==-1){fd=FS.nextfd()}stream.fd=fd;FS.streams[fd]=stream;return stream},closeStream(fd){FS.streams[fd]=null},dupStream(origStream,fd=-1){var stream=FS.createStream(origStream,fd);stream.stream_ops?.dup?.(stream);return stream},doSetAttr(stream,node,attr){var setattr=stream?.stream_ops.setattr;var arg=setattr?stream:node;setattr??=node.node_ops.setattr;FS.checkOpExists(setattr,63);setattr(arg,attr)},chrdev_stream_ops:{open(stream){var device=FS.getDevice(stream.node.rdev);stream.stream_ops=device.stream_ops;stream.stream_ops.open?.(stream)},llseek(){throw new FS.ErrnoError(70)}},major:dev=>dev>>8,minor:dev=>dev&255,makedev:(ma,mi)=>ma<<8|mi,registerDevice(dev,ops){FS.devices[dev]={stream_ops:ops}},getDevice:dev=>FS.devices[dev],getMounts(mount){var mounts=[];var check=[mount];while(check.length){var m=check.pop();mounts.push(m);check.push(...m.mounts)}return mounts},syncfs(populate,callback){if(typeof populate=="function"){callback=populate;populate=false}FS.syncFSRequests++;if(FS.syncFSRequests>1){err(`warning: ${FS.syncFSRequests} FS.syncfs operations in flight at once, probably just doing extra work`)}var mounts=FS.getMounts(FS.root.mount);var completed=0;function doCallback(errCode){FS.syncFSRequests--;return callback(errCode)}function done(errCode){if(errCode){if(!done.errored){done.errored=true;return doCallback(errCode)}return}if(++completed>=mounts.length){doCallback(null)}}for(var mount of mounts){if(mount.type.syncfs){mount.type.syncfs(mount,populate,done)}else{done(null)}}},mount(type,opts,mountpoint){var root=mountpoint==="/";var pseudo=!mountpoint;var node;if(root&&FS.root){throw new FS.ErrnoError(10)}else if(!root&&!pseudo){var lookup=FS.lookupPath(mountpoint,{follow_mount:false});mountpoint=lookup.path;node=lookup.node;if(FS.isMountpoint(node)){throw new FS.ErrnoError(10)}if(!FS.isDir(node.mode)){throw new FS.ErrnoError(54)}}var mount={type,opts,mountpoint,mounts:[]};var mountRoot=type.mount(mount);mountRoot.mount=mount;mount.root=mountRoot;if(root){FS.root=mountRoot}else if(node){node.mounted=mount;if(node.mount){node.mount.mounts.push(mount)}}return mountRoot},unmount(mountpoint){var lookup=FS.lookupPath(mountpoint,{follow_mount:false});if(!FS.isMountpoint(lookup.node)){throw new FS.ErrnoError(28)}var node=lookup.node;var mount=node.mounted;var mounts=FS.getMounts(mount);for(var[hash,current]of Object.entries(FS.nameTable)){while(current){var next=current.name_next;if(mounts.includes(current.mount)){FS.destroyNode(current)}current=next}}node.mounted=null;var idx=node.mount.mounts.indexOf(mount);node.mount.mounts.splice(idx,1)},lookup(parent,name){return parent.node_ops.lookup(parent,name)},mknod(path,mode,dev){var lookup=FS.lookupPath(path,{parent:true});var parent=lookup.node;var name=PATH.basename(path);if(!name){throw new FS.ErrnoError(28)}if(name==="."||name===".."){throw new FS.ErrnoError(20)}var errCode=FS.mayCreate(parent,name);if(errCode){throw new FS.ErrnoError(errCode)}if(!parent.node_ops.mknod){throw new FS.ErrnoError(63)}return parent.node_ops.mknod(parent,name,mode,dev)},statfs(path){return FS.statfsNode(FS.lookupPath(path,{follow:true}).node)},statfsStream(stream){return FS.statfsNode(stream.node)},statfsNode(node){var rtn={bsize:4096,frsize:4096,blocks:1e6,bfree:5e5,bavail:5e5,files:FS.nextInode,ffree:FS.nextInode-1,fsid:42,flags:2,namelen:255};if(node.node_ops.statfs){Object.assign(rtn,node.node_ops.statfs(node.mount.opts.root))}return rtn},create(path,mode=438){mode&=4095;mode|=32768;return FS.mknod(path,mode,0)},mkdir(path,mode=511){mode&=511|512;mode|=16384;return FS.mknod(path,mode,0)},mkdirTree(path,mode){var dirs=path.split("/");var d="";for(var dir of dirs){if(!dir)continue;if(d||PATH.isAbs(path))d+="/";d+=dir;try{FS.mkdir(d,mode)}catch(e){if(e.errno!=20)throw e}}},mkdev(path,mode,dev){if(typeof dev=="undefined"){dev=mode;mode=438}mode|=8192;return FS.mknod(path,mode,dev)},symlink(oldpath,newpath){if(!PATH_FS.resolve(oldpath)){throw new FS.ErrnoError(44)}var lookup=FS.lookupPath(newpath,{parent:true});var parent=lookup.node;if(!parent){throw new FS.ErrnoError(44)}var newname=PATH.basename(newpath);var errCode=FS.mayCreate(parent,newname);if(errCode){throw new FS.ErrnoError(errCode)}if(!parent.node_ops.symlink){throw new FS.ErrnoError(63)}return parent.node_ops.symlink(parent,newname,oldpath)},rename(old_path,new_path){var old_dirname=PATH.dirname(old_path);var new_dirname=PATH.dirname(new_path);var old_name=PATH.basename(old_path);var new_name=PATH.basename(new_path);var lookup,old_dir,new_dir;lookup=FS.lookupPath(old_path,{parent:true});old_dir=lookup.node;lookup=FS.lookupPath(new_path,{parent:true});new_dir=lookup.node;if(!old_dir||!new_dir)throw new FS.ErrnoError(44);if(old_dir.mount!==new_dir.mount){throw new FS.ErrnoError(75)}var old_node=FS.lookupNode(old_dir,old_name);var relative=PATH_FS.relative(old_path,new_dirname);if(relative.charAt(0)!=="."){throw new FS.ErrnoError(28)}relative=PATH_FS.relative(new_path,old_dirname);if(relative.charAt(0)!=="."){throw new FS.ErrnoError(55)}var new_node;try{new_node=FS.lookupNode(new_dir,new_name)}catch(e){}if(old_node===new_node){return}var isdir=FS.isDir(old_node.mode);var errCode=FS.mayDelete(old_dir,old_name,isdir);if(errCode){throw new FS.ErrnoError(errCode)}errCode=new_node?FS.mayDelete(new_dir,new_name,isdir):FS.mayCreate(new_dir,new_name);if(errCode){throw new FS.ErrnoError(errCode)}if(!old_dir.node_ops.rename){throw new FS.ErrnoError(63)}if(FS.isMountpoint(old_node)||new_node&&FS.isMountpoint(new_node)){throw new FS.ErrnoError(10)}if(new_dir!==old_dir){errCode=FS.nodePermissions(old_dir,"w");if(errCode){throw new FS.ErrnoError(errCode)}}FS.hashRemoveNode(old_node);try{old_dir.node_ops.rename(old_node,new_dir,new_name);old_node.parent=new_dir}catch(e){throw e}finally{FS.hashAddNode(old_node)}},rmdir(path){var lookup=FS.lookupPath(path,{parent:true});var parent=lookup.node;var name=PATH.basename(path);var node=FS.lookupNode(parent,name);var errCode=FS.mayDelete(parent,name,true);if(errCode){throw new FS.ErrnoError(errCode)}if(!parent.node_ops.rmdir){throw new FS.ErrnoError(63)}if(FS.isMountpoint(node)){throw new FS.ErrnoError(10)}parent.node_ops.rmdir(parent,name);FS.destroyNode(node)},readdir(path){var lookup=FS.lookupPath(path,{follow:true});var node=lookup.node;var readdir=FS.checkOpExists(node.node_ops.readdir,54);return readdir(node)},unlink(path){var lookup=FS.lookupPath(path,{parent:true});var parent=lookup.node;if(!parent){throw new FS.ErrnoError(44)}var name=PATH.basename(path);var node=FS.lookupNode(parent,name);var errCode=FS.mayDelete(parent,name,false);if(errCode){throw new FS.ErrnoError(errCode)}if(!parent.node_ops.unlink){throw new FS.ErrnoError(63)}if(FS.isMountpoint(node)){throw new FS.ErrnoError(10)}parent.node_ops.unlink(parent,name);FS.destroyNode(node)},readlink(path){var lookup=FS.lookupPath(path);var link=lookup.node;if(!link){throw new FS.ErrnoError(44)}if(!link.node_ops.readlink){throw new FS.ErrnoError(28)}return link.node_ops.readlink(link)},stat(path,dontFollow){var lookup=FS.lookupPath(path,{follow:!dontFollow});var node=lookup.node;var getattr=FS.checkOpExists(node.node_ops.getattr,63);return getattr(node)},fstat(fd){var stream=FS.getStreamChecked(fd);var node=stream.node;var getattr=stream.stream_ops.getattr;var arg=getattr?stream:node;getattr??=node.node_ops.getattr;FS.checkOpExists(getattr,63);return getattr(arg)},lstat(path){return FS.stat(path,true)},doChmod(stream,node,mode,dontFollow){FS.doSetAttr(stream,node,{mode:mode&4095|node.mode&~4095,ctime:Date.now(),dontFollow})},chmod(path,mode,dontFollow){var node;if(typeof path=="string"){var lookup=FS.lookupPath(path,{follow:!dontFollow});node=lookup.node}else{node=path}FS.doChmod(null,node,mode,dontFollow)},lchmod(path,mode){FS.chmod(path,mode,true)},fchmod(fd,mode){var stream=FS.getStreamChecked(fd);FS.doChmod(stream,stream.node,mode,false)},doChown(stream,node,dontFollow){FS.doSetAttr(stream,node,{timestamp:Date.now(),dontFollow})},chown(path,uid,gid,dontFollow){var node;if(typeof path=="string"){var lookup=FS.lookupPath(path,{follow:!dontFollow});node=lookup.node}else{node=path}FS.doChown(null,node,dontFollow)},lchown(path,uid,gid){FS.chown(path,uid,gid,true)},fchown(fd,uid,gid){var stream=FS.getStreamChecked(fd);FS.doChown(stream,stream.node,false)},doTruncate(stream,node,len){if(FS.isDir(node.mode)){throw new FS.ErrnoError(31)}if(!FS.isFile(node.mode)){throw new FS.ErrnoError(28)}var errCode=FS.nodePermissions(node,"w");if(errCode){throw new FS.ErrnoError(errCode)}FS.doSetAttr(stream,node,{size:len,timestamp:Date.now()})},truncate(path,len){if(len<0){throw new FS.ErrnoError(28)}var node;if(typeof path=="string"){var lookup=FS.lookupPath(path,{follow:true});node=lookup.node}else{node=path}FS.doTruncate(null,node,len)},ftruncate(fd,len){var stream=FS.getStreamChecked(fd);if(len<0||(stream.flags&2097155)===0){throw new FS.ErrnoError(28)}FS.doTruncate(stream,stream.node,len)},utime(path,atime,mtime){var lookup=FS.lookupPath(path,{follow:true});var node=lookup.node;var setattr=FS.checkOpExists(node.node_ops.setattr,63);setattr(node,{atime,mtime})},open(path,flags,mode=438){if(path===""){throw new FS.ErrnoError(44)}flags=typeof flags=="string"?FS_modeStringToFlags(flags):flags;if(flags&64){mode=mode&4095|32768}else{mode=0}var node;var isDirPath;if(typeof path=="object"){node=path}else{isDirPath=path.endsWith("/");var lookup=FS.lookupPath(path,{follow:!(flags&131072),noent_okay:true});node=lookup.node;path=lookup.path}var created=false;if(flags&64){if(node){if(flags&128){throw new FS.ErrnoError(20)}}else if(isDirPath){throw new FS.ErrnoError(31)}else{node=FS.mknod(path,mode|511,0);created=true}}if(!node){throw new FS.ErrnoError(44)}if(FS.isChrdev(node.mode)){flags&=~512}if(flags&65536&&!FS.isDir(node.mode)){throw new FS.ErrnoError(54)}if(!created){var errCode=FS.mayOpen(node,flags);if(errCode){throw new FS.ErrnoError(errCode)}}if(flags&512&&!created){FS.truncate(node,0)}flags&=~(128|512|131072);var stream=FS.createStream({node,path:FS.getPath(node),flags,seekable:true,position:0,stream_ops:node.stream_ops,ungotten:[],error:false});if(stream.stream_ops.open){stream.stream_ops.open(stream)}if(created){FS.chmod(node,mode&511)}if(Module["logReadFiles"]&&!(flags&1)){if(!(path in FS.readFiles)){FS.readFiles[path]=1}}return stream},close(stream){if(FS.isClosed(stream)){throw new FS.ErrnoError(8)}if(stream.getdents)stream.getdents=null;try{if(stream.stream_ops.close){stream.stream_ops.close(stream)}}catch(e){throw e}finally{FS.closeStream(stream.fd)}stream.fd=null},isClosed(stream){return stream.fd===null},llseek(stream,offset,whence){if(FS.isClosed(stream)){throw new FS.ErrnoError(8)}if(!stream.seekable||!stream.stream_ops.llseek){throw new FS.ErrnoError(70)}if(whence!=0&&whence!=1&&whence!=2){throw new FS.ErrnoError(28)}stream.position=stream.stream_ops.llseek(stream,offset,whence);stream.ungotten=[];return stream.position},read(stream,buffer,offset,length,position){if(length<0||position<0){throw new FS.ErrnoError(28)}if(FS.isClosed(stream)){throw new FS.ErrnoError(8)}if((stream.flags&2097155)===1){throw new FS.ErrnoError(8)}if(FS.isDir(stream.node.mode)){throw new FS.ErrnoError(31)}if(!stream.stream_ops.read){throw new FS.ErrnoError(28)}var seeking=typeof position!="undefined";if(!seeking){position=stream.position}else if(!stream.seekable){throw new FS.ErrnoError(70)}var bytesRead=stream.stream_ops.read(stream,buffer,offset,length,position);if(!seeking)stream.position+=bytesRead;return bytesRead},write(stream,buffer,offset,length,position,canOwn){if(length<0||position<0){throw new FS.ErrnoError(28)}if(FS.isClosed(stream)){throw new FS.ErrnoError(8)}if((stream.flags&2097155)===0){throw new FS.ErrnoError(8)}if(FS.isDir(stream.node.mode)){throw new FS.ErrnoError(31)}if(!stream.stream_ops.write){throw new FS.ErrnoError(28)}if(stream.seekable&&stream.flags&1024){FS.llseek(stream,0,2)}var seeking=typeof position!="undefined";if(!seeking){position=stream.position}else if(!stream.seekable){throw new FS.ErrnoError(70)}var bytesWritten=stream.stream_ops.write(stream,buffer,offset,length,position,canOwn);if(!seeking)stream.position+=bytesWritten;return bytesWritten},mmap(stream,length,position,prot,flags){if((prot&2)!==0&&(flags&2)===0&&(stream.flags&2097155)!==2){throw new FS.ErrnoError(2)}if((stream.flags&2097155)===1){throw new FS.ErrnoError(2)}if(!stream.stream_ops.mmap){throw new FS.ErrnoError(43)}if(!length){throw new FS.ErrnoError(28)}return stream.stream_ops.mmap(stream,length,position,prot,flags)},msync(stream,buffer,offset,length,mmapFlags){if(!stream.stream_ops.msync){return 0}return stream.stream_ops.msync(stream,buffer,offset,length,mmapFlags)},ioctl(stream,cmd,arg){if(!stream.stream_ops.ioctl){throw new FS.ErrnoError(59)}return stream.stream_ops.ioctl(stream,cmd,arg)},readFile(path,opts={}){opts.flags=opts.flags||0;opts.encoding=opts.encoding||"binary";if(opts.encoding!=="utf8"&&opts.encoding!=="binary"){abort(`Invalid encoding type "${opts.encoding}"`)}var stream=FS.open(path,opts.flags);var stat=FS.stat(path);var length=stat.size;var buf=new Uint8Array(length);FS.read(stream,buf,0,length,0);if(opts.encoding==="utf8"){buf=UTF8ArrayToString(buf)}FS.close(stream);return buf},writeFile(path,data,opts={}){opts.flags=opts.flags||577;var stream=FS.open(path,opts.flags,opts.mode);if(typeof data=="string"){data=new Uint8Array(intArrayFromString(data,true))}if(ArrayBuffer.isView(data)){FS.write(stream,data,0,data.byteLength,undefined,opts.canOwn)}else{abort("Unsupported data type")}FS.close(stream)},cwd:()=>FS.currentPath,chdir(path){var lookup=FS.lookupPath(path,{follow:true});if(lookup.node===null){throw new FS.ErrnoError(44)}if(!FS.isDir(lookup.node.mode)){throw new FS.ErrnoError(54)}var errCode=FS.nodePermissions(lookup.node,"x");if(errCode){throw new FS.ErrnoError(errCode)}FS.currentPath=lookup.path},createDefaultDirectories(){FS.mkdir("/tmp");FS.mkdir("/home");FS.mkdir("/home/web_user")},createDefaultDevices(){FS.mkdir("/dev");FS.registerDevice(FS.makedev(1,3),{read:()=>0,write:(stream,buffer,offset,length,pos)=>length,llseek:()=>0});FS.mkdev("/dev/null",FS.makedev(1,3));TTY.register(FS.makedev(5,0),TTY.default_tty_ops);TTY.register(FS.makedev(6,0),TTY.default_tty1_ops);FS.mkdev("/dev/tty",FS.makedev(5,0));FS.mkdev("/dev/tty1",FS.makedev(6,0));var randomBuffer=new Uint8Array(1024),randomLeft=0;var randomByte=()=>{if(randomLeft===0){randomFill(randomBuffer);randomLeft=randomBuffer.byteLength}return randomBuffer[--randomLeft]};FS.createDevice("/dev","random",randomByte);FS.createDevice("/dev","urandom",randomByte);FS.mkdir("/dev/shm");FS.mkdir("/dev/shm/tmp")},createSpecialDirectories(){FS.mkdir("/proc");var proc_self=FS.mkdir("/proc/self");FS.mkdir("/proc/self/fd");FS.mount({mount(){var node=FS.createNode(proc_self,"fd",16895,73);node.stream_ops={llseek:MEMFS.stream_ops.llseek};node.node_ops={lookup(parent,name){var fd=+name;var stream=FS.getStreamChecked(fd);var ret={parent:null,mount:{mountpoint:"fake"},node_ops:{readlink:()=>stream.path},id:fd+1};ret.parent=ret;return ret},readdir(){return Array.from(FS.streams.entries()).filter(([k,v])=>v).map(([k,v])=>k.toString())}};return node}},{},"/proc/self/fd")},createStandardStreams(input,output,error){if(input){FS.createDevice("/dev","stdin",input)}else{FS.symlink("/dev/tty","/dev/stdin")}if(output){FS.createDevice("/dev","stdout",null,output)}else{FS.symlink("/dev/tty","/dev/stdout")}if(error){FS.createDevice("/dev","stderr",null,error)}else{FS.symlink("/dev/tty1","/dev/stderr")}var stdin=FS.open("/dev/stdin",0);var stdout=FS.open("/dev/stdout",1);var stderr=FS.open("/dev/stderr",1)},staticInit(){FS.nameTable=new Array(4096);FS.mount(MEMFS,{},"/");FS.createDefaultDirectories();FS.createDefaultDevices();FS.createSpecialDirectories();FS.filesystems={MEMFS}},init(input,output,error){FS.initialized=true;input??=Module["stdin"];output??=Module["stdout"];error??=Module["stderr"];FS.createStandardStreams(input,output,error)},quit(){FS.initialized=false;for(var stream of FS.streams){if(stream){FS.close(stream)}}},findObject(path,dontResolveLastLink){var ret=FS.analyzePath(path,dontResolveLastLink);if(!ret.exists){return null}return ret.object},analyzePath(path,dontResolveLastLink){try{var lookup=FS.lookupPath(path,{follow:!dontResolveLastLink});path=lookup.path}catch(e){}var ret={isRoot:false,exists:false,error:0,name:null,path:null,object:null,parentExists:false,parentPath:null,parentObject:null};try{var lookup=FS.lookupPath(path,{parent:true});ret.parentExists=true;ret.parentPath=lookup.path;ret.parentObject=lookup.node;ret.name=PATH.basename(path);lookup=FS.lookupPath(path,{follow:!dontResolveLastLink});ret.exists=true;ret.path=lookup.path;ret.object=lookup.node;ret.name=lookup.node.name;ret.isRoot=lookup.path==="/"}catch(e){ret.error=e.errno}return ret},createPath(parent,path,canRead,canWrite){parent=typeof parent=="string"?parent:FS.getPath(parent);var parts=path.split("/").reverse();while(parts.length){var part=parts.pop();if(!part)continue;var current=PATH.join2(parent,part);try{FS.mkdir(current)}catch(e){if(e.errno!=20)throw e}parent=current}return current},createFile(parent,name,properties,canRead,canWrite){var path=PATH.join2(typeof parent=="string"?parent:FS.getPath(parent),name);var mode=FS_getMode(canRead,canWrite);return FS.create(path,mode)},createDataFile(parent,name,data,canRead,canWrite,canOwn){var path=name;if(parent){parent=typeof parent=="string"?parent:FS.getPath(parent);path=name?PATH.join2(parent,name):parent}var mode=FS_getMode(canRead,canWrite);var node=FS.create(path,mode);if(data){if(typeof data=="string"){var arr=new Array(data.length);for(var i=0,len=data.length;ithis.length-1||idx<0){return undefined}var chunkOffset=idx%this.chunkSize;var chunkNum=idx/this.chunkSize|0;return this.getter(chunkNum)[chunkOffset]}setDataGetter(getter){this.getter=getter}cacheLength(){var xhr=new XMLHttpRequest;xhr.open("HEAD",url,false);xhr.send(null);if(!(xhr.status>=200&&xhr.status<300||xhr.status===304))abort("Couldn't load "+url+". Status: "+xhr.status);var datalength=Number(xhr.getResponseHeader("Content-length"));var header;var hasByteServing=(header=xhr.getResponseHeader("Accept-Ranges"))&&header==="bytes";var usesGzip=(header=xhr.getResponseHeader("Content-Encoding"))&&header==="gzip";var chunkSize=1024*1024;if(!hasByteServing)chunkSize=datalength;var doXHR=(from,to)=>{if(from>to)abort("invalid range ("+from+", "+to+") or no bytes requested!");if(to>datalength-1)abort("only "+datalength+" bytes available! programmer error!");var xhr=new XMLHttpRequest;xhr.open("GET",url,false);if(datalength!==chunkSize)xhr.setRequestHeader("Range","bytes="+from+"-"+to);xhr.responseType="arraybuffer";if(xhr.overrideMimeType){xhr.overrideMimeType("text/plain; charset=x-user-defined")}xhr.send(null);if(!(xhr.status>=200&&xhr.status<300||xhr.status===304))abort("Couldn't load "+url+". Status: "+xhr.status);if(xhr.response!==undefined){return new Uint8Array(xhr.response||[])}return intArrayFromString(xhr.responseText||"",true)};var lazyArray=this;lazyArray.setDataGetter(chunkNum=>{var start=chunkNum*chunkSize;var end=(chunkNum+1)*chunkSize-1;end=Math.min(end,datalength-1);if(typeof lazyArray.chunks[chunkNum]=="undefined"){lazyArray.chunks[chunkNum]=doXHR(start,end)}if(typeof lazyArray.chunks[chunkNum]=="undefined")abort("doXHR failed!");return lazyArray.chunks[chunkNum]});if(usesGzip||!datalength){chunkSize=datalength=1;datalength=this.getter(0).length;chunkSize=datalength;out("LazyFiles on gzip forces download of the whole file when length is accessed")}this._length=datalength;this._chunkSize=chunkSize;this.lengthKnown=true}get length(){if(!this.lengthKnown){this.cacheLength()}return this._length}get chunkSize(){if(!this.lengthKnown){this.cacheLength()}return this._chunkSize}}if(globalThis.XMLHttpRequest){if(!ENVIRONMENT_IS_WORKER)abort("Cannot do synchronous binary XHRs outside webworkers in modern browsers. Use --embed-file or --preload-file in emcc");var lazyArray=new LazyUint8Array;var properties={isDevice:false,contents:lazyArray}}else{var properties={isDevice:false,url}}var node=FS.createFile(parent,name,properties,canRead,canWrite);if(properties.contents){node.contents=properties.contents}else if(properties.url){node.contents=null;node.url=properties.url}Object.defineProperties(node,{usedBytes:{get:function(){return this.contents.length}}});var stream_ops={};for(const[key,fn]of Object.entries(node.stream_ops)){stream_ops[key]=(...args)=>{FS.forceLoadFile(node);return fn(...args)}}function writeChunks(stream,buffer,offset,length,position){var contents=stream.node.contents;if(position>=contents.length)return 0;var size=Math.min(contents.length-position,length);if(contents.slice){for(var i=0;i{FS.forceLoadFile(node);return writeChunks(stream,buffer,offset,length,position)};stream_ops.mmap=(stream,length,position,prot,flags)=>{FS.forceLoadFile(node);var ptr=mmapAlloc(length);if(!ptr){throw new FS.ErrnoError(48)}writeChunks(stream,HEAP8,ptr,length,position);return{ptr,allocated:true}};node.stream_ops=stream_ops;return node}};var UTF8ToString=(ptr,maxBytesToRead,ignoreNul)=>ptr?UTF8ArrayToString(HEAPU8,ptr,maxBytesToRead,ignoreNul):"";var SYSCALLS={DEFAULT_POLLMASK:5,calculateAt(dirfd,path,allowEmpty){if(PATH.isAbs(path)){return path}var dir;if(dirfd===-100){dir=FS.cwd()}else{var dirstream=SYSCALLS.getStreamFromFD(dirfd);dir=dirstream.path}if(path.length==0){if(!allowEmpty){throw new FS.ErrnoError(44)}return dir}return dir+"/"+path},writeStat(buf,stat){HEAPU32[buf>>2]=stat.dev;HEAPU32[buf+4>>2]=stat.mode;HEAPU32[buf+8>>2]=stat.nlink;HEAPU32[buf+12>>2]=stat.uid;HEAPU32[buf+16>>2]=stat.gid;HEAPU32[buf+20>>2]=stat.rdev;HEAP64[buf+24>>3]=BigInt(stat.size);HEAP32[buf+32>>2]=4096;HEAP32[buf+36>>2]=stat.blocks;var atime=stat.atime.getTime();var mtime=stat.mtime.getTime();var ctime=stat.ctime.getTime();HEAP64[buf+40>>3]=BigInt(Math.floor(atime/1e3));HEAPU32[buf+48>>2]=atime%1e3*1e3*1e3;HEAP64[buf+56>>3]=BigInt(Math.floor(mtime/1e3));HEAPU32[buf+64>>2]=mtime%1e3*1e3*1e3;HEAP64[buf+72>>3]=BigInt(Math.floor(ctime/1e3));HEAPU32[buf+80>>2]=ctime%1e3*1e3*1e3;HEAP64[buf+88>>3]=BigInt(stat.ino);return 0},writeStatFs(buf,stats){HEAPU32[buf+4>>2]=stats.bsize;HEAPU32[buf+60>>2]=stats.bsize;HEAP64[buf+8>>3]=BigInt(stats.blocks);HEAP64[buf+16>>3]=BigInt(stats.bfree);HEAP64[buf+24>>3]=BigInt(stats.bavail);HEAP64[buf+32>>3]=BigInt(stats.files);HEAP64[buf+40>>3]=BigInt(stats.ffree);HEAPU32[buf+48>>2]=stats.fsid;HEAPU32[buf+64>>2]=stats.flags;HEAPU32[buf+56>>2]=stats.namelen},doMsync(addr,stream,len,flags,offset){if(!FS.isFile(stream.node.mode)){throw new FS.ErrnoError(43)}if(flags&2){return 0}var buffer=HEAPU8.slice(addr,addr+len);FS.msync(stream,buffer,offset,len,flags)},getStreamFromFD(fd){var stream=FS.getStreamChecked(fd);return stream},varargs:undefined,getStr(ptr){var ret=UTF8ToString(ptr);return ret}};function ___syscall_fcntl64(fd,cmd,varargs){SYSCALLS.varargs=varargs;try{var stream=SYSCALLS.getStreamFromFD(fd);switch(cmd){case 0:{var arg=syscallGetVarargI();if(arg<0){return-28}while(FS.streams[arg]){arg++}var newStream;newStream=FS.dupStream(stream,arg);return newStream.fd}case 1:case 2:return 0;case 3:return stream.flags;case 4:{var arg=syscallGetVarargI();stream.flags|=arg;return 0}case 12:{var arg=syscallGetVarargP();var offset=0;HEAP16[arg+offset>>1]=2;return 0}case 13:case 14:return 0}return-28}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_fstat64(fd,buf){try{return SYSCALLS.writeStat(buf,FS.fstat(fd))}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_ioctl(fd,op,varargs){SYSCALLS.varargs=varargs;try{var stream=SYSCALLS.getStreamFromFD(fd);switch(op){case 21509:{if(!stream.tty)return-59;return 0}case 21505:{if(!stream.tty)return-59;if(stream.tty.ops.ioctl_tcgets){var termios=stream.tty.ops.ioctl_tcgets(stream);var argp=syscallGetVarargP();HEAP32[argp>>2]=termios.c_iflag||0;HEAP32[argp+4>>2]=termios.c_oflag||0;HEAP32[argp+8>>2]=termios.c_cflag||0;HEAP32[argp+12>>2]=termios.c_lflag||0;for(var i=0;i<32;i++){HEAP8[argp+i+17]=termios.c_cc[i]||0}return 0}return 0}case 21510:case 21511:case 21512:{if(!stream.tty)return-59;return 0}case 21506:case 21507:case 21508:{if(!stream.tty)return-59;if(stream.tty.ops.ioctl_tcsets){var argp=syscallGetVarargP();var c_iflag=HEAP32[argp>>2];var c_oflag=HEAP32[argp+4>>2];var c_cflag=HEAP32[argp+8>>2];var c_lflag=HEAP32[argp+12>>2];var c_cc=[];for(var i=0;i<32;i++){c_cc.push(HEAP8[argp+i+17])}return stream.tty.ops.ioctl_tcsets(stream.tty,op,{c_iflag,c_oflag,c_cflag,c_lflag,c_cc})}return 0}case 21519:{if(!stream.tty)return-59;var argp=syscallGetVarargP();HEAP32[argp>>2]=0;return 0}case 21520:{if(!stream.tty)return-59;return-28}case 21537:case 21531:{var argp=syscallGetVarargP();return FS.ioctl(stream,op,argp)}case 21523:{if(!stream.tty)return-59;if(stream.tty.ops.ioctl_tiocgwinsz){var winsize=stream.tty.ops.ioctl_tiocgwinsz(stream.tty);var argp=syscallGetVarargP();HEAP16[argp>>1]=winsize[0];HEAP16[argp+2>>1]=winsize[1]}return 0}case 21524:{if(!stream.tty)return-59;return 0}case 21515:{if(!stream.tty)return-59;return 0}default:return-28}}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_newfstatat(dirfd,path,buf,flags){try{path=SYSCALLS.getStr(path);var nofollow=flags&256;var allowEmpty=flags&4096;flags=flags&~6400;path=SYSCALLS.calculateAt(dirfd,path,allowEmpty);return SYSCALLS.writeStat(buf,nofollow?FS.lstat(path):FS.stat(path))}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_openat(dirfd,path,flags,varargs){SYSCALLS.varargs=varargs;try{path=SYSCALLS.getStr(path);path=SYSCALLS.calculateAt(dirfd,path);var mode=varargs?syscallGetVarargI():0;return FS.open(path,flags,mode).fd}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}var __abort_js=()=>abort("");var structRegistrations={};var runDestructors=destructors=>{while(destructors.length){var ptr=destructors.pop();var del=destructors.pop();del(ptr)}};function readPointer(pointer){return this.fromWireType(HEAPU32[pointer>>2])}var awaitingDependencies={};var registeredTypes={};var typeDependencies={};var InternalError=class InternalError extends Error{constructor(message){super(message);this.name="InternalError"}};var throwInternalError=message=>{throw new InternalError(message)};var whenDependentTypesAreResolved=(myTypes,dependentTypes,getTypeConverters)=>{myTypes.forEach(type=>typeDependencies[type]=dependentTypes);function onComplete(typeConverters){var myTypeConverters=getTypeConverters(typeConverters);if(myTypeConverters.length!==myTypes.length){throwInternalError("Mismatched type converter count")}for(var i=0;i{typeConverters[i]=registeredTypes[dt];++registered;if(registered===unregisteredTypes.length){onComplete(typeConverters)}})}}if(0===unregisteredTypes.length){onComplete(typeConverters)}};var __embind_finalize_value_object=structType=>{var reg=structRegistrations[structType];delete structRegistrations[structType];var rawConstructor=reg.rawConstructor;var rawDestructor=reg.rawDestructor;var fieldRecords=reg.fields;var fieldTypes=fieldRecords.map(field=>field.getterReturnType).concat(fieldRecords.map(field=>field.setterArgumentType));whenDependentTypesAreResolved([structType],fieldTypes,fieldTypes=>{var fields={};for(var[i,field]of fieldRecords.entries()){const getterReturnType=fieldTypes[i];const getter=field.getter;const getterContext=field.getterContext;const setterArgumentType=fieldTypes[i+fieldRecords.length];const setter=field.setter;const setterContext=field.setterContext;fields[field.fieldName]={read:ptr=>getterReturnType.fromWireType(getter(getterContext,ptr)),write:(ptr,o)=>{var destructors=[];setter(setterContext,ptr,setterArgumentType.toWireType(destructors,o));runDestructors(destructors)},optional:getterReturnType.optional}}return[{name:reg.name,fromWireType:ptr=>{var rv={};for(var i in fields){rv[i]=fields[i].read(ptr)}rawDestructor(ptr);return rv},toWireType:(destructors,o)=>{for(var fieldName in fields){if(!(fieldName in o)&&!fields[fieldName].optional){throw new TypeError(`Missing field: "${fieldName}"`)}}var ptr=rawConstructor();for(fieldName in fields){fields[fieldName].write(ptr,o[fieldName])}if(destructors!==null){destructors.push(rawDestructor,ptr)}return ptr},readValueFromPointer:readPointer,destructorFunction:rawDestructor}]})};var AsciiToString=ptr=>{var str="";while(1){var ch=HEAPU8[ptr++];if(!ch)return str;str+=String.fromCharCode(ch)}};var BindingError=class BindingError extends Error{constructor(message){super(message);this.name="BindingError"}};var throwBindingError=message=>{throw new BindingError(message)};function sharedRegisterType(rawType,registeredInstance,options={}){var name=registeredInstance.name;if(!rawType){throwBindingError(`type "${name}" must have a positive integer typeid pointer`)}if(registeredTypes.hasOwnProperty(rawType)){if(options.ignoreDuplicateRegistrations){return}else{throwBindingError(`Cannot register type '${name}' twice`)}}registeredTypes[rawType]=registeredInstance;delete typeDependencies[rawType];if(awaitingDependencies.hasOwnProperty(rawType)){var callbacks=awaitingDependencies[rawType];delete awaitingDependencies[rawType];callbacks.forEach(cb=>cb())}}function registerType(rawType,registeredInstance,options={}){return sharedRegisterType(rawType,registeredInstance,options)}var integerReadValueFromPointer=(name,width,signed)=>{switch(width){case 1:return signed?pointer=>HEAP8[pointer]:pointer=>HEAPU8[pointer];case 2:return signed?pointer=>HEAP16[pointer>>1]:pointer=>HEAPU16[pointer>>1];case 4:return signed?pointer=>HEAP32[pointer>>2]:pointer=>HEAPU32[pointer>>2];case 8:return signed?pointer=>HEAP64[pointer>>3]:pointer=>HEAPU64[pointer>>3];default:throw new TypeError(`invalid integer width (${width}): ${name}`)}};var __embind_register_bigint=(primitiveType,name,size,minRange,maxRange)=>{name=AsciiToString(name);const isUnsignedType=minRange===0n;let fromWireType=value=>value;if(isUnsignedType){const bitSize=size*8;fromWireType=value=>BigInt.asUintN(bitSize,value);maxRange=fromWireType(maxRange)}registerType(primitiveType,{name,fromWireType,toWireType:(destructors,value)=>{if(typeof value=="number"){value=BigInt(value)}return value},readValueFromPointer:integerReadValueFromPointer(name,size,!isUnsignedType),destructorFunction:null})};var __embind_register_bool=(rawType,name,trueValue,falseValue)=>{name=AsciiToString(name);registerType(rawType,{name,fromWireType:function(wt){return!!wt},toWireType:function(destructors,o){return o?trueValue:falseValue},readValueFromPointer:function(pointer){return this.fromWireType(HEAPU8[pointer])},destructorFunction:null})};var shallowCopyInternalPointer=o=>({count:o.count,deleteScheduled:o.deleteScheduled,preservePointerOnDelete:o.preservePointerOnDelete,ptr:o.ptr,ptrType:o.ptrType,smartPtr:o.smartPtr,smartPtrType:o.smartPtrType});var throwInstanceAlreadyDeleted=obj=>{function getInstanceTypeName(handle){return handle.$$.ptrType.registeredClass.name}throwBindingError(getInstanceTypeName(obj)+" instance already deleted")};var finalizationRegistry=false;var detachFinalizer=handle=>{};var runDestructor=$$=>{if($$.smartPtr){$$.smartPtrType.rawDestructor($$.smartPtr)}else{$$.ptrType.registeredClass.rawDestructor($$.ptr)}};var releaseClassHandle=$$=>{$$.count.value-=1;var toDelete=0===$$.count.value;if(toDelete){runDestructor($$)}};var attachFinalizer=handle=>{if(!globalThis.FinalizationRegistry){attachFinalizer=handle=>handle;return handle}finalizationRegistry=new FinalizationRegistry(info=>{releaseClassHandle(info.$$)});attachFinalizer=handle=>{var $$=handle.$$;var hasSmartPtr=!!$$.smartPtr;if(hasSmartPtr){var info={$$};finalizationRegistry.register(handle,info,handle)}return handle};detachFinalizer=handle=>finalizationRegistry.unregister(handle);return attachFinalizer(handle)};var deletionQueue=[];var flushPendingDeletes=()=>{while(deletionQueue.length){var obj=deletionQueue.pop();obj.$$.deleteScheduled=false;obj["delete"]()}};var delayFunction;var init_ClassHandle=()=>{let proto=ClassHandle.prototype;Object.assign(proto,{isAliasOf(other){if(!(this instanceof ClassHandle)){return false}if(!(other instanceof ClassHandle)){return false}var leftClass=this.$$.ptrType.registeredClass;var left=this.$$.ptr;other.$$=other.$$;var rightClass=other.$$.ptrType.registeredClass;var right=other.$$.ptr;while(leftClass.baseClass){left=leftClass.upcast(left);leftClass=leftClass.baseClass}while(rightClass.baseClass){right=rightClass.upcast(right);rightClass=rightClass.baseClass}return leftClass===rightClass&&left===right},clone(){if(!this.$$.ptr){throwInstanceAlreadyDeleted(this)}if(this.$$.preservePointerOnDelete){this.$$.count.value+=1;return this}else{var clone=attachFinalizer(Object.create(Object.getPrototypeOf(this),{$$:{value:shallowCopyInternalPointer(this.$$)}}));clone.$$.count.value+=1;clone.$$.deleteScheduled=false;return clone}},delete(){if(!this.$$.ptr){throwInstanceAlreadyDeleted(this)}if(this.$$.deleteScheduled&&!this.$$.preservePointerOnDelete){throwBindingError("Object already scheduled for deletion")}detachFinalizer(this);releaseClassHandle(this.$$);if(!this.$$.preservePointerOnDelete){this.$$.smartPtr=undefined;this.$$.ptr=undefined}},isDeleted(){return!this.$$.ptr},deleteLater(){if(!this.$$.ptr){throwInstanceAlreadyDeleted(this)}if(this.$$.deleteScheduled&&!this.$$.preservePointerOnDelete){throwBindingError("Object already scheduled for deletion")}deletionQueue.push(this);if(deletionQueue.length===1&&delayFunction){delayFunction(flushPendingDeletes)}this.$$.deleteScheduled=true;return this}});const symbolDispose=Symbol.dispose;if(symbolDispose){proto[symbolDispose]=proto["delete"]}};function ClassHandle(){}var createNamedFunction=(name,func)=>Object.defineProperty(func,"name",{value:name});var registeredPointers={};var ensureOverloadTable=(proto,methodName,humanName)=>{if(undefined===proto[methodName].overloadTable){var prevFunc=proto[methodName];proto[methodName]=function(...args){if(!proto[methodName].overloadTable.hasOwnProperty(args.length)){throwBindingError(`Function '${humanName}' called with an invalid number of arguments (${args.length}) - expects one of (${proto[methodName].overloadTable})!`)}return proto[methodName].overloadTable[args.length].apply(this,args)};proto[methodName].overloadTable=[];proto[methodName].overloadTable[prevFunc.argCount]=prevFunc}};var exposePublicSymbol=(name,value,numArguments)=>{if(Module.hasOwnProperty(name)){if(undefined===numArguments||undefined!==Module[name].overloadTable&&undefined!==Module[name].overloadTable[numArguments]){throwBindingError(`Cannot register public name '${name}' twice`)}ensureOverloadTable(Module,name,name);if(Module[name].overloadTable.hasOwnProperty(numArguments)){throwBindingError(`Cannot register multiple overloads of a function with the same number of arguments (${numArguments})!`)}Module[name].overloadTable[numArguments]=value}else{Module[name]=value;Module[name].argCount=numArguments}};var char_0=48;var char_9=57;var makeLegalFunctionName=name=>{name=name.replace(/[^a-zA-Z0-9_]/g,"$");var f=name.charCodeAt(0);if(f>=char_0&&f<=char_9){return`_${name}`}return name};function RegisteredClass(name,constructor,instancePrototype,rawDestructor,baseClass,getActualType,upcast,downcast){this.name=name;this.constructor=constructor;this.instancePrototype=instancePrototype;this.rawDestructor=rawDestructor;this.baseClass=baseClass;this.getActualType=getActualType;this.upcast=upcast;this.downcast=downcast;this.pureVirtualFunctions=[]}var upcastPointer=(ptr,ptrClass,desiredClass)=>{while(ptrClass!==desiredClass){if(!ptrClass.upcast){throwBindingError(`Expected null or instance of ${desiredClass.name}, got an instance of ${ptrClass.name}`)}ptr=ptrClass.upcast(ptr);ptrClass=ptrClass.baseClass}return ptr};var embindRepr=v=>{if(v===null){return"null"}var t=typeof v;if(t==="object"||t==="array"||t==="function"){return v.toString()}else{return""+v}};function constNoSmartPtrRawPointerToWireType(destructors,handle){if(handle===null){if(this.isReference){throwBindingError(`null is not a valid ${this.name}`)}return 0}if(!handle.$$){throwBindingError(`Cannot pass "${embindRepr(handle)}" as a ${this.name}`)}if(!handle.$$.ptr){throwBindingError(`Cannot pass deleted object as a pointer of type ${this.name}`)}var handleClass=handle.$$.ptrType.registeredClass;var ptr=upcastPointer(handle.$$.ptr,handleClass,this.registeredClass);return ptr}function genericPointerToWireType(destructors,handle){var ptr;if(handle===null){if(this.isReference){throwBindingError(`null is not a valid ${this.name}`)}if(this.isSmartPointer){ptr=this.rawConstructor();if(destructors!==null){destructors.push(this.rawDestructor,ptr)}return ptr}else{return 0}}if(!handle||!handle.$$){throwBindingError(`Cannot pass "${embindRepr(handle)}" as a ${this.name}`)}if(!handle.$$.ptr){throwBindingError(`Cannot pass deleted object as a pointer of type ${this.name}`)}if(!this.isConst&&handle.$$.ptrType.isConst){throwBindingError(`Cannot convert argument of type ${handle.$$.smartPtrType?handle.$$.smartPtrType.name:handle.$$.ptrType.name} to parameter type ${this.name}`)}var handleClass=handle.$$.ptrType.registeredClass;ptr=upcastPointer(handle.$$.ptr,handleClass,this.registeredClass);if(this.isSmartPointer){if(undefined===handle.$$.smartPtr){throwBindingError("Passing raw pointer to smart pointer is illegal")}switch(this.sharingPolicy){case 0:if(handle.$$.smartPtrType===this){ptr=handle.$$.smartPtr}else{throwBindingError(`Cannot convert argument of type ${handle.$$.smartPtrType?handle.$$.smartPtrType.name:handle.$$.ptrType.name} to parameter type ${this.name}`)}break;case 1:ptr=handle.$$.smartPtr;break;case 2:if(handle.$$.smartPtrType===this){ptr=handle.$$.smartPtr}else{var clonedHandle=handle["clone"]();ptr=this.rawShare(ptr,Emval.toHandle(()=>clonedHandle["delete"]()));if(destructors!==null){destructors.push(this.rawDestructor,ptr)}}break;default:throwBindingError("Unsupporting sharing policy")}}return ptr}function nonConstNoSmartPtrRawPointerToWireType(destructors,handle){if(handle===null){if(this.isReference){throwBindingError(`null is not a valid ${this.name}`)}return 0}if(!handle.$$){throwBindingError(`Cannot pass "${embindRepr(handle)}" as a ${this.name}`)}if(!handle.$$.ptr){throwBindingError(`Cannot pass deleted object as a pointer of type ${this.name}`)}if(handle.$$.ptrType.isConst){throwBindingError(`Cannot convert argument of type ${handle.$$.ptrType.name} to parameter type ${this.name}`)}var handleClass=handle.$$.ptrType.registeredClass;var ptr=upcastPointer(handle.$$.ptr,handleClass,this.registeredClass);return ptr}var downcastPointer=(ptr,ptrClass,desiredClass)=>{if(ptrClass===desiredClass){return ptr}if(undefined===desiredClass.baseClass){return null}var rv=downcastPointer(ptr,ptrClass,desiredClass.baseClass);if(rv===null){return null}return desiredClass.downcast(rv)};var registeredInstances={};var getBasestPointer=(class_,ptr)=>{if(ptr===undefined){throwBindingError("ptr should not be undefined")}while(class_.baseClass){ptr=class_.upcast(ptr);class_=class_.baseClass}return ptr};var getInheritedInstance=(class_,ptr)=>{ptr=getBasestPointer(class_,ptr);return registeredInstances[ptr]};var makeClassHandle=(prototype,record)=>{if(!record.ptrType||!record.ptr){throwInternalError("makeClassHandle requires ptr and ptrType")}var hasSmartPtrType=!!record.smartPtrType;var hasSmartPtr=!!record.smartPtr;if(hasSmartPtrType!==hasSmartPtr){throwInternalError("Both smartPtrType and smartPtr must be specified")}record.count={value:1};return attachFinalizer(Object.create(prototype,{$$:{value:record,writable:true}}))};function RegisteredPointer_fromWireType(ptr){var rawPointer=this.getPointee(ptr);if(!rawPointer){this.destructor(ptr);return null}var registeredInstance=getInheritedInstance(this.registeredClass,rawPointer);if(undefined!==registeredInstance){if(0===registeredInstance.$$.count.value){registeredInstance.$$.ptr=rawPointer;registeredInstance.$$.smartPtr=ptr;return registeredInstance["clone"]()}else{var rv=registeredInstance["clone"]();this.destructor(ptr);return rv}}function makeDefaultHandle(){if(this.isSmartPointer){return makeClassHandle(this.registeredClass.instancePrototype,{ptrType:this.pointeeType,ptr:rawPointer,smartPtrType:this,smartPtr:ptr})}else{return makeClassHandle(this.registeredClass.instancePrototype,{ptrType:this,ptr})}}var actualType=this.registeredClass.getActualType(rawPointer);var registeredPointerRecord=registeredPointers[actualType];if(!registeredPointerRecord){return makeDefaultHandle.call(this)}var toType;if(this.isConst){toType=registeredPointerRecord.constPointerType}else{toType=registeredPointerRecord.pointerType}var dp=downcastPointer(rawPointer,this.registeredClass,toType.registeredClass);if(dp===null){return makeDefaultHandle.call(this)}if(this.isSmartPointer){return makeClassHandle(toType.registeredClass.instancePrototype,{ptrType:toType,ptr:dp,smartPtrType:this,smartPtr:ptr})}else{return makeClassHandle(toType.registeredClass.instancePrototype,{ptrType:toType,ptr:dp})}}var init_RegisteredPointer=()=>{Object.assign(RegisteredPointer.prototype,{getPointee(ptr){if(this.rawGetPointee){ptr=this.rawGetPointee(ptr)}return ptr},destructor(ptr){this.rawDestructor?.(ptr)},readValueFromPointer:readPointer,fromWireType:RegisteredPointer_fromWireType})};function RegisteredPointer(name,registeredClass,isReference,isConst,isSmartPointer,pointeeType,sharingPolicy,rawGetPointee,rawConstructor,rawShare,rawDestructor){this.name=name;this.registeredClass=registeredClass;this.isReference=isReference;this.isConst=isConst;this.isSmartPointer=isSmartPointer;this.pointeeType=pointeeType;this.sharingPolicy=sharingPolicy;this.rawGetPointee=rawGetPointee;this.rawConstructor=rawConstructor;this.rawShare=rawShare;this.rawDestructor=rawDestructor;if(!isSmartPointer&®isteredClass.baseClass===undefined){if(isConst){this.toWireType=constNoSmartPtrRawPointerToWireType;this.destructorFunction=null}else{this.toWireType=nonConstNoSmartPtrRawPointerToWireType;this.destructorFunction=null}}else{this.toWireType=genericPointerToWireType}}var replacePublicSymbol=(name,value,numArguments)=>{if(!Module.hasOwnProperty(name)){throwInternalError("Replacing nonexistent public symbol")}if(undefined!==Module[name].overloadTable&&undefined!==numArguments){Module[name].overloadTable[numArguments]=value}else{Module[name]=value;Module[name].argCount=numArguments}};var wasmTableMirror=[];var getWasmTableEntry=funcPtr=>{var func=wasmTableMirror[funcPtr];if(!func){wasmTableMirror[funcPtr]=func=wasmTable.get(funcPtr)}return func};var embind__requireFunction=(signature,rawFunction,isAsync=false)=>{signature=AsciiToString(signature);function makeDynCaller(){var rtn=getWasmTableEntry(rawFunction);return rtn}var fp=makeDynCaller();if(typeof fp!="function"){throwBindingError(`unknown function pointer with signature ${signature}: ${rawFunction}`)}return fp};class UnboundTypeError extends Error{}var getTypeName=type=>{var ptr=___getTypeName(type);var rv=AsciiToString(ptr);_free(ptr);return rv};var throwUnboundTypeError=(message,types)=>{var unboundTypes=[];var seen={};function visit(type){if(seen[type]){return}if(registeredTypes[type]){return}if(typeDependencies[type]){typeDependencies[type].forEach(visit);return}unboundTypes.push(type);seen[type]=true}types.forEach(visit);throw new UnboundTypeError(`${message}: `+unboundTypes.map(getTypeName).join([", "]))};var __embind_register_class=(rawType,rawPointerType,rawConstPointerType,baseClassRawType,getActualTypeSignature,getActualType,upcastSignature,upcast,downcastSignature,downcast,name,destructorSignature,rawDestructor)=>{name=AsciiToString(name);getActualType=embind__requireFunction(getActualTypeSignature,getActualType);upcast&&=embind__requireFunction(upcastSignature,upcast);downcast&&=embind__requireFunction(downcastSignature,downcast);rawDestructor=embind__requireFunction(destructorSignature,rawDestructor);var legalFunctionName=makeLegalFunctionName(name);exposePublicSymbol(legalFunctionName,function(){throwUnboundTypeError(`Cannot construct ${name} due to unbound types`,[baseClassRawType])});whenDependentTypesAreResolved([rawType,rawPointerType,rawConstPointerType],baseClassRawType?[baseClassRawType]:[],base=>{base=base[0];var baseClass;var basePrototype;if(baseClassRawType){baseClass=base.registeredClass;basePrototype=baseClass.instancePrototype}else{basePrototype=ClassHandle.prototype}var constructor=createNamedFunction(name,function(...args){if(Object.getPrototypeOf(this)!==instancePrototype){throw new BindingError(`Use 'new' to construct ${name}`)}if(undefined===registeredClass.constructor_body){throw new BindingError(`${name} has no accessible constructor`)}var body=registeredClass.constructor_body[args.length];if(undefined===body){throw new BindingError(`Tried to invoke ctor of ${name} with invalid number of parameters (${args.length}) - expected (${Object.keys(registeredClass.constructor_body).toString()}) parameters instead!`)}return body.apply(this,args)});var instancePrototype=Object.create(basePrototype,{constructor:{value:constructor}});constructor.prototype=instancePrototype;var registeredClass=new RegisteredClass(name,constructor,instancePrototype,rawDestructor,baseClass,getActualType,upcast,downcast);if(registeredClass.baseClass){registeredClass.baseClass.__derivedClasses??=[];registeredClass.baseClass.__derivedClasses.push(registeredClass)}var referenceConverter=new RegisteredPointer(name,registeredClass,true,false,false);var pointerConverter=new RegisteredPointer(name+"*",registeredClass,false,false,false);var constPointerConverter=new RegisteredPointer(name+" const*",registeredClass,false,true,false);registeredPointers[rawType]={pointerType:pointerConverter,constPointerType:constPointerConverter};replacePublicSymbol(legalFunctionName,constructor);return[referenceConverter,pointerConverter,constPointerConverter]})};var heap32VectorToArray=(count,firstElement)=>{var array=[];for(var i=0;i>2])}return array};function usesDestructorStack(argTypes){for(var i=1;i{var rawArgTypes=heap32VectorToArray(argCount,rawArgTypesAddr);invoker=embind__requireFunction(invokerSignature,invoker);whenDependentTypesAreResolved([],[rawClassType],classType=>{classType=classType[0];var humanName=`constructor ${classType.name}`;if(undefined===classType.registeredClass.constructor_body){classType.registeredClass.constructor_body=[]}if(undefined!==classType.registeredClass.constructor_body[argCount-1]){throw new BindingError(`Cannot register multiple constructors with identical number of parameters (${argCount-1}) for class '${classType.name}'! Overload resolution is currently only performed using the parameter count, not actual type info!`)}classType.registeredClass.constructor_body[argCount-1]=()=>{throwUnboundTypeError(`Cannot construct ${classType.name} due to unbound types`,rawArgTypes)};whenDependentTypesAreResolved([],rawArgTypes,argTypes=>{argTypes.splice(1,0,null);classType.registeredClass.constructor_body[argCount-1]=craftInvokerFunction(humanName,argTypes,null,invoker,rawConstructor);return[]});return[]})};var getFunctionName=signature=>{signature=signature.trim();const argsIndex=signature.indexOf("(");if(argsIndex===-1)return signature;return signature.slice(0,argsIndex)};var __embind_register_class_function=(rawClassType,methodName,argCount,rawArgTypesAddr,invokerSignature,rawInvoker,context,isPureVirtual,isAsync,isNonnullReturn)=>{var rawArgTypes=heap32VectorToArray(argCount,rawArgTypesAddr);methodName=AsciiToString(methodName);methodName=getFunctionName(methodName);rawInvoker=embind__requireFunction(invokerSignature,rawInvoker,isAsync);whenDependentTypesAreResolved([],[rawClassType],classType=>{classType=classType[0];var humanName=`${classType.name}.${methodName}`;if(methodName.startsWith("@@")){methodName=Symbol[methodName.substring(2)]}if(isPureVirtual){classType.registeredClass.pureVirtualFunctions.push(methodName)}function unboundTypesHandler(){throwUnboundTypeError(`Cannot call ${humanName} due to unbound types`,rawArgTypes)}var proto=classType.registeredClass.instancePrototype;var method=proto[methodName];if(undefined===method||undefined===method.overloadTable&&method.className!==classType.name&&method.argCount===argCount-2){unboundTypesHandler.argCount=argCount-2;unboundTypesHandler.className=classType.name;proto[methodName]=unboundTypesHandler}else{ensureOverloadTable(proto,methodName,humanName);proto[methodName].overloadTable[argCount-2]=unboundTypesHandler}whenDependentTypesAreResolved([],rawArgTypes,argTypes=>{var memberFunction=craftInvokerFunction(humanName,argTypes,classType,rawInvoker,context,isAsync);if(undefined===proto[methodName].overloadTable){memberFunction.argCount=argCount-2;proto[methodName]=memberFunction}else{proto[methodName].overloadTable[argCount-2]=memberFunction}return[]});return[]})};var __embind_register_constant=(name,type,value)=>{name=AsciiToString(name);whenDependentTypesAreResolved([],[type],type=>{type=type[0];Module[name]=type.fromWireType(value);return[]})};var emval_freelist=[];var emval_handles=[0,1,,1,null,1,true,1,false,1];var __emval_decref=handle=>{if(handle>9&&0===--emval_handles[handle+1]){emval_handles[handle]=undefined;emval_freelist.push(handle)}};var Emval={toValue:handle=>{if(!handle){throwBindingError(`Cannot use deleted val. handle = ${handle}`)}return emval_handles[handle]},toHandle:value=>{switch(value){case undefined:return 2;case null:return 4;case true:return 6;case false:return 8;default:{const handle=emval_freelist.pop()||emval_handles.length;emval_handles[handle]=value;emval_handles[handle+1]=1;return handle}}}};var EmValType={name:"emscripten::val",fromWireType:handle=>{var rv=Emval.toValue(handle);__emval_decref(handle);return rv},toWireType:(destructors,value)=>Emval.toHandle(value),readValueFromPointer:readPointer,destructorFunction:null};var __embind_register_emval=rawType=>registerType(rawType,EmValType);var enumReadValueFromPointer=(name,width,signed)=>{switch(width){case 1:return signed?function(pointer){return this.fromWireType(HEAP8[pointer])}:function(pointer){return this.fromWireType(HEAPU8[pointer])};case 2:return signed?function(pointer){return this.fromWireType(HEAP16[pointer>>1])}:function(pointer){return this.fromWireType(HEAPU16[pointer>>1])};case 4:return signed?function(pointer){return this.fromWireType(HEAP32[pointer>>2])}:function(pointer){return this.fromWireType(HEAPU32[pointer>>2])};default:throw new TypeError(`invalid integer width (${width}): ${name}`)}};var __embind_register_enum=(rawType,name,size,isSigned)=>{name=AsciiToString(name);function ctor(){}ctor.values={};registerType(rawType,{name,constructor:ctor,fromWireType:function(c){return this.constructor.values[c]},toWireType:(destructors,c)=>c.value,readValueFromPointer:enumReadValueFromPointer(name,size,isSigned),destructorFunction:null});exposePublicSymbol(name,ctor)};var requireRegisteredType=(rawType,humanName)=>{var impl=registeredTypes[rawType];if(undefined===impl){throwBindingError(`${humanName} has unknown type ${getTypeName(rawType)}`)}return impl};var __embind_register_enum_value=(rawEnumType,name,enumValue)=>{var enumType=requireRegisteredType(rawEnumType,"enum");name=AsciiToString(name);var Enum=enumType.constructor;var Value=Object.create(enumType.constructor.prototype,{value:{value:enumValue},constructor:{value:createNamedFunction(`${enumType.name}_${name}`,function(){})}});Enum.values[enumValue]=Value;Enum[name]=Value};var floatReadValueFromPointer=(name,width)=>{switch(width){case 4:return function(pointer){return this.fromWireType(HEAPF32[pointer>>2])};case 8:return function(pointer){return this.fromWireType(HEAPF64[pointer>>3])};default:throw new TypeError(`invalid float width (${width}): ${name}`)}};var __embind_register_float=(rawType,name,size)=>{name=AsciiToString(name);registerType(rawType,{name,fromWireType:value=>value,toWireType:(destructors,value)=>value,readValueFromPointer:floatReadValueFromPointer(name,size),destructorFunction:null})};var __embind_register_function=(name,argCount,rawArgTypesAddr,signature,rawInvoker,fn,isAsync,isNonnullReturn)=>{var argTypes=heap32VectorToArray(argCount,rawArgTypesAddr);name=AsciiToString(name);name=getFunctionName(name);rawInvoker=embind__requireFunction(signature,rawInvoker,isAsync);exposePublicSymbol(name,function(){throwUnboundTypeError(`Cannot call ${name} due to unbound types`,argTypes)},argCount-1);whenDependentTypesAreResolved([],argTypes,argTypes=>{var invokerArgsArray=[argTypes[0],null].concat(argTypes.slice(1));replacePublicSymbol(name,craftInvokerFunction(name,invokerArgsArray,null,rawInvoker,fn,isAsync),argCount-1);return[]})};var __embind_register_integer=(primitiveType,name,size,minRange,maxRange)=>{name=AsciiToString(name);const isUnsignedType=minRange===0;let fromWireType=value=>value;if(isUnsignedType){var bitshift=32-8*size;fromWireType=value=>value<>>bitshift;maxRange=fromWireType(maxRange)}registerType(primitiveType,{name,fromWireType,toWireType:(destructors,value)=>value,readValueFromPointer:integerReadValueFromPointer(name,size,minRange!==0),destructorFunction:null})};var __embind_register_memory_view=(rawType,dataTypeIndex,name)=>{var typeMapping=[Int8Array,Uint8Array,Int16Array,Uint16Array,Int32Array,Uint32Array,Float32Array,Float64Array,BigInt64Array,BigUint64Array];var TA=typeMapping[dataTypeIndex];function decodeMemoryView(handle){var size=HEAPU32[handle>>2];var data=HEAPU32[handle+4>>2];return new TA(HEAP8.buffer,data,size)}name=AsciiToString(name);registerType(rawType,{name,fromWireType:decodeMemoryView,readValueFromPointer:decodeMemoryView},{ignoreDuplicateRegistrations:true})};var stringToUTF8=(str,outPtr,maxBytesToWrite)=>stringToUTF8Array(str,HEAPU8,outPtr,maxBytesToWrite);var __embind_register_std_string=(rawType,name)=>{name=AsciiToString(name);var stdStringIsUTF8=true;registerType(rawType,{name,fromWireType(value){var length=HEAPU32[value>>2];var payload=value+4;var str;if(stdStringIsUTF8){str=UTF8ToString(payload,length,true)}else{str="";for(var i=0;i>2]=length;if(valueIsOfTypeString){if(stdStringIsUTF8){stringToUTF8(value,ptr,length+1)}else{for(var i=0;i255){_free(base);throwBindingError("String has UTF-16 code units that do not fit in 8 bits")}HEAPU8[ptr+i]=charCode}}}else{HEAPU8.set(value,ptr)}if(destructors!==null){destructors.push(_free,base)}return base},readValueFromPointer:readPointer,destructorFunction(ptr){_free(ptr)}})};var UTF16Decoder=globalThis.TextDecoder?new TextDecoder("utf-16le"):undefined;var UTF16ToString=(ptr,maxBytesToRead,ignoreNul)=>{var idx=ptr>>1;var endIdx=findStringEnd(HEAPU16,idx,maxBytesToRead/2,ignoreNul);if(endIdx-idx>16&&UTF16Decoder)return UTF16Decoder.decode(HEAPU16.subarray(idx,endIdx));var str="";for(var i=idx;i{maxBytesToWrite??=2147483647;if(maxBytesToWrite<2)return 0;maxBytesToWrite-=2;var startPtr=outPtr;var numCharsToWrite=maxBytesToWrite>1]=codeUnit;outPtr+=2}HEAP16[outPtr>>1]=0;return outPtr-startPtr};var lengthBytesUTF16=str=>str.length*2;var UTF32ToString=(ptr,maxBytesToRead,ignoreNul)=>{var str="";var startIdx=ptr>>2;for(var i=0;!(i>=maxBytesToRead/4);i++){var utf32=HEAPU32[startIdx+i];if(!utf32&&!ignoreNul)break;str+=String.fromCodePoint(utf32)}return str};var stringToUTF32=(str,outPtr,maxBytesToWrite)=>{maxBytesToWrite??=2147483647;if(maxBytesToWrite<4)return 0;var startPtr=outPtr;var endPtr=startPtr+maxBytesToWrite-4;for(var i=0;i65535){i++}HEAP32[outPtr>>2]=codePoint;outPtr+=4;if(outPtr+4>endPtr)break}HEAP32[outPtr>>2]=0;return outPtr-startPtr};var lengthBytesUTF32=str=>{var len=0;for(var i=0;i65535){i++}len+=4}return len};var __embind_register_std_wstring=(rawType,charSize,name)=>{name=AsciiToString(name);var decodeString,encodeString,lengthBytesUTF;if(charSize===2){decodeString=UTF16ToString;encodeString=stringToUTF16;lengthBytesUTF=lengthBytesUTF16}else{decodeString=UTF32ToString;encodeString=stringToUTF32;lengthBytesUTF=lengthBytesUTF32}registerType(rawType,{name,fromWireType:value=>{var length=HEAPU32[value>>2];var str=decodeString(value+4,length*charSize,true);_free(value);return str},toWireType:(destructors,value)=>{if(!(typeof value=="string")){throwBindingError(`Cannot pass non-string to C++ string type ${name}`)}var length=lengthBytesUTF(value);var ptr=_malloc(4+length+charSize);HEAPU32[ptr>>2]=length/charSize;encodeString(value,ptr+4,length+charSize);if(destructors!==null){destructors.push(_free,ptr)}return ptr},readValueFromPointer:readPointer,destructorFunction(ptr){_free(ptr)}})};var __embind_register_value_object=(rawType,name,constructorSignature,rawConstructor,destructorSignature,rawDestructor)=>{structRegistrations[rawType]={name:AsciiToString(name),rawConstructor:embind__requireFunction(constructorSignature,rawConstructor),rawDestructor:embind__requireFunction(destructorSignature,rawDestructor),fields:[]}};var __embind_register_value_object_field=(structType,fieldName,getterReturnType,getterSignature,getter,getterContext,setterArgumentType,setterSignature,setter,setterContext)=>{structRegistrations[structType].fields.push({fieldName:AsciiToString(fieldName),getterReturnType,getter:embind__requireFunction(getterSignature,getter),getterContext,setterArgumentType,setter:embind__requireFunction(setterSignature,setter),setterContext})};var __embind_register_void=(rawType,name)=>{name=AsciiToString(name);registerType(rawType,{isVoid:true,name,fromWireType:()=>undefined,toWireType:(destructors,o)=>undefined})};var __emscripten_throw_longjmp=()=>{throw Infinity};var emval_methodCallers=[];var emval_addMethodCaller=caller=>{var id=emval_methodCallers.length;emval_methodCallers.push(caller);return id};var emval_lookupTypes=(argCount,argTypes)=>{var a=new Array(argCount);for(var i=0;i>2],`parameter ${i}`)}return a};var emval_returnValue=(toReturnWire,destructorsRef,handle)=>{var destructors=[];var result=toReturnWire(destructors,handle);if(destructors.length){HEAPU32[destructorsRef>>2]=Emval.toHandle(destructors)}return result};var emval_symbols={};var getStringOrSymbol=address=>{var symbol=emval_symbols[address];if(symbol===undefined){return AsciiToString(address)}return symbol};var __emval_create_invoker=(argCount,argTypesPtr,kind)=>{var GenericWireTypeSize=8;var[retType,...argTypes]=emval_lookupTypes(argCount,argTypesPtr);var toReturnWire=retType.toWireType.bind(retType);var argFromPtr=argTypes.map(type=>type.readValueFromPointer.bind(type));argCount--;var captures={toValue:Emval.toValue};var args=argFromPtr.map((argFromPtr,i)=>{var captureName=`argFromPtr${i}`;captures[captureName]=argFromPtr;return`${captureName}(args${i?"+"+i*GenericWireTypeSize:""})`});var functionBody;switch(kind){case 0:functionBody="toValue(handle)";break;case 2:functionBody="new (toValue(handle))";break;case 3:functionBody="";break;case 1:captures["getStringOrSymbol"]=getStringOrSymbol;functionBody="toValue(handle)[getStringOrSymbol(methodName)]";break}functionBody+=`(${args})`;if(!retType.isVoid){captures["toReturnWire"]=toReturnWire;captures["emval_returnValue"]=emval_returnValue;functionBody=`return emval_returnValue(toReturnWire, destructorsRef, ${functionBody})`}functionBody=`return function (handle, methodName, destructorsRef, args) {\n ${functionBody}\n }`;var invokerFunction=new Function(Object.keys(captures),functionBody)(...Object.values(captures));var functionName=`methodCaller<(${argTypes.map(t=>t.name)}) => ${retType.name}>`;return emval_addMethodCaller(createNamedFunction(functionName,invokerFunction))};var __emval_get_property=(handle,key)=>{handle=Emval.toValue(handle);key=Emval.toValue(key);return Emval.toHandle(handle[key])};var __emval_incref=handle=>{if(handle>9){emval_handles[handle+1]+=1}};var __emval_invoke=(caller,handle,methodName,destructorsRef,args)=>emval_methodCallers[caller](handle,methodName,destructorsRef,args);var __emval_new_cstring=v=>Emval.toHandle(getStringOrSymbol(v));var __emval_run_destructors=handle=>{var destructors=Emval.toValue(handle);runDestructors(destructors);__emval_decref(handle)};var INT53_MAX=9007199254740992;var INT53_MIN=-9007199254740992;var bigintToI53Checked=num=>numINT53_MAX?NaN:Number(num);function __mmap_js(len,prot,flags,fd,offset,allocated,addr){offset=bigintToI53Checked(offset);try{var stream=SYSCALLS.getStreamFromFD(fd);var res=FS.mmap(stream,len,offset,prot,flags);var ptr=res.ptr;HEAP32[allocated>>2]=res.allocated;HEAPU32[addr>>2]=ptr;return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function __munmap_js(addr,len,prot,flags,fd,offset){offset=bigintToI53Checked(offset);try{var stream=SYSCALLS.getStreamFromFD(fd);if(prot&2){SYSCALLS.doMsync(addr,stream,len,flags,offset)}}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}var __tzset_js=(timezone,daylight,std_name,dst_name)=>{var currentYear=(new Date).getFullYear();var winter=new Date(currentYear,0,1);var summer=new Date(currentYear,6,1);var winterOffset=winter.getTimezoneOffset();var summerOffset=summer.getTimezoneOffset();var stdTimezoneOffset=Math.max(winterOffset,summerOffset);HEAPU32[timezone>>2]=stdTimezoneOffset*60;HEAP32[daylight>>2]=Number(winterOffset!=summerOffset);var extractZone=timezoneOffset=>{var sign=timezoneOffset>=0?"-":"+";var absOffset=Math.abs(timezoneOffset);var hours=String(Math.floor(absOffset/60)).padStart(2,"0");var minutes=String(absOffset%60).padStart(2,"0");return`UTC${sign}${hours}${minutes}`};var winterName=extractZone(winterOffset);var summerName=extractZone(summerOffset);if(summerOffsetperformance.now();var _emscripten_date_now=()=>Date.now();var nowIsMonotonic=1;var checkWasiClock=clock_id=>clock_id>=0&&clock_id<=3;function _clock_time_get(clk_id,ignored_precision,ptime){ignored_precision=bigintToI53Checked(ignored_precision);if(!checkWasiClock(clk_id)){return 28}var now;if(clk_id===0){now=_emscripten_date_now()}else if(nowIsMonotonic){now=_emscripten_get_now()}else{return 52}var nsec=Math.round(now*1e3*1e3);HEAP64[ptime>>3]=BigInt(nsec);return 0}var getHeapMax=()=>2147483648;var _emscripten_get_heap_max=()=>getHeapMax();var growMemory=size=>{var oldHeapSize=wasmMemory.buffer.byteLength;var pages=(size-oldHeapSize+65535)/65536|0;try{wasmMemory.grow(pages);updateMemoryViews();return 1}catch(e){}};var _emscripten_resize_heap=requestedSize=>{var oldSize=HEAPU8.length;requestedSize>>>=0;var maxHeapSize=getHeapMax();if(requestedSize>maxHeapSize){return false}for(var cutDown=1;cutDown<=4;cutDown*=2){var overGrownHeapSize=oldSize*(1+.2/cutDown);overGrownHeapSize=Math.min(overGrownHeapSize,requestedSize+100663296);var newSize=Math.min(maxHeapSize,alignMemory(Math.max(requestedSize,overGrownHeapSize),65536));var replacement=growMemory(newSize);if(replacement){return true}}return false};var ENV={};var getExecutableName=()=>thisProgram||"./this.program";var getEnvStrings=()=>{if(!getEnvStrings.strings){var lang=(typeof navigator=="object"&&navigator.language||"C").replace("-","_")+".UTF-8";var env={USER:"web_user",LOGNAME:"web_user",PATH:"/",PWD:"/",HOME:"/home/web_user",LANG:lang,_:getExecutableName()};for(var x in ENV){if(ENV[x]===undefined)delete env[x];else env[x]=ENV[x]}var strings=[];for(var x in env){strings.push(`${x}=${env[x]}`)}getEnvStrings.strings=strings}return getEnvStrings.strings};var _environ_get=(__environ,environ_buf)=>{var bufSize=0;var envp=0;for(var string of getEnvStrings()){var ptr=environ_buf+bufSize;HEAPU32[__environ+envp>>2]=ptr;bufSize+=stringToUTF8(string,ptr,Infinity)+1;envp+=4}return 0};var _environ_sizes_get=(penviron_count,penviron_buf_size)=>{var strings=getEnvStrings();HEAPU32[penviron_count>>2]=strings.length;var bufSize=0;for(var string of strings){bufSize+=lengthBytesUTF8(string)+1}HEAPU32[penviron_buf_size>>2]=bufSize;return 0};function _fd_close(fd){try{var stream=SYSCALLS.getStreamFromFD(fd);FS.close(stream);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return e.errno}}var doReadv=(stream,iov,iovcnt,offset)=>{var ret=0;for(var i=0;i>2];var len=HEAPU32[iov+4>>2];iov+=8;var curr=FS.read(stream,HEAP8,ptr,len,offset);if(curr<0)return-1;ret+=curr;if(curr>2]=num;return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return e.errno}}function _fd_seek(fd,offset,whence,newOffset){offset=bigintToI53Checked(offset);try{if(isNaN(offset))return 61;var stream=SYSCALLS.getStreamFromFD(fd);FS.llseek(stream,offset,whence);HEAP64[newOffset>>3]=BigInt(stream.position);if(stream.getdents&&offset===0&&whence===0)stream.getdents=null;return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return e.errno}}var doWritev=(stream,iov,iovcnt,offset)=>{var ret=0;for(var i=0;i>2];var len=HEAPU32[iov+4>>2];iov+=8;var curr=FS.write(stream,HEAP8,ptr,len,offset);if(curr<0)return-1;ret+=curr;if(curr>2]=num;return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return e.errno}}FS.createPreloadedFile=FS_createPreloadedFile;FS.preloadFile=FS_preloadFile;FS.staticInit();init_ClassHandle();init_RegisteredPointer();{if(Module["noExitRuntime"])noExitRuntime=Module["noExitRuntime"];if(Module["preloadPlugins"])preloadPlugins=Module["preloadPlugins"];if(Module["print"])out=Module["print"];if(Module["printErr"])err=Module["printErr"];if(Module["wasmBinary"])wasmBinary=Module["wasmBinary"];if(Module["arguments"])arguments_=Module["arguments"];if(Module["thisProgram"])thisProgram=Module["thisProgram"];if(Module["preInit"]){if(typeof Module["preInit"]=="function")Module["preInit"]=[Module["preInit"]];while(Module["preInit"].length>0){Module["preInit"].shift()()}}}var ___getTypeName,_free,_malloc,_emscripten_builtin_memalign,_setThrew,__emscripten_stack_restore,_emscripten_stack_get_current,memory,__indirect_function_table,wasmMemory,wasmTable;function assignWasmExports(wasmExports){___getTypeName=wasmExports["aa"];_free=wasmExports["ba"];_malloc=wasmExports["da"];_emscripten_builtin_memalign=wasmExports["ea"];_setThrew=wasmExports["fa"];__emscripten_stack_restore=wasmExports["ga"];_emscripten_stack_get_current=wasmExports["ha"];memory=wasmMemory=wasmExports["_"];__indirect_function_table=wasmTable=wasmExports["ca"]}var wasmImports={s:___cxa_throw,z:___syscall_fcntl64,N:___syscall_fstat64,P:___syscall_ioctl,M:___syscall_newfstatat,A:___syscall_openat,S:__abort_js,r:__embind_finalize_value_object,C:__embind_register_bigint,Y:__embind_register_bool,u:__embind_register_class,t:__embind_register_class_constructor,a:__embind_register_class_function,d:__embind_register_constant,W:__embind_register_emval,l:__embind_register_enum,b:__embind_register_enum_value,B:__embind_register_float,f:__embind_register_function,m:__embind_register_integer,j:__embind_register_memory_view,X:__embind_register_std_string,w:__embind_register_std_wstring,p:__embind_register_value_object,c:__embind_register_value_object_field,Z:__embind_register_void,H:__emscripten_throw_longjmp,i:__emval_create_invoker,e:__emval_decref,n:__emval_get_property,k:__emval_incref,h:__emval_invoke,o:__emval_new_cstring,g:__emval_run_destructors,J:__mmap_js,K:__munmap_js,E:__tzset_js,R:_clock_time_get,Q:_emscripten_date_now,D:_emscripten_get_heap_max,x:_emscripten_get_now,I:_emscripten_resize_heap,F:_environ_get,G:_environ_sizes_get,v:_fd_close,O:_fd_read,L:_fd_seek,y:_fd_write,U:invoke_ii,q:invoke_vi,T:invoke_vii,V:invoke_viii};function invoke_vi(index,a1){var sp=stackSave();try{getWasmTableEntry(index)(a1)}catch(e){stackRestore(sp);if(e!==e+0)throw e;_setThrew(1,0)}}function invoke_viii(index,a1,a2,a3){var sp=stackSave();try{getWasmTableEntry(index)(a1,a2,a3)}catch(e){stackRestore(sp);if(e!==e+0)throw e;_setThrew(1,0)}}function invoke_ii(index,a1){var sp=stackSave();try{return getWasmTableEntry(index)(a1)}catch(e){stackRestore(sp);if(e!==e+0)throw e;_setThrew(1,0)}}function invoke_vii(index,a1,a2){var sp=stackSave();try{getWasmTableEntry(index)(a1,a2)}catch(e){stackRestore(sp);if(e!==e+0)throw e;_setThrew(1,0)}}function run(){if(runDependencies>0){dependenciesFulfilled=run;return}preRun();if(runDependencies>0){dependenciesFulfilled=run;return}function doRun(){Module["calledRun"]=true;if(ABORT)return;initRuntime();readyPromiseResolve?.(Module);Module["onRuntimeInitialized"]?.();postRun()}if(Module["setStatus"]){Module["setStatus"]("Running...");setTimeout(()=>{setTimeout(()=>Module["setStatus"](""),1);doRun()},1)}else{doRun()}}var wasmExports;wasmExports=await (createWasm());run();if(runtimeInitialized){moduleRtn=Module}else{moduleRtn=new Promise((resolve,reject)=>{readyPromiseResolve=resolve;readyPromiseReject=reject})} +;return moduleRtn}})();if(typeof exports==="object"&&typeof module==="object"){module.exports=BASIS;module.exports.default=BASIS}else if(typeof define==="function"&&define["amd"])define([],()=>BASIS); diff --git a/external/basis_universal/webgl/encoder/build/basis_encoder.wasm b/external/basis_universal/webgl/encoder/build/basis_encoder.wasm index 9e62a1bb6c..0e63325d0d 100644 Binary files a/external/basis_universal/webgl/encoder/build/basis_encoder.wasm and b/external/basis_universal/webgl/encoder/build/basis_encoder.wasm differ diff --git a/external/basis_universal/webgl/encoder/build_notes.txt b/external/basis_universal/webgl/encoder/build_notes.txt new file mode 100644 index 0000000000..2db46f941d --- /dev/null +++ b/external/basis_universal/webgl/encoder/build_notes.txt @@ -0,0 +1,26 @@ +# Prereq: activate emsdk first (so emcmake/em++ are on PATH) +# Linux/macOS: +source /path/to/emsdk/emsdk_env.sh +# Windows PowerShell: +# & "C:\path\to\emsdk\emsdk_env.ps1" + +# ===== Release (fast; same behavior as your original file) ===== +emcmake cmake -S . -B build-release -DCMAKE_BUILD_TYPE=Release +cmake --build build-release -j + +# ===== Debug (symbols + assertions) ===== +emcmake cmake -S . -B build-debug -DCMAKE_BUILD_TYPE=Debug +cmake --build build-debug -j + +# ===== SAN (ASan + UBSan; great for catching bugs) ===== +emcmake cmake -S . -B build-san -DCMAKE_BUILD_TYPE=SAN +cmake --build build-san -j + +# Build a single target (optional) instead of all three: +cmake --build build-release -j --target basis_encoder.js +cmake --build build-release -j --target basis_encoder_threads.js +cmake --build build-release -j --target basis_encoder_threads_wasm64.js + +# Toggle Zstd (OFF = smaller binary, no KTX2 Zstd compression) +emcmake cmake -S . -B build-release -DCMAKE_BUILD_TYPE=Release -DKTX2_ZSTANDARD=OFF +cmake --build build-release -j diff --git a/external/basis_universal/webgl/index.html b/external/basis_universal/webgl/index.html index d469dcd707..f998dfbb20 100644 --- a/external/basis_universal/webgl/index.html +++ b/external/basis_universal/webgl/index.html @@ -6,12 +6,12 @@

Basis Universal LDR/HDR WebGL demos

- Example web pages using the transcoder (compiled to WASM) to render - .KTX2/.basis textures in WebGL, and the encoder to encode .KTX2 files. + Example web pages using the transcoder library, compiled to WASM, to render + .KTX2/.basis textures in WebGL, and the compressor library to encode .KTX2 files.

diff --git a/external/basis_universal/webgl/ktx2_encode_test/assets/license.txt b/external/basis_universal/webgl/ktx2_encode_test/assets/license.txt new file mode 100644 index 0000000000..e06b8ffc57 --- /dev/null +++ b/external/basis_universal/webgl/ktx2_encode_test/assets/license.txt @@ -0,0 +1 @@ +The test images in this directory are not required to build or use the Basis Universal codec. They are not copyrighted or owned by Binomial LLC. diff --git a/external/basis_universal/webgl/ktx2_encode_test/index.html b/external/basis_universal/webgl/ktx2_encode_test/index.html index 8f98e924fc..4b3956f7d7 100644 --- a/external/basis_universal/webgl/ktx2_encode_test/index.html +++ b/external/basis_universal/webgl/ktx2_encode_test/index.html @@ -1,944 +1,1338 @@ - + + + - - -
-
- Basis Universal Multithreaded .KTX2 Supercompressed GPU Texture Encoding/Transcoding Testbed v0.67 -
- -
This demo uses the Basis Universal C++ transcoder (compiled to WebAssembly using Emscripten) to transcode a .ktx2 file to FORMAT -
It also supports encoding .PNG, .EXR or .HDR files to LDR or HDR .KTX2 files. -
Thanks to Evan Parker for providing webgl-texture-utils and this test bed. Go back. -
Enable your browser debug console (F12 on Chrome/Firefox) to see debug output. -
Note: The largest image resolution that can be compressed in the browser with -
this library is limited to around 6 megapixels due to 32-bit WASM memory -
constraints. Larger images risk running out of memory. - -

-
- Supported WebGL formats: -
- -
-
- Test -
- -
- -
- -
-
- Use Multithreading (if available) - -
Additional Worker Threads (Max 18): - -

- -
- .ktx2 file: - - -
- -
- .png/.jpg/.exr/.hdr file: - - -
-
- - -
- -
-

Drag and drop a PNG/JPG/EXR/HDR file here, or click to select a file.

- -
-
+ // HACK HACK - for testing uncompressed fallbacks. + //astcSupported = false; + //etcSupported = false; + //dxtSupported = false; + //bc7Supported = false; + //pvrtcSupported = false; + //bc6hSupported = false; + //astcHDRSupported = false; + //rgbaHalfSupported = false; + + console.log('astcSupported: ' + astcSupported); + console.log('etcSupported: ' + etcSupported); + console.log('dxtSupported: ' + dxtSupported); + console.log('bc7Supported: ' + bc7Supported); + console.log('pvrtcSupported: ' + pvrtcSupported); + console.log('bc6hSupported: ' + bc6hSupported); + console.log('astcHDRSupported: ' + astcHDRSupported); + console.log('rgbaHalfSupported: ' + rgbaHalfSupported); + + updateSupportedFormats(); + updateDisableButtons(); + } - + function validateNitMultiplier(input) + { + const warning = document.getElementById('ldr-to-hdr-warning'); + const value = parseFloat(input.value); -
+ if (isNaN(value) || value < 0.1 || value > 1000) + { + // Show warning if value is invalid + warning.style.display = 'inline'; + } + else + { + // Hide warning if value is valid + warning.style.display = 'none'; + } + } - Visualization/Display Options: + function getNitMultiplier() + { + const input = document.getElementById('ldr-to-hdr-multiplier'); + const value = parseFloat(input.value); -
- Disable ETC1S->BC7 Transcoder's Chroma Artifact Filtering: - + if (!isNaN(value) && value >= 0.1 && value <= 1000) + { + return value; + } + else + { + log('Invalid LDR to HDR Nit Multiplier value. Defaulting to 100.0'); + return 100.0; // Default value if invalid + } + } -
-
+ function getSelectedBasisTexFormat() + { + const selectElem = document.getElementById('basis-tex-format'); + const selectedValue = parseInt(selectElem.value, 10); - Higher quality ASTC 6x6 HDR->BC6H transcoding: - + return selectedValue; + } -

- - - + function getXUASTCLDRSyntax() + { + const selectElem = document.getElementById('xuastc_ldr_syntax'); + const selectedValue = parseInt(selectElem.value, 10); -
-
- Disabled + return selectedValue; + } -
+ function getASTC6x6RDOLambda() + { + const input = document.getElementById('astc6x6-rdo-lambda'); + const value = parseFloat(input.value); - - - 1 + if (!isNaN(value) && value >= 0.0 && value <= 1000000) + { + return value; + } + else + { + log('Invalid lambda value. Defaulting to 0.0'); + return 0.0; // Default value if invalid + } + } -
- -
+ function showBusyModal() + { + //console.log("Showing modal"); + document.getElementById('busy-modal').style.display = 'flex'; + } -
-
+ function hideBusyModal() + { + //console.log("Hiding modal"); + document.getElementById('busy-modal').style.display = 'none'; + } - KTX2 Texture Format to Encode: - -
+ function setDropdownValue(selectId, value) + { + const selectElement = document.getElementById(selectId); + if (!selectElement) + return; -
+ selectElement.value = value; + selectElement.dispatchEvent(new Event('change')); + } - ETC1S LDR Options: -
- ETC1S Quality: - - 255 + function globalInit() + { + elem('SRGB').checked = true; -
- - - 2 + var gl = elem('canvas').getContext('webgl'); + checkForGPUFormatSupport(gl); -
+ window.renderer = new Renderer(gl); - UASTC LDR 4x4 Options: -
- - - 1 + elem('file').addEventListener('keydown', function (e) + { + if (e.keyCode == 13) { + runLoadFile(); + } + }, false); -
+ elem('imagefile').addEventListener('keydown', function (e) + { + if (e.keyCode == 13) { + runEncodeImageFile(); + } + }, false); - UASTC LDR RDO: - + { + let etc1SLevelSlider = document.getElementById('etc1s-comp-level-slider'); + let etc1sLevelSliderValueDisplay = document.getElementById('etc1s-comp-level-slider-value'); + etc1SLevelSlider.oninput = function() + { + etc1sLevelSliderValueDisplay.textContent = this.value; + } + } - - - 1.0 + { + let uastcHDRSlider = document.getElementById('uastc-hdr-quality-slider'); + let qualityHDRValueDisplay = document.getElementById('uastc-hdr-quality-value'); + uastcHDRSlider.oninput = function() + { + qualityHDRValueDisplay.textContent = this.value; + } + } -
+ { + let astcHDR6x6Slider = document.getElementById('astc-hdr6x6-comp-level-slider'); + let compLevelValueDisplay = document.getElementById('astc-hdr6x6-comp-level-value'); + astcHDR6x6Slider.oninput = function() + { + compLevelValueDisplay.textContent = this.value; + } + } - UASTC HDR 4x4 Options: + { + let uastcLDRSlider = document.getElementById('uastc-ldr-quality-slider'); + let qualityLDRValueDisplay = document.getElementById('uastc-ldr-quality-value'); + uastcLDRSlider.oninput = function() + { + qualityLDRValueDisplay.textContent = this.value; + } + } -
- - - 0 -
+ { + let rdoSlider = document.getElementById('rdo-quality-slider'); + let rdoValueDisplay = document.getElementById('rdo-quality-value'); + rdoSlider.oninput = function() + { + rdoValueDisplay.textContent = parseFloat(this.value).toFixed(1); + } + } -
+ { + let etc1SQualitySlider = document.getElementById('EncodeQuality'); + let etc1SQualitySliderValue = document.getElementById('encode-quality-value'); + etc1SQualitySlider.oninput = function() + { + etc1SQualitySliderValue.textContent = parseFloat(this.value).toFixed(0); + } + } - ASTC HDR 6x6 Options: -
+ { + let xuastcLDREffortSlider = document.getElementById('xuastc_ldr_effort_level_slider'); + let xuastcLDREffortValue = document.getElementById('xuastc_ldr_effort_level_value'); + xuastcLDREffortSlider.oninput = function() + { + xuastcLDREffortValue.textContent = this.value; + } + } - - - 0 -
+ { + let xuastcLDRDCTSlider = document.getElementById('xuastc_ldr_dct_quality_slider'); + let xuastcLDRDCTValue = document.getElementById('xuastc_ldr_dct_quality_value'); + xuastcLDRDCTSlider.oninput = function() + { + xuastcLDRDCTValue.textContent = this.value; + } + } + + { + document.getElementById("unified-effort-slider").oninput = function() + { + document.getElementById("unified-effort-value").textContent = this.value; + } + + document.getElementById("unified-quality-slider").oninput = function() + { + document.getElementById("unified-quality-value").textContent = this.value; + } + } + + runLoadFile(); + } - RDO Quality (Lambda, 0-50k, try 0-5k, higher=smaller): - -
+ function updateErrorLine(message) + { + const errorLine = document.getElementById('error-line'); + errorLine.textContent = message; + errorLine.style.color = message.trim() ? 'red' : ''; + } - REC 2020 Colorspace: - + -
+ - LDR->HDR Upconversion Options: -
- Convert LDR images to linear light: - + -
- LDR to HDR Upconversion Nit Multiplier: - - -
+ -
+
+
+ +
+
+ Basis Universal .KTX2 Supercompressed GPU Texture Encoding/Transcoding Testbed v2.01 +
+ +
This simple demo uses the Basis Universal C++ transcoder (compiled to WebAssembly using Emscripten) to transcode a .ktx2 file to: +
FORMAT +
+
The viewer is implemented in WebGL and renders a single textured quad. It also supports encoding .PNG, .JPG, .EXR or .HDR files to LDR or HDR .KTX2 files. + Thanks to Evan Parker for providing webgl-texture-utils and this test bed. Go back. + Notes: Enable your browser debug console (F12 on Chrome/Firefox) to see debug output. + The largest image resolution that can be compressed in the browser with + this library is limited to either 12 megapixels or 4 megapixels (depending on format and WASM64/WASM32) to avoid running out of WASM memory. + +

+
+ Supported WebGL formats: +
+ +
+
+ Test +
+ +
+ +
+ +
+
+
+ Use Multithreading (if available) + +
Additional Worker Threads (Max 18): + +

+ +
+ .ktx2 file: + + +
+ +
+ .png/.jpg/.exr/.hdr file: + + +
+
+ + +
+ +
+

Drag and drop a .KTX2 or image file here, or click to select a file.

+ +
+
+ + + +
+ + KTX2 Texture Format to Encode: + + +
+
+ + Primary compression quality/effort options: + +
+ Use unified quality/effort options (overrides below low-level options): + + +
+ + + 2 + +

+ + + + 80 + +
+
+ + Transcoder Options (Decode Flags): + +
+ ETC1S: No BC7 Chroma Artifact Filtering (faster transcoding): + + +
+ XUASTC/ASTC LDR: Disable deblocking filtering (faster): + + +
+ XUASTC/ASTC LDR: Stronger deblocking filtering: + + +
+ XUASTC/ASTC LDR: Use deblocking on all block sizes (slower): + + +
+ XUASTC LDR 4x4/6x6/8x6: No direct BC7 transcoding (slower/higher quality): + + +
+ + Prefer higher quality transcoding when supported (slower): + + +
+
+ + Display/Visualization Options: +
+ + + + + +
+ +
+ Disabled + +
+ + + + 1 + +
+ +
+ + Low-level ETC1S LDR Options: +
+ ETC1S Quality: + + 255 + +
+ + + 1 + +
+ + Low-level UASTC LDR 4x4 Options: + +
+ UASTC LDR RDO: + + + + + 1.0 + +
+ + + 1 + +
+ +
+ + Low-level UASTC HDR 4x4 Options: + +
+ + + 0 +
+ +
+ + Low-level ASTC/UASTC HDR 6x6 Options: +
- Other Options: + + + 0 +
-
- Use sRGB/perceptual metrics: - -
- Generate mipmaps: - + RDO Quality (Lambda, 0-50k, try 0-5k, higher=smaller): + +
-
-
- Debug Output (See Dev Console): - - Compute Stats: - + REC 2020 Colorspace: + -
+
+ + LDR->HDR Upconversion Options: +
+ Convert LDR images to linear light: + + +
+ LDR to HDR Upconversion Nit Multiplier: + + +
- Log Output: +
+ Low-level XUASTC/ASTC LDR 4x4-12x12 Options: +
+ + XUASTC LDR Syntax: + -
+
- + + + 3 +
- + + + 80 - + diff --git a/external/basis_universal/webgl/ktx2_encode_test/renderer.js b/external/basis_universal/webgl/ktx2_encode_test/renderer.js index 8de7e58cf3..cfd1b005ff 100644 --- a/external/basis_universal/webgl/ktx2_encode_test/renderer.js +++ b/external/basis_universal/webgl/ktx2_encode_test/renderer.js @@ -10,7 +10,7 @@ var Renderer = function (gl) { * @private */ this.gl_ = gl; - + /** * The WebGLProgram. * @type {WebGLProgram} @@ -53,11 +53,19 @@ var Renderer = function (gl) { */ this.quadVertexBuffer_ = gl.createBuffer(); gl.bindBuffer(gl.ARRAY_BUFFER, this.quadVertexBuffer_); + var vertices = new Float32Array( [-1.0, -1.0, 0.0, 1.0, +1.0, -1.0, 1.0, 1.0, -1.0, +1.0, 0.0, 0.0, 1.0, +1.0, 1.0, 0.0]); + +// var vertices = new Float32Array( +// [-1.0, -1.0, 0.0, .5, +// +1.0, -1.0, .5, .5, +// -1.0, +1.0, 0.0, 0.0, +// 1.0, +1.0, .5, 0.0]); + gl.bufferData(gl.ARRAY_BUFFER, vertices, gl.STATIC_DRAW); @@ -90,7 +98,7 @@ var Renderer = function (gl) { Renderer.prototype.finishInit = function () { - this.draw(); + //this.draw(); }; @@ -156,12 +164,12 @@ Renderer.prototype.createHalfRGBATexture = function (data, width, height, format return tex; }; -// WebGL requires each row of rgb565Data to be aligned on a 4-byte boundary. +// WebGL requires each row of rgb565Data to be aligned on a 4-byte boundary. Renderer.prototype.createRgb565Texture = function (rgb565Data, width, height) { var gl = this.gl_; var tex = gl.createTexture(); gl.bindTexture(gl.TEXTURE_2D, tex); - + gl.texImage2D( gl.TEXTURE_2D, 0, @@ -203,7 +211,7 @@ Renderer.prototype.createRgbaTexture = function (rgbaData, width, height) { }; -Renderer.prototype.drawTexture = function (texture, width, height, mode, scale, linearToSRGBFlag) { +Renderer.prototype.drawTexture = function (texture, width, height, mode, scale, linearToSRGBFlag, useLinearFiltering) { var gl = this.gl_; // draw scene gl.clearColor(0, 0, 0, 1); @@ -213,6 +221,11 @@ Renderer.prototype.drawTexture = function (texture, width, height, mode, scale, gl.activeTexture(gl.TEXTURE0); gl.bindTexture(gl.TEXTURE_2D, texture); + + // Point vs. bilinear sampling (no mipmaps involved here) + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, useLinearFiltering ? gl.LINEAR : gl.NEAREST); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, useLinearFiltering ? gl.LINEAR : gl.NEAREST); + gl.uniform1i(this.uniformLocations_.texSampler, 0); var x = 0.0; @@ -250,7 +263,7 @@ Renderer.prototype.compileShader_ = function (shaderSource, type) { var shader = gl.createShader(type); gl.shaderSource(shader, shaderSource); gl.compileShader(shader); - + // Check for errors const compiled = gl.getShaderParameter(shader, gl.COMPILE_STATUS); if (!compiled) { @@ -259,7 +272,7 @@ Renderer.prototype.compileShader_ = function (shaderSource, type) { gl.deleteShader(shader); // Cleanup shader object throw new Error('Shader compilation failed'); } - + return shader; }; @@ -310,3 +323,4 @@ Renderer.fragmentShaderSource_ = [ ' gl_FragColor = c;', '}' ].join('\n'); + diff --git a/external/basis_universal/webgl/texture_test/assets/base.basis b/external/basis_universal/webgl/texture_test/assets/base.basis new file mode 100644 index 0000000000..e7e102e74f Binary files /dev/null and b/external/basis_universal/webgl/texture_test/assets/base.basis differ diff --git a/external/basis_universal/webgl/texture_test/assets/license.txt b/external/basis_universal/webgl/texture_test/assets/license.txt new file mode 100644 index 0000000000..e06b8ffc57 --- /dev/null +++ b/external/basis_universal/webgl/texture_test/assets/license.txt @@ -0,0 +1 @@ +The test images in this directory are not required to build or use the Basis Universal codec. They are not copyrighted or owned by Binomial LLC. diff --git a/external/basis_universal/webgl/transcoder/basis_wrappers.cpp b/external/basis_universal/webgl/transcoder/basis_wrappers.cpp index 091dd799ad..04349d8e03 100644 --- a/external/basis_universal/webgl/transcoder/basis_wrappers.cpp +++ b/external/basis_universal/webgl/transcoder/basis_wrappers.cpp @@ -11,11 +11,11 @@ // getFileDesc(), getImageDesc(), getImageLevelDesc(): These functions return low-level information about where compressed data is located for each image in a .basis file. // This is useful for when you want to extract the compressed data and embed it into your own file formats, for container independent transcoding. // -// 2. Encoding (optional): See class basis_encoder. Encodes LDR .PNG or 32bpp images, or HDR half-float/float or .EXR/.HDR images to .basis/.ktx2 files in memory. +// 2. Encoding (optional): See class basis_encoder. Encodes LDR .PNG or 32bpp images, or HDR half-float/float or .EXR/.HDR images to .basis/.ktx2 files in memory. // Must compile with BASISU_SUPPORT_ENCODING=1. // Requires basisu_transcoder.cpp as well as all the .cpp files in the "encoder" directory. Results in a larger WebAssembly executable. // -// 3. Low level transcoding/container independent transcoding: See class lowlevel_etc1s_image_transcoder or function transcodeUASTCImage(). +// 3. Low level transcoding/container independent transcoding: See class lowlevel_etc1s_image_transcoder or function transcodeUASTCImage(). // For transcoding raw compressed ETC1S/UASTC LDR/UASTC HDR texture data from non-.basis files (say from KTX2) to GPU texture data. // // 4. Helpers, transcoder texture format information: See functions getBytesPerBlockOrPixel(), formatHasAlpha(), etc. @@ -27,10 +27,20 @@ // Enable debug printf()'s in this module. #ifndef BASISU_DEBUG_PRINTF +// DO NOT CHECK IN #define BASISU_DEBUG_PRINTF 0 #endif -#define BASISU_ENCODER_MAX_SOURCE_IMAGE_PIXELS (6291456) +// This check can be removed, but you risk crashing on larger images in 32-bit WASM. Also, ETC1S/UASTC LDR 4x4 encoding uses way less memory than UASTC HDR 6x6 encoding, so you could boost this in those cases. +// 32-bit WASM limitation (TODO: remove for 64-bit), to prevent OOM crashes during HDR encoding in particular. +// TODO: Even WASM64 in Chrome has limits which seem too low for us. For now, just impose this limit. +#ifdef __wasm64__ + #define BASISU_ENCODER_MAX_SOURCE_IMAGE_PIXELS (1024*1024*12) + #define BASISU_ENCODER_MAX_SOURCE_IMAGE_PIXELS_HIGHER_LIMIT (1024*1024*12) +#else + #define BASISU_ENCODER_MAX_SOURCE_IMAGE_PIXELS (1024*1024*4) + #define BASISU_ENCODER_MAX_SOURCE_IMAGE_PIXELS_HIGHER_LIMIT (1024*1024*12) +#endif #include "basisu_transcoder.h" #include @@ -60,7 +70,14 @@ void basis_init() std::lock_guard lock(s_init_mutex); #if BASISU_DEBUG_PRINTF - printf("basis_init()\n"); + printf("basis_init() " BASISD_VERSION_STRING " "); +#ifdef __wasm64__ + printf("WASM64 "); +#endif +#ifdef WASM_THREADS_ENABLED + printf("PTHREADS"); +#endif + printf("\n"); #endif if (g_basis_initialized_flag) @@ -75,17 +92,24 @@ void basis_init() g_basis_initialized_flag = true; } -static void copy_from_jsbuffer(const emscripten::val& srcBuffer, basisu::vector& dstVec) +#if 0 +// Old copy methods, used in previous builds for plain WASM (not WASM64). + +// false if resize() fails +static bool copy_from_jsbuffer(const emscripten::val& srcBuffer, basisu::vector& dstVec) { unsigned int length = srcBuffer["length"].as(); - dstVec.resize(length); + if (!dstVec.try_resize(length)) + return false; emscripten::val memory = emscripten::val::module_property("HEAP8")["buffer"]; emscripten::val memoryView = srcBuffer["constructor"].new_(memory, reinterpret_cast(dstVec.data()), length); // Copy the bytes from the Javascript buffer. memoryView.call("set", srcBuffer); + + return true; } static bool copy_to_jsbuffer(const emscripten::val& dstBuffer, const basisu::vector& srcVec) @@ -118,6 +142,36 @@ static bool copy_to_jsbuffer(const emscripten::val& dstBuffer, const basisu::vec return true; } +#else +// New methods, compatible with WASM64. +static bool copy_from_jsbuffer(const emscripten::val& srcBuffer, basisu::vector& dstVec) +{ + const size_t length = srcBuffer["length"].as(); + if (!dstVec.try_resize(length)) + return false; + + // View over dstVec in WASM memory; copy from JS buffer into it. + emscripten::val dstView = emscripten::val(emscripten::typed_memory_view(length, dstVec.data())); + dstView.call("set", srcBuffer); + return true; +} + +// WASM -> JS +static bool copy_to_jsbuffer(const emscripten::val& dstBuffer, const basisu::vector& srcVec) +{ + if (srcVec.empty()) + return false; + + const size_t dstLen = dstBuffer["byteLength"].as(); + if (srcVec.size() > dstLen) + return false; + + // View over srcVec; copy into provided JS TypedArray. + emscripten::val srcView = emscripten::val(emscripten::typed_memory_view(srcVec.size(), const_cast(srcVec.data()))); + dstBuffer.call("set", srcView); + return true; +} +#endif const uint32_t BASIS_MAGIC = 0xD4ADBEA1; const uint32_t KTX2_MAGIC = 0xD4ADBEF2; @@ -133,7 +187,7 @@ struct basis_file_desc uint32_t m_userdata0; uint32_t m_userdata1; - // Type of texture (cETC1S, cUASTC4x4, cUASTC_HDR_4x4, etc.) + // Type of texture (cETC1S, cUASTC_LDR_4x4, cUASTC_HDR_4x4, etc.) uint32_t m_tex_format; // basis_tex_format bool m_y_flipped; @@ -190,10 +244,7 @@ struct basis_file basisu::vector m_file; basis_file(const emscripten::val& jsBuffer) - : m_file([&]() { - size_t byteLength = jsBuffer["byteLength"].as(); - return basisu::vector(byteLength); - }()) + : m_file(jsBuffer["byteLength"].as()) { if (!g_basis_initialized_flag) { @@ -204,11 +255,27 @@ struct basis_file return; } +#if 0 unsigned int length = jsBuffer["length"].as(); emscripten::val memory = emscripten::val::module_property("HEAP8")["buffer"]; emscripten::val memoryView = jsBuffer["constructor"].new_(memory, reinterpret_cast(m_file.data()), length); memoryView.call("set", jsBuffer); +#else + const size_t n = jsBuffer["byteLength"].as(); + if (!n) + { +#if BASISU_DEBUG_PRINTF + printf("basis_file::basis_file: zero size file\n"); +#endif + m_file.clear(); + return; + } + + emscripten::val dstView = emscripten::val(emscripten::typed_memory_view(n, m_file.data())); + dstView.call("set", jsBuffer); +#endif + if (!m_transcoder.validate_header(m_file.data(), m_file.size())) { #if BASISU_DEBUG_PRINTF @@ -291,7 +358,7 @@ struct basis_file return orig_height; } - // Returns a basis_tex_format (cETC1S, cUASTC, cUASTC_HDR_4x4, etc.) + // Returns a basis_tex_format (cETC1S, cUASTC_LDR_4x4, cUASTC_HDR_4x4, etc. - see basiu_file_headers.h) uint32_t getBasisTexFormat() { assert(m_magic == BASIS_MAGIC); @@ -302,7 +369,7 @@ struct basis_file return (uint32_t)fmt; } - // Currently 4 or 6 + // Returns 4-12 uint32_t getBlockWidth() const { assert(m_magic == BASIS_MAGIC); @@ -313,7 +380,7 @@ struct basis_file return basis_tex_format_get_block_width(fmt); } - // Currently 4 or 6 + // Returns 4-12 uint32_t getBlockHeight() { assert(m_magic == BASIS_MAGIC); @@ -423,12 +490,13 @@ struct basis_file return result; } + // format is transcoder_texture_format uint32_t getImageTranscodedSizeInBytes(uint32_t image_index, uint32_t level_index, uint32_t format) { assert(m_magic == BASIS_MAGIC); if (m_magic != BASIS_MAGIC) return 0; - + if (format >= (int)transcoder_texture_format::cTFTotalTextureFormats) { assert(0); @@ -436,7 +504,7 @@ struct basis_file } const transcoder_texture_format tex_format = static_cast(format); - + uint32_t orig_width, orig_height, total_src_blocks; if (!m_transcoder.get_image_level_desc(m_file.data(), m_file.size(), image_index, level_index, orig_width, orig_height, total_src_blocks)) { @@ -454,7 +522,7 @@ struct basis_file if (m_magic != BASIS_MAGIC) return false; - return m_transcoder.get_basis_tex_format(m_file.data(), m_file.size()) == basis_tex_format::cUASTC4x4; + return m_transcoder.get_basis_tex_format(m_file.data(), m_file.size()) == basis_tex_format::cUASTC_LDR_4x4; } bool isETC1S() @@ -474,9 +542,9 @@ struct basis_file return false; basis_tex_format fmt = m_transcoder.get_basis_tex_format(m_file.data(), m_file.size()); - return (fmt == basis_tex_format::cETC1S) || (fmt == basis_tex_format::cUASTC4x4); + return (fmt == basis_tex_format::cETC1S) || (fmt == basis_tex_format::cUASTC_LDR_4x4); } - + // True if the texture is UASTC HDR 4x4 or ASTC HDR 6x6. // In this case, it can only be transcoded to BC6H, ASTC HDR (of the same block dimensions, currently 4x4 or 6x6), RGB9E5 or half-float RGB/RGBA images. bool isHDR() @@ -488,7 +556,7 @@ struct basis_file basis_tex_format fmt = m_transcoder.get_basis_tex_format(m_file.data(), m_file.size()); return basis_tex_format_is_hdr(fmt); } - + bool isHDR4x4() { assert(m_magic == BASIS_MAGIC); @@ -506,9 +574,31 @@ struct basis_file return false; basis_tex_format fmt = m_transcoder.get_basis_tex_format(m_file.data(), m_file.size()); - return (fmt == basis_tex_format::cASTC_HDR_6x6) || (fmt == basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE); + return (fmt == basis_tex_format::cASTC_HDR_6x6) || (fmt == basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE); } + // True for plain ASTC LDR 4x4-12x12 + bool isASTC_LDR() + { + assert(m_magic == BASIS_MAGIC); + if (m_magic != BASIS_MAGIC) + return false; + + basis_tex_format fmt = m_transcoder.get_basis_tex_format(m_file.data(), m_file.size()); + return basis_tex_format_is_astc_ldr(fmt); + } + + // True for XUASTC LDR 4x4-12x12 + bool isXUASTC_LDR() + { + assert(m_magic == BASIS_MAGIC); + if (m_magic != BASIS_MAGIC) + return false; + + basis_tex_format fmt = m_transcoder.get_basis_tex_format(m_file.data(), m_file.size()); + return basis_tex_format_is_xuastc_ldr(fmt); + } + uint32_t startTranscoding() { assert(m_magic == BASIS_MAGIC); @@ -519,6 +609,7 @@ struct basis_file } // Here for backwards compat, prefer transcodeImageWithFlags(). + // format is transcoder_texture_format uint32_t transcodeImage(const emscripten::val& dst, uint32_t image_index, uint32_t level_index, uint32_t format, uint32_t unused, uint32_t get_alpha_for_opaque_formats) { (void)unused; @@ -531,7 +622,7 @@ struct basis_file return 0; const transcoder_texture_format transcoder_format = static_cast(format); - + uint32_t orig_width, orig_height, total_src_blocks; if (!m_transcoder.get_image_level_desc(m_file.data(), m_file.size(), image_index, level_index, orig_width, orig_height, total_src_blocks)) return 0; @@ -539,10 +630,11 @@ struct basis_file basisu::vector dst_data; uint32_t flags = get_alpha_for_opaque_formats ? cDecodeFlagsTranscodeAlphaDataToOpaqueFormats : 0; - + const uint32_t transcoded_size_in_bytes = getImageTranscodedSizeInBytes(image_index, level_index, format); - dst_data.resize(transcoded_size_in_bytes); + if (!dst_data.try_resize(transcoded_size_in_bytes)) + return 0; uint32_t status; @@ -568,15 +660,25 @@ struct basis_file flags); } +#if 0 emscripten::val memory = emscripten::val::module_property("HEAP8")["buffer"]; emscripten::val memoryView = emscripten::val::global("Uint8Array").new_(memory, reinterpret_cast(dst_data.data()), dst_data.size()); - dst.call("set", memoryView); +#else + if (!dst_data.empty()) + { + const size_t n = dst_data.size(); + emscripten::val srcView = emscripten::val(emscripten::typed_memory_view(n, dst_data.data())); + dst.call("set", srcView); // 'dst' is a JS Uint8Array + } +#endif + return status; } - - // Like transcodeImage(), but with fixed parameters. + + // Like transcodeImage(), but with updated parameters. // For flags, see cDecodeFlagsPVRTCDecodeToNextPow2 etc. + // format is transcoder_texture_format uint32_t transcodeImageWithFlags(const emscripten::val& dst, uint32_t image_index, uint32_t level_index, uint32_t format, uint32_t flags) { assert(m_magic == BASIS_MAGIC); @@ -587,16 +689,17 @@ struct basis_file return 0; const transcoder_texture_format transcoder_format = static_cast(format); - + uint32_t orig_width, orig_height, total_src_blocks; if (!m_transcoder.get_image_level_desc(m_file.data(), m_file.size(), image_index, level_index, orig_width, orig_height, total_src_blocks)) return 0; basisu::vector dst_data; - + const uint32_t transcoded_size_in_bytes = getImageTranscodedSizeInBytes(image_index, level_index, format); - dst_data.resize(transcoded_size_in_bytes); + if (!dst_data.try_resize(transcoded_size_in_bytes)) + return 0; uint32_t status; @@ -622,10 +725,16 @@ struct basis_file flags); } +#if 0 emscripten::val memory = emscripten::val::module_property("HEAP8")["buffer"]; emscripten::val memoryView = emscripten::val::global("Uint8Array").new_(memory, reinterpret_cast(dst_data.data()), dst_data.size()); - dst.call("set", memoryView); +#else + const size_t n = dst_data.size(); + emscripten::val srcView = emscripten::val(emscripten::typed_memory_view(n, dst_data.data())); + dst.call("set", srcView); // 'dst' is a JS Uint8Array +#endif + return status; } }; @@ -658,10 +767,7 @@ struct ktx2_file bool m_is_valid = false; ktx2_file(const emscripten::val& jsBuffer) - : m_file([&]() { - size_t byteLength = jsBuffer["byteLength"].as(); - return basisu::vector(byteLength); - }()) + : m_file(jsBuffer["byteLength"].as()) { if (!g_basis_initialized_flag) { @@ -672,10 +778,16 @@ struct ktx2_file return; } +#if 0 unsigned int length = jsBuffer["length"].as(); emscripten::val memory = emscripten::val::module_property("HEAP8")["buffer"]; emscripten::val memoryView = jsBuffer["constructor"].new_(memory, reinterpret_cast(m_file.data()), length); memoryView.call("set", jsBuffer); +#else + const size_t n = jsBuffer["byteLength"].as(); + emscripten::val dstView = emscripten::val(emscripten::typed_memory_view(n, m_file.data())); + dstView.call("set", jsBuffer); +#endif if (!m_transcoder.init(m_file.data(), m_file.size())) { @@ -764,8 +876,8 @@ struct ktx2_file hdr.m_kvd_byte_offset = h.m_kvd_byte_offset; hdr.m_kvd_byte_length = h.m_kvd_byte_length; - hdr.m_sgd_byte_offset = (uint32_t)h.m_sgd_byte_offset.get_uint64(); - hdr.m_sgd_byte_length = (uint32_t)h.m_sgd_byte_length.get_uint64(); + hdr.m_sgd_byte_offset = static_cast(h.m_sgd_byte_offset.get_uint64()); + hdr.m_sgd_byte_length = static_cast(h.m_sgd_byte_length.get_uint64()); return hdr; } @@ -823,6 +935,7 @@ struct ktx2_file return 1; } + // The image's original width, i.e. before being potentially expanded up to blocks. uint32_t getWidth() { assert(m_magic == KTX2_MAGIC); @@ -831,6 +944,7 @@ struct ktx2_file return m_transcoder.get_width(); } + // The image's original height, i.e. before being potentially expanded up to blocks. uint32_t getHeight() { assert(m_magic == KTX2_MAGIC); @@ -839,6 +953,7 @@ struct ktx2_file return m_transcoder.get_height(); } + // 4-12 uint32_t getBlockWidth() { assert(m_magic == KTX2_MAGIC); @@ -847,6 +962,7 @@ struct ktx2_file return m_transcoder.get_block_width(); } + // 4-12 uint32_t getBlockHeight() { assert(m_magic == KTX2_MAGIC); @@ -855,6 +971,7 @@ struct ktx2_file return m_transcoder.get_block_height(); } + // 2D or cubemaps uint32_t getFaces() { assert(m_magic == KTX2_MAGIC); @@ -863,6 +980,7 @@ struct ktx2_file return m_transcoder.get_faces(); } + // Layers for tex arrays uint32_t getLayers() { assert(m_magic == KTX2_MAGIC); @@ -871,6 +989,7 @@ struct ktx2_file return m_transcoder.get_layers(); } + // Mip-map levels uint32_t getLevels() { assert(m_magic == KTX2_MAGIC); @@ -879,7 +998,7 @@ struct ktx2_file return m_transcoder.get_levels(); } - // Returns a basis_tex_format: cETC1S, cUASTC4x4, or cUASTC_HDR_4x4, etc. + // Returns a basis_tex_format: cETC1S, cUASTC_LDR_4x4, or cUASTC_HDR_4x4, etc. - see basisu_file_headers.h uint32_t getBasisTexFormat() { assert(m_magic == KTX2_MAGIC); @@ -922,7 +1041,7 @@ struct ktx2_file return m_transcoder.is_etc1s(); } - // Returns true if the texture is UASTC HDR or ASTC HDR. In this case, it can only be transcoded to BC6H, ASTC HDR (of the same block dimensions), RGB9E5 or half-float RGB/RGBA images. + // Returns true if the texture is UASTC HDR or ASTC HDR. In this case, it can only be transcoded to BC6H, ASTC HDR (of the same block dimensions), RGB9E5 or half-float RGB/RGBA images. bool isHDR() { assert(m_magic == KTX2_MAGIC); @@ -947,6 +1066,22 @@ struct ktx2_file return m_transcoder.is_hdr_6x6(); } + bool isASTC_LDR() + { + assert(m_magic == KTX2_MAGIC); + if (m_magic != KTX2_MAGIC) + return false; + return m_transcoder.is_astc_ldr(); + } + + bool isXUASTC_LDR() + { + assert(m_magic == KTX2_MAGIC); + if (m_magic != KTX2_MAGIC) + return false; + return m_transcoder.is_xuastc_ldr(); + } + bool getHasAlpha() { assert(m_magic == KTX2_MAGIC); @@ -979,6 +1114,14 @@ struct ktx2_file return m_transcoder.get_dfd_transfer_func(); } + bool isSRGB() + { + assert(m_magic == KTX2_MAGIC); + if (m_magic != KTX2_MAGIC) + return 0; + return m_transcoder.is_srgb(); + } + uint32_t getDFDFlags() { assert(m_magic == KTX2_MAGIC); @@ -1020,6 +1163,7 @@ struct ktx2_file return m_transcoder.is_video(); } + // The linear light LDR->HDR upconversion multiplier used (def=100.0 nits) float getLDRHDRUpconversionNitMultiplier() { assert(m_magic == KTX2_MAGIC); @@ -1056,6 +1200,7 @@ struct ktx2_file return info; } + // format is transcoder_texture_format uint32_t getImageTranscodedSizeInBytes(uint32_t level_index, uint32_t layer_index, uint32_t face_index, uint32_t format) { assert(m_magic == KTX2_MAGIC); @@ -1069,7 +1214,7 @@ struct ktx2_file } const transcoder_texture_format tex_format = static_cast(format); - + ktx2_image_level_info info; if (!m_transcoder.get_image_level_info(info, level_index, layer_index, face_index)) { @@ -1094,6 +1239,7 @@ struct ktx2_file // Here for backwards compat, prefer transcodeImageWithFlags(). // get_alpha_for_opaque_formats defaults to false // channel0/channel1 default to -1 + // format is transcoder_texture_format uint32_t transcodeImage(const emscripten::val& dst, uint32_t level_index, uint32_t layer_index, uint32_t face_index, uint32_t format, uint32_t get_alpha_for_opaque_formats, int channel0, int channel1) { assert(m_magic == KTX2_MAGIC); @@ -1107,7 +1253,7 @@ struct ktx2_file const uint32_t dst_block_width = basis_get_block_width(transcoder_format); const uint32_t dst_block_height = basis_get_block_height(transcoder_format); - + ktx2_image_level_info info; if (!m_transcoder.get_image_level_info(info, level_index, layer_index, face_index)) return 0; @@ -1120,7 +1266,8 @@ struct ktx2_file const uint32_t transcoded_size_in_bytes = getImageTranscodedSizeInBytes(level_index, layer_index, face_index, format); - dst_data.resize(transcoded_size_in_bytes); + if (!dst_data.try_resize(transcoded_size_in_bytes)) + return 0; uint32_t status; @@ -1151,15 +1298,22 @@ struct ktx2_file nullptr); } +#if 0 emscripten::val memory = emscripten::val::module_property("HEAP8")["buffer"]; emscripten::val memoryView = emscripten::val::global("Uint8Array").new_(memory, reinterpret_cast(dst_data.data()), dst_data.size()); - dst.call("set", memoryView); +#else + const size_t n = dst_data.size(); + emscripten::val srcView = emscripten::val(emscripten::typed_memory_view(n, dst_data.data())); + dst.call("set", srcView); // 'dst' must be a Uint8Array (or compatible TypedArray) +#endif + return status; } - + // like transcodeImage(), but with fixed parameters (includes flags) // For flags, see cDecodeFlagsPVRTCDecodeToNextPow2 etc. + // format is transcoder_texture_format uint32_t transcodeImageWithFlags(const emscripten::val& dst, uint32_t level_index, uint32_t layer_index, uint32_t face_index, uint32_t format, uint32_t flags, int channel0, int channel1) { assert(m_magic == KTX2_MAGIC); @@ -1168,12 +1322,12 @@ struct ktx2_file if (format >= (int)transcoder_texture_format::cTFTotalTextureFormats) return 0; - + const transcoder_texture_format transcoder_format = static_cast(format); const uint32_t dst_block_width = basis_get_block_width(transcoder_format); const uint32_t dst_block_height = basis_get_block_height(transcoder_format); - + ktx2_image_level_info info; if (!m_transcoder.get_image_level_info(info, level_index, layer_index, face_index)) return 0; @@ -1215,10 +1369,16 @@ struct ktx2_file nullptr); } +#if 0 emscripten::val memory = emscripten::val::module_property("HEAP8")["buffer"]; emscripten::val memoryView = emscripten::val::global("Uint8Array").new_(memory, reinterpret_cast(dst_data.data()), dst_data.size()); - dst.call("set", memoryView); +#else + const size_t n = dst_data.size(); + emscripten::val srcView = emscripten::val(emscripten::typed_memory_view(n, dst_data.data())); + dst.call("set", srcView); // dst = JS Uint8Array +#endif + return status; } @@ -1234,10 +1394,19 @@ enum class ldr_image_type cJPGImage = 2 }; +enum xuastc_ldr_syntax +{ + cFullArith = (int)basist::astc_ldr_t::xuastc_ldr_syntax::cFullArith, + cHybridArithZStd = (int)basist::astc_ldr_t::xuastc_ldr_syntax::cHybridArithZStd, + cFullZStd = (int)basist::astc_ldr_t::xuastc_ldr_syntax::cFullZStd, + cTotal = 3 +}; + class basis_encoder { bool m_threading_enabled = false; uint32_t m_num_extra_worker_threads = 0; + float m_last_encode_mip0_rgba_psnr = 0.0f; public: basis_compressor_params m_params; @@ -1252,7 +1421,7 @@ class basis_encoder m_num_extra_worker_threads = num_extra_worker_threads; } - // Only works for LDR inputs. + // Only valid for LDR inputs. bool set_slice_source_image(uint32_t slice_index, const emscripten::val& src_image_js_val, uint32_t src_image_width, uint32_t src_image_height, ldr_image_type img_type) { // Resize the source_images array if necessary @@ -1261,7 +1430,8 @@ class basis_encoder // First copy the src image buffer to the heap. basisu::vector src_image_buf; - copy_from_jsbuffer(src_image_js_val, src_image_buf); + if (!copy_from_jsbuffer(src_image_js_val, src_image_buf)) + return false; // Now load the source image. image& src_img = m_params.m_source_images[slice_index]; @@ -1328,14 +1498,15 @@ class basis_encoder hdr_image_type img_type, bool ldr_srgb_to_linear_conversion, float ldr_to_hdr_nit_multiplier) { assert(ldr_to_hdr_nit_multiplier > 0.0f); - + // Resize the source_images_hdr array if necessary if (slice_index >= m_params.m_source_images_hdr.size()) m_params.m_source_images_hdr.resize(slice_index + 1); // First copy the src image buffer to the heap. basisu::vector src_image_buf; - copy_from_jsbuffer(src_image_js_val, src_image_buf); + if (!copy_from_jsbuffer(src_image_js_val, src_image_buf)) + return false; // Now load the source image. imagef& src_img = m_params.m_source_images_hdr[slice_index]; @@ -1345,14 +1516,14 @@ class basis_encoder if ((img_type == hdr_image_type::cHITPNGImage) || (img_type == hdr_image_type::cHITJPGImage)) { - // Because we're loading the image ourselves we need to add these tags so the UI knows how to tone map LDR upconverted outputs. + // Because we're loading the image ourselves we need to add these tags so the UI knows how to tone map LDR upconverted outputs. // Normally basis_compressor adds them when it loads the images itself from source files. basist::ktx2_add_key_value(m_params.m_ktx2_key_values, "LDRUpconversionMultiplier", fmt_string("{}", ldr_to_hdr_nit_multiplier)); if (ldr_srgb_to_linear_conversion) basist::ktx2_add_key_value(m_params.m_ktx2_key_values, "LDRUpconversionSRGBToLinear", "1"); } - + return true; } @@ -1370,7 +1541,7 @@ class basis_encoder // We don't use threading for now, but the compressor needs a job pool. uint32_t num_new_threads = 0; bool enable_threading = false; - + #if WASM_THREADS_ENABLED if ((emscripten_has_threading_support()) && (m_threading_enabled) && (m_num_extra_worker_threads)) { @@ -1384,20 +1555,29 @@ class basis_encoder // Initialize the compression parameters structure. This is the same structure that the command line tool fills in. basis_compressor_params ¶ms = m_params; - + // Check to see if we would risk running out of memory in 32-bit WASM. There's not much we can do about this limit until memory64 is available. uint64_t total_src_texels = 0; - + for (uint32_t i = 0; i < m_params.m_source_images.size(); i++) total_src_texels += m_params.m_source_images[i].get_total_pixels(); for (uint32_t i = 0; i < m_params.m_source_images_hdr.size(); i++) total_src_texels += m_params.m_source_images_hdr[i].get_total_pixels(); + + // Try to prevent running out of memory inside WASM. + uint32_t max_pixels_thresh = BASISU_ENCODER_MAX_SOURCE_IMAGE_PIXELS; - if (total_src_texels > BASISU_ENCODER_MAX_SOURCE_IMAGE_PIXELS) + // The simpler compressors need less temporary memory, so their threshold can be higher. + if (m_params.is_etc1s() || m_params.is_uastc_ldr_4x4() || m_params.is_uastc_hdr_4x4()) { - printf("ERROR: basis_encoder::encode(): The total number of source texels to compress is too large for 32-bit WASM (above BASISU_ENCODER_MAX_SOURCE_IMAGE_PIXELS in basis_wrappers.cpp)." - "This is not a fundamental limitation of the library, but of WASM. Processing images this large risks running out of memory until WASM memory64 is available.\n"); + max_pixels_thresh = BASISU_ENCODER_MAX_SOURCE_IMAGE_PIXELS_HIGHER_LIMIT; + } + + if (total_src_texels > max_pixels_thresh) + { + printf("ERROR: basis_encoder::encode(): The total number of source texels to compress %llu is greater than %u, which is likely too large for WASM (above BASISU_ENCODER_MAX_SOURCE_IMAGE_PIXELS in basis_wrappers.cpp).", + total_src_texels, max_pixels_thresh); return 0; } @@ -1438,6 +1618,17 @@ class basis_encoder #endif return 0; } + + m_last_encode_mip0_rgba_psnr = 0.0f; + if (comp.get_stats().size()) + { + float psnr = comp.get_stats()[0].m_basis_rgba_avg_psnr; + + if (psnr == 0.0f) + psnr = comp.get_stats()[0].m_basis_rgb_avg_psnr; // HDR, not RGBA though + + m_last_encode_mip0_rgba_psnr = psnr; + } if (params.m_create_ktx2_file) { @@ -1458,6 +1649,11 @@ class basis_encoder return (uint32_t)comp.get_output_basis_file().size(); } } + + float get_last_encode_mip0_rgba_psnr() const + { + return m_last_encode_mip0_rgba_psnr; + } }; #endif @@ -1474,8 +1670,10 @@ class lowlevel_etc1s_image_transcoder : public basisu_lowlevel_etc1s_transcoder bool decode_palettes(uint32_t num_endpoints, const emscripten::val& endpoint_data, uint32_t num_selectors, const emscripten::val& selector_data) { basisu::vector temp_endpoint_data, temp_selector_data; - copy_from_jsbuffer(endpoint_data, temp_endpoint_data); - copy_from_jsbuffer(selector_data, temp_selector_data); + if (!copy_from_jsbuffer(endpoint_data, temp_endpoint_data)) + return false; + if (!copy_from_jsbuffer(selector_data, temp_selector_data)) + return false; #if 0 printf("decode_palettes: %u %u %u %u, %u %u\n", @@ -1500,7 +1698,8 @@ class lowlevel_etc1s_image_transcoder : public basisu_lowlevel_etc1s_transcoder bool decode_tables(const emscripten::val& table_data) { basisu::vector temp_table_data; - copy_from_jsbuffer(table_data, temp_table_data); + if (!copy_from_jsbuffer(table_data, temp_table_data)) + return false; if (!temp_table_data.size()) { @@ -1529,7 +1728,7 @@ class lowlevel_etc1s_image_transcoder : public basisu_lowlevel_etc1s_transcoder if (!g_basis_initialized_flag) { #if BASISU_DEBUG_PRINTF - printf("transcode_etc1s_image: basis_init() must be called first\n"); + printf("lowlevel_etc1s_image_transcoder::transcode_image: basis_init() must be called first\n"); #endif assert(0); return false; @@ -1537,12 +1736,13 @@ class lowlevel_etc1s_image_transcoder : public basisu_lowlevel_etc1s_transcoder // FIXME: Access the JavaScript buffer directly vs. copying it. basisu::vector temp_comp_data; - copy_from_jsbuffer(compressed_data, temp_comp_data); + if (!copy_from_jsbuffer(compressed_data, temp_comp_data)) + return false; if (!temp_comp_data.size()) { #if BASISU_DEBUG_PRINTF - printf("transcode_etc1s_image: compressed_data is empty\n"); + printf("lowlevel_etc1s_image_transcoder::transcode_image: compressed_data is empty\n"); #endif assert(0); return false; @@ -1552,7 +1752,7 @@ class lowlevel_etc1s_image_transcoder : public basisu_lowlevel_etc1s_transcoder if (!output_blocks_len) { #if BASISU_DEBUG_PRINTF - printf("transcode_etc1s_image: output_blocks is empty\n"); + printf("lowlevel_etc1s_image_transcoder::transcode_image: output_blocks is empty\n"); #endif assert(0); return false; @@ -1576,7 +1776,7 @@ class lowlevel_etc1s_image_transcoder : public basisu_lowlevel_etc1s_transcoder if (!status) { #if BASISU_DEBUG_PRINTF - printf("transcode_etc1s_image: basisu_lowlevel_etc1s_transcoder::transcode_image failed\n"); + printf("lowlevel_etc1s_image_transcoder::transcode_image: basisu_lowlevel_etc1s_transcoder::transcode_image failed\n"); #endif assert(0); return false; @@ -1590,12 +1790,12 @@ class lowlevel_etc1s_image_transcoder : public basisu_lowlevel_etc1s_transcoder }; // Supports UASTC LDR 4x4, UASTC HDR 4x4, and ASTC HDR 6x6/intermediate (but not ETC1S). -bool transcode_uastc_image( - uint32_t basis_tex_format_int, +bool transcode_uastc_image2( + uint32_t basis_tex_format_int, bool use_astc_srgb_decode_profile, uint32_t target_format_int, // see transcoder_texture_format const emscripten::val& output_blocks, uint32_t output_blocks_buf_size_in_blocks_or_pixels, const emscripten::val& compressed_data, - uint32_t num_blocks_x, uint32_t num_blocks_y, uint32_t orig_width, uint32_t orig_height, uint32_t level_index, + uint32_t src_num_blocks_x, uint32_t src_num_blocks_y, uint32_t orig_width, uint32_t orig_height, uint32_t level_index, uint32_t slice_offset, uint32_t slice_length, uint32_t decode_flags, // see cDecodeFlagsPVRTCDecodeToNextPow2 etc. bool has_alpha, @@ -1607,13 +1807,13 @@ bool transcode_uastc_image( assert(basis_tex_format_int < (uint32_t)basis_tex_format::cTotalFormats); assert(target_format_int < (uint32_t)transcoder_texture_format::cTFTotalTextureFormats); + basis_tex_format src_tex_format = static_cast(basis_tex_format_int); transcoder_texture_format target_format = static_cast(target_format_int); - basis_tex_format tex_format = static_cast(basis_tex_format_int); - + if (!g_basis_initialized_flag) { #if BASISU_DEBUG_PRINTF - printf("transcode_uastc_image: basis_init() must be called first\n"); + printf("transcode_uastc_image2: basis_init() must be called first\n"); #endif assert(0); return false; @@ -1621,12 +1821,13 @@ bool transcode_uastc_image( // FIXME: Access the JavaScript buffer directly vs. copying it. basisu::vector temp_comp_data; - copy_from_jsbuffer(compressed_data, temp_comp_data); + if (!copy_from_jsbuffer(compressed_data, temp_comp_data)) + return false; if (!temp_comp_data.size()) { #if BASISU_DEBUG_PRINTF - printf("transcode_uastc_image: compressed_data is empty\n"); + printf("transcode_uastc_image2: compressed_data is empty\n"); #endif assert(0); return false; @@ -1636,7 +1837,7 @@ bool transcode_uastc_image( if (!output_blocks_len) { #if BASISU_DEBUG_PRINTF - printf("transcode_uastc_image: output_blocks is empty\n"); + printf("transcode_uastc_image2: output_blocks is empty\n"); #endif assert(0); return false; @@ -1646,7 +1847,7 @@ bool transcode_uastc_image( printf("format: %u\n", (uint32_t)target_format); printf("output_blocks size: %u buf size: %u\n", output_blocks_len, output_blocks_buf_size_in_blocks_or_pixels); printf("compressed_data size: %u\n", compressed_data["byteLength"].as()); - printf("%u %u %u %u %u\n", num_blocks_x, num_blocks_y, orig_width, orig_height, level_index); + printf("%u %u %u %u %u\n", src_num_blocks_x, src_num_blocks_y, orig_width, orig_height, level_index); printf("%u %u\n", slice_offset, slice_length); printf("%u\n", decode_flags); printf("has_alpha: %u is_video: %u\n", has_alpha, is_video); @@ -1654,8 +1855,27 @@ bool transcode_uastc_image( basisu::vector temp_output_blocks(output_blocks_len); - bool status = false; - if (tex_format == basis_tex_format::cUASTC_HDR_4x4) + bool status = false; + if (basis_tex_format_is_astc_ldr(src_tex_format) || basis_tex_format_is_xuastc_ldr(src_tex_format)) + { + basisu_lowlevel_xuastc_ldr_transcoder transcoder; + + status = transcoder.transcode_image( + src_tex_format, use_astc_srgb_decode_profile, + (transcoder_texture_format)target_format, + &temp_output_blocks[0], output_blocks_buf_size_in_blocks_or_pixels, + &temp_comp_data[0], temp_comp_data.size(), + src_num_blocks_x, src_num_blocks_y, orig_width, orig_height, level_index, + slice_offset, slice_length, + decode_flags, + has_alpha, + is_video, + output_row_pitch_in_blocks_or_pixels, + nullptr, + output_rows_in_pixels, + channel0, channel1); + } + else if (src_tex_format == basis_tex_format::cUASTC_HDR_4x4) { basisu_lowlevel_uastc_hdr_4x4_transcoder transcoder; @@ -1663,7 +1883,7 @@ bool transcode_uastc_image( (transcoder_texture_format)target_format, &temp_output_blocks[0], output_blocks_buf_size_in_blocks_or_pixels, &temp_comp_data[0], temp_comp_data.size(), - num_blocks_x, num_blocks_y, orig_width, orig_height, level_index, + src_num_blocks_x, src_num_blocks_y, orig_width, orig_height, level_index, slice_offset, slice_length, decode_flags, has_alpha, @@ -1673,7 +1893,7 @@ bool transcode_uastc_image( output_rows_in_pixels, channel0, channel1); } - else if (tex_format == basis_tex_format::cASTC_HDR_6x6) + else if (src_tex_format == basis_tex_format::cASTC_HDR_6x6) { basisu_lowlevel_astc_hdr_6x6_transcoder transcoder; @@ -1681,7 +1901,7 @@ bool transcode_uastc_image( (transcoder_texture_format)target_format, &temp_output_blocks[0], output_blocks_buf_size_in_blocks_or_pixels, &temp_comp_data[0], temp_comp_data.size(), - num_blocks_x, num_blocks_y, orig_width, orig_height, level_index, + src_num_blocks_x, src_num_blocks_y, orig_width, orig_height, level_index, slice_offset, slice_length, decode_flags, has_alpha, @@ -1691,15 +1911,15 @@ bool transcode_uastc_image( output_rows_in_pixels, channel0, channel1); } - else if (tex_format == basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE) + else if (src_tex_format == basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE) { - basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder transcoder; + basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder transcoder; status = transcoder.transcode_image( (transcoder_texture_format)target_format, &temp_output_blocks[0], output_blocks_buf_size_in_blocks_or_pixels, &temp_comp_data[0], temp_comp_data.size(), - num_blocks_x, num_blocks_y, orig_width, orig_height, level_index, + src_num_blocks_x, src_num_blocks_y, orig_width, orig_height, level_index, slice_offset, slice_length, decode_flags, has_alpha, @@ -1709,7 +1929,7 @@ bool transcode_uastc_image( output_rows_in_pixels, channel0, channel1); } - else if (tex_format == basis_tex_format::cUASTC4x4) + else if (src_tex_format == basis_tex_format::cUASTC_LDR_4x4) { basisu_lowlevel_uastc_ldr_4x4_transcoder transcoder; @@ -1717,7 +1937,7 @@ bool transcode_uastc_image( (transcoder_texture_format)target_format, &temp_output_blocks[0], output_blocks_buf_size_in_blocks_or_pixels, &temp_comp_data[0], temp_comp_data.size(), - num_blocks_x, num_blocks_y, orig_width, orig_height, level_index, + src_num_blocks_x, src_num_blocks_y, orig_width, orig_height, level_index, slice_offset, slice_length, decode_flags, has_alpha, @@ -1735,7 +1955,7 @@ bool transcode_uastc_image( if (!status) { #if BASISU_DEBUG_PRINTF - printf("transcode_uastc_image: basisu_lowlevel_uastc_transcoder::transcode_image failed\n"); + printf("transcode_uastc_image2: basisu_lowlevel_uastc_transcoder::transcode_image failed\n"); #endif assert(0); return false; @@ -1747,82 +1967,154 @@ bool transcode_uastc_image( return true; } +// Previous API - prefer transcode_uastc_image2(), which allows the caller to control the ASTC decode profile (srgb/linear) for XUASTC/ASTC. +bool transcode_uastc_image( + uint32_t basis_tex_format_int, + uint32_t target_format_int, // see transcoder_texture_format + const emscripten::val& output_blocks, uint32_t output_blocks_buf_size_in_blocks_or_pixels, + const emscripten::val& compressed_data, + uint32_t src_num_blocks_x, uint32_t src_num_blocks_y, uint32_t orig_width, uint32_t orig_height, uint32_t level_index, + uint32_t slice_offset, uint32_t slice_length, + uint32_t decode_flags, // see cDecodeFlagsPVRTCDecodeToNextPow2 etc. + bool has_alpha, + bool is_video, + uint32_t output_row_pitch_in_blocks_or_pixels, + uint32_t output_rows_in_pixels, + int channel0, int channel1) +{ + // Just assume sRGB decode profile - which is the compressor's default. + const bool use_astc_srgb_decode_profile = true; + + return transcode_uastc_image2( + basis_tex_format_int, use_astc_srgb_decode_profile, + target_format_int, // see transcoder_texture_format + output_blocks, output_blocks_buf_size_in_blocks_or_pixels, + compressed_data, + src_num_blocks_x, src_num_blocks_y, orig_width, orig_height, level_index, + slice_offset, slice_length, + decode_flags, // see cDecodeFlagsPVRTCDecodeToNextPow2 etc. + has_alpha, + is_video, + output_row_pitch_in_blocks_or_pixels, + output_rows_in_pixels, + channel0, channel1); +} + +// transcoder_tex_fmt is transcoder_texture_format uint32_t get_bytes_per_block_or_pixel(uint32_t transcoder_tex_fmt) { assert(transcoder_tex_fmt < (uint32_t)transcoder_texture_format::cTFTotalTextureFormats); return basis_get_bytes_per_block_or_pixel(static_cast(transcoder_tex_fmt)); } +// transcoder_tex_fmt is transcoder_texture_format bool format_has_alpha(uint32_t transcoder_tex_fmt) { assert(transcoder_tex_fmt < (uint32_t)transcoder_texture_format::cTFTotalTextureFormats); return basis_transcoder_format_has_alpha(static_cast(transcoder_tex_fmt)); } +// transcoder_tex_fmt is transcoder_texture_format bool format_is_hdr(uint32_t transcode_tex_fmt) { assert(transcode_tex_fmt < (uint32_t)transcoder_texture_format::cTFTotalTextureFormats); return basis_transcoder_format_is_hdr(static_cast(transcode_tex_fmt)); } +// transcoder_tex_fmt is transcoder_texture_format bool format_is_ldr(uint32_t transcode_tex_fmt) { assert(transcode_tex_fmt < (uint32_t)transcoder_texture_format::cTFTotalTextureFormats); return !basis_transcoder_format_is_hdr(static_cast(transcode_tex_fmt)); } +// transcoder_tex_fmt is transcoder_texture_format bool format_is_uncompressed(uint32_t transcoder_tex_fmt) { assert(transcoder_tex_fmt < (uint32_t)transcoder_texture_format::cTFTotalTextureFormats); return basis_transcoder_format_is_uncompressed(static_cast(transcoder_tex_fmt)); } +// transcoder_tex_fmt is transcoder_texture_format, file_fmt is basis_tex_fmt bool is_format_supported(uint32_t transcoder_tex_fmt, uint32_t file_fmt) { assert(transcoder_tex_fmt < (uint32_t)transcoder_texture_format::cTFTotalTextureFormats); + assert(file_fmt < (uint32_t)basis_tex_format::cTotalFormats); return basis_is_format_supported(static_cast(transcoder_tex_fmt), static_cast(file_fmt)); } -// transcoder_texture_format +// transcoder_tex_fmt is transcoder_texture_format uint32_t get_format_block_width(uint32_t transcoder_tex_fmt) { assert(transcoder_tex_fmt < (uint32_t)transcoder_texture_format::cTFTotalTextureFormats); return basis_get_block_width(static_cast(transcoder_tex_fmt)); } -// transcoder_texture_format +// fmt is transcoder_texture_format uint32_t get_format_block_height(uint32_t transcoder_tex_fmt) { assert(transcoder_tex_fmt < (uint32_t)transcoder_texture_format::cTFTotalTextureFormats); return basis_get_block_height(static_cast(transcoder_tex_fmt)); } -// basis_tex_format -uint32_t get_basis_tex_format_block_width(uint32_t fmt) +// file_fmt is basis_tex_format +uint32_t get_basis_tex_format_block_width(uint32_t file_fmt) +{ + assert(file_fmt < (uint32_t)basis_tex_format::cTotalFormats); + return basis_tex_format_get_block_width(static_cast(file_fmt)); +} + +// file_fmt is basis_tex_format +uint32_t get_basis_tex_format_block_height(uint32_t file_fmt) { - assert(fmt < (uint32_t)basis_tex_format::cTotalFormats); - return basis_tex_format_get_block_width(static_cast(fmt)); + assert(file_fmt < (uint32_t)basis_tex_format::cTotalFormats); + return basis_tex_format_get_block_height(static_cast(file_fmt)); } -// basis_tex_format -uint32_t get_basis_tex_format_block_height(uint32_t fmt) +// file_fmt is basis_tex_format +bool is_basis_tex_format_hdr(uint32_t file_fmt) { - assert(fmt < (uint32_t)basis_tex_format::cTotalFormats); - return basis_tex_format_get_block_height(static_cast(fmt)); + assert(file_fmt < (uint32_t)basis_tex_format::cTotalFormats); + return basis_tex_format_is_hdr((basis_tex_format)file_fmt); } -// basis_tex_format -bool is_basis_tex_format_hdr(uint32_t fmt) +// file_fmt is basis_tex_format +bool is_basis_tex_format_ldr(uint32_t file_fmt) { - assert(fmt < (uint32_t)basis_tex_format::cTotalFormats); - return ((basis_tex_format)fmt == basis_tex_format::cUASTC_HDR_4x4) || ((basis_tex_format)fmt == basis_tex_format::cASTC_HDR_6x6) || ((basis_tex_format)fmt == basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE); + assert(file_fmt < (uint32_t)basis_tex_format::cTotalFormats); + return basis_tex_format_is_ldr((basis_tex_format)file_fmt); } -// basis_tex_format -bool is_basis_tex_format_ldr(uint32_t fmt) +// file_fmt is basis_tex_format +bool is_basis_tex_format_xuastc_ldr(uint32_t file_fmt) { - assert(fmt < (uint32_t)basis_tex_format::cTotalFormats); - return ((basis_tex_format)fmt == basis_tex_format::cETC1S) || ((basis_tex_format)fmt == basis_tex_format::cUASTC4x4); + assert(file_fmt < (uint32_t)basis_tex_format::cTotalFormats); + return basis_tex_format_is_xuastc_ldr((basis_tex_format)file_fmt); +} + +// file_fmt is basis_tex_format +bool is_basis_tex_format_astc_ldr(uint32_t file_fmt) +{ + assert(file_fmt < (uint32_t)basis_tex_format::cTotalFormats); + return basis_tex_format_is_astc_ldr((basis_tex_format)file_fmt); +} + +// Returns transcoder_texture_format, file_fmt is basis_tex_format. +// // Returns the best ASTC texture format to use given any basis_tex_format (the one with the proper block size). +// Use get_transcoder_texture_format_from_basis_tex_format() instead (same thing). Here for backwards compat. +uint32_t get_transcoder_texture_format_from_xuastc_or_astc_ldr_basis_tex_format(uint32_t file_fmt) +{ + assert(file_fmt < (uint32_t)basis_tex_format::cTotalFormats); + return (uint32_t)basis_get_transcoder_texture_format_from_xuastc_or_astc_ldr_basis_tex_format(static_cast(file_fmt)); +} + +// Same as get_transcoder_texture_format_from_xuastc_or_astc_ldr_basis_tex_format(), just a smaller name, works with any basis_tex_format. +// Returns the best ASTC texture format to use given any basis_tex_format (the one with the proper block size). +// Returns transcoder_texture_format, file_fmt is basis_tex_format. +uint32_t get_transcoder_texture_format_from_basis_tex_format(uint32_t file_fmt) +{ + assert(file_fmt < (uint32_t)basis_tex_format::cTotalFormats); + return (uint32_t)basis_get_transcoder_texture_format_from_basis_tex_format(static_cast(file_fmt)); } uint32_t convert_float_to_half(float f) @@ -1848,10 +2140,10 @@ uint32_t get_debug_flags_wrapper() EMSCRIPTEN_BINDINGS(basis_codec) { function("initializeBasis", &basis_init); - + function("setDebugFlags", &set_debug_flags_wrapper); function("getDebugFlags", &get_debug_flags_wrapper); - + // Expose BasisFileDesc structure value_object("BasisFileDesc") .field("version", &basis_file_desc::m_version) @@ -1926,6 +2218,19 @@ EMSCRIPTEN_BINDINGS(basis_codec) { .value("cTFRGBA_HALF", transcoder_texture_format::cTFRGBA_HALF) .value("cTFRGB_9E5", transcoder_texture_format::cTFRGB_9E5) .value("cTFASTC_HDR_6x6_RGBA", transcoder_texture_format::cTFASTC_HDR_6x6_RGBA) + .value("cTFASTC_LDR_5x4_RGBA", transcoder_texture_format::cTFASTC_LDR_5x4_RGBA) + .value("cTFASTC_LDR_5x5_RGBA", transcoder_texture_format::cTFASTC_LDR_5x5_RGBA) + .value("cTFASTC_LDR_6x5_RGBA", transcoder_texture_format::cTFASTC_LDR_6x5_RGBA) + .value("cTFASTC_LDR_6x6_RGBA", transcoder_texture_format::cTFASTC_LDR_6x6_RGBA) + .value("cTFASTC_LDR_8x5_RGBA", transcoder_texture_format::cTFASTC_LDR_8x5_RGBA) + .value("cTFASTC_LDR_8x6_RGBA", transcoder_texture_format::cTFASTC_LDR_8x6_RGBA) + .value("cTFASTC_LDR_10x5_RGBA", transcoder_texture_format::cTFASTC_LDR_10x5_RGBA) + .value("cTFASTC_LDR_10x6_RGBA", transcoder_texture_format::cTFASTC_LDR_10x6_RGBA) + .value("cTFASTC_LDR_8x8_RGBA", transcoder_texture_format::cTFASTC_LDR_8x8_RGBA) + .value("cTFASTC_LDR_10x8_RGBA", transcoder_texture_format::cTFASTC_LDR_10x8_RGBA) + .value("cTFASTC_LDR_10x10_RGBA", transcoder_texture_format::cTFASTC_LDR_10x10_RGBA) + .value("cTFASTC_LDR_12x10_RGBA", transcoder_texture_format::cTFASTC_LDR_12x10_RGBA) + .value("cTFASTC_LDR_12x12_RGBA", transcoder_texture_format::cTFASTC_LDR_12x12_RGBA) .value("cTFTotalTextureFormats", transcoder_texture_format::cTFTotalTextureFormats) ; @@ -1944,7 +2249,11 @@ EMSCRIPTEN_BINDINGS(basis_codec) { function("isBasisTexFormatHDR", &is_basis_tex_format_hdr); function("isBasisTexFormatLDR", &is_basis_tex_format_ldr); - + function("isBasisTexFormatXUASTCLDR", &is_basis_tex_format_xuastc_ldr); + function("isBasisTexFormatASTCLDR", &is_basis_tex_format_astc_ldr); + function("getTranscoderTextureFormatFromXUASTCOrASTCLDRBasisTexFormat", &get_transcoder_texture_format_from_xuastc_or_astc_ldr_basis_tex_format); + function("getTranscoderTextureFormatFromBasisTexFormat", &get_transcoder_texture_format_from_basis_tex_format); + function("convertFloatToHalf", &convert_float_to_half); function("convertHalfToFloat", &convert_half_to_float); @@ -1957,13 +2266,46 @@ EMSCRIPTEN_BINDINGS(basis_codec) { .value("cBASISTexTypeVolume", cBASISTexTypeVolume) ; - // Expose enum basis_tex_format + // Expose enum basis_tex_format - supported KTX2/.basis texture types. enum_("basis_tex_format") .value("cETC1S", basis_tex_format::cETC1S) - .value("cUASTC4x4", basis_tex_format::cUASTC4x4) + .value("cUASTC4x4", basis_tex_format::cUASTC_LDR_4x4) // name has changed, keeping for backwards compat + .value("cUASTC_LDR_4x4", basis_tex_format::cUASTC_LDR_4x4) .value("cUASTC_HDR_4x4", basis_tex_format::cUASTC_HDR_4x4) .value("cASTC_HDR_6x6", basis_tex_format::cASTC_HDR_6x6) - .value("cASTC_HDR_6x6_INTERMEDIATE", basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE) + .value("cUASTC_HDR_6x6_INTERMEDIATE", basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE) + .value("cUASTC_HDR_6x6", basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE) // the correct name + .value("cASTC_HDR_6x6_INTERMEDIATE", basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE) // was misnamed in previous release, keeping for backwards compat + // XUASTC LDR 4x4-12x12 + .value("cXUASTC_LDR_4x4", basis_tex_format::cXUASTC_LDR_4x4) + .value("cXUASTC_LDR_5x4", basis_tex_format::cXUASTC_LDR_5x4) + .value("cXUASTC_LDR_5x5", basis_tex_format::cXUASTC_LDR_5x5) + .value("cXUASTC_LDR_6x5", basis_tex_format::cXUASTC_LDR_6x5) + .value("cXUASTC_LDR_6x6", basis_tex_format::cXUASTC_LDR_6x6) + .value("cXUASTC_LDR_8x5", basis_tex_format::cXUASTC_LDR_8x5) + .value("cXUASTC_LDR_8x6", basis_tex_format::cXUASTC_LDR_8x6) + .value("cXUASTC_LDR_10x5", basis_tex_format::cXUASTC_LDR_10x5) + .value("cXUASTC_LDR_10x6", basis_tex_format::cXUASTC_LDR_10x6) + .value("cXUASTC_LDR_8x8", basis_tex_format::cXUASTC_LDR_8x8) + .value("cXUASTC_LDR_10x8", basis_tex_format::cXUASTC_LDR_10x8) + .value("cXUASTC_LDR_10x10", basis_tex_format::cXUASTC_LDR_10x10) + .value("cXUASTC_LDR_12x10", basis_tex_format::cXUASTC_LDR_12x10) + .value("cXUASTC_LDR_12x12", basis_tex_format::cXUASTC_LDR_12x12) + // ASTC LDR 4x4-12x12 + .value("cASTC_LDR_4x4", basis_tex_format::cASTC_LDR_4x4) + .value("cASTC_LDR_5x4", basis_tex_format::cASTC_LDR_5x4) + .value("cASTC_LDR_5x5", basis_tex_format::cASTC_LDR_5x5) + .value("cASTC_LDR_6x5", basis_tex_format::cASTC_LDR_6x5) + .value("cASTC_LDR_6x6", basis_tex_format::cASTC_LDR_6x6) + .value("cASTC_LDR_8x5", basis_tex_format::cASTC_LDR_8x5) + .value("cASTC_LDR_8x6", basis_tex_format::cASTC_LDR_8x6) + .value("cASTC_LDR_10x5", basis_tex_format::cASTC_LDR_10x5) + .value("cASTC_LDR_10x6", basis_tex_format::cASTC_LDR_10x6) + .value("cASTC_LDR_8x8", basis_tex_format::cASTC_LDR_8x8) + .value("cASTC_LDR_10x8", basis_tex_format::cASTC_LDR_10x8) + .value("cASTC_LDR_10x10", basis_tex_format::cASTC_LDR_10x10) + .value("cASTC_LDR_12x10", basis_tex_format::cASTC_LDR_12x10) + .value("cASTC_LDR_12x12", basis_tex_format::cASTC_LDR_12x12) ; // .basis file transcoder object. If all you want to do is transcode already encoded .basis files, this is all you really need. @@ -1999,6 +2341,12 @@ EMSCRIPTEN_BINDINGS(basis_codec) { .function("isLDR", optional_override([](basis_file& self) { return self.isLDR(); })) + .function("isASTC_LDR", optional_override([](basis_file& self) { + return self.isASTC_LDR(); + })) + .function("isXUASTC_LDR", optional_override([](basis_file& self) { + return self.isXUASTC_LDR(); + })) .function("getNumImages", optional_override([](basis_file& self) { return self.getNumImages(); })) @@ -2055,6 +2403,10 @@ EMSCRIPTEN_BINDINGS(basis_codec) { .value("cDecodeFlagsOutputHasAlphaIndices", cDecodeFlagsOutputHasAlphaIndices) .value("cDecodeFlagsHighQuality", cDecodeFlagsHighQuality) .value("cDecodeFlagsNoETC1SChromaFiltering", cDecodeFlagsNoETC1SChromaFiltering) + .value("cDecodeFlagsNoDeblockFiltering", cDecodeFlagsNoDeblockFiltering) + .value("cDecodeFlagsStrongerDeblockFiltering", cDecodeFlagsStrongerDeblockFiltering) + .value("cDecodeFlagsForceDeblockFiltering", cDecodeFlagsForceDeblockFiltering) + .value("cDecodeFlagXUASTCLDRDisableFastBC7Transcoding", cDecodeFlagXUASTCLDRDisableFastBC7Transcoding) ; // The low-level ETC1S transcoder is a class because it has persistent state (such as the endpoint/selector codebooks and Huffman tables, and transcoder state for video) @@ -2067,10 +2419,11 @@ EMSCRIPTEN_BINDINGS(basis_codec) { // The low-level UASTC transcoder (for UASTC LDR 4x4, HDR 4x4, or ASTC HDR 6x6) is a single function. function("transcodeUASTCImage", &transcode_uastc_image); + function("transcodeUASTCImage2", &transcode_uastc_image2); function("transcoderSupportsKTX2", &basisu_transcoder_supports_ktx2); function("transcoderSupportsKTX2Zstd", &basisu_transcoder_supports_ktx2_zstd); - + #if BASISD_SUPPORT_KTX2 // KTX2 enums/constants enum_("ktx2_supercompression") @@ -2080,10 +2433,11 @@ EMSCRIPTEN_BINDINGS(basis_codec) { ; constant("KTX2_VK_FORMAT_UNDEFINED", KTX2_VK_FORMAT_UNDEFINED); - constant("KTX2_KDF_DF_MODEL_UASTC", KTX2_KDF_DF_MODEL_UASTC_LDR_4X4); constant("KTX2_KDF_DF_MODEL_ETC1S", KTX2_KDF_DF_MODEL_ETC1S); - constant("KTX2_KDF_DF_MODEL_ASTC_HDR_6X6_INTERMEDIATE", KTX2_KDF_DF_MODEL_ASTC_HDR_6X6_INTERMEDIATE); - + constant("KTX2_KDF_DF_MODEL_UASTC", KTX2_KDF_DF_MODEL_UASTC_LDR_4X4); + constant("KTX2_KDF_DF_MODEL_UASTC_HDR_6X6_INTERMEDIATE", KTX2_KDF_DF_MODEL_UASTC_HDR_6X6_INTERMEDIATE); + constant("KTX2_KDF_DF_MODEL_XUASTC_LDR_INTERMEDIATE", KTX2_KDF_DF_MODEL_XUASTC_LDR_INTERMEDIATE); + constant("KTX2_IMAGE_IS_P_FRAME", KTX2_IMAGE_IS_P_FRAME); constant("KTX2_UASTC_BLOCK_SIZE", KTX2_UASTC_BLOCK_SIZE); constant("KTX2_MAX_SUPPORTED_LEVEL_COUNT", KTX2_MAX_SUPPORTED_LEVEL_COUNT); @@ -2185,10 +2539,13 @@ EMSCRIPTEN_BINDINGS(basis_codec) { .function("isHDR6x6", &ktx2_file::isHDR6x6) .function("isLDR", &ktx2_file::isLDR) .function("isETC1S", &ktx2_file::isETC1S) + .function("isASTC_LDR", &ktx2_file::isASTC_LDR) + .function("isXUASTC_LDR", &ktx2_file::isXUASTC_LDR) .function("getHasAlpha", &ktx2_file::getHasAlpha) .function("getDFDColorModel", &ktx2_file::getDFDColorModel) .function("getDFDColorPrimaries", &ktx2_file::getDFDColorPrimaries) .function("getDFDTransferFunc", &ktx2_file::getDFDTransferFunc) + .function("isSRGB", &ktx2_file::isSRGB) .function("getDFDFlags", &ktx2_file::getDFDFlags) .function("getDFDTotalSamples", &ktx2_file::getDFDTotalSamples) .function("getDFDChannelID0", &ktx2_file::getDFDChannelID0) @@ -2219,6 +2576,8 @@ EMSCRIPTEN_BINDINGS(basis_codec) { constant("BASISU_MAX_IMAGE_DIMENSION", BASISU_MAX_IMAGE_DIMENSION); constant("BASISU_QUALITY_MIN", BASISU_QUALITY_MIN); constant("BASISU_QUALITY_MAX", BASISU_QUALITY_MAX); + constant("BASISU_XUASTC_QUALITY_MIN", BASISU_XUASTC_QUALITY_MIN); + constant("BASISU_XUASTC_QUALITY_MAX", BASISU_XUASTC_QUALITY_MAX); constant("BASISU_MAX_ENDPOINT_CLUSTERS", BASISU_MAX_ENDPOINT_CLUSTERS); constant("BASISU_MAX_SELECTOR_CLUSTERS", BASISU_MAX_SELECTOR_CLUSTERS); constant("BASISU_MAX_SLICES", BASISU_MAX_SLICES); @@ -2226,10 +2585,10 @@ EMSCRIPTEN_BINDINGS(basis_codec) { constant("BASISU_RDO_UASTC_DICT_SIZE_MIN", BASISU_RDO_UASTC_DICT_SIZE_MIN); constant("BASISU_RDO_UASTC_DICT_SIZE_MAX", BASISU_RDO_UASTC_DICT_SIZE_MAX); constant("BASISU_MAX_RESAMPLER_FILTERS", g_num_resample_filters); - constant("BASISU_DEFAULT_COMPRESSION_LEVEL", BASISU_DEFAULT_COMPRESSION_LEVEL); - constant("BASISU_MAX_COMPRESSION_LEVEL", BASISU_MAX_COMPRESSION_LEVEL); + constant("BASISU_DEFAULT_ETC1S_COMPRESSION_LEVEL", BASISU_DEFAULT_ETC1S_COMPRESSION_LEVEL); + constant("BASISU_MAX_ETC1S_COMPRESSION_LEVEL", BASISU_MAX_ETC1S_COMPRESSION_LEVEL); - // The maximum representable floating point value in a UASTC HDR or ASTC HDR texture (any larger values will get clamped and a warning issued). + // The maximum representable floating point value in a UASTC HDR or ASTC HDR texture (any larger values will get clamped and a warning issued). constant("ASTC_HDR_MAX_VAL", basist::ASTC_HDR_MAX_VAL); // UASTC LDR/HDR flags/options @@ -2244,12 +2603,12 @@ EMSCRIPTEN_BINDINGS(basis_codec) { constant("cPackUASTCETC1FasterHints", cPackUASTCETC1FasterHints); constant("cPackUASTCETC1FastestHints", cPackUASTCETC1FastestHints); constant("cPackUASTCETC1DisableFlipAndIndividual", cPackUASTCETC1DisableFlipAndIndividual); - + constant("UASTC_RDO_DEFAULT_MAX_ALLOWED_RMS_INCREASE_RATIO", UASTC_RDO_DEFAULT_MAX_ALLOWED_RMS_INCREASE_RATIO); constant("UASTC_RDO_DEFAULT_SKIP_BLOCK_RMS_THRESH", UASTC_RDO_DEFAULT_SKIP_BLOCK_RMS_THRESH); constant("cPackASTC6x6MaxUserCompLevel", ::astc_6x6_hdr::ASTC_HDR_6X6_MAX_USER_COMP_LEVEL); - + enum_("hdr_image_type") .value("cHITRGBAHalfFloat", hdr_image_type::cHITRGBAHalfFloat) .value("cHITRGBAFloat", hdr_image_type::cHITRGBAFloat) @@ -2265,6 +2624,13 @@ EMSCRIPTEN_BINDINGS(basis_codec) { .value("cJPGImage", ldr_image_type::cJPGImage) ; + enum_("xuastc_ldr_syntax") + .value("cFullArith", xuastc_ldr_syntax::cFullArith) + .value("cHybridArithZStd", xuastc_ldr_syntax::cHybridArithZStd) + .value("cFullZStd", xuastc_ldr_syntax::cFullZStd) + .value("cTotal", xuastc_ldr_syntax::cTotal) + ; + // Compression/encoding object. // You create this object, call the set() methods to fill in the parameters/source images/options, call encode(), and you get back a .basis or .KTX2 file. // You can call .encode() multiple times, changing the parameters/options in between calls. @@ -2278,6 +2644,10 @@ EMSCRIPTEN_BINDINGS(basis_codec) { return self.encode(dst_basis_file_js_val); })) + .function("getLastEncodeMip0RGBAPSNR", optional_override([](basis_encoder& self) { + return self.get_last_encode_mip0_rgba_psnr(); + })) + // Sets the slice's source image, either from a PNG/JPG file or from a raw 32-bit RGBA raster image. // If the input is a raster image, the buffer must be width*height*4 bytes in size. The raster image is stored in top down scanline order. // The first texel is the top-left texel. The texel byte order in memory is R,G,B,A (R first at offset 0, A last at offset 3). @@ -2287,18 +2657,19 @@ EMSCRIPTEN_BINDINGS(basis_codec) { return self.set_slice_source_image(slice_index, src_image_js_val, width, height, (ldr_image_type)img_type); })) + // If true threaded compression will be used with X *extra* helper threads. .function("controlThreading", optional_override([](basis_encoder& self, bool enable_threading, uint32_t num_extra_worker_threads) { return self.control_threading(enable_threading, num_extra_worker_threads); })) // HDR targets only - .function("setSliceSourceImageHDR", optional_override([](basis_encoder& self, uint32_t slice_index, const emscripten::val& src_image_js_val, uint32_t width, uint32_t height, uint32_t img_type, + .function("setSliceSourceImageHDR", optional_override([](basis_encoder& self, uint32_t slice_index, const emscripten::val& src_image_js_val, uint32_t width, uint32_t height, uint32_t img_type, bool ldr_srgb_to_linear_conversion, float ldr_to_hdr_nit_multiplier) { return self.set_slice_source_image_hdr(slice_index, src_image_js_val, width, height, (hdr_image_type)img_type, ldr_srgb_to_linear_conversion, ldr_to_hdr_nit_multiplier); })) - // Sets the desired encoding format. This is the preferred way to control which format the encoder creates. - // tex_format is a basis_tex_format (cETC1s, cUASTC4x4, cUASTC_HDR_4x4 etc.) + // Sets the desired encoding format. This is the preferred way to control which format/ASTC block size the encoder creates. + // tex_format is a basis_tex_format (cETC1s, cUASTC_LDR_4x4, cUASTC_HDR_4x4 etc.) - see basisu_file_headers.h. // This can be used instead of the older setUASTC(), setHDR() etc. methods. // All formats .function("setFormatMode", optional_override([](basis_encoder& self, int tex_format) { @@ -2306,12 +2677,23 @@ EMSCRIPTEN_BINDINGS(basis_codec) { self.m_params.set_format_mode((basis_tex_format)tex_format); })) + // setFormatModeAndEffortQuality() is like setFormatMode(), except it also sets the effort [0,10] and quality [0,100] parameters to (hopefully) reasonable values for the selected format. + // If effort==-1, no effort related parameters will be modified. + // If quality==-1, no quality related parameters will be modified. + // These values directly correspond to the command line tool's "-effort X" and "-quality X" unified codec compression options. + .function("setFormatModeAndQualityEffort", optional_override([](basis_encoder& self, int tex_format, int quality, int effort, bool set_defaults) { + assert((tex_format >= 0) && (tex_format < (uint32_t)basis_tex_format::cTotalFormats)); + assert((effort >= -1) && (effort <= 10)); + assert((quality >= -1) && (quality <= 100)); + self.m_params.set_format_mode_and_quality_effort((basis_tex_format)tex_format, quality, effort, set_defaults); + })) + // If true, the encoder will output a UASTC LDR 4x4 texture, otherwise a ETC1S texture. // (This is for backwards compatibility, prefer setFormatMode() instead.) // All formats .function("setUASTC", optional_override([](basis_encoder& self, bool uastc_flag) { if (uastc_flag) - self.m_params.set_format_mode(basis_tex_format::cUASTC4x4); + self.m_params.set_format_mode(basis_tex_format::cUASTC_LDR_4x4); else self.m_params.set_format_mode(basis_tex_format::cETC1S); })) @@ -2325,8 +2707,9 @@ EMSCRIPTEN_BINDINGS(basis_codec) { else self.m_params.set_format_mode(basis_tex_format::cETC1S); // don't really know what to set })) - - // Sets the UASTC HDR 4x4 quality vs. encoder performance tradeoff (0-4, default is 1). Higher=slower but better quality. + + // Sets the UASTC HDR 4x4 quality/effort vs. encoder performance tradeoff (0-4, default is 1). Higher=slower but better quality. + // TODO: Rename, this is really a compressor "effort" level. // UASTC HDR 4x4 .function("setUASTCHDRQualityLevel", optional_override([](basis_encoder& self, int level) { assert((level >= uastc_hdr_4x4_codec_options::cMinLevel) && (level <= uastc_hdr_4x4_codec_options::cMaxLevel)); @@ -2348,6 +2731,7 @@ EMSCRIPTEN_BINDINGS(basis_codec) { // If true, the input is assumed to be in sRGB space. Be sure to set this correctly! (Examples: True on photos, albedo/spec maps, and false on normal maps.) // In HDR mode, if perceptual is true R and G are weighted higher (2.0, 3.0) than B (1.0). Otherwise the encoder uses equal weightings for each channel. + // Importantly, also see setKTX2SRGBTransferFunc() and setMipSRGB(). // ETC1S, UASTC LDR 4x4, UASTC HDR 4x4 .function("setPerceptual", optional_override([](basis_encoder& self, bool perceptual_flag) { self.m_params.m_perceptual = perceptual_flag; @@ -2382,8 +2766,9 @@ EMSCRIPTEN_BINDINGS(basis_codec) { .function("setLambda", optional_override([](basis_encoder& self, float rdo_quality) { self.m_params.m_astc_hdr_6x6_options.m_lambda = rdo_quality; })) - - // ASTC HDR 6x6: Enables REC 2020 delta E ITP vs. REC 709 in the encoder. + + // ASTC HDR 6x6: Enables REC 2020 delta E ITP vs. REC 709 in the encoder (and sets the colorspace in the KTX2 header). + // Note this colorspace always goes into the KTX2 header (DFD), for all modes (ETC1S, UASTC LDR 4x4, etc.) .function("setRec2020", optional_override([](basis_encoder& self, bool rec2020) { self.m_params.m_astc_hdr_6x6_options.m_rec2020_bt2100_color_gamut = rec2020; })) @@ -2471,22 +2856,23 @@ EMSCRIPTEN_BINDINGS(basis_codec) { self.m_params.m_etc1s_max_selector_clusters = max_selector_clusters; })) - // Sets the ETC1S encoder's quality level, which controls the file size vs. quality tradeoff. - // Default is -1 (meaning unused - the compressor will use m_max_endpoint_clusters/m_max_selector_clusters instead to control the codebook sizes). - // Range is [1,BASISU_QUALITY_MAX] - // ETC1S mode + // Sets the ETC1S or XUASTC LDR 4x4-12x12 encoder's quality level, which controls the file size vs. quality tradeoff. + // Default is -1 (meaning unused - the compressor will use m_max_endpoint_clusters/m_max_selector_clusters instead to control the codebook sizes in ETC1S mode, or no DCT in XUASTC LDR 4x4-12x12 mode). + // Range is [1,BASISU_QUALITY_MAX] (ETC1S) or [1,100] (XUASTC LDR 4x4-12x12) + // For XUASTC LDR, you also need to enable DCT usage, below. + // ETC1S mode or XUASTC LDR 4x4-12x12 .function("setQualityLevel", optional_override([](basis_encoder& self, int quality_level) { assert(quality_level >= -1 && quality_level <= BASISU_QUALITY_MAX); - self.m_params.m_etc1s_quality_level = quality_level; + self.m_params.m_quality_level = quality_level; })) // The compression_level parameter controls the encoder perf vs. file size tradeoff for ETC1S files. // It does not directly control file size vs. quality - see quality_level(). // Default is BASISU_DEFAULT_COMPRESSION_LEVEL, range is [0,BASISU_MAX_COMPRESSION_LEVEL] // ETC1S mode - .function("setCompressionLevel", optional_override([](basis_encoder& self, int comp_level) { - assert(comp_level >= 0 && comp_level <= BASISU_MAX_COMPRESSION_LEVEL); - self.m_params.m_compression_level = comp_level; + .function("setETC1SCompressionLevel", optional_override([](basis_encoder& self, int comp_level) { + assert(comp_level >= 0 && comp_level <= BASISU_MAX_ETC1S_COMPRESSION_LEVEL); + self.m_params.m_etc1s_compression_level = comp_level; })) // setNormalMapMode is the same as the basisu.exe "-normal_map" option. It tunes several codec parameters so compression works better on normal maps. @@ -2526,16 +2912,16 @@ EMSCRIPTEN_BINDINGS(basis_codec) { .function("setKTX2UASTCSupercompression", optional_override([](basis_encoder& self, bool use_zstandard) { self.m_params.m_ktx2_uastc_supercompression = use_zstandard ? basist::KTX2_SS_ZSTANDARD : basist::KTX2_SS_NONE; })) + + // TODO: Expose KTX2 key value array, other options to JavaScript. See encoder/basisu_comp.h. +#endif - // KTX2: Use sRGB transfer func in the file's DFD. Default is FALSE. This should very probably match the "perceptual" setting. + // KTX2/.basis: Use sRGB transfer func in the file's header/DFD. Default is FALSE. This should very probably match the "perceptual" and mipRGB settings. // All formats - .function("setKTX2SRGBTransferFunc", optional_override([](basis_encoder& self, bool srgb_transfer_func) { - self.m_params.m_ktx2_srgb_transfer_func = srgb_transfer_func; + .function("setKTX2AndBasisSRGBTransferFunc", optional_override([](basis_encoder& self, bool srgb_transfer_func) { + self.m_params.m_ktx2_and_basis_srgb_transfer_function = srgb_transfer_func; })) - // TODO: Expose KTX2 key value array, other options to JavaScript. See encoder/basisu_comp.h. -#endif - // --- Mip-map options (format independent) // If true mipmaps will be generated from the source images @@ -2621,7 +3007,7 @@ EMSCRIPTEN_BINDINGS(basis_codec) { .function("setRDOUASTCQualityScalar", optional_override([](basis_encoder& self, float rdo_quality) { self.m_params.m_rdo_uastc_ldr_4x4_quality_scalar = rdo_quality; })) - + // Default is BASISU_RDO_UASTC_DICT_SIZE_DEFAULT, range is [BASISU_RDO_UASTC_DICT_SIZE_MIN, BASISU_RDO_UASTC_DICT_SIZE_MAX] // UASTC LDR 4x4 .function("setRDOUASTCDictSize", optional_override([](basis_encoder& self, int dict_size) { @@ -2641,6 +3027,65 @@ EMSCRIPTEN_BINDINGS(basis_codec) { self.m_params.m_rdo_uastc_ldr_4x4_skip_block_rms_thresh = rdo_uastc_skip_block_rms_thresh; })) + // XUASTC/ASTC LDR 4x4-12x12 specific options + + // Enable XUASTC LDR DCT usage. Recommended to also enabled lossy supercompression for more compression. + // DCT quality [1,100] is set via setQualityLevel() above. + .function("setXUASTCLDRUseDCT", optional_override([](basis_encoder& self, bool xuastc_use_dct) { + self.m_params.m_xuastc_ldr_use_dct = xuastc_use_dct; + })) + + // Enables lossy XUASTC LDR supercompression (bounded distortion/windowed RDO) + .function("setXUASTCLDRUseLossySupercompression", optional_override([](basis_encoder& self, bool xuastc_use_lossy_supercompression) { + self.m_params.m_xuastc_ldr_use_lossy_supercompression = xuastc_use_lossy_supercompression; + })) + + // XUASTC LDR: Disable 2-3 subset usage, independent of effort level (for lower quality, for faster transcoding to BC7) + .function("setXUASTCLDRForceDisableSubsets", optional_override([](basis_encoder& self, bool flag) { + self.m_params.m_xuastc_ldr_force_disable_subsets = flag; + })) + + // XUASTC LDR: Disable RGB dual plane usage, indepdnent of effort level (for lower quality, for faster transcoding to BC7) + .function("setXUASTCLDRForceDisableRGBDualPlane", optional_override([](basis_encoder& self, bool flag) { + self.m_params.m_xuastc_ldr_force_disable_rgb_dual_plane = flag; + })) + + // Sets the XUASTC LDR syntax: see the xuastc_ldr_syntax enum. + .function("setXUASTCLDRSyntax", optional_override([](basis_encoder& self, int syntax) { + self.m_params.m_xuastc_ldr_syntax = syntax; + })) + + // Sets the ASTC/XUASTC LDR: compressor effort level [0,10] (encoding time vs. max achievable quality tradeoff, higher=slower) + // This is like setCompressionLevel() above, but for only ASTC/UASTC LDR 4x4-12x12, and has a different range. + .function("setASTCOrXUASTCLDREffortLevel", optional_override([](basis_encoder& self, int effort_level) { + self.m_params.m_xuastc_ldr_effort_level = effort_level; + })) + + // Sets the ASTC/XUASTC LDR channel weights + .function("setASTCOrXUASTCLDRWeights", optional_override([](basis_encoder& self, uint32_t x, uint32_t y, uint32_t z, uint32_t w) { + self.m_params.m_xuastc_ldr_channel_weights[0] = x; + self.m_params.m_xuastc_ldr_channel_weights[1] = y; + self.m_params.m_xuastc_ldr_channel_weights[2] = z; + self.m_params.m_xuastc_ldr_channel_weights[3] = w; + })) + + // Sets XUASTC LDR lossy supercompression (bounded/windows RDO) parameters. + // Must be enabled via setXUASTCLDRUseLossySupercompression(). + .function("setXUASTCLDRBoundedRDOParam", optional_override([](basis_encoder& self, uint32_t idx, float value) { + switch (idx) + { + case 0: self.m_params.m_ls_min_psnr = value; break; + case 1: self.m_params.m_ls_min_alpha_psnr = value; break; + case 2: self.m_params.m_ls_thresh_psnr = value; break; + case 3: self.m_params.m_ls_thresh_alpha_psnr = value; break; + case 4: self.m_params.m_ls_thresh_edge_psnr = value; break; + case 5: self.m_params.m_ls_thresh_edge_alpha_psnr = value; break; + default: + assert(0); + break; + } + })) + // --- Low level options // Disables ETC1S selector RDO diff --git a/external/basis_universal/webgl/transcoder/build/basis_transcoder.js b/external/basis_universal/webgl/transcoder/build/basis_transcoder.js index cc0f78ab93..bb00f4d781 100644 --- a/external/basis_universal/webgl/transcoder/build/basis_transcoder.js +++ b/external/basis_universal/webgl/transcoder/build/basis_transcoder.js @@ -1,2 +1,2 @@ -var BASIS=(()=>{var _scriptName=globalThis.document?.currentScript?.src;return async function(moduleArg={}){var moduleRtn;var Module=moduleArg;var ENVIRONMENT_IS_WEB=!!globalThis.window;var ENVIRONMENT_IS_WORKER=!!globalThis.WorkerGlobalScope;var ENVIRONMENT_IS_NODE=globalThis.process?.versions?.node&&globalThis.process?.type!="renderer";var arguments_=[];var thisProgram="./this.program";var quit_=(status,toThrow)=>{throw toThrow};if(typeof __filename!="undefined"){_scriptName=__filename}else if(ENVIRONMENT_IS_WORKER){_scriptName=self.location.href}var scriptDirectory="";function locateFile(path){if(Module["locateFile"]){return Module["locateFile"](path,scriptDirectory)}return scriptDirectory+path}var readAsync,readBinary;if(ENVIRONMENT_IS_NODE){var fs=require("fs");scriptDirectory=__dirname+"/";readBinary=filename=>{filename=isFileURI(filename)?new URL(filename):filename;var ret=fs.readFileSync(filename);return ret};readAsync=async(filename,binary=true)=>{filename=isFileURI(filename)?new URL(filename):filename;var ret=fs.readFileSync(filename,binary?undefined:"utf8");return ret};if(process.argv.length>1){thisProgram=process.argv[1].replace(/\\/g,"/")}arguments_=process.argv.slice(2);quit_=(status,toThrow)=>{process.exitCode=status;throw toThrow}}else if(ENVIRONMENT_IS_WEB||ENVIRONMENT_IS_WORKER){try{scriptDirectory=new URL(".",_scriptName).href}catch{}{if(ENVIRONMENT_IS_WORKER){readBinary=url=>{var xhr=new XMLHttpRequest;xhr.open("GET",url,false);xhr.responseType="arraybuffer";xhr.send(null);return new Uint8Array(xhr.response)}}readAsync=async url=>{if(isFileURI(url)){return new Promise((resolve,reject)=>{var xhr=new XMLHttpRequest;xhr.open("GET",url,true);xhr.responseType="arraybuffer";xhr.onload=()=>{if(xhr.status==200||xhr.status==0&&xhr.response){resolve(xhr.response);return}reject(xhr.status)};xhr.onerror=reject;xhr.send(null)})}var response=await fetch(url,{credentials:"same-origin"});if(response.ok){return response.arrayBuffer()}throw new Error(response.status+" : "+response.url)}}}else{}var out=console.log.bind(console);var err=console.error.bind(console);var wasmBinary;var ABORT=false;var isFileURI=filename=>filename.startsWith("file://");var readyPromiseResolve,readyPromiseReject;var HEAP8,HEAPU8,HEAP16,HEAPU16,HEAP32,HEAPU32,HEAPF32,HEAPF64;var HEAP64,HEAPU64;var runtimeInitialized=false;function updateMemoryViews(){var b=wasmMemory.buffer;Module["HEAP8"]=HEAP8=new Int8Array(b);HEAP16=new Int16Array(b);HEAPU8=new Uint8Array(b);HEAPU16=new Uint16Array(b);HEAP32=new Int32Array(b);HEAPU32=new Uint32Array(b);HEAPF32=new Float32Array(b);HEAPF64=new Float64Array(b);HEAP64=new BigInt64Array(b);HEAPU64=new BigUint64Array(b)}function preRun(){if(Module["preRun"]){if(typeof Module["preRun"]=="function")Module["preRun"]=[Module["preRun"]];while(Module["preRun"].length){addOnPreRun(Module["preRun"].shift())}}callRuntimeCallbacks(onPreRuns)}function initRuntime(){runtimeInitialized=true;wasmExports["J"]()}function postRun(){if(Module["postRun"]){if(typeof Module["postRun"]=="function")Module["postRun"]=[Module["postRun"]];while(Module["postRun"].length){addOnPostRun(Module["postRun"].shift())}}callRuntimeCallbacks(onPostRuns)}function abort(what){Module["onAbort"]?.(what);what="Aborted("+what+")";err(what);ABORT=true;what+=". Build with -sASSERTIONS for more info.";var e=new WebAssembly.RuntimeError(what);readyPromiseReject?.(e);throw e}var wasmBinaryFile;function findWasmBinary(){return locateFile("basis_transcoder.wasm")}function getBinarySync(file){if(file==wasmBinaryFile&&wasmBinary){return new Uint8Array(wasmBinary)}if(readBinary){return readBinary(file)}throw"both async and sync fetching of the wasm failed"}async function getWasmBinary(binaryFile){if(!wasmBinary){try{var response=await readAsync(binaryFile);return new Uint8Array(response)}catch{}}return getBinarySync(binaryFile)}async function instantiateArrayBuffer(binaryFile,imports){try{var binary=await getWasmBinary(binaryFile);var instance=await WebAssembly.instantiate(binary,imports);return instance}catch(reason){err(`failed to asynchronously prepare wasm: ${reason}`);abort(reason)}}async function instantiateAsync(binary,binaryFile,imports){if(!binary&&!isFileURI(binaryFile)&&!ENVIRONMENT_IS_NODE){try{var response=fetch(binaryFile,{credentials:"same-origin"});var instantiationResult=await WebAssembly.instantiateStreaming(response,imports);return instantiationResult}catch(reason){err(`wasm streaming compile failed: ${reason}`);err("falling back to ArrayBuffer instantiation")}}return instantiateArrayBuffer(binaryFile,imports)}function getWasmImports(){var imports={a:wasmImports};return imports}async function createWasm(){function receiveInstance(instance,module){wasmExports=instance.exports;assignWasmExports(wasmExports);updateMemoryViews();return wasmExports}function receiveInstantiationResult(result){return receiveInstance(result["instance"])}var info=getWasmImports();if(Module["instantiateWasm"]){return new Promise((resolve,reject)=>{Module["instantiateWasm"](info,(inst,mod)=>{resolve(receiveInstance(inst,mod))})})}wasmBinaryFile??=findWasmBinary();var result=await instantiateAsync(wasmBinary,wasmBinaryFile,info);var exports=receiveInstantiationResult(result);return exports}class ExitStatus{name="ExitStatus";constructor(status){this.message=`Program terminated with exit(${status})`;this.status=status}}var callRuntimeCallbacks=callbacks=>{while(callbacks.length>0){callbacks.shift()(Module)}};var onPostRuns=[];var addOnPostRun=cb=>onPostRuns.push(cb);var onPreRuns=[];var addOnPreRun=cb=>onPreRuns.push(cb);var noExitRuntime=true;class ExceptionInfo{constructor(excPtr){this.excPtr=excPtr;this.ptr=excPtr-24}set_type(type){HEAPU32[this.ptr+4>>2]=type}get_type(){return HEAPU32[this.ptr+4>>2]}set_destructor(destructor){HEAPU32[this.ptr+8>>2]=destructor}get_destructor(){return HEAPU32[this.ptr+8>>2]}set_caught(caught){caught=caught?1:0;HEAP8[this.ptr+12]=caught}get_caught(){return HEAP8[this.ptr+12]!=0}set_rethrown(rethrown){rethrown=rethrown?1:0;HEAP8[this.ptr+13]=rethrown}get_rethrown(){return HEAP8[this.ptr+13]!=0}init(type,destructor){this.set_adjusted_ptr(0);this.set_type(type);this.set_destructor(destructor)}set_adjusted_ptr(adjustedPtr){HEAPU32[this.ptr+16>>2]=adjustedPtr}get_adjusted_ptr(){return HEAPU32[this.ptr+16>>2]}}var exceptionLast=0;var uncaughtExceptionCount=0;var ___cxa_throw=(ptr,type,destructor)=>{var info=new ExceptionInfo(ptr);info.init(type,destructor);exceptionLast=ptr;uncaughtExceptionCount++;throw exceptionLast};var __abort_js=()=>abort("");var structRegistrations={};var runDestructors=destructors=>{while(destructors.length){var ptr=destructors.pop();var del=destructors.pop();del(ptr)}};function readPointer(pointer){return this.fromWireType(HEAPU32[pointer>>2])}var awaitingDependencies={};var registeredTypes={};var typeDependencies={};var InternalError=class InternalError extends Error{constructor(message){super(message);this.name="InternalError"}};var throwInternalError=message=>{throw new InternalError(message)};var whenDependentTypesAreResolved=(myTypes,dependentTypes,getTypeConverters)=>{myTypes.forEach(type=>typeDependencies[type]=dependentTypes);function onComplete(typeConverters){var myTypeConverters=getTypeConverters(typeConverters);if(myTypeConverters.length!==myTypes.length){throwInternalError("Mismatched type converter count")}for(var i=0;i{typeConverters[i]=registeredTypes[dt];++registered;if(registered===unregisteredTypes.length){onComplete(typeConverters)}})}}if(0===unregisteredTypes.length){onComplete(typeConverters)}};var __embind_finalize_value_object=structType=>{var reg=structRegistrations[structType];delete structRegistrations[structType];var rawConstructor=reg.rawConstructor;var rawDestructor=reg.rawDestructor;var fieldRecords=reg.fields;var fieldTypes=fieldRecords.map(field=>field.getterReturnType).concat(fieldRecords.map(field=>field.setterArgumentType));whenDependentTypesAreResolved([structType],fieldTypes,fieldTypes=>{var fields={};for(var[i,field]of fieldRecords.entries()){const getterReturnType=fieldTypes[i];const getter=field.getter;const getterContext=field.getterContext;const setterArgumentType=fieldTypes[i+fieldRecords.length];const setter=field.setter;const setterContext=field.setterContext;fields[field.fieldName]={read:ptr=>getterReturnType.fromWireType(getter(getterContext,ptr)),write:(ptr,o)=>{var destructors=[];setter(setterContext,ptr,setterArgumentType.toWireType(destructors,o));runDestructors(destructors)},optional:getterReturnType.optional}}return[{name:reg.name,fromWireType:ptr=>{var rv={};for(var i in fields){rv[i]=fields[i].read(ptr)}rawDestructor(ptr);return rv},toWireType:(destructors,o)=>{for(var fieldName in fields){if(!(fieldName in o)&&!fields[fieldName].optional){throw new TypeError(`Missing field: "${fieldName}"`)}}var ptr=rawConstructor();for(fieldName in fields){fields[fieldName].write(ptr,o[fieldName])}if(destructors!==null){destructors.push(rawDestructor,ptr)}return ptr},readValueFromPointer:readPointer,destructorFunction:rawDestructor}]})};var AsciiToString=ptr=>{var str="";while(1){var ch=HEAPU8[ptr++];if(!ch)return str;str+=String.fromCharCode(ch)}};var BindingError=class BindingError extends Error{constructor(message){super(message);this.name="BindingError"}};var throwBindingError=message=>{throw new BindingError(message)};function sharedRegisterType(rawType,registeredInstance,options={}){var name=registeredInstance.name;if(!rawType){throwBindingError(`type "${name}" must have a positive integer typeid pointer`)}if(registeredTypes.hasOwnProperty(rawType)){if(options.ignoreDuplicateRegistrations){return}else{throwBindingError(`Cannot register type '${name}' twice`)}}registeredTypes[rawType]=registeredInstance;delete typeDependencies[rawType];if(awaitingDependencies.hasOwnProperty(rawType)){var callbacks=awaitingDependencies[rawType];delete awaitingDependencies[rawType];callbacks.forEach(cb=>cb())}}function registerType(rawType,registeredInstance,options={}){return sharedRegisterType(rawType,registeredInstance,options)}var integerReadValueFromPointer=(name,width,signed)=>{switch(width){case 1:return signed?pointer=>HEAP8[pointer]:pointer=>HEAPU8[pointer];case 2:return signed?pointer=>HEAP16[pointer>>1]:pointer=>HEAPU16[pointer>>1];case 4:return signed?pointer=>HEAP32[pointer>>2]:pointer=>HEAPU32[pointer>>2];case 8:return signed?pointer=>HEAP64[pointer>>3]:pointer=>HEAPU64[pointer>>3];default:throw new TypeError(`invalid integer width (${width}): ${name}`)}};var __embind_register_bigint=(primitiveType,name,size,minRange,maxRange)=>{name=AsciiToString(name);const isUnsignedType=minRange===0n;let fromWireType=value=>value;if(isUnsignedType){const bitSize=size*8;fromWireType=value=>BigInt.asUintN(bitSize,value);maxRange=fromWireType(maxRange)}registerType(primitiveType,{name,fromWireType,toWireType:(destructors,value)=>{if(typeof value=="number"){value=BigInt(value)}return value},readValueFromPointer:integerReadValueFromPointer(name,size,!isUnsignedType),destructorFunction:null})};var __embind_register_bool=(rawType,name,trueValue,falseValue)=>{name=AsciiToString(name);registerType(rawType,{name,fromWireType:function(wt){return!!wt},toWireType:function(destructors,o){return o?trueValue:falseValue},readValueFromPointer:function(pointer){return this.fromWireType(HEAPU8[pointer])},destructorFunction:null})};var shallowCopyInternalPointer=o=>({count:o.count,deleteScheduled:o.deleteScheduled,preservePointerOnDelete:o.preservePointerOnDelete,ptr:o.ptr,ptrType:o.ptrType,smartPtr:o.smartPtr,smartPtrType:o.smartPtrType});var throwInstanceAlreadyDeleted=obj=>{function getInstanceTypeName(handle){return handle.$$.ptrType.registeredClass.name}throwBindingError(getInstanceTypeName(obj)+" instance already deleted")};var finalizationRegistry=false;var detachFinalizer=handle=>{};var runDestructor=$$=>{if($$.smartPtr){$$.smartPtrType.rawDestructor($$.smartPtr)}else{$$.ptrType.registeredClass.rawDestructor($$.ptr)}};var releaseClassHandle=$$=>{$$.count.value-=1;var toDelete=0===$$.count.value;if(toDelete){runDestructor($$)}};var attachFinalizer=handle=>{if(!globalThis.FinalizationRegistry){attachFinalizer=handle=>handle;return handle}finalizationRegistry=new FinalizationRegistry(info=>{releaseClassHandle(info.$$)});attachFinalizer=handle=>{var $$=handle.$$;var hasSmartPtr=!!$$.smartPtr;if(hasSmartPtr){var info={$$};finalizationRegistry.register(handle,info,handle)}return handle};detachFinalizer=handle=>finalizationRegistry.unregister(handle);return attachFinalizer(handle)};var deletionQueue=[];var flushPendingDeletes=()=>{while(deletionQueue.length){var obj=deletionQueue.pop();obj.$$.deleteScheduled=false;obj["delete"]()}};var delayFunction;var init_ClassHandle=()=>{let proto=ClassHandle.prototype;Object.assign(proto,{isAliasOf(other){if(!(this instanceof ClassHandle)){return false}if(!(other instanceof ClassHandle)){return false}var leftClass=this.$$.ptrType.registeredClass;var left=this.$$.ptr;other.$$=other.$$;var rightClass=other.$$.ptrType.registeredClass;var right=other.$$.ptr;while(leftClass.baseClass){left=leftClass.upcast(left);leftClass=leftClass.baseClass}while(rightClass.baseClass){right=rightClass.upcast(right);rightClass=rightClass.baseClass}return leftClass===rightClass&&left===right},clone(){if(!this.$$.ptr){throwInstanceAlreadyDeleted(this)}if(this.$$.preservePointerOnDelete){this.$$.count.value+=1;return this}else{var clone=attachFinalizer(Object.create(Object.getPrototypeOf(this),{$$:{value:shallowCopyInternalPointer(this.$$)}}));clone.$$.count.value+=1;clone.$$.deleteScheduled=false;return clone}},delete(){if(!this.$$.ptr){throwInstanceAlreadyDeleted(this)}if(this.$$.deleteScheduled&&!this.$$.preservePointerOnDelete){throwBindingError("Object already scheduled for deletion")}detachFinalizer(this);releaseClassHandle(this.$$);if(!this.$$.preservePointerOnDelete){this.$$.smartPtr=undefined;this.$$.ptr=undefined}},isDeleted(){return!this.$$.ptr},deleteLater(){if(!this.$$.ptr){throwInstanceAlreadyDeleted(this)}if(this.$$.deleteScheduled&&!this.$$.preservePointerOnDelete){throwBindingError("Object already scheduled for deletion")}deletionQueue.push(this);if(deletionQueue.length===1&&delayFunction){delayFunction(flushPendingDeletes)}this.$$.deleteScheduled=true;return this}});const symbolDispose=Symbol.dispose;if(symbolDispose){proto[symbolDispose]=proto["delete"]}};function ClassHandle(){}var createNamedFunction=(name,func)=>Object.defineProperty(func,"name",{value:name});var registeredPointers={};var ensureOverloadTable=(proto,methodName,humanName)=>{if(undefined===proto[methodName].overloadTable){var prevFunc=proto[methodName];proto[methodName]=function(...args){if(!proto[methodName].overloadTable.hasOwnProperty(args.length)){throwBindingError(`Function '${humanName}' called with an invalid number of arguments (${args.length}) - expects one of (${proto[methodName].overloadTable})!`)}return proto[methodName].overloadTable[args.length].apply(this,args)};proto[methodName].overloadTable=[];proto[methodName].overloadTable[prevFunc.argCount]=prevFunc}};var exposePublicSymbol=(name,value,numArguments)=>{if(Module.hasOwnProperty(name)){if(undefined===numArguments||undefined!==Module[name].overloadTable&&undefined!==Module[name].overloadTable[numArguments]){throwBindingError(`Cannot register public name '${name}' twice`)}ensureOverloadTable(Module,name,name);if(Module[name].overloadTable.hasOwnProperty(numArguments)){throwBindingError(`Cannot register multiple overloads of a function with the same number of arguments (${numArguments})!`)}Module[name].overloadTable[numArguments]=value}else{Module[name]=value;Module[name].argCount=numArguments}};var char_0=48;var char_9=57;var makeLegalFunctionName=name=>{name=name.replace(/[^a-zA-Z0-9_]/g,"$");var f=name.charCodeAt(0);if(f>=char_0&&f<=char_9){return`_${name}`}return name};function RegisteredClass(name,constructor,instancePrototype,rawDestructor,baseClass,getActualType,upcast,downcast){this.name=name;this.constructor=constructor;this.instancePrototype=instancePrototype;this.rawDestructor=rawDestructor;this.baseClass=baseClass;this.getActualType=getActualType;this.upcast=upcast;this.downcast=downcast;this.pureVirtualFunctions=[]}var upcastPointer=(ptr,ptrClass,desiredClass)=>{while(ptrClass!==desiredClass){if(!ptrClass.upcast){throwBindingError(`Expected null or instance of ${desiredClass.name}, got an instance of ${ptrClass.name}`)}ptr=ptrClass.upcast(ptr);ptrClass=ptrClass.baseClass}return ptr};var embindRepr=v=>{if(v===null){return"null"}var t=typeof v;if(t==="object"||t==="array"||t==="function"){return v.toString()}else{return""+v}};function constNoSmartPtrRawPointerToWireType(destructors,handle){if(handle===null){if(this.isReference){throwBindingError(`null is not a valid ${this.name}`)}return 0}if(!handle.$$){throwBindingError(`Cannot pass "${embindRepr(handle)}" as a ${this.name}`)}if(!handle.$$.ptr){throwBindingError(`Cannot pass deleted object as a pointer of type ${this.name}`)}var handleClass=handle.$$.ptrType.registeredClass;var ptr=upcastPointer(handle.$$.ptr,handleClass,this.registeredClass);return ptr}function genericPointerToWireType(destructors,handle){var ptr;if(handle===null){if(this.isReference){throwBindingError(`null is not a valid ${this.name}`)}if(this.isSmartPointer){ptr=this.rawConstructor();if(destructors!==null){destructors.push(this.rawDestructor,ptr)}return ptr}else{return 0}}if(!handle||!handle.$$){throwBindingError(`Cannot pass "${embindRepr(handle)}" as a ${this.name}`)}if(!handle.$$.ptr){throwBindingError(`Cannot pass deleted object as a pointer of type ${this.name}`)}if(!this.isConst&&handle.$$.ptrType.isConst){throwBindingError(`Cannot convert argument of type ${handle.$$.smartPtrType?handle.$$.smartPtrType.name:handle.$$.ptrType.name} to parameter type ${this.name}`)}var handleClass=handle.$$.ptrType.registeredClass;ptr=upcastPointer(handle.$$.ptr,handleClass,this.registeredClass);if(this.isSmartPointer){if(undefined===handle.$$.smartPtr){throwBindingError("Passing raw pointer to smart pointer is illegal")}switch(this.sharingPolicy){case 0:if(handle.$$.smartPtrType===this){ptr=handle.$$.smartPtr}else{throwBindingError(`Cannot convert argument of type ${handle.$$.smartPtrType?handle.$$.smartPtrType.name:handle.$$.ptrType.name} to parameter type ${this.name}`)}break;case 1:ptr=handle.$$.smartPtr;break;case 2:if(handle.$$.smartPtrType===this){ptr=handle.$$.smartPtr}else{var clonedHandle=handle["clone"]();ptr=this.rawShare(ptr,Emval.toHandle(()=>clonedHandle["delete"]()));if(destructors!==null){destructors.push(this.rawDestructor,ptr)}}break;default:throwBindingError("Unsupporting sharing policy")}}return ptr}function nonConstNoSmartPtrRawPointerToWireType(destructors,handle){if(handle===null){if(this.isReference){throwBindingError(`null is not a valid ${this.name}`)}return 0}if(!handle.$$){throwBindingError(`Cannot pass "${embindRepr(handle)}" as a ${this.name}`)}if(!handle.$$.ptr){throwBindingError(`Cannot pass deleted object as a pointer of type ${this.name}`)}if(handle.$$.ptrType.isConst){throwBindingError(`Cannot convert argument of type ${handle.$$.ptrType.name} to parameter type ${this.name}`)}var handleClass=handle.$$.ptrType.registeredClass;var ptr=upcastPointer(handle.$$.ptr,handleClass,this.registeredClass);return ptr}var downcastPointer=(ptr,ptrClass,desiredClass)=>{if(ptrClass===desiredClass){return ptr}if(undefined===desiredClass.baseClass){return null}var rv=downcastPointer(ptr,ptrClass,desiredClass.baseClass);if(rv===null){return null}return desiredClass.downcast(rv)};var registeredInstances={};var getBasestPointer=(class_,ptr)=>{if(ptr===undefined){throwBindingError("ptr should not be undefined")}while(class_.baseClass){ptr=class_.upcast(ptr);class_=class_.baseClass}return ptr};var getInheritedInstance=(class_,ptr)=>{ptr=getBasestPointer(class_,ptr);return registeredInstances[ptr]};var makeClassHandle=(prototype,record)=>{if(!record.ptrType||!record.ptr){throwInternalError("makeClassHandle requires ptr and ptrType")}var hasSmartPtrType=!!record.smartPtrType;var hasSmartPtr=!!record.smartPtr;if(hasSmartPtrType!==hasSmartPtr){throwInternalError("Both smartPtrType and smartPtr must be specified")}record.count={value:1};return attachFinalizer(Object.create(prototype,{$$:{value:record,writable:true}}))};function RegisteredPointer_fromWireType(ptr){var rawPointer=this.getPointee(ptr);if(!rawPointer){this.destructor(ptr);return null}var registeredInstance=getInheritedInstance(this.registeredClass,rawPointer);if(undefined!==registeredInstance){if(0===registeredInstance.$$.count.value){registeredInstance.$$.ptr=rawPointer;registeredInstance.$$.smartPtr=ptr;return registeredInstance["clone"]()}else{var rv=registeredInstance["clone"]();this.destructor(ptr);return rv}}function makeDefaultHandle(){if(this.isSmartPointer){return makeClassHandle(this.registeredClass.instancePrototype,{ptrType:this.pointeeType,ptr:rawPointer,smartPtrType:this,smartPtr:ptr})}else{return makeClassHandle(this.registeredClass.instancePrototype,{ptrType:this,ptr})}}var actualType=this.registeredClass.getActualType(rawPointer);var registeredPointerRecord=registeredPointers[actualType];if(!registeredPointerRecord){return makeDefaultHandle.call(this)}var toType;if(this.isConst){toType=registeredPointerRecord.constPointerType}else{toType=registeredPointerRecord.pointerType}var dp=downcastPointer(rawPointer,this.registeredClass,toType.registeredClass);if(dp===null){return makeDefaultHandle.call(this)}if(this.isSmartPointer){return makeClassHandle(toType.registeredClass.instancePrototype,{ptrType:toType,ptr:dp,smartPtrType:this,smartPtr:ptr})}else{return makeClassHandle(toType.registeredClass.instancePrototype,{ptrType:toType,ptr:dp})}}var init_RegisteredPointer=()=>{Object.assign(RegisteredPointer.prototype,{getPointee(ptr){if(this.rawGetPointee){ptr=this.rawGetPointee(ptr)}return ptr},destructor(ptr){this.rawDestructor?.(ptr)},readValueFromPointer:readPointer,fromWireType:RegisteredPointer_fromWireType})};function RegisteredPointer(name,registeredClass,isReference,isConst,isSmartPointer,pointeeType,sharingPolicy,rawGetPointee,rawConstructor,rawShare,rawDestructor){this.name=name;this.registeredClass=registeredClass;this.isReference=isReference;this.isConst=isConst;this.isSmartPointer=isSmartPointer;this.pointeeType=pointeeType;this.sharingPolicy=sharingPolicy;this.rawGetPointee=rawGetPointee;this.rawConstructor=rawConstructor;this.rawShare=rawShare;this.rawDestructor=rawDestructor;if(!isSmartPointer&®isteredClass.baseClass===undefined){if(isConst){this.toWireType=constNoSmartPtrRawPointerToWireType;this.destructorFunction=null}else{this.toWireType=nonConstNoSmartPtrRawPointerToWireType;this.destructorFunction=null}}else{this.toWireType=genericPointerToWireType}}var replacePublicSymbol=(name,value,numArguments)=>{if(!Module.hasOwnProperty(name)){throwInternalError("Replacing nonexistent public symbol")}if(undefined!==Module[name].overloadTable&&undefined!==numArguments){Module[name].overloadTable[numArguments]=value}else{Module[name]=value;Module[name].argCount=numArguments}};var wasmTableMirror=[];var getWasmTableEntry=funcPtr=>{var func=wasmTableMirror[funcPtr];if(!func){wasmTableMirror[funcPtr]=func=wasmTable.get(funcPtr)}return func};var embind__requireFunction=(signature,rawFunction,isAsync=false)=>{signature=AsciiToString(signature);function makeDynCaller(){var rtn=getWasmTableEntry(rawFunction);return rtn}var fp=makeDynCaller();if(typeof fp!="function"){throwBindingError(`unknown function pointer with signature ${signature}: ${rawFunction}`)}return fp};class UnboundTypeError extends Error{}var getTypeName=type=>{var ptr=___getTypeName(type);var rv=AsciiToString(ptr);_free(ptr);return rv};var throwUnboundTypeError=(message,types)=>{var unboundTypes=[];var seen={};function visit(type){if(seen[type]){return}if(registeredTypes[type]){return}if(typeDependencies[type]){typeDependencies[type].forEach(visit);return}unboundTypes.push(type);seen[type]=true}types.forEach(visit);throw new UnboundTypeError(`${message}: `+unboundTypes.map(getTypeName).join([", "]))};var __embind_register_class=(rawType,rawPointerType,rawConstPointerType,baseClassRawType,getActualTypeSignature,getActualType,upcastSignature,upcast,downcastSignature,downcast,name,destructorSignature,rawDestructor)=>{name=AsciiToString(name);getActualType=embind__requireFunction(getActualTypeSignature,getActualType);upcast&&=embind__requireFunction(upcastSignature,upcast);downcast&&=embind__requireFunction(downcastSignature,downcast);rawDestructor=embind__requireFunction(destructorSignature,rawDestructor);var legalFunctionName=makeLegalFunctionName(name);exposePublicSymbol(legalFunctionName,function(){throwUnboundTypeError(`Cannot construct ${name} due to unbound types`,[baseClassRawType])});whenDependentTypesAreResolved([rawType,rawPointerType,rawConstPointerType],baseClassRawType?[baseClassRawType]:[],base=>{base=base[0];var baseClass;var basePrototype;if(baseClassRawType){baseClass=base.registeredClass;basePrototype=baseClass.instancePrototype}else{basePrototype=ClassHandle.prototype}var constructor=createNamedFunction(name,function(...args){if(Object.getPrototypeOf(this)!==instancePrototype){throw new BindingError(`Use 'new' to construct ${name}`)}if(undefined===registeredClass.constructor_body){throw new BindingError(`${name} has no accessible constructor`)}var body=registeredClass.constructor_body[args.length];if(undefined===body){throw new BindingError(`Tried to invoke ctor of ${name} with invalid number of parameters (${args.length}) - expected (${Object.keys(registeredClass.constructor_body).toString()}) parameters instead!`)}return body.apply(this,args)});var instancePrototype=Object.create(basePrototype,{constructor:{value:constructor}});constructor.prototype=instancePrototype;var registeredClass=new RegisteredClass(name,constructor,instancePrototype,rawDestructor,baseClass,getActualType,upcast,downcast);if(registeredClass.baseClass){registeredClass.baseClass.__derivedClasses??=[];registeredClass.baseClass.__derivedClasses.push(registeredClass)}var referenceConverter=new RegisteredPointer(name,registeredClass,true,false,false);var pointerConverter=new RegisteredPointer(name+"*",registeredClass,false,false,false);var constPointerConverter=new RegisteredPointer(name+" const*",registeredClass,false,true,false);registeredPointers[rawType]={pointerType:pointerConverter,constPointerType:constPointerConverter};replacePublicSymbol(legalFunctionName,constructor);return[referenceConverter,pointerConverter,constPointerConverter]})};var heap32VectorToArray=(count,firstElement)=>{var array=[];for(var i=0;i>2])}return array};function usesDestructorStack(argTypes){for(var i=1;i{var rawArgTypes=heap32VectorToArray(argCount,rawArgTypesAddr);invoker=embind__requireFunction(invokerSignature,invoker);whenDependentTypesAreResolved([],[rawClassType],classType=>{classType=classType[0];var humanName=`constructor ${classType.name}`;if(undefined===classType.registeredClass.constructor_body){classType.registeredClass.constructor_body=[]}if(undefined!==classType.registeredClass.constructor_body[argCount-1]){throw new BindingError(`Cannot register multiple constructors with identical number of parameters (${argCount-1}) for class '${classType.name}'! Overload resolution is currently only performed using the parameter count, not actual type info!`)}classType.registeredClass.constructor_body[argCount-1]=()=>{throwUnboundTypeError(`Cannot construct ${classType.name} due to unbound types`,rawArgTypes)};whenDependentTypesAreResolved([],rawArgTypes,argTypes=>{argTypes.splice(1,0,null);classType.registeredClass.constructor_body[argCount-1]=craftInvokerFunction(humanName,argTypes,null,invoker,rawConstructor);return[]});return[]})};var getFunctionName=signature=>{signature=signature.trim();const argsIndex=signature.indexOf("(");if(argsIndex===-1)return signature;return signature.slice(0,argsIndex)};var __embind_register_class_function=(rawClassType,methodName,argCount,rawArgTypesAddr,invokerSignature,rawInvoker,context,isPureVirtual,isAsync,isNonnullReturn)=>{var rawArgTypes=heap32VectorToArray(argCount,rawArgTypesAddr);methodName=AsciiToString(methodName);methodName=getFunctionName(methodName);rawInvoker=embind__requireFunction(invokerSignature,rawInvoker,isAsync);whenDependentTypesAreResolved([],[rawClassType],classType=>{classType=classType[0];var humanName=`${classType.name}.${methodName}`;if(methodName.startsWith("@@")){methodName=Symbol[methodName.substring(2)]}if(isPureVirtual){classType.registeredClass.pureVirtualFunctions.push(methodName)}function unboundTypesHandler(){throwUnboundTypeError(`Cannot call ${humanName} due to unbound types`,rawArgTypes)}var proto=classType.registeredClass.instancePrototype;var method=proto[methodName];if(undefined===method||undefined===method.overloadTable&&method.className!==classType.name&&method.argCount===argCount-2){unboundTypesHandler.argCount=argCount-2;unboundTypesHandler.className=classType.name;proto[methodName]=unboundTypesHandler}else{ensureOverloadTable(proto,methodName,humanName);proto[methodName].overloadTable[argCount-2]=unboundTypesHandler}whenDependentTypesAreResolved([],rawArgTypes,argTypes=>{var memberFunction=craftInvokerFunction(humanName,argTypes,classType,rawInvoker,context,isAsync);if(undefined===proto[methodName].overloadTable){memberFunction.argCount=argCount-2;proto[methodName]=memberFunction}else{proto[methodName].overloadTable[argCount-2]=memberFunction}return[]});return[]})};var __embind_register_constant=(name,type,value)=>{name=AsciiToString(name);whenDependentTypesAreResolved([],[type],type=>{type=type[0];Module[name]=type.fromWireType(value);return[]})};var emval_freelist=[];var emval_handles=[0,1,,1,null,1,true,1,false,1];var __emval_decref=handle=>{if(handle>9&&0===--emval_handles[handle+1]){emval_handles[handle]=undefined;emval_freelist.push(handle)}};var Emval={toValue:handle=>{if(!handle){throwBindingError(`Cannot use deleted val. handle = ${handle}`)}return emval_handles[handle]},toHandle:value=>{switch(value){case undefined:return 2;case null:return 4;case true:return 6;case false:return 8;default:{const handle=emval_freelist.pop()||emval_handles.length;emval_handles[handle]=value;emval_handles[handle+1]=1;return handle}}}};var EmValType={name:"emscripten::val",fromWireType:handle=>{var rv=Emval.toValue(handle);__emval_decref(handle);return rv},toWireType:(destructors,value)=>Emval.toHandle(value),readValueFromPointer:readPointer,destructorFunction:null};var __embind_register_emval=rawType=>registerType(rawType,EmValType);var enumReadValueFromPointer=(name,width,signed)=>{switch(width){case 1:return signed?function(pointer){return this.fromWireType(HEAP8[pointer])}:function(pointer){return this.fromWireType(HEAPU8[pointer])};case 2:return signed?function(pointer){return this.fromWireType(HEAP16[pointer>>1])}:function(pointer){return this.fromWireType(HEAPU16[pointer>>1])};case 4:return signed?function(pointer){return this.fromWireType(HEAP32[pointer>>2])}:function(pointer){return this.fromWireType(HEAPU32[pointer>>2])};default:throw new TypeError(`invalid integer width (${width}): ${name}`)}};var __embind_register_enum=(rawType,name,size,isSigned)=>{name=AsciiToString(name);function ctor(){}ctor.values={};registerType(rawType,{name,constructor:ctor,fromWireType:function(c){return this.constructor.values[c]},toWireType:(destructors,c)=>c.value,readValueFromPointer:enumReadValueFromPointer(name,size,isSigned),destructorFunction:null});exposePublicSymbol(name,ctor)};var requireRegisteredType=(rawType,humanName)=>{var impl=registeredTypes[rawType];if(undefined===impl){throwBindingError(`${humanName} has unknown type ${getTypeName(rawType)}`)}return impl};var __embind_register_enum_value=(rawEnumType,name,enumValue)=>{var enumType=requireRegisteredType(rawEnumType,"enum");name=AsciiToString(name);var Enum=enumType.constructor;var Value=Object.create(enumType.constructor.prototype,{value:{value:enumValue},constructor:{value:createNamedFunction(`${enumType.name}_${name}`,function(){})}});Enum.values[enumValue]=Value;Enum[name]=Value};var floatReadValueFromPointer=(name,width)=>{switch(width){case 4:return function(pointer){return this.fromWireType(HEAPF32[pointer>>2])};case 8:return function(pointer){return this.fromWireType(HEAPF64[pointer>>3])};default:throw new TypeError(`invalid float width (${width}): ${name}`)}};var __embind_register_float=(rawType,name,size)=>{name=AsciiToString(name);registerType(rawType,{name,fromWireType:value=>value,toWireType:(destructors,value)=>value,readValueFromPointer:floatReadValueFromPointer(name,size),destructorFunction:null})};var __embind_register_function=(name,argCount,rawArgTypesAddr,signature,rawInvoker,fn,isAsync,isNonnullReturn)=>{var argTypes=heap32VectorToArray(argCount,rawArgTypesAddr);name=AsciiToString(name);name=getFunctionName(name);rawInvoker=embind__requireFunction(signature,rawInvoker,isAsync);exposePublicSymbol(name,function(){throwUnboundTypeError(`Cannot call ${name} due to unbound types`,argTypes)},argCount-1);whenDependentTypesAreResolved([],argTypes,argTypes=>{var invokerArgsArray=[argTypes[0],null].concat(argTypes.slice(1));replacePublicSymbol(name,craftInvokerFunction(name,invokerArgsArray,null,rawInvoker,fn,isAsync),argCount-1);return[]})};var __embind_register_integer=(primitiveType,name,size,minRange,maxRange)=>{name=AsciiToString(name);const isUnsignedType=minRange===0;let fromWireType=value=>value;if(isUnsignedType){var bitshift=32-8*size;fromWireType=value=>value<>>bitshift;maxRange=fromWireType(maxRange)}registerType(primitiveType,{name,fromWireType,toWireType:(destructors,value)=>value,readValueFromPointer:integerReadValueFromPointer(name,size,minRange!==0),destructorFunction:null})};var __embind_register_memory_view=(rawType,dataTypeIndex,name)=>{var typeMapping=[Int8Array,Uint8Array,Int16Array,Uint16Array,Int32Array,Uint32Array,Float32Array,Float64Array,BigInt64Array,BigUint64Array];var TA=typeMapping[dataTypeIndex];function decodeMemoryView(handle){var size=HEAPU32[handle>>2];var data=HEAPU32[handle+4>>2];return new TA(HEAP8.buffer,data,size)}name=AsciiToString(name);registerType(rawType,{name,fromWireType:decodeMemoryView,readValueFromPointer:decodeMemoryView},{ignoreDuplicateRegistrations:true})};var stringToUTF8Array=(str,heap,outIdx,maxBytesToWrite)=>{if(!(maxBytesToWrite>0))return 0;var startIdx=outIdx;var endIdx=outIdx+maxBytesToWrite-1;for(var i=0;i=endIdx)break;heap[outIdx++]=u}else if(u<=2047){if(outIdx+1>=endIdx)break;heap[outIdx++]=192|u>>6;heap[outIdx++]=128|u&63}else if(u<=65535){if(outIdx+2>=endIdx)break;heap[outIdx++]=224|u>>12;heap[outIdx++]=128|u>>6&63;heap[outIdx++]=128|u&63}else{if(outIdx+3>=endIdx)break;heap[outIdx++]=240|u>>18;heap[outIdx++]=128|u>>12&63;heap[outIdx++]=128|u>>6&63;heap[outIdx++]=128|u&63;i++}}heap[outIdx]=0;return outIdx-startIdx};var stringToUTF8=(str,outPtr,maxBytesToWrite)=>stringToUTF8Array(str,HEAPU8,outPtr,maxBytesToWrite);var lengthBytesUTF8=str=>{var len=0;for(var i=0;i=55296&&c<=57343){len+=4;++i}else{len+=3}}return len};var UTF8Decoder=globalThis.TextDecoder&&new TextDecoder;var findStringEnd=(heapOrArray,idx,maxBytesToRead,ignoreNul)=>{var maxIdx=idx+maxBytesToRead;if(ignoreNul)return maxIdx;while(heapOrArray[idx]&&!(idx>=maxIdx))++idx;return idx};var UTF8ArrayToString=(heapOrArray,idx=0,maxBytesToRead,ignoreNul)=>{var endPtr=findStringEnd(heapOrArray,idx,maxBytesToRead,ignoreNul);if(endPtr-idx>16&&heapOrArray.buffer&&UTF8Decoder){return UTF8Decoder.decode(heapOrArray.subarray(idx,endPtr))}var str="";while(idx>10,56320|ch&1023)}}return str};var UTF8ToString=(ptr,maxBytesToRead,ignoreNul)=>ptr?UTF8ArrayToString(HEAPU8,ptr,maxBytesToRead,ignoreNul):"";var __embind_register_std_string=(rawType,name)=>{name=AsciiToString(name);var stdStringIsUTF8=true;registerType(rawType,{name,fromWireType(value){var length=HEAPU32[value>>2];var payload=value+4;var str;if(stdStringIsUTF8){str=UTF8ToString(payload,length,true)}else{str="";for(var i=0;i>2]=length;if(valueIsOfTypeString){if(stdStringIsUTF8){stringToUTF8(value,ptr,length+1)}else{for(var i=0;i255){_free(base);throwBindingError("String has UTF-16 code units that do not fit in 8 bits")}HEAPU8[ptr+i]=charCode}}}else{HEAPU8.set(value,ptr)}if(destructors!==null){destructors.push(_free,base)}return base},readValueFromPointer:readPointer,destructorFunction(ptr){_free(ptr)}})};var UTF16Decoder=globalThis.TextDecoder?new TextDecoder("utf-16le"):undefined;var UTF16ToString=(ptr,maxBytesToRead,ignoreNul)=>{var idx=ptr>>1;var endIdx=findStringEnd(HEAPU16,idx,maxBytesToRead/2,ignoreNul);if(endIdx-idx>16&&UTF16Decoder)return UTF16Decoder.decode(HEAPU16.subarray(idx,endIdx));var str="";for(var i=idx;i{maxBytesToWrite??=2147483647;if(maxBytesToWrite<2)return 0;maxBytesToWrite-=2;var startPtr=outPtr;var numCharsToWrite=maxBytesToWrite>1]=codeUnit;outPtr+=2}HEAP16[outPtr>>1]=0;return outPtr-startPtr};var lengthBytesUTF16=str=>str.length*2;var UTF32ToString=(ptr,maxBytesToRead,ignoreNul)=>{var str="";var startIdx=ptr>>2;for(var i=0;!(i>=maxBytesToRead/4);i++){var utf32=HEAPU32[startIdx+i];if(!utf32&&!ignoreNul)break;str+=String.fromCodePoint(utf32)}return str};var stringToUTF32=(str,outPtr,maxBytesToWrite)=>{maxBytesToWrite??=2147483647;if(maxBytesToWrite<4)return 0;var startPtr=outPtr;var endPtr=startPtr+maxBytesToWrite-4;for(var i=0;i65535){i++}HEAP32[outPtr>>2]=codePoint;outPtr+=4;if(outPtr+4>endPtr)break}HEAP32[outPtr>>2]=0;return outPtr-startPtr};var lengthBytesUTF32=str=>{var len=0;for(var i=0;i65535){i++}len+=4}return len};var __embind_register_std_wstring=(rawType,charSize,name)=>{name=AsciiToString(name);var decodeString,encodeString,lengthBytesUTF;if(charSize===2){decodeString=UTF16ToString;encodeString=stringToUTF16;lengthBytesUTF=lengthBytesUTF16}else{decodeString=UTF32ToString;encodeString=stringToUTF32;lengthBytesUTF=lengthBytesUTF32}registerType(rawType,{name,fromWireType:value=>{var length=HEAPU32[value>>2];var str=decodeString(value+4,length*charSize,true);_free(value);return str},toWireType:(destructors,value)=>{if(!(typeof value=="string")){throwBindingError(`Cannot pass non-string to C++ string type ${name}`)}var length=lengthBytesUTF(value);var ptr=_malloc(4+length+charSize);HEAPU32[ptr>>2]=length/charSize;encodeString(value,ptr+4,length+charSize);if(destructors!==null){destructors.push(_free,ptr)}return ptr},readValueFromPointer:readPointer,destructorFunction(ptr){_free(ptr)}})};var __embind_register_value_object=(rawType,name,constructorSignature,rawConstructor,destructorSignature,rawDestructor)=>{structRegistrations[rawType]={name:AsciiToString(name),rawConstructor:embind__requireFunction(constructorSignature,rawConstructor),rawDestructor:embind__requireFunction(destructorSignature,rawDestructor),fields:[]}};var __embind_register_value_object_field=(structType,fieldName,getterReturnType,getterSignature,getter,getterContext,setterArgumentType,setterSignature,setter,setterContext)=>{structRegistrations[structType].fields.push({fieldName:AsciiToString(fieldName),getterReturnType,getter:embind__requireFunction(getterSignature,getter),getterContext,setterArgumentType,setter:embind__requireFunction(setterSignature,setter),setterContext})};var __embind_register_void=(rawType,name)=>{name=AsciiToString(name);registerType(rawType,{isVoid:true,name,fromWireType:()=>undefined,toWireType:(destructors,o)=>undefined})};var emval_methodCallers=[];var emval_addMethodCaller=caller=>{var id=emval_methodCallers.length;emval_methodCallers.push(caller);return id};var emval_lookupTypes=(argCount,argTypes)=>{var a=new Array(argCount);for(var i=0;i>2],`parameter ${i}`)}return a};var emval_returnValue=(toReturnWire,destructorsRef,handle)=>{var destructors=[];var result=toReturnWire(destructors,handle);if(destructors.length){HEAPU32[destructorsRef>>2]=Emval.toHandle(destructors)}return result};var emval_symbols={};var getStringOrSymbol=address=>{var symbol=emval_symbols[address];if(symbol===undefined){return AsciiToString(address)}return symbol};var __emval_create_invoker=(argCount,argTypesPtr,kind)=>{var GenericWireTypeSize=8;var[retType,...argTypes]=emval_lookupTypes(argCount,argTypesPtr);var toReturnWire=retType.toWireType.bind(retType);var argFromPtr=argTypes.map(type=>type.readValueFromPointer.bind(type));argCount--;var captures={toValue:Emval.toValue};var args=argFromPtr.map((argFromPtr,i)=>{var captureName=`argFromPtr${i}`;captures[captureName]=argFromPtr;return`${captureName}(args${i?"+"+i*GenericWireTypeSize:""})`});var functionBody;switch(kind){case 0:functionBody="toValue(handle)";break;case 2:functionBody="new (toValue(handle))";break;case 3:functionBody="";break;case 1:captures["getStringOrSymbol"]=getStringOrSymbol;functionBody="toValue(handle)[getStringOrSymbol(methodName)]";break}functionBody+=`(${args})`;if(!retType.isVoid){captures["toReturnWire"]=toReturnWire;captures["emval_returnValue"]=emval_returnValue;functionBody=`return emval_returnValue(toReturnWire, destructorsRef, ${functionBody})`}functionBody=`return function (handle, methodName, destructorsRef, args) {\n ${functionBody}\n }`;var invokerFunction=new Function(Object.keys(captures),functionBody)(...Object.values(captures));var functionName=`methodCaller<(${argTypes.map(t=>t.name)}) => ${retType.name}>`;return emval_addMethodCaller(createNamedFunction(functionName,invokerFunction))};var __emval_get_global=name=>{if(!name){return Emval.toHandle(globalThis)}name=getStringOrSymbol(name);return Emval.toHandle(globalThis[name])};var __emval_get_module_property=name=>{name=getStringOrSymbol(name);return Emval.toHandle(Module[name])};var __emval_get_property=(handle,key)=>{handle=Emval.toValue(handle);key=Emval.toValue(key);return Emval.toHandle(handle[key])};var __emval_incref=handle=>{if(handle>9){emval_handles[handle+1]+=1}};var __emval_invoke=(caller,handle,methodName,destructorsRef,args)=>emval_methodCallers[caller](handle,methodName,destructorsRef,args);var __emval_new_cstring=v=>Emval.toHandle(getStringOrSymbol(v));var __emval_run_destructors=handle=>{var destructors=Emval.toValue(handle);runDestructors(destructors);__emval_decref(handle)};var getHeapMax=()=>2147483648;var alignMemory=(size,alignment)=>Math.ceil(size/alignment)*alignment;var growMemory=size=>{var oldHeapSize=wasmMemory.buffer.byteLength;var pages=(size-oldHeapSize+65535)/65536|0;try{wasmMemory.grow(pages);updateMemoryViews();return 1}catch(e){}};var _emscripten_resize_heap=requestedSize=>{var oldSize=HEAPU8.length;requestedSize>>>=0;var maxHeapSize=getHeapMax();if(requestedSize>maxHeapSize){return false}for(var cutDown=1;cutDown<=4;cutDown*=2){var overGrownHeapSize=oldSize*(1+.2/cutDown);overGrownHeapSize=Math.min(overGrownHeapSize,requestedSize+100663296);var newSize=Math.min(maxHeapSize,alignMemory(Math.max(requestedSize,overGrownHeapSize),65536));var replacement=growMemory(newSize);if(replacement){return true}}return false};var _fd_close=fd=>52;var INT53_MAX=9007199254740992;var INT53_MIN=-9007199254740992;var bigintToI53Checked=num=>numINT53_MAX?NaN:Number(num);function _fd_seek(fd,offset,whence,newOffset){offset=bigintToI53Checked(offset);return 70}var printCharBuffers=[null,[],[]];var printChar=(stream,curr)=>{var buffer=printCharBuffers[stream];if(curr===0||curr===10){(stream===1?out:err)(UTF8ArrayToString(buffer));buffer.length=0}else{buffer.push(curr)}};var _fd_write=(fd,iov,iovcnt,pnum)=>{var num=0;for(var i=0;i>2];var len=HEAPU32[iov+4>>2];iov+=8;for(var j=0;j>2]=num;return 0};init_ClassHandle();init_RegisteredPointer();{if(Module["noExitRuntime"])noExitRuntime=Module["noExitRuntime"];if(Module["print"])out=Module["print"];if(Module["printErr"])err=Module["printErr"];if(Module["wasmBinary"])wasmBinary=Module["wasmBinary"];if(Module["arguments"])arguments_=Module["arguments"];if(Module["thisProgram"])thisProgram=Module["thisProgram"];if(Module["preInit"]){if(typeof Module["preInit"]=="function")Module["preInit"]=[Module["preInit"]];while(Module["preInit"].length>0){Module["preInit"].shift()()}}}var ___getTypeName,_malloc,_free,memory,__indirect_function_table,wasmMemory,wasmTable;function assignWasmExports(wasmExports){___getTypeName=wasmExports["K"];_malloc=wasmExports["L"];_free=wasmExports["N"];memory=wasmMemory=wasmExports["I"];__indirect_function_table=wasmTable=wasmExports["M"]}var wasmImports={H:___cxa_throw,z:__abort_js,r:__embind_finalize_value_object,y:__embind_register_bigint,F:__embind_register_bool,v:__embind_register_class,u:__embind_register_class_constructor,c:__embind_register_class_function,n:__embind_register_constant,D:__embind_register_emval,p:__embind_register_enum,a:__embind_register_enum_value,x:__embind_register_float,i:__embind_register_function,m:__embind_register_integer,j:__embind_register_memory_view,E:__embind_register_std_string,t:__embind_register_std_wstring,s:__embind_register_value_object,d:__embind_register_value_object_field,G:__embind_register_void,h:__emval_create_invoker,b:__emval_decref,q:__emval_get_global,o:__emval_get_module_property,k:__emval_get_property,g:__emval_incref,f:__emval_invoke,l:__emval_new_cstring,e:__emval_run_destructors,A:_emscripten_resize_heap,C:_fd_close,B:_fd_seek,w:_fd_write};function run(){preRun();function doRun(){Module["calledRun"]=true;if(ABORT)return;initRuntime();readyPromiseResolve?.(Module);Module["onRuntimeInitialized"]?.();postRun()}if(Module["setStatus"]){Module["setStatus"]("Running...");setTimeout(()=>{setTimeout(()=>Module["setStatus"](""),1);doRun()},1)}else{doRun()}}var wasmExports;wasmExports=await (createWasm());run();if(runtimeInitialized){moduleRtn=Module}else{moduleRtn=new Promise((resolve,reject)=>{readyPromiseResolve=resolve;readyPromiseReject=reject})} +var BASIS=(()=>{var _scriptName=globalThis.document?.currentScript?.src;return async function(moduleArg={}){var moduleRtn;var Module=moduleArg;var ENVIRONMENT_IS_WEB=!!globalThis.window;var ENVIRONMENT_IS_WORKER=!!globalThis.WorkerGlobalScope;var ENVIRONMENT_IS_NODE=globalThis.process?.versions?.node&&globalThis.process?.type!="renderer";var arguments_=[];var thisProgram="./this.program";var quit_=(status,toThrow)=>{throw toThrow};if(typeof __filename!="undefined"){_scriptName=__filename}else if(ENVIRONMENT_IS_WORKER){_scriptName=self.location.href}var scriptDirectory="";function locateFile(path){if(Module["locateFile"]){return Module["locateFile"](path,scriptDirectory)}return scriptDirectory+path}var readAsync,readBinary;if(ENVIRONMENT_IS_NODE){var fs=require("fs");scriptDirectory=__dirname+"/";readBinary=filename=>{filename=isFileURI(filename)?new URL(filename):filename;var ret=fs.readFileSync(filename);return ret};readAsync=async(filename,binary=true)=>{filename=isFileURI(filename)?new URL(filename):filename;var ret=fs.readFileSync(filename,binary?undefined:"utf8");return ret};if(process.argv.length>1){thisProgram=process.argv[1].replace(/\\/g,"/")}arguments_=process.argv.slice(2);quit_=(status,toThrow)=>{process.exitCode=status;throw toThrow}}else if(ENVIRONMENT_IS_WEB||ENVIRONMENT_IS_WORKER){try{scriptDirectory=new URL(".",_scriptName).href}catch{}{if(ENVIRONMENT_IS_WORKER){readBinary=url=>{var xhr=new XMLHttpRequest;xhr.open("GET",url,false);xhr.responseType="arraybuffer";xhr.send(null);return new Uint8Array(xhr.response)}}readAsync=async url=>{if(isFileURI(url)){return new Promise((resolve,reject)=>{var xhr=new XMLHttpRequest;xhr.open("GET",url,true);xhr.responseType="arraybuffer";xhr.onload=()=>{if(xhr.status==200||xhr.status==0&&xhr.response){resolve(xhr.response);return}reject(xhr.status)};xhr.onerror=reject;xhr.send(null)})}var response=await fetch(url,{credentials:"same-origin"});if(response.ok){return response.arrayBuffer()}throw new Error(response.status+" : "+response.url)}}}else{}var out=console.log.bind(console);var err=console.error.bind(console);var wasmBinary;var ABORT=false;var isFileURI=filename=>filename.startsWith("file://");var readyPromiseResolve,readyPromiseReject;var HEAP8,HEAPU8,HEAP16,HEAPU16,HEAP32,HEAPU32,HEAPF32,HEAPF64;var HEAP64,HEAPU64;var runtimeInitialized=false;function updateMemoryViews(){var b=wasmMemory.buffer;Module["HEAP8"]=HEAP8=new Int8Array(b);HEAP16=new Int16Array(b);HEAPU8=new Uint8Array(b);HEAPU16=new Uint16Array(b);HEAP32=new Int32Array(b);HEAPU32=new Uint32Array(b);HEAPF32=new Float32Array(b);HEAPF64=new Float64Array(b);HEAP64=new BigInt64Array(b);HEAPU64=new BigUint64Array(b)}function preRun(){if(Module["preRun"]){if(typeof Module["preRun"]=="function")Module["preRun"]=[Module["preRun"]];while(Module["preRun"].length){addOnPreRun(Module["preRun"].shift())}}callRuntimeCallbacks(onPreRuns)}function initRuntime(){runtimeInitialized=true;wasmExports["H"]()}function postRun(){if(Module["postRun"]){if(typeof Module["postRun"]=="function")Module["postRun"]=[Module["postRun"]];while(Module["postRun"].length){addOnPostRun(Module["postRun"].shift())}}callRuntimeCallbacks(onPostRuns)}function abort(what){Module["onAbort"]?.(what);what="Aborted("+what+")";err(what);ABORT=true;what+=". Build with -sASSERTIONS for more info.";var e=new WebAssembly.RuntimeError(what);readyPromiseReject?.(e);throw e}var wasmBinaryFile;function findWasmBinary(){return locateFile("basis_transcoder.wasm")}function getBinarySync(file){if(file==wasmBinaryFile&&wasmBinary){return new Uint8Array(wasmBinary)}if(readBinary){return readBinary(file)}throw"both async and sync fetching of the wasm failed"}async function getWasmBinary(binaryFile){if(!wasmBinary){try{var response=await readAsync(binaryFile);return new Uint8Array(response)}catch{}}return getBinarySync(binaryFile)}async function instantiateArrayBuffer(binaryFile,imports){try{var binary=await getWasmBinary(binaryFile);var instance=await WebAssembly.instantiate(binary,imports);return instance}catch(reason){err(`failed to asynchronously prepare wasm: ${reason}`);abort(reason)}}async function instantiateAsync(binary,binaryFile,imports){if(!binary&&!isFileURI(binaryFile)&&!ENVIRONMENT_IS_NODE){try{var response=fetch(binaryFile,{credentials:"same-origin"});var instantiationResult=await WebAssembly.instantiateStreaming(response,imports);return instantiationResult}catch(reason){err(`wasm streaming compile failed: ${reason}`);err("falling back to ArrayBuffer instantiation")}}return instantiateArrayBuffer(binaryFile,imports)}function getWasmImports(){var imports={a:wasmImports};return imports}async function createWasm(){function receiveInstance(instance,module){wasmExports=instance.exports;assignWasmExports(wasmExports);updateMemoryViews();return wasmExports}function receiveInstantiationResult(result){return receiveInstance(result["instance"])}var info=getWasmImports();if(Module["instantiateWasm"]){return new Promise((resolve,reject)=>{Module["instantiateWasm"](info,(inst,mod)=>{resolve(receiveInstance(inst,mod))})})}wasmBinaryFile??=findWasmBinary();var result=await instantiateAsync(wasmBinary,wasmBinaryFile,info);var exports=receiveInstantiationResult(result);return exports}class ExitStatus{name="ExitStatus";constructor(status){this.message=`Program terminated with exit(${status})`;this.status=status}}var callRuntimeCallbacks=callbacks=>{while(callbacks.length>0){callbacks.shift()(Module)}};var onPostRuns=[];var addOnPostRun=cb=>onPostRuns.push(cb);var onPreRuns=[];var addOnPreRun=cb=>onPreRuns.push(cb);var noExitRuntime=true;class ExceptionInfo{constructor(excPtr){this.excPtr=excPtr;this.ptr=excPtr-24}set_type(type){HEAPU32[this.ptr+4>>2]=type}get_type(){return HEAPU32[this.ptr+4>>2]}set_destructor(destructor){HEAPU32[this.ptr+8>>2]=destructor}get_destructor(){return HEAPU32[this.ptr+8>>2]}set_caught(caught){caught=caught?1:0;HEAP8[this.ptr+12]=caught}get_caught(){return HEAP8[this.ptr+12]!=0}set_rethrown(rethrown){rethrown=rethrown?1:0;HEAP8[this.ptr+13]=rethrown}get_rethrown(){return HEAP8[this.ptr+13]!=0}init(type,destructor){this.set_adjusted_ptr(0);this.set_type(type);this.set_destructor(destructor)}set_adjusted_ptr(adjustedPtr){HEAPU32[this.ptr+16>>2]=adjustedPtr}get_adjusted_ptr(){return HEAPU32[this.ptr+16>>2]}}var exceptionLast=0;var uncaughtExceptionCount=0;var ___cxa_throw=(ptr,type,destructor)=>{var info=new ExceptionInfo(ptr);info.init(type,destructor);exceptionLast=ptr;uncaughtExceptionCount++;throw exceptionLast};var __abort_js=()=>abort("");var structRegistrations={};var runDestructors=destructors=>{while(destructors.length){var ptr=destructors.pop();var del=destructors.pop();del(ptr)}};function readPointer(pointer){return this.fromWireType(HEAPU32[pointer>>2])}var awaitingDependencies={};var registeredTypes={};var typeDependencies={};var InternalError=class InternalError extends Error{constructor(message){super(message);this.name="InternalError"}};var throwInternalError=message=>{throw new InternalError(message)};var whenDependentTypesAreResolved=(myTypes,dependentTypes,getTypeConverters)=>{myTypes.forEach(type=>typeDependencies[type]=dependentTypes);function onComplete(typeConverters){var myTypeConverters=getTypeConverters(typeConverters);if(myTypeConverters.length!==myTypes.length){throwInternalError("Mismatched type converter count")}for(var i=0;i{typeConverters[i]=registeredTypes[dt];++registered;if(registered===unregisteredTypes.length){onComplete(typeConverters)}})}}if(0===unregisteredTypes.length){onComplete(typeConverters)}};var __embind_finalize_value_object=structType=>{var reg=structRegistrations[structType];delete structRegistrations[structType];var rawConstructor=reg.rawConstructor;var rawDestructor=reg.rawDestructor;var fieldRecords=reg.fields;var fieldTypes=fieldRecords.map(field=>field.getterReturnType).concat(fieldRecords.map(field=>field.setterArgumentType));whenDependentTypesAreResolved([structType],fieldTypes,fieldTypes=>{var fields={};for(var[i,field]of fieldRecords.entries()){const getterReturnType=fieldTypes[i];const getter=field.getter;const getterContext=field.getterContext;const setterArgumentType=fieldTypes[i+fieldRecords.length];const setter=field.setter;const setterContext=field.setterContext;fields[field.fieldName]={read:ptr=>getterReturnType.fromWireType(getter(getterContext,ptr)),write:(ptr,o)=>{var destructors=[];setter(setterContext,ptr,setterArgumentType.toWireType(destructors,o));runDestructors(destructors)},optional:getterReturnType.optional}}return[{name:reg.name,fromWireType:ptr=>{var rv={};for(var i in fields){rv[i]=fields[i].read(ptr)}rawDestructor(ptr);return rv},toWireType:(destructors,o)=>{for(var fieldName in fields){if(!(fieldName in o)&&!fields[fieldName].optional){throw new TypeError(`Missing field: "${fieldName}"`)}}var ptr=rawConstructor();for(fieldName in fields){fields[fieldName].write(ptr,o[fieldName])}if(destructors!==null){destructors.push(rawDestructor,ptr)}return ptr},readValueFromPointer:readPointer,destructorFunction:rawDestructor}]})};var AsciiToString=ptr=>{var str="";while(1){var ch=HEAPU8[ptr++];if(!ch)return str;str+=String.fromCharCode(ch)}};var BindingError=class BindingError extends Error{constructor(message){super(message);this.name="BindingError"}};var throwBindingError=message=>{throw new BindingError(message)};function sharedRegisterType(rawType,registeredInstance,options={}){var name=registeredInstance.name;if(!rawType){throwBindingError(`type "${name}" must have a positive integer typeid pointer`)}if(registeredTypes.hasOwnProperty(rawType)){if(options.ignoreDuplicateRegistrations){return}else{throwBindingError(`Cannot register type '${name}' twice`)}}registeredTypes[rawType]=registeredInstance;delete typeDependencies[rawType];if(awaitingDependencies.hasOwnProperty(rawType)){var callbacks=awaitingDependencies[rawType];delete awaitingDependencies[rawType];callbacks.forEach(cb=>cb())}}function registerType(rawType,registeredInstance,options={}){return sharedRegisterType(rawType,registeredInstance,options)}var integerReadValueFromPointer=(name,width,signed)=>{switch(width){case 1:return signed?pointer=>HEAP8[pointer]:pointer=>HEAPU8[pointer];case 2:return signed?pointer=>HEAP16[pointer>>1]:pointer=>HEAPU16[pointer>>1];case 4:return signed?pointer=>HEAP32[pointer>>2]:pointer=>HEAPU32[pointer>>2];case 8:return signed?pointer=>HEAP64[pointer>>3]:pointer=>HEAPU64[pointer>>3];default:throw new TypeError(`invalid integer width (${width}): ${name}`)}};var __embind_register_bigint=(primitiveType,name,size,minRange,maxRange)=>{name=AsciiToString(name);const isUnsignedType=minRange===0n;let fromWireType=value=>value;if(isUnsignedType){const bitSize=size*8;fromWireType=value=>BigInt.asUintN(bitSize,value);maxRange=fromWireType(maxRange)}registerType(primitiveType,{name,fromWireType,toWireType:(destructors,value)=>{if(typeof value=="number"){value=BigInt(value)}return value},readValueFromPointer:integerReadValueFromPointer(name,size,!isUnsignedType),destructorFunction:null})};var __embind_register_bool=(rawType,name,trueValue,falseValue)=>{name=AsciiToString(name);registerType(rawType,{name,fromWireType:function(wt){return!!wt},toWireType:function(destructors,o){return o?trueValue:falseValue},readValueFromPointer:function(pointer){return this.fromWireType(HEAPU8[pointer])},destructorFunction:null})};var shallowCopyInternalPointer=o=>({count:o.count,deleteScheduled:o.deleteScheduled,preservePointerOnDelete:o.preservePointerOnDelete,ptr:o.ptr,ptrType:o.ptrType,smartPtr:o.smartPtr,smartPtrType:o.smartPtrType});var throwInstanceAlreadyDeleted=obj=>{function getInstanceTypeName(handle){return handle.$$.ptrType.registeredClass.name}throwBindingError(getInstanceTypeName(obj)+" instance already deleted")};var finalizationRegistry=false;var detachFinalizer=handle=>{};var runDestructor=$$=>{if($$.smartPtr){$$.smartPtrType.rawDestructor($$.smartPtr)}else{$$.ptrType.registeredClass.rawDestructor($$.ptr)}};var releaseClassHandle=$$=>{$$.count.value-=1;var toDelete=0===$$.count.value;if(toDelete){runDestructor($$)}};var attachFinalizer=handle=>{if(!globalThis.FinalizationRegistry){attachFinalizer=handle=>handle;return handle}finalizationRegistry=new FinalizationRegistry(info=>{releaseClassHandle(info.$$)});attachFinalizer=handle=>{var $$=handle.$$;var hasSmartPtr=!!$$.smartPtr;if(hasSmartPtr){var info={$$};finalizationRegistry.register(handle,info,handle)}return handle};detachFinalizer=handle=>finalizationRegistry.unregister(handle);return attachFinalizer(handle)};var deletionQueue=[];var flushPendingDeletes=()=>{while(deletionQueue.length){var obj=deletionQueue.pop();obj.$$.deleteScheduled=false;obj["delete"]()}};var delayFunction;var init_ClassHandle=()=>{let proto=ClassHandle.prototype;Object.assign(proto,{isAliasOf(other){if(!(this instanceof ClassHandle)){return false}if(!(other instanceof ClassHandle)){return false}var leftClass=this.$$.ptrType.registeredClass;var left=this.$$.ptr;other.$$=other.$$;var rightClass=other.$$.ptrType.registeredClass;var right=other.$$.ptr;while(leftClass.baseClass){left=leftClass.upcast(left);leftClass=leftClass.baseClass}while(rightClass.baseClass){right=rightClass.upcast(right);rightClass=rightClass.baseClass}return leftClass===rightClass&&left===right},clone(){if(!this.$$.ptr){throwInstanceAlreadyDeleted(this)}if(this.$$.preservePointerOnDelete){this.$$.count.value+=1;return this}else{var clone=attachFinalizer(Object.create(Object.getPrototypeOf(this),{$$:{value:shallowCopyInternalPointer(this.$$)}}));clone.$$.count.value+=1;clone.$$.deleteScheduled=false;return clone}},delete(){if(!this.$$.ptr){throwInstanceAlreadyDeleted(this)}if(this.$$.deleteScheduled&&!this.$$.preservePointerOnDelete){throwBindingError("Object already scheduled for deletion")}detachFinalizer(this);releaseClassHandle(this.$$);if(!this.$$.preservePointerOnDelete){this.$$.smartPtr=undefined;this.$$.ptr=undefined}},isDeleted(){return!this.$$.ptr},deleteLater(){if(!this.$$.ptr){throwInstanceAlreadyDeleted(this)}if(this.$$.deleteScheduled&&!this.$$.preservePointerOnDelete){throwBindingError("Object already scheduled for deletion")}deletionQueue.push(this);if(deletionQueue.length===1&&delayFunction){delayFunction(flushPendingDeletes)}this.$$.deleteScheduled=true;return this}});const symbolDispose=Symbol.dispose;if(symbolDispose){proto[symbolDispose]=proto["delete"]}};function ClassHandle(){}var createNamedFunction=(name,func)=>Object.defineProperty(func,"name",{value:name});var registeredPointers={};var ensureOverloadTable=(proto,methodName,humanName)=>{if(undefined===proto[methodName].overloadTable){var prevFunc=proto[methodName];proto[methodName]=function(...args){if(!proto[methodName].overloadTable.hasOwnProperty(args.length)){throwBindingError(`Function '${humanName}' called with an invalid number of arguments (${args.length}) - expects one of (${proto[methodName].overloadTable})!`)}return proto[methodName].overloadTable[args.length].apply(this,args)};proto[methodName].overloadTable=[];proto[methodName].overloadTable[prevFunc.argCount]=prevFunc}};var exposePublicSymbol=(name,value,numArguments)=>{if(Module.hasOwnProperty(name)){if(undefined===numArguments||undefined!==Module[name].overloadTable&&undefined!==Module[name].overloadTable[numArguments]){throwBindingError(`Cannot register public name '${name}' twice`)}ensureOverloadTable(Module,name,name);if(Module[name].overloadTable.hasOwnProperty(numArguments)){throwBindingError(`Cannot register multiple overloads of a function with the same number of arguments (${numArguments})!`)}Module[name].overloadTable[numArguments]=value}else{Module[name]=value;Module[name].argCount=numArguments}};var char_0=48;var char_9=57;var makeLegalFunctionName=name=>{name=name.replace(/[^a-zA-Z0-9_]/g,"$");var f=name.charCodeAt(0);if(f>=char_0&&f<=char_9){return`_${name}`}return name};function RegisteredClass(name,constructor,instancePrototype,rawDestructor,baseClass,getActualType,upcast,downcast){this.name=name;this.constructor=constructor;this.instancePrototype=instancePrototype;this.rawDestructor=rawDestructor;this.baseClass=baseClass;this.getActualType=getActualType;this.upcast=upcast;this.downcast=downcast;this.pureVirtualFunctions=[]}var upcastPointer=(ptr,ptrClass,desiredClass)=>{while(ptrClass!==desiredClass){if(!ptrClass.upcast){throwBindingError(`Expected null or instance of ${desiredClass.name}, got an instance of ${ptrClass.name}`)}ptr=ptrClass.upcast(ptr);ptrClass=ptrClass.baseClass}return ptr};var embindRepr=v=>{if(v===null){return"null"}var t=typeof v;if(t==="object"||t==="array"||t==="function"){return v.toString()}else{return""+v}};function constNoSmartPtrRawPointerToWireType(destructors,handle){if(handle===null){if(this.isReference){throwBindingError(`null is not a valid ${this.name}`)}return 0}if(!handle.$$){throwBindingError(`Cannot pass "${embindRepr(handle)}" as a ${this.name}`)}if(!handle.$$.ptr){throwBindingError(`Cannot pass deleted object as a pointer of type ${this.name}`)}var handleClass=handle.$$.ptrType.registeredClass;var ptr=upcastPointer(handle.$$.ptr,handleClass,this.registeredClass);return ptr}function genericPointerToWireType(destructors,handle){var ptr;if(handle===null){if(this.isReference){throwBindingError(`null is not a valid ${this.name}`)}if(this.isSmartPointer){ptr=this.rawConstructor();if(destructors!==null){destructors.push(this.rawDestructor,ptr)}return ptr}else{return 0}}if(!handle||!handle.$$){throwBindingError(`Cannot pass "${embindRepr(handle)}" as a ${this.name}`)}if(!handle.$$.ptr){throwBindingError(`Cannot pass deleted object as a pointer of type ${this.name}`)}if(!this.isConst&&handle.$$.ptrType.isConst){throwBindingError(`Cannot convert argument of type ${handle.$$.smartPtrType?handle.$$.smartPtrType.name:handle.$$.ptrType.name} to parameter type ${this.name}`)}var handleClass=handle.$$.ptrType.registeredClass;ptr=upcastPointer(handle.$$.ptr,handleClass,this.registeredClass);if(this.isSmartPointer){if(undefined===handle.$$.smartPtr){throwBindingError("Passing raw pointer to smart pointer is illegal")}switch(this.sharingPolicy){case 0:if(handle.$$.smartPtrType===this){ptr=handle.$$.smartPtr}else{throwBindingError(`Cannot convert argument of type ${handle.$$.smartPtrType?handle.$$.smartPtrType.name:handle.$$.ptrType.name} to parameter type ${this.name}`)}break;case 1:ptr=handle.$$.smartPtr;break;case 2:if(handle.$$.smartPtrType===this){ptr=handle.$$.smartPtr}else{var clonedHandle=handle["clone"]();ptr=this.rawShare(ptr,Emval.toHandle(()=>clonedHandle["delete"]()));if(destructors!==null){destructors.push(this.rawDestructor,ptr)}}break;default:throwBindingError("Unsupporting sharing policy")}}return ptr}function nonConstNoSmartPtrRawPointerToWireType(destructors,handle){if(handle===null){if(this.isReference){throwBindingError(`null is not a valid ${this.name}`)}return 0}if(!handle.$$){throwBindingError(`Cannot pass "${embindRepr(handle)}" as a ${this.name}`)}if(!handle.$$.ptr){throwBindingError(`Cannot pass deleted object as a pointer of type ${this.name}`)}if(handle.$$.ptrType.isConst){throwBindingError(`Cannot convert argument of type ${handle.$$.ptrType.name} to parameter type ${this.name}`)}var handleClass=handle.$$.ptrType.registeredClass;var ptr=upcastPointer(handle.$$.ptr,handleClass,this.registeredClass);return ptr}var downcastPointer=(ptr,ptrClass,desiredClass)=>{if(ptrClass===desiredClass){return ptr}if(undefined===desiredClass.baseClass){return null}var rv=downcastPointer(ptr,ptrClass,desiredClass.baseClass);if(rv===null){return null}return desiredClass.downcast(rv)};var registeredInstances={};var getBasestPointer=(class_,ptr)=>{if(ptr===undefined){throwBindingError("ptr should not be undefined")}while(class_.baseClass){ptr=class_.upcast(ptr);class_=class_.baseClass}return ptr};var getInheritedInstance=(class_,ptr)=>{ptr=getBasestPointer(class_,ptr);return registeredInstances[ptr]};var makeClassHandle=(prototype,record)=>{if(!record.ptrType||!record.ptr){throwInternalError("makeClassHandle requires ptr and ptrType")}var hasSmartPtrType=!!record.smartPtrType;var hasSmartPtr=!!record.smartPtr;if(hasSmartPtrType!==hasSmartPtr){throwInternalError("Both smartPtrType and smartPtr must be specified")}record.count={value:1};return attachFinalizer(Object.create(prototype,{$$:{value:record,writable:true}}))};function RegisteredPointer_fromWireType(ptr){var rawPointer=this.getPointee(ptr);if(!rawPointer){this.destructor(ptr);return null}var registeredInstance=getInheritedInstance(this.registeredClass,rawPointer);if(undefined!==registeredInstance){if(0===registeredInstance.$$.count.value){registeredInstance.$$.ptr=rawPointer;registeredInstance.$$.smartPtr=ptr;return registeredInstance["clone"]()}else{var rv=registeredInstance["clone"]();this.destructor(ptr);return rv}}function makeDefaultHandle(){if(this.isSmartPointer){return makeClassHandle(this.registeredClass.instancePrototype,{ptrType:this.pointeeType,ptr:rawPointer,smartPtrType:this,smartPtr:ptr})}else{return makeClassHandle(this.registeredClass.instancePrototype,{ptrType:this,ptr})}}var actualType=this.registeredClass.getActualType(rawPointer);var registeredPointerRecord=registeredPointers[actualType];if(!registeredPointerRecord){return makeDefaultHandle.call(this)}var toType;if(this.isConst){toType=registeredPointerRecord.constPointerType}else{toType=registeredPointerRecord.pointerType}var dp=downcastPointer(rawPointer,this.registeredClass,toType.registeredClass);if(dp===null){return makeDefaultHandle.call(this)}if(this.isSmartPointer){return makeClassHandle(toType.registeredClass.instancePrototype,{ptrType:toType,ptr:dp,smartPtrType:this,smartPtr:ptr})}else{return makeClassHandle(toType.registeredClass.instancePrototype,{ptrType:toType,ptr:dp})}}var init_RegisteredPointer=()=>{Object.assign(RegisteredPointer.prototype,{getPointee(ptr){if(this.rawGetPointee){ptr=this.rawGetPointee(ptr)}return ptr},destructor(ptr){this.rawDestructor?.(ptr)},readValueFromPointer:readPointer,fromWireType:RegisteredPointer_fromWireType})};function RegisteredPointer(name,registeredClass,isReference,isConst,isSmartPointer,pointeeType,sharingPolicy,rawGetPointee,rawConstructor,rawShare,rawDestructor){this.name=name;this.registeredClass=registeredClass;this.isReference=isReference;this.isConst=isConst;this.isSmartPointer=isSmartPointer;this.pointeeType=pointeeType;this.sharingPolicy=sharingPolicy;this.rawGetPointee=rawGetPointee;this.rawConstructor=rawConstructor;this.rawShare=rawShare;this.rawDestructor=rawDestructor;if(!isSmartPointer&®isteredClass.baseClass===undefined){if(isConst){this.toWireType=constNoSmartPtrRawPointerToWireType;this.destructorFunction=null}else{this.toWireType=nonConstNoSmartPtrRawPointerToWireType;this.destructorFunction=null}}else{this.toWireType=genericPointerToWireType}}var replacePublicSymbol=(name,value,numArguments)=>{if(!Module.hasOwnProperty(name)){throwInternalError("Replacing nonexistent public symbol")}if(undefined!==Module[name].overloadTable&&undefined!==numArguments){Module[name].overloadTable[numArguments]=value}else{Module[name]=value;Module[name].argCount=numArguments}};var wasmTableMirror=[];var getWasmTableEntry=funcPtr=>{var func=wasmTableMirror[funcPtr];if(!func){wasmTableMirror[funcPtr]=func=wasmTable.get(funcPtr)}return func};var embind__requireFunction=(signature,rawFunction,isAsync=false)=>{signature=AsciiToString(signature);function makeDynCaller(){var rtn=getWasmTableEntry(rawFunction);return rtn}var fp=makeDynCaller();if(typeof fp!="function"){throwBindingError(`unknown function pointer with signature ${signature}: ${rawFunction}`)}return fp};class UnboundTypeError extends Error{}var getTypeName=type=>{var ptr=___getTypeName(type);var rv=AsciiToString(ptr);_free(ptr);return rv};var throwUnboundTypeError=(message,types)=>{var unboundTypes=[];var seen={};function visit(type){if(seen[type]){return}if(registeredTypes[type]){return}if(typeDependencies[type]){typeDependencies[type].forEach(visit);return}unboundTypes.push(type);seen[type]=true}types.forEach(visit);throw new UnboundTypeError(`${message}: `+unboundTypes.map(getTypeName).join([", "]))};var __embind_register_class=(rawType,rawPointerType,rawConstPointerType,baseClassRawType,getActualTypeSignature,getActualType,upcastSignature,upcast,downcastSignature,downcast,name,destructorSignature,rawDestructor)=>{name=AsciiToString(name);getActualType=embind__requireFunction(getActualTypeSignature,getActualType);upcast&&=embind__requireFunction(upcastSignature,upcast);downcast&&=embind__requireFunction(downcastSignature,downcast);rawDestructor=embind__requireFunction(destructorSignature,rawDestructor);var legalFunctionName=makeLegalFunctionName(name);exposePublicSymbol(legalFunctionName,function(){throwUnboundTypeError(`Cannot construct ${name} due to unbound types`,[baseClassRawType])});whenDependentTypesAreResolved([rawType,rawPointerType,rawConstPointerType],baseClassRawType?[baseClassRawType]:[],base=>{base=base[0];var baseClass;var basePrototype;if(baseClassRawType){baseClass=base.registeredClass;basePrototype=baseClass.instancePrototype}else{basePrototype=ClassHandle.prototype}var constructor=createNamedFunction(name,function(...args){if(Object.getPrototypeOf(this)!==instancePrototype){throw new BindingError(`Use 'new' to construct ${name}`)}if(undefined===registeredClass.constructor_body){throw new BindingError(`${name} has no accessible constructor`)}var body=registeredClass.constructor_body[args.length];if(undefined===body){throw new BindingError(`Tried to invoke ctor of ${name} with invalid number of parameters (${args.length}) - expected (${Object.keys(registeredClass.constructor_body).toString()}) parameters instead!`)}return body.apply(this,args)});var instancePrototype=Object.create(basePrototype,{constructor:{value:constructor}});constructor.prototype=instancePrototype;var registeredClass=new RegisteredClass(name,constructor,instancePrototype,rawDestructor,baseClass,getActualType,upcast,downcast);if(registeredClass.baseClass){registeredClass.baseClass.__derivedClasses??=[];registeredClass.baseClass.__derivedClasses.push(registeredClass)}var referenceConverter=new RegisteredPointer(name,registeredClass,true,false,false);var pointerConverter=new RegisteredPointer(name+"*",registeredClass,false,false,false);var constPointerConverter=new RegisteredPointer(name+" const*",registeredClass,false,true,false);registeredPointers[rawType]={pointerType:pointerConverter,constPointerType:constPointerConverter};replacePublicSymbol(legalFunctionName,constructor);return[referenceConverter,pointerConverter,constPointerConverter]})};var heap32VectorToArray=(count,firstElement)=>{var array=[];for(var i=0;i>2])}return array};function usesDestructorStack(argTypes){for(var i=1;i{var rawArgTypes=heap32VectorToArray(argCount,rawArgTypesAddr);invoker=embind__requireFunction(invokerSignature,invoker);whenDependentTypesAreResolved([],[rawClassType],classType=>{classType=classType[0];var humanName=`constructor ${classType.name}`;if(undefined===classType.registeredClass.constructor_body){classType.registeredClass.constructor_body=[]}if(undefined!==classType.registeredClass.constructor_body[argCount-1]){throw new BindingError(`Cannot register multiple constructors with identical number of parameters (${argCount-1}) for class '${classType.name}'! Overload resolution is currently only performed using the parameter count, not actual type info!`)}classType.registeredClass.constructor_body[argCount-1]=()=>{throwUnboundTypeError(`Cannot construct ${classType.name} due to unbound types`,rawArgTypes)};whenDependentTypesAreResolved([],rawArgTypes,argTypes=>{argTypes.splice(1,0,null);classType.registeredClass.constructor_body[argCount-1]=craftInvokerFunction(humanName,argTypes,null,invoker,rawConstructor);return[]});return[]})};var getFunctionName=signature=>{signature=signature.trim();const argsIndex=signature.indexOf("(");if(argsIndex===-1)return signature;return signature.slice(0,argsIndex)};var __embind_register_class_function=(rawClassType,methodName,argCount,rawArgTypesAddr,invokerSignature,rawInvoker,context,isPureVirtual,isAsync,isNonnullReturn)=>{var rawArgTypes=heap32VectorToArray(argCount,rawArgTypesAddr);methodName=AsciiToString(methodName);methodName=getFunctionName(methodName);rawInvoker=embind__requireFunction(invokerSignature,rawInvoker,isAsync);whenDependentTypesAreResolved([],[rawClassType],classType=>{classType=classType[0];var humanName=`${classType.name}.${methodName}`;if(methodName.startsWith("@@")){methodName=Symbol[methodName.substring(2)]}if(isPureVirtual){classType.registeredClass.pureVirtualFunctions.push(methodName)}function unboundTypesHandler(){throwUnboundTypeError(`Cannot call ${humanName} due to unbound types`,rawArgTypes)}var proto=classType.registeredClass.instancePrototype;var method=proto[methodName];if(undefined===method||undefined===method.overloadTable&&method.className!==classType.name&&method.argCount===argCount-2){unboundTypesHandler.argCount=argCount-2;unboundTypesHandler.className=classType.name;proto[methodName]=unboundTypesHandler}else{ensureOverloadTable(proto,methodName,humanName);proto[methodName].overloadTable[argCount-2]=unboundTypesHandler}whenDependentTypesAreResolved([],rawArgTypes,argTypes=>{var memberFunction=craftInvokerFunction(humanName,argTypes,classType,rawInvoker,context,isAsync);if(undefined===proto[methodName].overloadTable){memberFunction.argCount=argCount-2;proto[methodName]=memberFunction}else{proto[methodName].overloadTable[argCount-2]=memberFunction}return[]});return[]})};var __embind_register_constant=(name,type,value)=>{name=AsciiToString(name);whenDependentTypesAreResolved([],[type],type=>{type=type[0];Module[name]=type.fromWireType(value);return[]})};var emval_freelist=[];var emval_handles=[0,1,,1,null,1,true,1,false,1];var __emval_decref=handle=>{if(handle>9&&0===--emval_handles[handle+1]){emval_handles[handle]=undefined;emval_freelist.push(handle)}};var Emval={toValue:handle=>{if(!handle){throwBindingError(`Cannot use deleted val. handle = ${handle}`)}return emval_handles[handle]},toHandle:value=>{switch(value){case undefined:return 2;case null:return 4;case true:return 6;case false:return 8;default:{const handle=emval_freelist.pop()||emval_handles.length;emval_handles[handle]=value;emval_handles[handle+1]=1;return handle}}}};var EmValType={name:"emscripten::val",fromWireType:handle=>{var rv=Emval.toValue(handle);__emval_decref(handle);return rv},toWireType:(destructors,value)=>Emval.toHandle(value),readValueFromPointer:readPointer,destructorFunction:null};var __embind_register_emval=rawType=>registerType(rawType,EmValType);var enumReadValueFromPointer=(name,width,signed)=>{switch(width){case 1:return signed?function(pointer){return this.fromWireType(HEAP8[pointer])}:function(pointer){return this.fromWireType(HEAPU8[pointer])};case 2:return signed?function(pointer){return this.fromWireType(HEAP16[pointer>>1])}:function(pointer){return this.fromWireType(HEAPU16[pointer>>1])};case 4:return signed?function(pointer){return this.fromWireType(HEAP32[pointer>>2])}:function(pointer){return this.fromWireType(HEAPU32[pointer>>2])};default:throw new TypeError(`invalid integer width (${width}): ${name}`)}};var __embind_register_enum=(rawType,name,size,isSigned)=>{name=AsciiToString(name);function ctor(){}ctor.values={};registerType(rawType,{name,constructor:ctor,fromWireType:function(c){return this.constructor.values[c]},toWireType:(destructors,c)=>c.value,readValueFromPointer:enumReadValueFromPointer(name,size,isSigned),destructorFunction:null});exposePublicSymbol(name,ctor)};var requireRegisteredType=(rawType,humanName)=>{var impl=registeredTypes[rawType];if(undefined===impl){throwBindingError(`${humanName} has unknown type ${getTypeName(rawType)}`)}return impl};var __embind_register_enum_value=(rawEnumType,name,enumValue)=>{var enumType=requireRegisteredType(rawEnumType,"enum");name=AsciiToString(name);var Enum=enumType.constructor;var Value=Object.create(enumType.constructor.prototype,{value:{value:enumValue},constructor:{value:createNamedFunction(`${enumType.name}_${name}`,function(){})}});Enum.values[enumValue]=Value;Enum[name]=Value};var floatReadValueFromPointer=(name,width)=>{switch(width){case 4:return function(pointer){return this.fromWireType(HEAPF32[pointer>>2])};case 8:return function(pointer){return this.fromWireType(HEAPF64[pointer>>3])};default:throw new TypeError(`invalid float width (${width}): ${name}`)}};var __embind_register_float=(rawType,name,size)=>{name=AsciiToString(name);registerType(rawType,{name,fromWireType:value=>value,toWireType:(destructors,value)=>value,readValueFromPointer:floatReadValueFromPointer(name,size),destructorFunction:null})};var __embind_register_function=(name,argCount,rawArgTypesAddr,signature,rawInvoker,fn,isAsync,isNonnullReturn)=>{var argTypes=heap32VectorToArray(argCount,rawArgTypesAddr);name=AsciiToString(name);name=getFunctionName(name);rawInvoker=embind__requireFunction(signature,rawInvoker,isAsync);exposePublicSymbol(name,function(){throwUnboundTypeError(`Cannot call ${name} due to unbound types`,argTypes)},argCount-1);whenDependentTypesAreResolved([],argTypes,argTypes=>{var invokerArgsArray=[argTypes[0],null].concat(argTypes.slice(1));replacePublicSymbol(name,craftInvokerFunction(name,invokerArgsArray,null,rawInvoker,fn,isAsync),argCount-1);return[]})};var __embind_register_integer=(primitiveType,name,size,minRange,maxRange)=>{name=AsciiToString(name);const isUnsignedType=minRange===0;let fromWireType=value=>value;if(isUnsignedType){var bitshift=32-8*size;fromWireType=value=>value<>>bitshift;maxRange=fromWireType(maxRange)}registerType(primitiveType,{name,fromWireType,toWireType:(destructors,value)=>value,readValueFromPointer:integerReadValueFromPointer(name,size,minRange!==0),destructorFunction:null})};var __embind_register_memory_view=(rawType,dataTypeIndex,name)=>{var typeMapping=[Int8Array,Uint8Array,Int16Array,Uint16Array,Int32Array,Uint32Array,Float32Array,Float64Array,BigInt64Array,BigUint64Array];var TA=typeMapping[dataTypeIndex];function decodeMemoryView(handle){var size=HEAPU32[handle>>2];var data=HEAPU32[handle+4>>2];return new TA(HEAP8.buffer,data,size)}name=AsciiToString(name);registerType(rawType,{name,fromWireType:decodeMemoryView,readValueFromPointer:decodeMemoryView},{ignoreDuplicateRegistrations:true})};var stringToUTF8Array=(str,heap,outIdx,maxBytesToWrite)=>{if(!(maxBytesToWrite>0))return 0;var startIdx=outIdx;var endIdx=outIdx+maxBytesToWrite-1;for(var i=0;i=endIdx)break;heap[outIdx++]=u}else if(u<=2047){if(outIdx+1>=endIdx)break;heap[outIdx++]=192|u>>6;heap[outIdx++]=128|u&63}else if(u<=65535){if(outIdx+2>=endIdx)break;heap[outIdx++]=224|u>>12;heap[outIdx++]=128|u>>6&63;heap[outIdx++]=128|u&63}else{if(outIdx+3>=endIdx)break;heap[outIdx++]=240|u>>18;heap[outIdx++]=128|u>>12&63;heap[outIdx++]=128|u>>6&63;heap[outIdx++]=128|u&63;i++}}heap[outIdx]=0;return outIdx-startIdx};var stringToUTF8=(str,outPtr,maxBytesToWrite)=>stringToUTF8Array(str,HEAPU8,outPtr,maxBytesToWrite);var lengthBytesUTF8=str=>{var len=0;for(var i=0;i=55296&&c<=57343){len+=4;++i}else{len+=3}}return len};var UTF8Decoder=globalThis.TextDecoder&&new TextDecoder;var findStringEnd=(heapOrArray,idx,maxBytesToRead,ignoreNul)=>{var maxIdx=idx+maxBytesToRead;if(ignoreNul)return maxIdx;while(heapOrArray[idx]&&!(idx>=maxIdx))++idx;return idx};var UTF8ArrayToString=(heapOrArray,idx=0,maxBytesToRead,ignoreNul)=>{var endPtr=findStringEnd(heapOrArray,idx,maxBytesToRead,ignoreNul);if(endPtr-idx>16&&heapOrArray.buffer&&UTF8Decoder){return UTF8Decoder.decode(heapOrArray.subarray(idx,endPtr))}var str="";while(idx>10,56320|ch&1023)}}return str};var UTF8ToString=(ptr,maxBytesToRead,ignoreNul)=>ptr?UTF8ArrayToString(HEAPU8,ptr,maxBytesToRead,ignoreNul):"";var __embind_register_std_string=(rawType,name)=>{name=AsciiToString(name);var stdStringIsUTF8=true;registerType(rawType,{name,fromWireType(value){var length=HEAPU32[value>>2];var payload=value+4;var str;if(stdStringIsUTF8){str=UTF8ToString(payload,length,true)}else{str="";for(var i=0;i>2]=length;if(valueIsOfTypeString){if(stdStringIsUTF8){stringToUTF8(value,ptr,length+1)}else{for(var i=0;i255){_free(base);throwBindingError("String has UTF-16 code units that do not fit in 8 bits")}HEAPU8[ptr+i]=charCode}}}else{HEAPU8.set(value,ptr)}if(destructors!==null){destructors.push(_free,base)}return base},readValueFromPointer:readPointer,destructorFunction(ptr){_free(ptr)}})};var UTF16Decoder=globalThis.TextDecoder?new TextDecoder("utf-16le"):undefined;var UTF16ToString=(ptr,maxBytesToRead,ignoreNul)=>{var idx=ptr>>1;var endIdx=findStringEnd(HEAPU16,idx,maxBytesToRead/2,ignoreNul);if(endIdx-idx>16&&UTF16Decoder)return UTF16Decoder.decode(HEAPU16.subarray(idx,endIdx));var str="";for(var i=idx;i{maxBytesToWrite??=2147483647;if(maxBytesToWrite<2)return 0;maxBytesToWrite-=2;var startPtr=outPtr;var numCharsToWrite=maxBytesToWrite>1]=codeUnit;outPtr+=2}HEAP16[outPtr>>1]=0;return outPtr-startPtr};var lengthBytesUTF16=str=>str.length*2;var UTF32ToString=(ptr,maxBytesToRead,ignoreNul)=>{var str="";var startIdx=ptr>>2;for(var i=0;!(i>=maxBytesToRead/4);i++){var utf32=HEAPU32[startIdx+i];if(!utf32&&!ignoreNul)break;str+=String.fromCodePoint(utf32)}return str};var stringToUTF32=(str,outPtr,maxBytesToWrite)=>{maxBytesToWrite??=2147483647;if(maxBytesToWrite<4)return 0;var startPtr=outPtr;var endPtr=startPtr+maxBytesToWrite-4;for(var i=0;i65535){i++}HEAP32[outPtr>>2]=codePoint;outPtr+=4;if(outPtr+4>endPtr)break}HEAP32[outPtr>>2]=0;return outPtr-startPtr};var lengthBytesUTF32=str=>{var len=0;for(var i=0;i65535){i++}len+=4}return len};var __embind_register_std_wstring=(rawType,charSize,name)=>{name=AsciiToString(name);var decodeString,encodeString,lengthBytesUTF;if(charSize===2){decodeString=UTF16ToString;encodeString=stringToUTF16;lengthBytesUTF=lengthBytesUTF16}else{decodeString=UTF32ToString;encodeString=stringToUTF32;lengthBytesUTF=lengthBytesUTF32}registerType(rawType,{name,fromWireType:value=>{var length=HEAPU32[value>>2];var str=decodeString(value+4,length*charSize,true);_free(value);return str},toWireType:(destructors,value)=>{if(!(typeof value=="string")){throwBindingError(`Cannot pass non-string to C++ string type ${name}`)}var length=lengthBytesUTF(value);var ptr=_malloc(4+length+charSize);HEAPU32[ptr>>2]=length/charSize;encodeString(value,ptr+4,length+charSize);if(destructors!==null){destructors.push(_free,ptr)}return ptr},readValueFromPointer:readPointer,destructorFunction(ptr){_free(ptr)}})};var __embind_register_value_object=(rawType,name,constructorSignature,rawConstructor,destructorSignature,rawDestructor)=>{structRegistrations[rawType]={name:AsciiToString(name),rawConstructor:embind__requireFunction(constructorSignature,rawConstructor),rawDestructor:embind__requireFunction(destructorSignature,rawDestructor),fields:[]}};var __embind_register_value_object_field=(structType,fieldName,getterReturnType,getterSignature,getter,getterContext,setterArgumentType,setterSignature,setter,setterContext)=>{structRegistrations[structType].fields.push({fieldName:AsciiToString(fieldName),getterReturnType,getter:embind__requireFunction(getterSignature,getter),getterContext,setterArgumentType,setter:embind__requireFunction(setterSignature,setter),setterContext})};var __embind_register_void=(rawType,name)=>{name=AsciiToString(name);registerType(rawType,{isVoid:true,name,fromWireType:()=>undefined,toWireType:(destructors,o)=>undefined})};var emval_methodCallers=[];var emval_addMethodCaller=caller=>{var id=emval_methodCallers.length;emval_methodCallers.push(caller);return id};var emval_lookupTypes=(argCount,argTypes)=>{var a=new Array(argCount);for(var i=0;i>2],`parameter ${i}`)}return a};var emval_returnValue=(toReturnWire,destructorsRef,handle)=>{var destructors=[];var result=toReturnWire(destructors,handle);if(destructors.length){HEAPU32[destructorsRef>>2]=Emval.toHandle(destructors)}return result};var emval_symbols={};var getStringOrSymbol=address=>{var symbol=emval_symbols[address];if(symbol===undefined){return AsciiToString(address)}return symbol};var __emval_create_invoker=(argCount,argTypesPtr,kind)=>{var GenericWireTypeSize=8;var[retType,...argTypes]=emval_lookupTypes(argCount,argTypesPtr);var toReturnWire=retType.toWireType.bind(retType);var argFromPtr=argTypes.map(type=>type.readValueFromPointer.bind(type));argCount--;var captures={toValue:Emval.toValue};var args=argFromPtr.map((argFromPtr,i)=>{var captureName=`argFromPtr${i}`;captures[captureName]=argFromPtr;return`${captureName}(args${i?"+"+i*GenericWireTypeSize:""})`});var functionBody;switch(kind){case 0:functionBody="toValue(handle)";break;case 2:functionBody="new (toValue(handle))";break;case 3:functionBody="";break;case 1:captures["getStringOrSymbol"]=getStringOrSymbol;functionBody="toValue(handle)[getStringOrSymbol(methodName)]";break}functionBody+=`(${args})`;if(!retType.isVoid){captures["toReturnWire"]=toReturnWire;captures["emval_returnValue"]=emval_returnValue;functionBody=`return emval_returnValue(toReturnWire, destructorsRef, ${functionBody})`}functionBody=`return function (handle, methodName, destructorsRef, args) {\n ${functionBody}\n }`;var invokerFunction=new Function(Object.keys(captures),functionBody)(...Object.values(captures));var functionName=`methodCaller<(${argTypes.map(t=>t.name)}) => ${retType.name}>`;return emval_addMethodCaller(createNamedFunction(functionName,invokerFunction))};var __emval_get_property=(handle,key)=>{handle=Emval.toValue(handle);key=Emval.toValue(key);return Emval.toHandle(handle[key])};var __emval_incref=handle=>{if(handle>9){emval_handles[handle+1]+=1}};var __emval_invoke=(caller,handle,methodName,destructorsRef,args)=>emval_methodCallers[caller](handle,methodName,destructorsRef,args);var __emval_new_cstring=v=>Emval.toHandle(getStringOrSymbol(v));var __emval_run_destructors=handle=>{var destructors=Emval.toValue(handle);runDestructors(destructors);__emval_decref(handle)};var getHeapMax=()=>2147483648;var alignMemory=(size,alignment)=>Math.ceil(size/alignment)*alignment;var growMemory=size=>{var oldHeapSize=wasmMemory.buffer.byteLength;var pages=(size-oldHeapSize+65535)/65536|0;try{wasmMemory.grow(pages);updateMemoryViews();return 1}catch(e){}};var _emscripten_resize_heap=requestedSize=>{var oldSize=HEAPU8.length;requestedSize>>>=0;var maxHeapSize=getHeapMax();if(requestedSize>maxHeapSize){return false}for(var cutDown=1;cutDown<=4;cutDown*=2){var overGrownHeapSize=oldSize*(1+.2/cutDown);overGrownHeapSize=Math.min(overGrownHeapSize,requestedSize+100663296);var newSize=Math.min(maxHeapSize,alignMemory(Math.max(requestedSize,overGrownHeapSize),65536));var replacement=growMemory(newSize);if(replacement){return true}}return false};var _fd_close=fd=>52;var INT53_MAX=9007199254740992;var INT53_MIN=-9007199254740992;var bigintToI53Checked=num=>numINT53_MAX?NaN:Number(num);function _fd_seek(fd,offset,whence,newOffset){offset=bigintToI53Checked(offset);return 70}var printCharBuffers=[null,[],[]];var printChar=(stream,curr)=>{var buffer=printCharBuffers[stream];if(curr===0||curr===10){(stream===1?out:err)(UTF8ArrayToString(buffer));buffer.length=0}else{buffer.push(curr)}};var _fd_write=(fd,iov,iovcnt,pnum)=>{var num=0;for(var i=0;i>2];var len=HEAPU32[iov+4>>2];iov+=8;for(var j=0;j>2]=num;return 0};init_ClassHandle();init_RegisteredPointer();{if(Module["noExitRuntime"])noExitRuntime=Module["noExitRuntime"];if(Module["print"])out=Module["print"];if(Module["printErr"])err=Module["printErr"];if(Module["wasmBinary"])wasmBinary=Module["wasmBinary"];if(Module["arguments"])arguments_=Module["arguments"];if(Module["thisProgram"])thisProgram=Module["thisProgram"];if(Module["preInit"]){if(typeof Module["preInit"]=="function")Module["preInit"]=[Module["preInit"]];while(Module["preInit"].length>0){Module["preInit"].shift()()}}}var ___getTypeName,_malloc,_free,memory,__indirect_function_table,wasmMemory,wasmTable;function assignWasmExports(wasmExports){___getTypeName=wasmExports["I"];_malloc=wasmExports["J"];_free=wasmExports["L"];memory=wasmMemory=wasmExports["G"];__indirect_function_table=wasmTable=wasmExports["K"]}var wasmImports={F:___cxa_throw,x:__abort_js,q:__embind_finalize_value_object,w:__embind_register_bigint,D:__embind_register_bool,t:__embind_register_class,s:__embind_register_class_constructor,b:__embind_register_class_function,k:__embind_register_constant,B:__embind_register_emval,o:__embind_register_enum,a:__embind_register_enum_value,v:__embind_register_float,e:__embind_register_function,l:__embind_register_integer,i:__embind_register_memory_view,C:__embind_register_std_string,r:__embind_register_std_wstring,p:__embind_register_value_object,c:__embind_register_value_object_field,E:__embind_register_void,h:__emval_create_invoker,d:__emval_decref,m:__emval_get_property,j:__emval_incref,g:__emval_invoke,n:__emval_new_cstring,f:__emval_run_destructors,y:_emscripten_resize_heap,A:_fd_close,z:_fd_seek,u:_fd_write};function run(){preRun();function doRun(){Module["calledRun"]=true;if(ABORT)return;initRuntime();readyPromiseResolve?.(Module);Module["onRuntimeInitialized"]?.();postRun()}if(Module["setStatus"]){Module["setStatus"]("Running...");setTimeout(()=>{setTimeout(()=>Module["setStatus"](""),1);doRun()},1)}else{doRun()}}var wasmExports;wasmExports=await (createWasm());run();if(runtimeInitialized){moduleRtn=Module}else{moduleRtn=new Promise((resolve,reject)=>{readyPromiseResolve=resolve;readyPromiseReject=reject})} ;return moduleRtn}})();if(typeof exports==="object"&&typeof module==="object"){module.exports=BASIS;module.exports.default=BASIS}else if(typeof define==="function"&&define["amd"])define([],()=>BASIS); diff --git a/external/basis_universal/webgl/transcoder/build/basis_transcoder.wasm b/external/basis_universal/webgl/transcoder/build/basis_transcoder.wasm index 28fff9aced..aedeb8baa5 100644 Binary files a/external/basis_universal/webgl/transcoder/build/basis_transcoder.wasm and b/external/basis_universal/webgl/transcoder/build/basis_transcoder.wasm differ diff --git a/external/basis_universal/webgl/webserver_cross_origin.py b/external/basis_universal/webgl/webserver_cross_origin.py index 8af0eda80f..d251101e04 100644 --- a/external/basis_universal/webgl/webserver_cross_origin.py +++ b/external/basis_universal/webgl/webserver_cross_origin.py @@ -7,7 +7,7 @@ def end_headers(self): self.send_header("Cross-Origin-Embedder-Policy", "require-corp") super().end_headers() -PORT = 8080 +PORT = 8081 with socketserver.TCPServer(("", PORT), CORSRequestHandler) as httpd: print(f"Serving at http://localhost:{PORT}") httpd.serve_forever() diff --git a/external/basis_universal/zstd/zstd.c b/external/basis_universal/zstd/zstd.c index d47e6dde4a..43958d8234 100644 --- a/external/basis_universal/zstd/zstd.c +++ b/external/basis_universal/zstd/zstd.c @@ -4,11 +4,11 @@ * * Generate using: * \code - * combine.sh -r ../../lib -o zstd.c zstd-in.c + * python combine.py -r ../../lib -x legacy/zstd_legacy.h -o zstd.c zstd-in.c * \endcode */ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -25,9 +25,13 @@ * Note: MEM_MODULE stops xxhash redefining BYTE, U16, etc., which are also * defined in mem.h (breaking C99 compatibility). * - * Note: the undefs for xxHash allow Zstd's implementation to coinside with with + * Note: the undefs for xxHash allow Zstd's implementation to coincide with * standalone xxHash usage (with global defines). * + * Note: if you enable ZSTD_LEGACY_SUPPORT the combine.py script will need + * re-running without the "-x legacy/zstd_legacy.h" option (it excludes the + * legacy support at the source level). + * * Note: multithreading is enabled for all platforms apart from Emscripten. */ #define DEBUGLEVEL 0 @@ -43,13 +47,15 @@ #define ZSTD_MULTITHREAD #endif #define ZSTD_TRACE 0 +/* TODO: Can't amalgamate ASM function */ +#define ZSTD_DISABLE_ASM 1 /* Include zstd_deps.h first with all the options we need enabled. */ #define ZSTD_DEPS_NEED_MALLOC #define ZSTD_DEPS_NEED_MATH64 /**** start inlining common/zstd_deps.h ****/ /* - * Copyright (c) 2016-2021, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -74,6 +80,18 @@ #ifndef ZSTD_DEPS_COMMON #define ZSTD_DEPS_COMMON +/* Even though we use qsort_r only for the dictionary builder, the macro + * _GNU_SOURCE has to be declared *before* the inclusion of any standard + * header and the script 'combine.sh' combines the whole zstd source code + * in a single file. + */ +#if defined(__linux) || defined(__linux__) || defined(linux) || defined(__gnu_linux__) || \ + defined(__CYGWIN__) || defined(__MSYS__) +#if !defined(_GNU_SOURCE) && !defined(__ANDROID__) /* NDK doesn't ship qsort_r(). */ +#define _GNU_SOURCE +#endif +#endif + #include #include #include @@ -165,7 +183,7 @@ /* ****************************************************************** * debug * Part of FSE library - * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -187,7 +205,7 @@ /* ****************************************************************** * debug * Part of FSE library - * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -218,10 +236,6 @@ #ifndef DEBUG_H_12987983217 #define DEBUG_H_12987983217 -#if defined (__cplusplus) -extern "C" { -#endif - /* static assert is triggered at compile time, leaving no runtime artefact. * static assert only works with compile-time constants. @@ -271,34 +285,44 @@ extern int g_debuglevel; /* the variable is only declared, It's useful when enabling very verbose levels on selective conditions (such as position in src) */ -# define RAWLOG(l, ...) { \ - if (l<=g_debuglevel) { \ - ZSTD_DEBUG_PRINT(__VA_ARGS__); \ - } } -# define DEBUGLOG(l, ...) { \ - if (l<=g_debuglevel) { \ - ZSTD_DEBUG_PRINT(__FILE__ ": " __VA_ARGS__); \ - ZSTD_DEBUG_PRINT(" \n"); \ - } } -#else -# define RAWLOG(l, ...) {} /* disabled */ -# define DEBUGLOG(l, ...) {} /* disabled */ -#endif +# define RAWLOG(l, ...) \ + do { \ + if (l<=g_debuglevel) { \ + ZSTD_DEBUG_PRINT(__VA_ARGS__); \ + } \ + } while (0) +#define STRINGIFY(x) #x +#define TOSTRING(x) STRINGIFY(x) +#define LINE_AS_STRING TOSTRING(__LINE__) -#if defined (__cplusplus) -} +# define DEBUGLOG(l, ...) \ + do { \ + if (l<=g_debuglevel) { \ + ZSTD_DEBUG_PRINT(__FILE__ ":" LINE_AS_STRING ": " __VA_ARGS__); \ + ZSTD_DEBUG_PRINT(" \n"); \ + } \ + } while (0) +#else +# define RAWLOG(l, ...) do { } while (0) /* disabled */ +# define DEBUGLOG(l, ...) do { } while (0) /* disabled */ #endif #endif /* DEBUG_H_12987983217 */ /**** ended inlining debug.h ****/ +#if !defined(ZSTD_LINUX_KERNEL) || (DEBUGLEVEL>=2) +/* We only use this when DEBUGLEVEL>=2, but we get -Werror=pedantic errors if a + * translation unit is empty. So remove this from Linux kernel builds, but + * otherwise just leave it in. + */ int g_debuglevel = DEBUGLEVEL; +#endif /**** ended inlining common/debug.c ****/ /**** start inlining common/entropy_common.c ****/ /* ****************************************************************** * Common functions of New Generation Entropy library - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -315,7 +339,7 @@ int g_debuglevel = DEBUGLEVEL; ***************************************/ /**** start inlining mem.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -327,17 +351,13 @@ int g_debuglevel = DEBUGLEVEL; #ifndef MEM_H_MODULE #define MEM_H_MODULE -#if defined (__cplusplus) -extern "C" { -#endif - /*-**************************************** * Dependencies ******************************************/ #include /* size_t, ptrdiff_t */ /**** start inlining compiler.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -349,6 +369,182 @@ extern "C" { #ifndef ZSTD_COMPILER_H #define ZSTD_COMPILER_H +#include + +/**** start inlining portability_macros.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_PORTABILITY_MACROS_H +#define ZSTD_PORTABILITY_MACROS_H + +/** + * This header file contains macro definitions to support portability. + * This header is shared between C and ASM code, so it MUST only + * contain macro definitions. It MUST not contain any C code. + * + * This header ONLY defines macros to detect platforms/feature support. + * + */ + + +/* compat. with non-clang compilers */ +#ifndef __has_attribute + #define __has_attribute(x) 0 +#endif + +/* compat. with non-clang compilers */ +#ifndef __has_builtin +# define __has_builtin(x) 0 +#endif + +/* compat. with non-clang compilers */ +#ifndef __has_feature +# define __has_feature(x) 0 +#endif + +/* detects whether we are being compiled under msan */ +#ifndef ZSTD_MEMORY_SANITIZER +# if __has_feature(memory_sanitizer) +# define ZSTD_MEMORY_SANITIZER 1 +# else +# define ZSTD_MEMORY_SANITIZER 0 +# endif +#endif + +/* detects whether we are being compiled under asan */ +#ifndef ZSTD_ADDRESS_SANITIZER +# if __has_feature(address_sanitizer) +# define ZSTD_ADDRESS_SANITIZER 1 +# elif defined(__SANITIZE_ADDRESS__) +# define ZSTD_ADDRESS_SANITIZER 1 +# else +# define ZSTD_ADDRESS_SANITIZER 0 +# endif +#endif + +/* detects whether we are being compiled under dfsan */ +#ifndef ZSTD_DATAFLOW_SANITIZER +# if __has_feature(dataflow_sanitizer) +# define ZSTD_DATAFLOW_SANITIZER 1 +# else +# define ZSTD_DATAFLOW_SANITIZER 0 +# endif +#endif + +/* Mark the internal assembly functions as hidden */ +#ifdef __ELF__ +# define ZSTD_HIDE_ASM_FUNCTION(func) .hidden func +#elif defined(__APPLE__) +# define ZSTD_HIDE_ASM_FUNCTION(func) .private_extern func +#else +# define ZSTD_HIDE_ASM_FUNCTION(func) +#endif + +/* Compile time determination of BMI2 support */ +#ifndef STATIC_BMI2 +# if defined(__BMI2__) +# define STATIC_BMI2 1 +# elif defined(_MSC_VER) && defined(__AVX2__) +# define STATIC_BMI2 1 /* MSVC does not have a BMI2 specific flag, but every CPU that supports AVX2 also supports BMI2 */ +# endif +#endif + +#ifndef STATIC_BMI2 +# define STATIC_BMI2 0 +#endif + +/* Enable runtime BMI2 dispatch based on the CPU. + * Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default. + */ +#ifndef DYNAMIC_BMI2 +# if ((defined(__clang__) && __has_attribute(__target__)) \ + || (defined(__GNUC__) \ + && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \ + && (defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) || defined(_M_X64)) \ + && !defined(__BMI2__) +# define DYNAMIC_BMI2 1 +# else +# define DYNAMIC_BMI2 0 +# endif +#endif + +/** + * Only enable assembly for GNU C compatible compilers, + * because other platforms may not support GAS assembly syntax. + * + * Only enable assembly for Linux / MacOS / Win32, other platforms may + * work, but they haven't been tested. This could likely be + * extended to BSD systems. + * + * Disable assembly when MSAN is enabled, because MSAN requires + * 100% of code to be instrumented to work. + */ +#if defined(__GNUC__) +# if defined(__linux__) || defined(__linux) || defined(__APPLE__) || defined(_WIN32) +# if ZSTD_MEMORY_SANITIZER +# define ZSTD_ASM_SUPPORTED 0 +# elif ZSTD_DATAFLOW_SANITIZER +# define ZSTD_ASM_SUPPORTED 0 +# else +# define ZSTD_ASM_SUPPORTED 1 +# endif +# else +# define ZSTD_ASM_SUPPORTED 0 +# endif +#else +# define ZSTD_ASM_SUPPORTED 0 +#endif + +/** + * Determines whether we should enable assembly for x86-64 + * with BMI2. + * + * Enable if all of the following conditions hold: + * - ASM hasn't been explicitly disabled by defining ZSTD_DISABLE_ASM + * - Assembly is supported + * - We are compiling for x86-64 and either: + * - DYNAMIC_BMI2 is enabled + * - BMI2 is supported at compile time + */ +#if !defined(ZSTD_DISABLE_ASM) && \ + ZSTD_ASM_SUPPORTED && \ + defined(__x86_64__) && \ + (DYNAMIC_BMI2 || defined(__BMI2__)) +# define ZSTD_ENABLE_ASM_X86_64_BMI2 1 +#else +# define ZSTD_ENABLE_ASM_X86_64_BMI2 0 +#endif + +/* + * For x86 ELF targets, add .note.gnu.property section for Intel CET in + * assembly sources when CET is enabled. + * + * Additionally, any function that may be called indirectly must begin + * with ZSTD_CET_ENDBRANCH. + */ +#if defined(__ELF__) && (defined(__x86_64__) || defined(__i386__)) \ + && defined(__has_include) +# if __has_include() +# include +# define ZSTD_CET_ENDBRANCH _CET_ENDBR +# endif +#endif + +#ifndef ZSTD_CET_ENDBRANCH +# define ZSTD_CET_ENDBRANCH +#endif + +#endif /* ZSTD_PORTABILITY_MACROS_H */ +/**** ended inlining portability_macros.h ****/ + /*-******************************************************* * Compiler specifics *********************************************************/ @@ -361,7 +557,7 @@ extern "C" { # define INLINE_KEYWORD #endif -#if defined(__GNUC__) || defined(__ICCARM__) +#if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__) # define FORCE_INLINE_ATTR __attribute__((always_inline)) #elif defined(_MSC_VER) # define FORCE_INLINE_ATTR __forceinline @@ -378,7 +574,7 @@ extern "C" { /** On MSVC qsort requires that functions passed into it use the __cdecl calling conversion(CC). - This explictly marks such functions as __cdecl so that the code will still compile + This explicitly marks such functions as __cdecl so that the code will still compile if a CC other than __cdecl has been made the default. */ #if defined(_MSC_VER) @@ -387,12 +583,19 @@ extern "C" { # define WIN_CDECL #endif +/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */ +#if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__) +# define UNUSED_ATTR __attribute__((unused)) +#else +# define UNUSED_ATTR +#endif + /** * FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant * parameters. They must be inlined for the compiler to eliminate the constant * branches. */ -#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR +#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR UNUSED_ATTR /** * HINT_INLINE is used to help the compiler generate better code. It is *not* * used for "templates", so it can be tweaked based on the compilers @@ -407,21 +610,37 @@ extern "C" { #if !defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 8 && __GNUC__ < 5 # define HINT_INLINE static INLINE_KEYWORD #else -# define HINT_INLINE static INLINE_KEYWORD FORCE_INLINE_ATTR +# define HINT_INLINE FORCE_INLINE_TEMPLATE #endif -/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */ +/* "soft" inline : + * The compiler is free to select if it's a good idea to inline or not. + * The main objective is to silence compiler warnings + * when a defined function in included but not used. + * + * Note : this macro is prefixed `MEM_` because it used to be provided by `mem.h` unit. + * Updating the prefix is probably preferable, but requires a fairly large codemod, + * since this name is used everywhere. + */ +#ifndef MEM_STATIC /* already defined in Linux Kernel mem.h */ #if defined(__GNUC__) -# define UNUSED_ATTR __attribute__((unused)) +# define MEM_STATIC static __inline UNUSED_ATTR +#elif defined(__IAR_SYSTEMS_ICC__) +# define MEM_STATIC static inline UNUSED_ATTR +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define MEM_STATIC static inline +#elif defined(_MSC_VER) +# define MEM_STATIC static __inline #else -# define UNUSED_ATTR +# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif #endif /* force no inlining */ #ifdef _MSC_VER # define FORCE_NOINLINE static __declspec(noinline) #else -# if defined(__GNUC__) || defined(__ICCARM__) +# if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__) # define FORCE_NOINLINE static __attribute__((__noinline__)) # else # define FORCE_NOINLINE static @@ -430,68 +649,56 @@ extern "C" { /* target attribute */ -#ifndef __has_attribute - #define __has_attribute(x) 0 /* Compatibility with non-clang compilers. */ -#endif -#if defined(__GNUC__) || defined(__ICCARM__) +#if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__) # define TARGET_ATTRIBUTE(target) __attribute__((__target__(target))) #else # define TARGET_ATTRIBUTE(target) #endif -/* Enable runtime BMI2 dispatch based on the CPU. - * Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default. +/* Target attribute for BMI2 dynamic dispatch. + * Enable lzcnt, bmi, and bmi2. + * We test for bmi1 & bmi2. lzcnt is included in bmi1. */ -#ifndef DYNAMIC_BMI2 - #if ((defined(__clang__) && __has_attribute(__target__)) \ - || (defined(__GNUC__) \ - && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \ - && (defined(__x86_64__) || defined(_M_X86)) \ - && !defined(__BMI2__) - # define DYNAMIC_BMI2 1 - #else - # define DYNAMIC_BMI2 0 - #endif -#endif +#define BMI2_TARGET_ATTRIBUTE TARGET_ATTRIBUTE("lzcnt,bmi,bmi2") /* prefetch * can be disabled, by declaring NO_PREFETCH build macro */ #if defined(NO_PREFETCH) -# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */ -# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */ +# define PREFETCH_L1(ptr) do { (void)(ptr); } while (0) /* disabled */ +# define PREFETCH_L2(ptr) do { (void)(ptr); } while (0) /* disabled */ #else -# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */ -//rg: fix for ARM64EC compilation -//# include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ -# include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ +# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) && !defined(_M_ARM64EC) /* _mm_prefetch() is not defined outside of x86/x64 */ +# include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ # define PREFETCH_L1(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) # define PREFETCH_L2(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T1) # elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) # define PREFETCH_L1(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) # define PREFETCH_L2(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */) # elif defined(__aarch64__) -# define PREFETCH_L1(ptr) __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(ptr))) -# define PREFETCH_L2(ptr) __asm__ __volatile__("prfm pldl2keep, %0" ::"Q"(*(ptr))) +# define PREFETCH_L1(ptr) do { __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(ptr))); } while (0) +# define PREFETCH_L2(ptr) do { __asm__ __volatile__("prfm pldl2keep, %0" ::"Q"(*(ptr))); } while (0) # else -# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */ -# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */ +# define PREFETCH_L1(ptr) do { (void)(ptr); } while (0) /* disabled */ +# define PREFETCH_L2(ptr) do { (void)(ptr); } while (0) /* disabled */ # endif #endif /* NO_PREFETCH */ #define CACHELINE_SIZE 64 -#define PREFETCH_AREA(p, s) { \ - const char* const _ptr = (const char*)(p); \ - size_t const _size = (size_t)(s); \ - size_t _pos; \ - for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) { \ - PREFETCH_L2(_ptr + _pos); \ - } \ -} +#define PREFETCH_AREA(p, s) \ + do { \ + const char* const _ptr = (const char*)(p); \ + size_t const _size = (size_t)(s); \ + size_t _pos; \ + for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) { \ + PREFETCH_L2(_ptr + _pos); \ + } \ + } while (0) /* vectorization - * older GCC (pre gcc-4.3 picked as the cutoff) uses a different syntax */ -#if !defined(__INTEL_COMPILER) && !defined(__clang__) && defined(__GNUC__) + * older GCC (pre gcc-4.3 picked as the cutoff) uses a different syntax, + * and some compilers, like Intel ICC and MCST LCC, do not support it at all. */ +#if !defined(__INTEL_COMPILER) && !defined(__clang__) && defined(__GNUC__) && !defined(__LCC__) # if (__GNUC__ == 4 && __GNUC_MINOR__ > 3) || (__GNUC__ >= 5) # define DONT_VECTORIZE __attribute__((optimize("no-tree-vectorize"))) # else @@ -514,6 +721,12 @@ extern "C" { #define UNLIKELY(x) (x) #endif +#if __has_builtin(__builtin_unreachable) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5))) +# define ZSTD_UNREACHABLE do { assert(0), __builtin_unreachable(); } while (0) +#else +# define ZSTD_UNREACHABLE do { assert(0); } while (0) +#endif + /* disable warnings */ #ifdef _MSC_VER /* Visual Studio */ # include /* For Visual 2005 */ @@ -524,39 +737,198 @@ extern "C" { # pragma warning(disable : 4324) /* disable: C4324: padded structure */ #endif -/*Like DYNAMIC_BMI2 but for compile time determination of BMI2 support*/ -#ifndef STATIC_BMI2 -# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) -# ifdef __AVX2__ //MSVC does not have a BMI2 specific flag, but every CPU that supports AVX2 also supports BMI2 -# define STATIC_BMI2 1 -# endif +/* compile time determination of SIMD support */ +#if !defined(ZSTD_NO_INTRINSICS) +# if defined(__AVX2__) +# define ZSTD_ARCH_X86_AVX2 +# endif +# if defined(__SSE2__) || defined(_M_X64) || (defined (_M_IX86) && defined(_M_IX86_FP) && (_M_IX86_FP >= 2)) +# define ZSTD_ARCH_X86_SSE2 +# endif +# if defined(__ARM_NEON) || defined(_M_ARM64) +# define ZSTD_ARCH_ARM_NEON +# endif +# +# if defined(ZSTD_ARCH_X86_AVX2) +# include +# endif +# if defined(ZSTD_ARCH_X86_SSE2) +# include +# elif defined(ZSTD_ARCH_ARM_NEON) +# include # endif #endif -#ifndef STATIC_BMI2 - #define STATIC_BMI2 0 +/* C-language Attributes are added in C23. */ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201710L) && defined(__has_c_attribute) +# define ZSTD_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) +#else +# define ZSTD_HAS_C_ATTRIBUTE(x) 0 #endif -/* compat. with non-clang compilers */ -#ifndef __has_builtin -# define __has_builtin(x) 0 +/* Only use C++ attributes in C++. Some compilers report support for C++ + * attributes when compiling with C. + */ +#if defined(__cplusplus) && defined(__has_cpp_attribute) +# define ZSTD_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) +#else +# define ZSTD_HAS_CPP_ATTRIBUTE(x) 0 +#endif + +/* Define ZSTD_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute. + * - C23: https://en.cppreference.com/w/c/language/attributes/fallthrough + * - CPP17: https://en.cppreference.com/w/cpp/language/attributes/fallthrough + * - Else: __attribute__((__fallthrough__)) + */ +#ifndef ZSTD_FALLTHROUGH +# if ZSTD_HAS_C_ATTRIBUTE(fallthrough) +# define ZSTD_FALLTHROUGH [[fallthrough]] +# elif ZSTD_HAS_CPP_ATTRIBUTE(fallthrough) +# define ZSTD_FALLTHROUGH [[fallthrough]] +# elif __has_attribute(__fallthrough__) +/* Leading semicolon is to satisfy gcc-11 with -pedantic. Without the semicolon + * gcc complains about: a label can only be part of a statement and a declaration is not a statement. + */ +# define ZSTD_FALLTHROUGH ; __attribute__((__fallthrough__)) +# else +# define ZSTD_FALLTHROUGH +# endif #endif -/* compat. with non-clang compilers */ -#ifndef __has_feature -# define __has_feature(x) 0 -#endif +/*-************************************************************** +* Alignment +*****************************************************************/ -/* detects whether we are being compiled under msan */ -#ifndef ZSTD_MEMORY_SANITIZER -# if __has_feature(memory_sanitizer) -# define ZSTD_MEMORY_SANITIZER 1 +/* @return 1 if @u is a 2^n value, 0 otherwise + * useful to check a value is valid for alignment restrictions */ +MEM_STATIC int ZSTD_isPower2(size_t u) { + return (u & (u-1)) == 0; +} + +/* this test was initially positioned in mem.h, + * but this file is removed (or replaced) for linux kernel + * so it's now hosted in compiler.h, + * which remains valid for both user & kernel spaces. + */ + +#ifndef ZSTD_ALIGNOF +# if defined(__GNUC__) || defined(_MSC_VER) +/* covers gcc, clang & MSVC */ +/* note : this section must come first, before C11, + * due to a limitation in the kernel source generator */ +# define ZSTD_ALIGNOF(T) __alignof(T) + +# elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) +/* C11 support */ +# include +# define ZSTD_ALIGNOF(T) alignof(T) + +# else +/* No known support for alignof() - imperfect backup */ +# define ZSTD_ALIGNOF(T) (sizeof(void*) < sizeof(T) ? sizeof(void*) : sizeof(T)) + +# endif +#endif /* ZSTD_ALIGNOF */ + +#ifndef ZSTD_ALIGNED +/* C90-compatible alignment macro (GCC/Clang). Adjust for other compilers if needed. */ +# if defined(__GNUC__) || defined(__clang__) +# define ZSTD_ALIGNED(a) __attribute__((aligned(a))) +# elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */ +# define ZSTD_ALIGNED(a) _Alignas(a) +#elif defined(_MSC_VER) +# define ZSTD_ALIGNED(n) __declspec(align(n)) +# else + /* this compiler will require its own alignment instruction */ +# define ZSTD_ALIGNED(...) +# endif +#endif /* ZSTD_ALIGNED */ + + +/*-************************************************************** +* Sanitizer +*****************************************************************/ + +/** + * Zstd relies on pointer overflow in its decompressor. + * We add this attribute to functions that rely on pointer overflow. + */ +#ifndef ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +# if __has_attribute(no_sanitize) +# if !defined(__clang__) && defined(__GNUC__) && __GNUC__ < 8 + /* gcc < 8 only has signed-integer-overlow which triggers on pointer overflow */ +# define ZSTD_ALLOW_POINTER_OVERFLOW_ATTR __attribute__((no_sanitize("signed-integer-overflow"))) +# else + /* older versions of clang [3.7, 5.0) will warn that pointer-overflow is ignored. */ +# define ZSTD_ALLOW_POINTER_OVERFLOW_ATTR __attribute__((no_sanitize("pointer-overflow"))) +# endif # else -# define ZSTD_MEMORY_SANITIZER 0 +# define ZSTD_ALLOW_POINTER_OVERFLOW_ATTR # endif #endif -#if ZSTD_MEMORY_SANITIZER +/** + * Helper function to perform a wrapped pointer difference without triggering + * UBSAN. + * + * @returns lhs - rhs with wrapping + */ +MEM_STATIC +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +ptrdiff_t ZSTD_wrappedPtrDiff(unsigned char const* lhs, unsigned char const* rhs) +{ + return lhs - rhs; +} + +/** + * Helper function to perform a wrapped pointer add without triggering UBSAN. + * + * @return ptr + add with wrapping + */ +MEM_STATIC +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +unsigned char const* ZSTD_wrappedPtrAdd(unsigned char const* ptr, ptrdiff_t add) +{ + return ptr + add; +} + +/** + * Helper function to perform a wrapped pointer subtraction without triggering + * UBSAN. + * + * @return ptr - sub with wrapping + */ +MEM_STATIC +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +unsigned char const* ZSTD_wrappedPtrSub(unsigned char const* ptr, ptrdiff_t sub) +{ + return ptr - sub; +} + +/** + * Helper function to add to a pointer that works around C's undefined behavior + * of adding 0 to NULL. + * + * @returns `ptr + add` except it defines `NULL + 0 == NULL`. + */ +MEM_STATIC +unsigned char* ZSTD_maybeNullPtrAdd(unsigned char* ptr, ptrdiff_t add) +{ + return add > 0 ? ptr + add : ptr; +} + +/* Issue #3240 reports an ASAN failure on an llvm-mingw build. Out of an + * abundance of caution, disable our custom poisoning on mingw. */ +#ifdef __MINGW32__ +#ifndef ZSTD_ASAN_DONT_POISON_WORKSPACE +#define ZSTD_ASAN_DONT_POISON_WORKSPACE 1 +#endif +#ifndef ZSTD_MSAN_DONT_POISON_WORKSPACE +#define ZSTD_MSAN_DONT_POISON_WORKSPACE 1 +#endif +#endif + +#if ZSTD_MEMORY_SANITIZER && !defined(ZSTD_MSAN_DONT_POISON_WORKSPACE) /* Not all platforms that support msan provide sanitizers/msan_interface.h. * We therefore declare the functions we need ourselves, rather than trying to * include the header file... */ @@ -575,20 +947,13 @@ void __msan_poison(const volatile void *a, size_t size); /* Returns the offset of the first (at least partially) poisoned byte in the memory range, or -1 if the whole range is good. */ intptr_t __msan_test_shadow(const volatile void *x, size_t size); -#endif -/* detects whether we are being compiled under asan */ -#ifndef ZSTD_ADDRESS_SANITIZER -# if __has_feature(address_sanitizer) -# define ZSTD_ADDRESS_SANITIZER 1 -# elif defined(__SANITIZE_ADDRESS__) -# define ZSTD_ADDRESS_SANITIZER 1 -# else -# define ZSTD_ADDRESS_SANITIZER 0 -# endif +/* Print shadow and origin for the memory range to stderr in a human-readable + format. */ +void __msan_print_shadow(const volatile void *x, size_t size); #endif -#if ZSTD_ADDRESS_SANITIZER +#if ZSTD_ADDRESS_SANITIZER && !defined(ZSTD_ASAN_DONT_POISON_WORKSPACE) /* Not all platforms that support asan provide sanitizers/asan_interface.h. * We therefore declare the functions we need ourselves, rather than trying to * include the header file... */ @@ -638,15 +1003,8 @@ void __asan_unpoison_memory_region(void const volatile *addr, size_t size); #if defined(_MSC_VER) /* Visual Studio */ # include /* _byteswap_ulong */ # include /* _byteswap_* */ -#endif -#if defined(__GNUC__) -# define MEM_STATIC static __inline __attribute__((unused)) -#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define MEM_STATIC static inline -#elif defined(_MSC_VER) -# define MEM_STATIC static __inline -#else -# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#elif defined(__ICCARM__) +# include #endif /*-************************************************************** @@ -659,6 +1017,8 @@ void __asan_unpoison_memory_region(void const volatile *addr, size_t size); # include /* intptr_t */ # endif typedef uint8_t BYTE; + typedef uint8_t U8; + typedef int8_t S8; typedef uint16_t U16; typedef int16_t S16; typedef uint32_t U32; @@ -671,6 +1031,8 @@ void __asan_unpoison_memory_region(void const volatile *addr, size_t size); # error "this implementation requires char to be exactly 8-bit type" #endif typedef unsigned char BYTE; + typedef unsigned char U8; + typedef signed char S8; #if USHRT_MAX != 65535 # error "this implementation requires short to be exactly 16-bit type" #endif @@ -687,7 +1049,6 @@ void __asan_unpoison_memory_region(void const volatile *addr, size_t size); typedef signed long long S64; #endif - /*-************************************************************** * Memory I/O API *****************************************************************/ @@ -737,23 +1098,15 @@ MEM_STATIC size_t MEM_swapST(size_t in); /*-************************************************************** * Memory I/O Implementation *****************************************************************/ -/* MEM_FORCE_MEMORY_ACCESS : - * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. - * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. - * The below switch allow to select different access method for improved performance. - * Method 0 (default) : use `memcpy()`. Safe and portable. - * Method 1 : `__packed` statement. It depends on compiler extension (i.e., not portable). - * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. +/* MEM_FORCE_MEMORY_ACCESS : For accessing unaligned memory: + * Method 0 : always use `memcpy()`. Safe and portable. + * Method 1 : Use compiler extension to set unaligned access. * Method 2 : direct access. This method is portable but violate C standard. * It can generate buggy code on targets depending on alignment. - * In some circumstances, it's the only known way to get the most performance (i.e. GCC + ARMv6) - * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. - * Prefer these methods in priority order (0 > 1 > 2) + * Default : method 1 if supported, else method 0 */ #ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ -# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) -# define MEM_FORCE_MEMORY_ACCESS 2 -# elif defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__) +# ifdef __GNUC__ # define MEM_FORCE_MEMORY_ACCESS 1 # endif #endif @@ -763,8 +1116,24 @@ MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; } MEM_STATIC unsigned MEM_isLittleEndian(void) { +#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + return 1; +#elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + return 0; +#elif defined(__clang__) && __LITTLE_ENDIAN__ + return 1; +#elif defined(__clang__) && __BIG_ENDIAN__ + return 0; +#elif defined(_MSC_VER) && (_M_X64 || _M_IX86) + return 1; +#elif defined(__DMC__) && defined(_M_IX86) + return 1; +#elif defined(__IAR_SYSTEMS_ICC__) && __LITTLE_ENDIAN__ + return 1; +#else const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ return one.c[0]; +#endif } #if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2) @@ -782,30 +1151,19 @@ MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; } #elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1) -/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ -/* currently only defined for gcc and icc */ -#if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32)) - __pragma( pack(push, 1) ) - typedef struct { U16 v; } unalign16; - typedef struct { U32 v; } unalign32; - typedef struct { U64 v; } unalign64; - typedef struct { size_t v; } unalignArch; - __pragma( pack(pop) ) -#else - typedef struct { U16 v; } __attribute__((packed)) unalign16; - typedef struct { U32 v; } __attribute__((packed)) unalign32; - typedef struct { U64 v; } __attribute__((packed)) unalign64; - typedef struct { size_t v; } __attribute__((packed)) unalignArch; -#endif +typedef __attribute__((aligned(1))) U16 unalign16; +typedef __attribute__((aligned(1))) U32 unalign32; +typedef __attribute__((aligned(1))) U64 unalign64; +typedef __attribute__((aligned(1))) size_t unalignArch; -MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign16*)ptr)->v; } -MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign32*)ptr)->v; } -MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign64*)ptr)->v; } -MEM_STATIC size_t MEM_readST(const void* ptr) { return ((const unalignArch*)ptr)->v; } +MEM_STATIC U16 MEM_read16(const void* ptr) { return *(const unalign16*)ptr; } +MEM_STATIC U32 MEM_read32(const void* ptr) { return *(const unalign32*)ptr; } +MEM_STATIC U64 MEM_read64(const void* ptr) { return *(const unalign64*)ptr; } +MEM_STATIC size_t MEM_readST(const void* ptr) { return *(const unalignArch*)ptr; } -MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign16*)memPtr)->v = value; } -MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign32*)memPtr)->v = value; } -MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign64*)memPtr)->v = value; } +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(unalign16*)memPtr = value; } +MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(unalign32*)memPtr = value; } +MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(unalign64*)memPtr = value; } #else @@ -849,6 +1207,14 @@ MEM_STATIC void MEM_write64(void* memPtr, U64 value) #endif /* MEM_FORCE_MEMORY_ACCESS */ +MEM_STATIC U32 MEM_swap32_fallback(U32 in) +{ + return ((in << 24) & 0xff000000 ) | + ((in << 8) & 0x00ff0000 ) | + ((in >> 8) & 0x0000ff00 ) | + ((in >> 24) & 0x000000ff ); +} + MEM_STATIC U32 MEM_swap32(U32 in) { #if defined(_MSC_VER) /* Visual Studio */ @@ -856,23 +1222,16 @@ MEM_STATIC U32 MEM_swap32(U32 in) #elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \ || (defined(__clang__) && __has_builtin(__builtin_bswap32)) return __builtin_bswap32(in); +#elif defined(__ICCARM__) + return __REV(in); #else - return ((in << 24) & 0xff000000 ) | - ((in << 8) & 0x00ff0000 ) | - ((in >> 8) & 0x0000ff00 ) | - ((in >> 24) & 0x000000ff ); + return MEM_swap32_fallback(in); #endif } -MEM_STATIC U64 MEM_swap64(U64 in) +MEM_STATIC U64 MEM_swap64_fallback(U64 in) { -#if defined(_MSC_VER) /* Visual Studio */ - return _byteswap_uint64(in); -#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \ - || (defined(__clang__) && __has_builtin(__builtin_bswap64)) - return __builtin_bswap64(in); -#else - return ((in << 56) & 0xff00000000000000ULL) | + return ((in << 56) & 0xff00000000000000ULL) | ((in << 40) & 0x00ff000000000000ULL) | ((in << 24) & 0x0000ff0000000000ULL) | ((in << 8) & 0x000000ff00000000ULL) | @@ -880,6 +1239,17 @@ MEM_STATIC U64 MEM_swap64(U64 in) ((in >> 24) & 0x0000000000ff0000ULL) | ((in >> 40) & 0x000000000000ff00ULL) | ((in >> 56) & 0x00000000000000ffULL); +} + +MEM_STATIC U64 MEM_swap64(U64 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_uint64(in); +#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \ + || (defined(__clang__) && __has_builtin(__builtin_bswap64)) + return __builtin_bswap64(in); +#else + return MEM_swap64_fallback(in); #endif } @@ -916,7 +1286,7 @@ MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val) MEM_STATIC U32 MEM_readLE24(const void* memPtr) { - return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16); + return (U32)MEM_readLE16(memPtr) + ((U32)(((const BYTE*)memPtr)[2]) << 16); } MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val) @@ -1026,16 +1396,11 @@ MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val) /* code only tested on 32 and 64 bits systems */ MEM_STATIC void MEM_check(void) { DEBUG_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); } - -#if defined (__cplusplus) -} -#endif - #endif /* MEM_H_MODULE */ /**** ended inlining mem.h ****/ /**** start inlining error_private.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -1049,18 +1414,12 @@ MEM_STATIC void MEM_check(void) { DEBUG_STATIC_ASSERT((sizeof(size_t)==4) || (si #ifndef ERROR_H_MODULE #define ERROR_H_MODULE -#if defined (__cplusplus) -extern "C" { -#endif - - /* **************************************** * Dependencies ******************************************/ -/**** skipping file: zstd_deps.h ****/ -/**** start inlining zstd_errors.h ****/ +/**** start inlining ../zstd_errors.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -1076,24 +1435,32 @@ extern "C" { extern "C" { #endif -/*===== dependency =====*/ -#include /* size_t */ - - /* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */ -#ifndef ZSTDERRORLIB_VISIBILITY -# if defined(__GNUC__) && (__GNUC__ >= 4) -# define ZSTDERRORLIB_VISIBILITY __attribute__ ((visibility ("default"))) +#ifndef ZSTDERRORLIB_VISIBLE + /* Backwards compatibility with old macro name */ +# ifdef ZSTDERRORLIB_VISIBILITY +# define ZSTDERRORLIB_VISIBLE ZSTDERRORLIB_VISIBILITY +# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDERRORLIB_VISIBLE __attribute__ ((visibility ("default"))) +# else +# define ZSTDERRORLIB_VISIBLE +# endif +#endif + +#ifndef ZSTDERRORLIB_HIDDEN +# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDERRORLIB_HIDDEN __attribute__ ((visibility ("hidden"))) # else -# define ZSTDERRORLIB_VISIBILITY +# define ZSTDERRORLIB_HIDDEN # endif #endif + #if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) -# define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBILITY +# define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBLE #elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) -# define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +# define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ #else -# define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBILITY +# define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBLE #endif /*-********************************************* @@ -1119,14 +1486,18 @@ typedef enum { ZSTD_error_frameParameter_windowTooLarge = 16, ZSTD_error_corruption_detected = 20, ZSTD_error_checksum_wrong = 22, + ZSTD_error_literals_headerWrong = 24, ZSTD_error_dictionary_corrupted = 30, ZSTD_error_dictionary_wrong = 32, ZSTD_error_dictionaryCreation_failed = 34, ZSTD_error_parameter_unsupported = 40, + ZSTD_error_parameter_combination_unsupported = 41, ZSTD_error_parameter_outOfBound = 42, ZSTD_error_tableLog_tooLarge = 44, ZSTD_error_maxSymbolValue_tooLarge = 46, ZSTD_error_maxSymbolValue_tooSmall = 48, + ZSTD_error_cannotProduce_uncompressedBlock = 49, + ZSTD_error_stabilityCondition_notRespected = 50, ZSTD_error_stage_wrong = 60, ZSTD_error_init_missing = 62, ZSTD_error_memory_allocation = 64, @@ -1134,18 +1505,18 @@ typedef enum { ZSTD_error_dstSize_tooSmall = 70, ZSTD_error_srcSize_wrong = 72, ZSTD_error_dstBuffer_null = 74, + ZSTD_error_noForwardProgress_destFull = 80, + ZSTD_error_noForwardProgress_inputEmpty = 82, /* following error codes are __NOT STABLE__, they can be removed or changed in future versions */ ZSTD_error_frameIndex_tooLarge = 100, ZSTD_error_seekableIO = 102, ZSTD_error_dstBuffer_wrong = 104, ZSTD_error_srcBuffer_wrong = 105, + ZSTD_error_sequenceProducer_failed = 106, + ZSTD_error_externalSequences_invalid = 107, ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */ } ZSTD_ErrorCode; -/*! ZSTD_getErrorCode() : - convert a `size_t` function result into a `ZSTD_ErrorCode` enum type, - which can be used to compare with enum list published above */ -ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); /**< Same as ZSTD_getErrorName, but using a `ZSTD_ErrorCode` enum argument */ @@ -1154,8 +1525,10 @@ ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); /**< Sa #endif #endif /* ZSTD_ERRORS_H_398273423 */ -/**** ended inlining zstd_errors.h ****/ - +/**** ended inlining ../zstd_errors.h ****/ +/**** skipping file: compiler.h ****/ +/**** skipping file: debug.h ****/ +/**** skipping file: zstd_deps.h ****/ /* **************************************** * Compiler-specific @@ -1190,8 +1563,13 @@ ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } /* check and forward error code */ -#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return e -#define CHECK_F(f) { CHECK_V_F(_var_err__, f); } +#define CHECK_V_F(e, f) \ + size_t const e = f; \ + do { \ + if (ERR_isError(e)) \ + return e; \ + } while (0) +#define CHECK_F(f) do { CHECK_V_F(_var_err__, f); } while (0) /*-**************************************** @@ -1205,9 +1583,86 @@ ERR_STATIC const char* ERR_getErrorName(size_t code) return ERR_getErrorString(ERR_getErrorCode(code)); } -#if defined (__cplusplus) +/** + * Ignore: this is an internal helper. + * + * This is a helper function to help force C99-correctness during compilation. + * Under strict compilation modes, variadic macro arguments can't be empty. + * However, variadic function arguments can be. Using a function therefore lets + * us statically check that at least one (string) argument was passed, + * independent of the compilation flags. + */ +static INLINE_KEYWORD UNUSED_ATTR +void _force_has_format_string(const char *format, ...) { + (void)format; } -#endif + +/** + * Ignore: this is an internal helper. + * + * We want to force this function invocation to be syntactically correct, but + * we don't want to force runtime evaluation of its arguments. + */ +#define _FORCE_HAS_FORMAT_STRING(...) \ + do { \ + if (0) { \ + _force_has_format_string(__VA_ARGS__); \ + } \ + } while (0) + +#define ERR_QUOTE(str) #str + +/** + * Return the specified error if the condition evaluates to true. + * + * In debug modes, prints additional information. + * In order to do that (particularly, printing the conditional that failed), + * this can't just wrap RETURN_ERROR(). + */ +#define RETURN_ERROR_IF(cond, err, ...) \ + do { \ + if (cond) { \ + RAWLOG(3, "%s:%d: ERROR!: check %s failed, returning %s", \ + __FILE__, __LINE__, ERR_QUOTE(cond), ERR_QUOTE(ERROR(err))); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return ERROR(err); \ + } \ + } while (0) + +/** + * Unconditionally return the specified error. + * + * In debug modes, prints additional information. + */ +#define RETURN_ERROR(err, ...) \ + do { \ + RAWLOG(3, "%s:%d: ERROR!: unconditional check failed, returning %s", \ + __FILE__, __LINE__, ERR_QUOTE(ERROR(err))); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return ERROR(err); \ + } while(0) + +/** + * If the provided expression evaluates to an error code, returns that error code. + * + * In debug modes, prints additional information. + */ +#define FORWARD_IF_ERROR(err, ...) \ + do { \ + size_t const err_code = (err); \ + if (ERR_isError(err_code)) { \ + RAWLOG(3, "%s:%d: ERROR!: forwarding error in %s: %s", \ + __FILE__, __LINE__, ERR_QUOTE(err), ERR_getErrorName(err_code)); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return err_code; \ + } \ + } while(0) #endif /* ERROR_H_MODULE */ /**** ended inlining error_private.h ****/ @@ -1216,7 +1671,7 @@ ERR_STATIC const char* ERR_getErrorName(size_t code) /* ****************************************************************** * FSE : Finite State Entropy codec * Public Prototypes declaration - * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -1226,11 +1681,6 @@ ERR_STATIC const char* ERR_getErrorName(size_t code) * in the COPYING file in the root directory of this source tree). * You may select, at your option, one of the above-listed licenses. ****************************************************************** */ - -#if defined (__cplusplus) -extern "C" { -#endif - #ifndef FSE_H #define FSE_H @@ -1240,7 +1690,6 @@ extern "C" { ******************************************/ /**** skipping file: zstd_deps.h ****/ - /*-***************************************** * FSE_PUBLIC_API : control library symbols visibility ******************************************/ @@ -1268,34 +1717,6 @@ extern "C" { FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */ -/*-**************************************** -* FSE simple functions -******************************************/ -/*! FSE_compress() : - Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'. - 'dst' buffer must be already allocated. Compression runs faster is dstCapacity >= FSE_compressBound(srcSize). - @return : size of compressed data (<= dstCapacity). - Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! - if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead. - if FSE_isError(return), compression failed (more details using FSE_getErrorName()) -*/ -FSE_PUBLIC_API size_t FSE_compress(void* dst, size_t dstCapacity, - const void* src, size_t srcSize); - -/*! FSE_decompress(): - Decompress FSE data from buffer 'cSrc', of size 'cSrcSize', - into already allocated destination buffer 'dst', of size 'dstCapacity'. - @return : size of regenerated data (<= maxDstSize), - or an error code, which can be tested using FSE_isError() . - - ** Important ** : FSE_decompress() does not decompress non-compressible nor RLE data !!! - Why ? : making this distinction requires a header. - Header management is intentionally delegated to the user layer, which can better manage special cases. -*/ -FSE_PUBLIC_API size_t FSE_decompress(void* dst, size_t dstCapacity, - const void* cSrc, size_t cSrcSize); - - /*-***************************************** * Tool functions ******************************************/ @@ -1306,20 +1727,6 @@ FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return FSE_PUBLIC_API const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */ -/*-***************************************** -* FSE advanced functions -******************************************/ -/*! FSE_compress2() : - Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog' - Both parameters can be defined as '0' to mean : use default value - @return : size of compressed data - Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!! - if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression. - if FSE_isError(return), it's an error code. -*/ -FSE_PUBLIC_API size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); - - /*-***************************************** * FSE detailed API ******************************************/ @@ -1379,8 +1786,6 @@ FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize, /*! Constructor and Destructor of FSE_CTable. Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */ typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */ -FSE_PUBLIC_API FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog); -FSE_PUBLIC_API void FSE_freeCTable (FSE_CTable* ct); /*! FSE_buildCTable(): Builds `ct`, which must be already allocated, using FSE_createCTable(). @@ -1456,23 +1861,7 @@ FSE_PUBLIC_API size_t FSE_readNCount_bmi2(short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize, int bmi2); -/*! Constructor and Destructor of FSE_DTable. - Note that its size depends on 'tableLog' */ typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */ -FSE_PUBLIC_API FSE_DTable* FSE_createDTable(unsigned tableLog); -FSE_PUBLIC_API void FSE_freeDTable(FSE_DTable* dt); - -/*! FSE_buildDTable(): - Builds 'dt', which must be already allocated, using FSE_createDTable(). - return : 0, or an errorCode, which can be tested using FSE_isError() */ -FSE_PUBLIC_API size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); - -/*! FSE_decompress_usingDTable(): - Decompress compressed source `cSrc` of size `cSrcSize` using `dt` - into `dst` which must be already allocated. - @return : size of regenerated data (necessarily <= `dstCapacity`), - or an errorCode, which can be tested using FSE_isError() */ -FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt); /*! Tutorial : @@ -1504,15 +1893,14 @@ If there is an error, the function will return an error code, which can be teste #endif /* FSE_H */ + #if defined(FSE_STATIC_LINKING_ONLY) && !defined(FSE_H_FSE_STATIC_LINKING_ONLY) #define FSE_H_FSE_STATIC_LINKING_ONLY - -/* *** Dependency *** */ /**** start inlining bitstream.h ****/ /* ****************************************************************** * bitstream * Part of FSE library - * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -1525,9 +1913,6 @@ If there is an error, the function will return an error code, which can be teste #ifndef BITSTREAM_H_MODULE #define BITSTREAM_H_MODULE -#if defined (__cplusplus) -extern "C" { -#endif /* * This API consists of small unitary functions, which must be inlined for best performance. * Since link-time-optimization is not available for all compilers, @@ -1541,14 +1926,220 @@ extern "C" { /**** skipping file: compiler.h ****/ /**** skipping file: debug.h ****/ /**** skipping file: error_private.h ****/ +/**** start inlining bits.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_BITS_H +#define ZSTD_BITS_H +/**** skipping file: mem.h ****/ + +MEM_STATIC unsigned ZSTD_countTrailingZeros32_fallback(U32 val) +{ + assert(val != 0); + { + static const U32 DeBruijnBytePos[32] = {0, 1, 28, 2, 29, 14, 24, 3, + 30, 22, 20, 15, 25, 17, 4, 8, + 31, 27, 13, 23, 21, 19, 16, 7, + 26, 12, 18, 6, 11, 5, 10, 9}; + return DeBruijnBytePos[((U32) ((val & -(S32) val) * 0x077CB531U)) >> 27]; + } +} + +MEM_STATIC unsigned ZSTD_countTrailingZeros32(U32 val) +{ + assert(val != 0); +#if defined(_MSC_VER) +# if STATIC_BMI2 + return (unsigned)_tzcnt_u32(val); +# else + if (val != 0) { + unsigned long r; + _BitScanForward(&r, val); + return (unsigned)r; + } else { + __assume(0); /* Should not reach this code path */ + } +# endif +#elif defined(__GNUC__) && (__GNUC__ >= 4) + return (unsigned)__builtin_ctz(val); +#elif defined(__ICCARM__) + return (unsigned)__builtin_ctz(val); +#else + return ZSTD_countTrailingZeros32_fallback(val); +#endif +} + +MEM_STATIC unsigned ZSTD_countLeadingZeros32_fallback(U32 val) +{ + assert(val != 0); + { + static const U32 DeBruijnClz[32] = {0, 9, 1, 10, 13, 21, 2, 29, + 11, 14, 16, 18, 22, 25, 3, 30, + 8, 12, 20, 28, 15, 17, 24, 7, + 19, 27, 23, 6, 26, 5, 4, 31}; + val |= val >> 1; + val |= val >> 2; + val |= val >> 4; + val |= val >> 8; + val |= val >> 16; + return 31 - DeBruijnClz[(val * 0x07C4ACDDU) >> 27]; + } +} + +MEM_STATIC unsigned ZSTD_countLeadingZeros32(U32 val) +{ + assert(val != 0); +#if defined(_MSC_VER) +# if STATIC_BMI2 + return (unsigned)_lzcnt_u32(val); +# else + if (val != 0) { + unsigned long r; + _BitScanReverse(&r, val); + return (unsigned)(31 - r); + } else { + __assume(0); /* Should not reach this code path */ + } +# endif +#elif defined(__GNUC__) && (__GNUC__ >= 4) + return (unsigned)__builtin_clz(val); +#elif defined(__ICCARM__) + return (unsigned)__builtin_clz(val); +#else + return ZSTD_countLeadingZeros32_fallback(val); +#endif +} + +MEM_STATIC unsigned ZSTD_countTrailingZeros64(U64 val) +{ + assert(val != 0); +#if defined(_MSC_VER) && defined(_WIN64) +# if STATIC_BMI2 + return (unsigned)_tzcnt_u64(val); +# else + if (val != 0) { + unsigned long r; + _BitScanForward64(&r, val); + return (unsigned)r; + } else { + __assume(0); /* Should not reach this code path */ + } +# endif +#elif defined(__GNUC__) && (__GNUC__ >= 4) && defined(__LP64__) + return (unsigned)__builtin_ctzll(val); +#elif defined(__ICCARM__) + return (unsigned)__builtin_ctzll(val); +#else + { + U32 mostSignificantWord = (U32)(val >> 32); + U32 leastSignificantWord = (U32)val; + if (leastSignificantWord == 0) { + return 32 + ZSTD_countTrailingZeros32(mostSignificantWord); + } else { + return ZSTD_countTrailingZeros32(leastSignificantWord); + } + } +#endif +} + +MEM_STATIC unsigned ZSTD_countLeadingZeros64(U64 val) +{ + assert(val != 0); +#if defined(_MSC_VER) && defined(_WIN64) +# if STATIC_BMI2 + return (unsigned)_lzcnt_u64(val); +# else + if (val != 0) { + unsigned long r; + _BitScanReverse64(&r, val); + return (unsigned)(63 - r); + } else { + __assume(0); /* Should not reach this code path */ + } +# endif +#elif defined(__GNUC__) && (__GNUC__ >= 4) + return (unsigned)(__builtin_clzll(val)); +#elif defined(__ICCARM__) + return (unsigned)(__builtin_clzll(val)); +#else + { + U32 mostSignificantWord = (U32)(val >> 32); + U32 leastSignificantWord = (U32)val; + if (mostSignificantWord == 0) { + return 32 + ZSTD_countLeadingZeros32(leastSignificantWord); + } else { + return ZSTD_countLeadingZeros32(mostSignificantWord); + } + } +#endif +} + +MEM_STATIC unsigned ZSTD_NbCommonBytes(size_t val) +{ + if (MEM_isLittleEndian()) { + if (MEM_64bits()) { + return ZSTD_countTrailingZeros64((U64)val) >> 3; + } else { + return ZSTD_countTrailingZeros32((U32)val) >> 3; + } + } else { /* Big Endian CPU */ + if (MEM_64bits()) { + return ZSTD_countLeadingZeros64((U64)val) >> 3; + } else { + return ZSTD_countLeadingZeros32((U32)val) >> 3; + } + } +} + +MEM_STATIC unsigned ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus */ +{ + assert(val != 0); + return 31 - ZSTD_countLeadingZeros32(val); +} + +/* ZSTD_rotateRight_*(): + * Rotates a bitfield to the right by "count" bits. + * https://en.wikipedia.org/w/index.php?title=Circular_shift&oldid=991635599#Implementing_circular_shifts + */ +MEM_STATIC +U64 ZSTD_rotateRight_U64(U64 const value, U32 count) { + assert(count < 64); + count &= 0x3F; /* for fickle pattern recognition */ + return (value >> count) | (U64)(value << ((0U - count) & 0x3F)); +} + +MEM_STATIC +U32 ZSTD_rotateRight_U32(U32 const value, U32 count) { + assert(count < 32); + count &= 0x1F; /* for fickle pattern recognition */ + return (value >> count) | (U32)(value << ((0U - count) & 0x1F)); +} + +MEM_STATIC +U16 ZSTD_rotateRight_U16(U16 const value, U32 count) { + assert(count < 16); + count &= 0x0F; /* for fickle pattern recognition */ + return (value >> count) | (U16)(value << ((0U - count) & 0x0F)); +} + +#endif /* ZSTD_BITS_H */ +/**** ended inlining bits.h ****/ /*========================================= * Target specific =========================================*/ #ifndef ZSTD_NO_INTRINSICS -# if defined(__BMI__) && defined(__GNUC__) -# include /* support for bextr (experimental) */ +# if (defined(__BMI__) || defined(__BMI2__)) && defined(__GNUC__) +# include /* support for bextr (experimental)/bzhi */ # elif defined(__ICCARM__) # include # endif @@ -1562,12 +2153,13 @@ extern "C" { /*-****************************************** * bitStream encoding API (write forward) ********************************************/ +typedef size_t BitContainerType; /* bitStream can mix input from multiple sources. * A critical property of these streams is that they encode and decode in **reverse** direction. * So the first bit sequence you add will be the last to be read, like a LIFO stack. */ typedef struct { - size_t bitContainer; + BitContainerType bitContainer; unsigned bitPos; char* startPtr; char* ptr; @@ -1575,7 +2167,7 @@ typedef struct { } BIT_CStream_t; MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity); -MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits); +MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, BitContainerType value, unsigned nbBits); MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC); MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC); @@ -1584,7 +2176,7 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC); * `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code. * * bits are first added to a local register. -* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems. +* Local register is BitContainerType, 64-bits on 64-bits systems, or 32-bits on 32-bits systems. * Writing data into memory is an explicit operation, performed by the flushBits function. * Hence keep track how many bits are potentially stored into local register to avoid register overflow. * After a flushBits, a maximum of 7 bits might still be stored into local register. @@ -1601,28 +2193,28 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC); * bitStream decoding API (read backward) **********************************************/ typedef struct { - size_t bitContainer; + BitContainerType bitContainer; unsigned bitsConsumed; const char* ptr; const char* start; const char* limitPtr; } BIT_DStream_t; -typedef enum { BIT_DStream_unfinished = 0, - BIT_DStream_endOfBuffer = 1, - BIT_DStream_completed = 2, - BIT_DStream_overflow = 3 } BIT_DStream_status; /* result of BIT_reloadDStream() */ - /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */ +typedef enum { BIT_DStream_unfinished = 0, /* fully refilled */ + BIT_DStream_endOfBuffer = 1, /* still some bits left in bitstream */ + BIT_DStream_completed = 2, /* bitstream entirely consumed, bit-exact */ + BIT_DStream_overflow = 3 /* user requested more bits than present in bitstream */ + } BIT_DStream_status; /* result of BIT_reloadDStream() */ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize); -MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits); +MEM_STATIC BitContainerType BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits); MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD); MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD); /* Start by invoking BIT_initDStream(). * A chunk of the bitStream is then stored into a local register. -* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). +* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (BitContainerType). * You can then retrieve bitFields stored into the local register, **in reverse order**. * Local register is explicitly reloaded from memory by the BIT_reloadDStream() method. * A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished. @@ -1634,7 +2226,7 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD); /*-**************************************** * unsafe API ******************************************/ -MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits); +MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, BitContainerType value, unsigned nbBits); /* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */ MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC); @@ -1643,42 +2235,6 @@ MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC); MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits); /* faster, but works only if nbBits >= 1 */ - - -/*-************************************************************** -* Internal functions -****************************************************************/ -MEM_STATIC unsigned BIT_highbit32 (U32 val) -{ - assert(val != 0); - { -# if defined(_MSC_VER) /* Visual */ -# if STATIC_BMI2 == 1 - return _lzcnt_u32(val) ^ 31; -# else - unsigned long r = 0; - return _BitScanReverse(&r, val) ? (unsigned)r : 0; -# endif -# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */ - return __builtin_clz (val) ^ 31; -# elif defined(__ICCARM__) /* IAR Intrinsic */ - return 31 - __CLZ(val); -# else /* Software version */ - static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, - 11, 14, 16, 18, 22, 25, 3, 30, - 8, 12, 20, 28, 15, 17, 24, 7, - 19, 27, 23, 6, 26, 5, 4, 31 }; - U32 v = val; - v |= v >> 1; - v |= v >> 2; - v |= v >> 4; - v |= v >> 8; - v |= v >> 16; - return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27]; -# endif - } -} - /*===== Local Constants =====*/ static const unsigned BIT_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, @@ -1708,16 +2264,31 @@ MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, return 0; } +FORCE_INLINE_TEMPLATE BitContainerType BIT_getLowerBits(BitContainerType bitContainer, U32 const nbBits) +{ +#if STATIC_BMI2 && !defined(ZSTD_NO_INTRINSICS) +# if (defined(__x86_64__) || defined(_M_X64)) && !defined(__ILP32__) + return _bzhi_u64(bitContainer, nbBits); +# else + DEBUG_STATIC_ASSERT(sizeof(bitContainer) == sizeof(U32)); + return _bzhi_u32(bitContainer, nbBits); +# endif +#else + assert(nbBits < BIT_MASK_SIZE); + return bitContainer & BIT_mask[nbBits]; +#endif +} + /*! BIT_addBits() : * can add up to 31 bits into `bitC`. * Note : does not check for register overflow ! */ MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, - size_t value, unsigned nbBits) + BitContainerType value, unsigned nbBits) { DEBUG_STATIC_ASSERT(BIT_MASK_SIZE == 32); assert(nbBits < BIT_MASK_SIZE); assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8); - bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos; + bitC->bitContainer |= BIT_getLowerBits(value, nbBits) << bitC->bitPos; bitC->bitPos += nbBits; } @@ -1725,7 +2296,7 @@ MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, * works only if `value` is _clean_, * meaning all high bits above nbBits are 0 */ MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, - size_t value, unsigned nbBits) + BitContainerType value, unsigned nbBits) { assert((value>>nbBits) == 0); assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8); @@ -1772,7 +2343,7 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC) BIT_addBitsFast(bitC, 1, 1); /* endMark */ BIT_flushBits(bitC); if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */ - return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0); + return (size_t)(bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0); } @@ -1796,35 +2367,35 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer); bitD->bitContainer = MEM_readLEST(bitD->ptr); { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; - bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */ + bitD->bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */ if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } } else { bitD->ptr = bitD->start; bitD->bitContainer = *(const BYTE*)(bitD->start); switch(srcSize) { - case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16); - /* fall-through */ + case 7: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16); + ZSTD_FALLTHROUGH; - case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24); - /* fall-through */ + case 6: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24); + ZSTD_FALLTHROUGH; - case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32); - /* fall-through */ + case 5: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32); + ZSTD_FALLTHROUGH; - case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24; - /* fall-through */ + case 4: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[3]) << 24; + ZSTD_FALLTHROUGH; - case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16; - /* fall-through */ + case 3: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[2]) << 16; + ZSTD_FALLTHROUGH; - case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8; - /* fall-through */ + case 2: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[1]) << 8; + ZSTD_FALLTHROUGH; default: break; } { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; - bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; + bitD->bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0; if (lastByte == 0) return ERROR(corruption_detected); /* endMark not present */ } bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8; @@ -1833,26 +2404,25 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si return srcSize; } -MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getUpperBits(size_t bitContainer, U32 const start) +FORCE_INLINE_TEMPLATE BitContainerType BIT_getUpperBits(BitContainerType bitContainer, U32 const start) { return bitContainer >> start; } -MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) +FORCE_INLINE_TEMPLATE BitContainerType BIT_getMiddleBits(BitContainerType bitContainer, U32 const start, U32 const nbBits) { U32 const regMask = sizeof(bitContainer)*8 - 1; /* if start > regMask, bitstream is corrupted, and result is undefined */ assert(nbBits < BIT_MASK_SIZE); - return (bitContainer >> (start & regMask)) & BIT_mask[nbBits]; -} - -MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) -{ -#if defined(STATIC_BMI2) && STATIC_BMI2 == 1 - return _bzhi_u64(bitContainer, nbBits); + /* x86 transform & ((1 << nbBits) - 1) to bzhi instruction, it is better + * than accessing memory. When bmi2 instruction is not present, we consider + * such cpus old (pre-Haswell, 2013) and their performance is not of that + * importance. + */ +#if defined(__x86_64__) || defined(_M_X64) + return (bitContainer >> (start & regMask)) & ((((U64)1) << nbBits) - 1); #else - assert(nbBits < BIT_MASK_SIZE); - return bitContainer & BIT_mask[nbBits]; + return (bitContainer >> (start & regMask)) & BIT_mask[nbBits]; #endif } @@ -1862,7 +2432,7 @@ MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getLowerBits(size_t bitContainer, U32 co * On 32-bits, maxNbBits==24. * On 64-bits, maxNbBits==56. * @return : value extracted */ -MEM_STATIC FORCE_INLINE_ATTR size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits) +FORCE_INLINE_TEMPLATE BitContainerType BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits) { /* arbitrate between double-shift and shift+mask */ #if 1 @@ -1878,14 +2448,14 @@ MEM_STATIC FORCE_INLINE_ATTR size_t BIT_lookBits(const BIT_DStream_t* bitD, U3 /*! BIT_lookBitsFast() : * unsafe version; only works if nbBits >= 1 */ -MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits) +MEM_STATIC BitContainerType BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits) { U32 const regMask = sizeof(bitD->bitContainer)*8 - 1; assert(nbBits >= 1); return (bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> (((regMask+1)-nbBits) & regMask); } -MEM_STATIC FORCE_INLINE_ATTR void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) +FORCE_INLINE_TEMPLATE void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) { bitD->bitsConsumed += nbBits; } @@ -1894,23 +2464,38 @@ MEM_STATIC FORCE_INLINE_ATTR void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) * Read (consume) next n bits from local register and update. * Pay attention to not read more than nbBits contained into local register. * @return : extracted value. */ -MEM_STATIC FORCE_INLINE_ATTR size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits) +FORCE_INLINE_TEMPLATE BitContainerType BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits) { - size_t const value = BIT_lookBits(bitD, nbBits); + BitContainerType const value = BIT_lookBits(bitD, nbBits); BIT_skipBits(bitD, nbBits); return value; } /*! BIT_readBitsFast() : - * unsafe version; only works only if nbBits >= 1 */ -MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits) + * unsafe version; only works if nbBits >= 1 */ +MEM_STATIC BitContainerType BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits) { - size_t const value = BIT_lookBitsFast(bitD, nbBits); + BitContainerType const value = BIT_lookBitsFast(bitD, nbBits); assert(nbBits >= 1); BIT_skipBits(bitD, nbBits); return value; } +/*! BIT_reloadDStream_internal() : + * Simple variant of BIT_reloadDStream(), with two conditions: + * 1. bitstream is valid : bitsConsumed <= sizeof(bitD->bitContainer)*8 + * 2. look window is valid after shifted down : bitD->ptr >= bitD->start + */ +MEM_STATIC BIT_DStream_status BIT_reloadDStream_internal(BIT_DStream_t* bitD) +{ + assert(bitD->bitsConsumed <= sizeof(bitD->bitContainer)*8); + bitD->ptr -= bitD->bitsConsumed >> 3; + assert(bitD->ptr >= bitD->start); + bitD->bitsConsumed &= 7; + bitD->bitContainer = MEM_readLEST(bitD->ptr); + return BIT_DStream_unfinished; +} + /*! BIT_reloadDStreamFast() : * Similar to BIT_reloadDStream(), but with two differences: * 1. bitsConsumed <= sizeof(bitD->bitContainer)*8 must hold! @@ -1921,31 +2506,35 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStreamFast(BIT_DStream_t* bitD) { if (UNLIKELY(bitD->ptr < bitD->limitPtr)) return BIT_DStream_overflow; - assert(bitD->bitsConsumed <= sizeof(bitD->bitContainer)*8); - bitD->ptr -= bitD->bitsConsumed >> 3; - bitD->bitsConsumed &= 7; - bitD->bitContainer = MEM_readLEST(bitD->ptr); - return BIT_DStream_unfinished; + return BIT_reloadDStream_internal(bitD); } /*! BIT_reloadDStream() : * Refill `bitD` from buffer previously set in BIT_initDStream() . - * This function is safe, it guarantees it will not read beyond src buffer. + * This function is safe, it guarantees it will not never beyond src buffer. * @return : status of `BIT_DStream_t` internal register. * when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */ -MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) +FORCE_INLINE_TEMPLATE BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) { - if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* overflow detected, like end of stream */ + /* note : once in overflow mode, a bitstream remains in this mode until it's reset */ + if (UNLIKELY(bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8))) { + static const BitContainerType zeroFilled = 0; + bitD->ptr = (const char*)&zeroFilled; /* aliasing is allowed for char */ + /* overflow detected, erroneous scenario or end of stream: no update */ return BIT_DStream_overflow; + } + + assert(bitD->ptr >= bitD->start); if (bitD->ptr >= bitD->limitPtr) { - return BIT_reloadDStreamFast(bitD); + return BIT_reloadDStream_internal(bitD); } if (bitD->ptr == bitD->start) { + /* reached end of bitStream => no update */ if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer; return BIT_DStream_completed; } - /* start < ptr < limitPtr */ + /* start < ptr < limitPtr => cautious update */ { U32 nbBytes = bitD->bitsConsumed >> 3; BIT_DStream_status result = BIT_DStream_unfinished; if (bitD->ptr - nbBytes < bitD->start) { @@ -1967,14 +2556,9 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream) return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8)); } -#if defined (__cplusplus) -} -#endif - #endif /* BITSTREAM_H_MODULE */ /**** ended inlining bitstream.h ****/ - /* ***************************************** * Static allocation *******************************************/ @@ -1999,24 +2583,15 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream) unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus); /**< same as FSE_optimalTableLog(), which used `minus==2` */ -/* FSE_compress_wksp() : - * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`). - * FSE_COMPRESS_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable. - */ -#define FSE_COMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + ((maxTableLog > 12) ? (1 << (maxTableLog - 2)) : 1024) ) -size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); - -size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits); -/**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */ - size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue); /**< build a fake FSE_CTable, designed to compress always the same symbolValue */ /* FSE_buildCTable_wksp() : * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). * `wkspSize` must be >= `FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)` of `unsigned`. + * See FSE_buildCTable_wksp() for breakdown of workspace usage. */ -#define FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog) (maxSymbolValue + 2 + (1ull << (tableLog - 2))) +#define FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog) (((maxSymbolValue + 2) + (1ull << (tableLog)))/2 + sizeof(U64)/sizeof(U32) /* additional 8 bytes for potential table overwrite */) #define FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) (sizeof(unsigned) * FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)) size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); @@ -2025,19 +2600,11 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsi FSE_PUBLIC_API size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< Same as FSE_buildDTable(), using an externally allocated `workspace` produced with `FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxSymbolValue)` */ -size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits); -/**< build a fake FSE_DTable, designed to read a flat distribution where each symbol uses nbBits */ - -size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue); -/**< build a fake FSE_DTable, designed to always generate the same symbolValue */ - -#define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue)) +#define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + 1 + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) + (FSE_MAX_SYMBOL_VALUE + 1) / 2 + 1) #define FSE_DECOMPRESS_WKSP_SIZE(maxTableLog, maxSymbolValue) (FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(unsigned)) -size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize); -/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)` */ - size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2); -/**< Same as FSE_decompress_wksp() but with dynamic BMI2 support. Pass 1 if your CPU supports BMI2 or 0 if it doesn't. */ +/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)`. + * Set bmi2 to 1 if your CPU supports BMI2 or 0 if it doesn't */ typedef enum { FSE_repeat_none, /**< Cannot use the previous table */ @@ -2220,20 +2787,20 @@ MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, un FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; const U16* const stateTable = (const U16*)(statePtr->stateTable); U32 const nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16); - BIT_addBits(bitC, statePtr->value, nbBitsOut); + BIT_addBits(bitC, (BitContainerType)statePtr->value, nbBitsOut); statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; } MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr) { - BIT_addBits(bitC, statePtr->value, statePtr->stateLog); + BIT_addBits(bitC, (BitContainerType)statePtr->value, statePtr->stateLog); BIT_flushBits(bitC); } /* FSE_getMaxNbBits() : * Approximate maximum cost of a symbol, in bits. - * Fractional get rounded up (i.e : a symbol with a normalized frequency of 3 gives the same result as a frequency of 2) + * Fractional get rounded up (i.e. a symbol with a normalized frequency of 3 gives the same result as a frequency of 2) * note 1 : assume symbolValue is valid (<= maxSymbolValue) * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */ MEM_STATIC U32 FSE_getMaxNbBits(const void* symbolTTPtr, U32 symbolValue) @@ -2386,20 +2953,13 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) #define FSE_TABLESTEP(tableSize) (((tableSize)>>1) + ((tableSize)>>3) + 3) - #endif /* FSE_STATIC_LINKING_ONLY */ - - -#if defined (__cplusplus) -} -#endif /**** ended inlining fse.h ****/ -#define HUF_STATIC_LINKING_ONLY /* HUF_TABLELOG_ABSOLUTEMAX */ /**** start inlining huf.h ****/ /* ****************************************************************** * huff0 huffman codec, * part of Finite State Entropy library - * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -2410,115 +2970,33 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) * You may select, at your option, one of the above-listed licenses. ****************************************************************** */ -#if defined (__cplusplus) -extern "C" { -#endif - #ifndef HUF_H_298734234 #define HUF_H_298734234 /* *** Dependencies *** */ /**** skipping file: zstd_deps.h ****/ - - -/* *** library symbols visibility *** */ -/* Note : when linking with -fvisibility=hidden on gcc, or by default on Visual, - * HUF symbols remain "private" (internal symbols for library only). - * Set macro FSE_DLL_EXPORT to 1 if you want HUF symbols visible on DLL interface */ -#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4) -# define HUF_PUBLIC_API __attribute__ ((visibility ("default"))) -#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */ -# define HUF_PUBLIC_API __declspec(dllexport) -#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1) -# define HUF_PUBLIC_API __declspec(dllimport) /* not required, just to generate faster code (saves a function pointer load from IAT and an indirect jump) */ -#else -# define HUF_PUBLIC_API -#endif - - -/* ========================== */ -/* *** simple functions *** */ -/* ========================== */ - -/** HUF_compress() : - * Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'. - * 'dst' buffer must be already allocated. - * Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize). - * `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB. - * @return : size of compressed data (<= `dstCapacity`). - * Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! - * if HUF_isError(return), compression failed (more details using HUF_getErrorName()) - */ -HUF_PUBLIC_API size_t HUF_compress(void* dst, size_t dstCapacity, - const void* src, size_t srcSize); - -/** HUF_decompress() : - * Decompress HUF data from buffer 'cSrc', of size 'cSrcSize', - * into already allocated buffer 'dst', of minimum size 'dstSize'. - * `originalSize` : **must** be the ***exact*** size of original (uncompressed) data. - * Note : in contrast with FSE, HUF_decompress can regenerate - * RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data, - * because it knows size to regenerate (originalSize). - * @return : size of regenerated data (== originalSize), - * or an error code, which can be tested using HUF_isError() - */ -HUF_PUBLIC_API size_t HUF_decompress(void* dst, size_t originalSize, - const void* cSrc, size_t cSrcSize); - +/**** skipping file: mem.h ****/ +#define FSE_STATIC_LINKING_ONLY +/**** skipping file: fse.h ****/ /* *** Tool functions *** */ -#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */ -HUF_PUBLIC_API size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ +#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */ +size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ /* Error Management */ -HUF_PUBLIC_API unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ -HUF_PUBLIC_API const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ - - -/* *** Advanced function *** */ +unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ +const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ -/** HUF_compress2() : - * Same as HUF_compress(), but offers control over `maxSymbolValue` and `tableLog`. - * `maxSymbolValue` must be <= HUF_SYMBOLVALUE_MAX . - * `tableLog` must be `<= HUF_TABLELOG_MAX` . */ -HUF_PUBLIC_API size_t HUF_compress2 (void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned tableLog); - -/** HUF_compress4X_wksp() : - * Same as HUF_compress2(), but uses externally allocated `workSpace`. - * `workspace` must have minimum alignment of 4, and be at least as large as HUF_WORKSPACE_SIZE */ -#define HUF_WORKSPACE_SIZE ((6 << 10) + 256) -#define HUF_WORKSPACE_SIZE_U32 (HUF_WORKSPACE_SIZE / sizeof(U32)) -HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned tableLog, - void* workSpace, size_t wkspSize); - -#endif /* HUF_H_298734234 */ - -/* ****************************************************************** - * WARNING !! - * The following section contains advanced and experimental definitions - * which shall never be used in the context of a dynamic library, - * because they are not guaranteed to remain stable in the future. - * Only consider them in association with static linking. - * *****************************************************************/ -#if defined(HUF_STATIC_LINKING_ONLY) && !defined(HUF_H_HUF_STATIC_LINKING_ONLY) -#define HUF_H_HUF_STATIC_LINKING_ONLY - -/* *** Dependencies *** */ -/**** skipping file: mem.h ****/ -#define FSE_STATIC_LINKING_ONLY -/**** skipping file: fse.h ****/ +#define HUF_WORKSPACE_SIZE ((8 << 10) + 512 /* sorting scratch space */) +#define HUF_WORKSPACE_SIZE_U64 (HUF_WORKSPACE_SIZE / sizeof(U64)) /* *** Constants *** */ -#define HUF_TABLELOG_MAX 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */ +#define HUF_TABLELOG_MAX 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_TABLELOG_ABSOLUTEMAX */ #define HUF_TABLELOG_DEFAULT 11 /* default tableLog value when none specified */ #define HUF_SYMBOLVALUE_MAX 255 -#define HUF_TABLELOG_ABSOLUTEMAX 15 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ +#define HUF_TABLELOG_ABSOLUTEMAX 12 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ #if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX) # error "HUF_TABLELOG_MAX is too large !" #endif @@ -2534,15 +3012,11 @@ HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity, /* static allocation of HUF's Compression Table */ /* this is a private definition, just exposed for allocation and strict aliasing purpose. never EVER access its members directly */ -struct HUF_CElt_s { - U16 val; - BYTE nbBits; -}; /* typedef'd to HUF_CElt */ -typedef struct HUF_CElt_s HUF_CElt; /* consider it an incomplete type */ -#define HUF_CTABLE_SIZE_U32(maxSymbolValue) ((maxSymbolValue)+1) /* Use tables of U32, for proper alignment */ -#define HUF_CTABLE_SIZE(maxSymbolValue) (HUF_CTABLE_SIZE_U32(maxSymbolValue) * sizeof(U32)) +typedef size_t HUF_CElt; /* consider it an incomplete type */ +#define HUF_CTABLE_SIZE_ST(maxSymbolValue) ((maxSymbolValue)+2) /* Use tables of size_t, for proper alignment */ +#define HUF_CTABLE_SIZE(maxSymbolValue) (HUF_CTABLE_SIZE_ST(maxSymbolValue) * sizeof(size_t)) #define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \ - HUF_CElt name[HUF_CTABLE_SIZE_U32(maxSymbolValue)] /* no final ; */ + HUF_CElt name[HUF_CTABLE_SIZE_ST(maxSymbolValue)] /* no final ; */ /* static allocation of HUF's DTable */ typedef U32 HUF_DTable; @@ -2556,25 +3030,49 @@ typedef U32 HUF_DTable; /* **************************************** * Advanced decompression functions ******************************************/ -size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ -#endif -size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */ -size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */ -size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< considers RLE and uncompressed as errors */ -size_t HUF_decompress4X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ -size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */ -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ -size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */ -#endif +/** + * Huffman flags bitset. + * For all flags, 0 is the default value. + */ +typedef enum { + /** + * If compiled with DYNAMIC_BMI2: Set flag only if the CPU supports BMI2 at runtime. + * Otherwise: Ignored. + */ + HUF_flags_bmi2 = (1 << 0), + /** + * If set: Test possible table depths to find the one that produces the smallest header + encoded size. + * If unset: Use heuristic to find the table depth. + */ + HUF_flags_optimalDepth = (1 << 1), + /** + * If set: If the previous table can encode the input, always reuse the previous table. + * If unset: If the previous table can encode the input, reuse the previous table if it results in a smaller output. + */ + HUF_flags_preferRepeat = (1 << 2), + /** + * If set: Sample the input and check if the sample is uncompressible, if it is then don't attempt to compress. + * If unset: Always histogram the entire input. + */ + HUF_flags_suspectUncompressible = (1 << 3), + /** + * If set: Don't use assembly implementations + * If unset: Allow using assembly implementations + */ + HUF_flags_disableAsm = (1 << 4), + /** + * If set: Don't use the fast decoding loop, always use the fallback decoding loop. + * If unset: Use the fast decoding loop when possible. + */ + HUF_flags_disableFast = (1 << 5) +} HUF_flags_e; /* **************************************** * HUF detailed API * ****************************************/ +#define HUF_OPTIMAL_DEPTH_THRESHOLD ZSTD_btultra /*! HUF_compress() does the following: * 1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h") @@ -2587,10 +3085,12 @@ size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, * For example, it's possible to compress several blocks using the same 'CTable', * or to save and regenerate 'CTable' using external methods. */ -unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); -size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); /* @return : maxNbBits; CTable and count can overlap. In which case, CTable will overwrite count content */ -size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog); -size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); +unsigned HUF_minTableLog(unsigned symbolCardinality); +unsigned HUF_cardinality(const unsigned* count, unsigned maxSymbolValue); +unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace, + size_t wkspSize, HUF_CElt* table, const unsigned* count, int flags); /* table is used as scratch space for building and testing tables, not a return value */ +size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, void* workspace, size_t workspaceSize); +size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags); size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue); int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue); @@ -2599,22 +3099,24 @@ typedef enum { HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */ HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */ } HUF_repeat; + /** HUF_compress4X_repeat() : * Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. * If it uses hufTable it does not modify hufTable or repeat. * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. - * If preferRepeat then the old table will always be used if valid. */ + * If preferRepeat then the old table will always be used if valid. + * If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */ size_t HUF_compress4X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */ - HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2); + HUF_CElt* hufTable, HUF_repeat* repeat, int flags); /** HUF_buildCTable_wksp() : * Same as HUF_buildCTable(), but using externally allocated scratch buffer. * `workSpace` must be aligned on 4-bytes boundaries, and its size must be >= HUF_CTABLE_WORKSPACE_SIZE. */ -#define HUF_CTABLE_WORKSPACE_SIZE_U32 (2*HUF_SYMBOLVALUE_MAX +1 +1) +#define HUF_CTABLE_WORKSPACE_SIZE_U32 ((4 * (HUF_SYMBOLVALUE_MAX + 1)) + 192) #define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned)) size_t HUF_buildCTable_wksp (HUF_CElt* tree, const unsigned* count, U32 maxSymbolValue, U32 maxNbBits, @@ -2640,17 +3142,29 @@ size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, const void* src, size_t srcSize, void* workspace, size_t wkspSize, - int bmi2); + int flags); /** HUF_readCTable() : * Loading a CTable saved with HUF_writeCTable() */ size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned *hasZeroWeights); -/** HUF_getNbBits() : +/** HUF_getNbBitsFromCTable() : * Read nbBits from CTable symbolTable, for symbol `symbolValue` presumed <= HUF_SYMBOLVALUE_MAX - * Note 1 : is not inlined, as HUF_CElt definition is private - * Note 2 : const void* used, so that it can provide a statically allocated table as argument (which uses type U32) */ -U32 HUF_getNbBits(const void* symbolTable, U32 symbolValue); + * Note 1 : If symbolValue > HUF_readCTableHeader(symbolTable).maxSymbolValue, returns 0 + * Note 2 : is not inlined, as HUF_CElt definition is private + */ +U32 HUF_getNbBitsFromCTable(const HUF_CElt* symbolTable, U32 symbolValue); + +typedef struct { + BYTE tableLog; + BYTE maxSymbolValue; + BYTE unused[sizeof(size_t) - 2]; +} HUF_CTableHeader; + +/** HUF_readCTableHeader() : + * @returns The header from the CTable specifying the tableLog and the maxSymbolValue. + */ +HUF_CTableHeader HUF_readCTableHeader(HUF_CElt const* ctable); /* * HUF_decompress() does the following: @@ -2676,88 +3190,51 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize); * a required workspace size greater than that specified in the following * macro. */ -#define HUF_DECOMPRESS_WORKSPACE_SIZE (2 << 10) +#define HUF_DECOMPRESS_WORKSPACE_SIZE ((2 << 10) + (1 << 9)) #define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32)) -#ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_readDTableX1 (HUF_DTable* DTable, const void* src, size_t srcSize); -size_t HUF_readDTableX1_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize); -#endif -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize); -size_t HUF_readDTableX2_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize); -#endif - -size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); -#ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_decompress4X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); -#endif -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); -#endif - /* ====================== */ /* single stream variants */ /* ====================== */ -size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); -size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */ -size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags); /** HUF_compress1X_repeat() : * Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. * If it uses hufTable it does not modify hufTable or repeat. * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. - * If preferRepeat then the old table will always be used if valid. */ + * If preferRepeat then the old table will always be used if valid. + * If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */ size_t HUF_compress1X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */ - HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2); - -size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */ -#endif + HUF_CElt* hufTable, HUF_repeat* repeat, int flags); -size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); -size_t HUF_decompress1X_DCtx_wksp (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); -#ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_decompress1X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ -size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */ -#endif -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ -size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */ -#endif - -size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */ -#ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_decompress1X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); -#endif +size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); #ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); /**< double-symbols decoder */ #endif /* BMI2 variants. * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0. */ -size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2); +size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags); #ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2); +size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); #endif -size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2); -size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2); +size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags); +size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); #ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2); +size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags); #endif - -#endif /* HUF_STATIC_LINKING_ONLY */ - -#if defined (__cplusplus) -} +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags); #endif + +#endif /* HUF_H_298734234 */ /**** ended inlining huf.h ****/ +/**** skipping file: bits.h ****/ /*=== Version ===*/ @@ -2775,28 +3252,6 @@ const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); } /*-************************************************************** * FSE NCount encoding-decoding ****************************************************************/ -static U32 FSE_ctz(U32 val) -{ - assert(val != 0); - { -# if defined(_MSC_VER) /* Visual */ - unsigned long r=0; - return _BitScanForward(&r, val) ? (unsigned)r : 0; -# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */ - return __builtin_ctz(val); -# elif defined(__ICCARM__) /* IAR Intrinsic */ - return __CTZ(val); -# else /* Software version */ - U32 count = 0; - while ((val & 1) == 0) { - val >>= 1; - ++count; - } - return count; -# endif - } -} - FORCE_INLINE_TEMPLATE size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, const void* headerBuffer, size_t hbSize) @@ -2844,7 +3299,7 @@ size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigne * repeat. * Avoid UB by setting the high bit to 1. */ - int repeats = FSE_ctz(~bitStream | 0x80000000) >> 1; + int repeats = ZSTD_countTrailingZeros32(~bitStream | 0x80000000) >> 1; while (repeats >= 12) { charnum += 3 * 12; if (LIKELY(ip <= iend-7)) { @@ -2855,7 +3310,7 @@ size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigne ip = iend - 4; } bitStream = MEM_readLE32(ip) >> bitCount; - repeats = FSE_ctz(~bitStream | 0x80000000) >> 1; + repeats = ZSTD_countTrailingZeros32(~bitStream | 0x80000000) >> 1; } charnum += 3 * repeats; bitStream >>= 2 * repeats; @@ -2920,7 +3375,7 @@ size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigne * know that threshold > 1. */ if (remaining <= 1) break; - nbBits = BIT_highbit32(remaining) + 1; + nbBits = ZSTD_highbit32(remaining) + 1; threshold = 1 << (nbBits - 1); } if (charnum >= maxSV1) break; @@ -2954,7 +3409,7 @@ static size_t FSE_readNCount_body_default( } #if DYNAMIC_BMI2 -TARGET_ATTRIBUTE("bmi2") static size_t FSE_readNCount_body_bmi2( +BMI2_TARGET_ATTRIBUTE static size_t FSE_readNCount_body_bmi2( short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, const void* headerBuffer, size_t hbSize) { @@ -2995,7 +3450,7 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, const void* src, size_t srcSize) { U32 wksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; - return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* bmi2 */ 0); + return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* flags */ 0); } FORCE_INLINE_TEMPLATE size_t @@ -3036,21 +3491,21 @@ HUF_readStats_body(BYTE* huffWeight, size_t hwSize, U32* rankStats, ZSTD_memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32)); weightTotal = 0; { U32 n; for (n=0; n= HUF_TABLELOG_MAX) return ERROR(corruption_detected); + if (huffWeight[n] > HUF_TABLELOG_MAX) return ERROR(corruption_detected); rankStats[huffWeight[n]]++; weightTotal += (1 << huffWeight[n]) >> 1; } } if (weightTotal == 0) return ERROR(corruption_detected); /* get last non-null symbol weight (implied, total must be 2^n) */ - { U32 const tableLog = BIT_highbit32(weightTotal) + 1; + { U32 const tableLog = ZSTD_highbit32(weightTotal) + 1; if (tableLog > HUF_TABLELOG_MAX) return ERROR(corruption_detected); *tableLogPtr = tableLog; /* determine last weight */ { U32 const total = 1 << tableLog; U32 const rest = total - weightTotal; - U32 const verif = 1 << BIT_highbit32(rest); - U32 const lastWeight = BIT_highbit32(rest) + 1; + U32 const verif = 1 << ZSTD_highbit32(rest); + U32 const lastWeight = ZSTD_highbit32(rest) + 1; if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ huffWeight[oSize] = (BYTE)lastWeight; rankStats[lastWeight]++; @@ -3074,7 +3529,7 @@ static size_t HUF_readStats_body_default(BYTE* huffWeight, size_t hwSize, U32* r } #if DYNAMIC_BMI2 -static TARGET_ATTRIBUTE("bmi2") size_t HUF_readStats_body_bmi2(BYTE* huffWeight, size_t hwSize, U32* rankStats, +static BMI2_TARGET_ATTRIBUTE size_t HUF_readStats_body_bmi2(BYTE* huffWeight, size_t hwSize, U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, const void* src, size_t srcSize, void* workSpace, size_t wkspSize) @@ -3087,20 +3542,20 @@ size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, - int bmi2) + int flags) { #if DYNAMIC_BMI2 - if (bmi2) { + if (flags & HUF_flags_bmi2) { return HUF_readStats_body_bmi2(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize); } #endif - (void)bmi2; + (void)flags; return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize); } /**** ended inlining common/entropy_common.c ****/ /**** start inlining common/error_private.c ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -3128,9 +3583,11 @@ const char* ERR_getErrorString(ERR_enum code) case PREFIX(version_unsupported): return "Version not supported"; case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding"; - case PREFIX(corruption_detected): return "Corrupted block detected"; + case PREFIX(corruption_detected): return "Data corruption detected"; case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; + case PREFIX(literals_headerWrong): return "Header of Literals' block doesn't respect format specification"; case PREFIX(parameter_unsupported): return "Unsupported parameter"; + case PREFIX(parameter_combination_unsupported): return "Unsupported combination of parameters"; case PREFIX(parameter_outOfBound): return "Parameter is out of bound"; case PREFIX(init_missing): return "Context should be init first"; case PREFIX(memory_allocation): return "Allocation error : not enough memory"; @@ -3139,17 +3596,23 @@ const char* ERR_getErrorString(ERR_enum code) case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; + case PREFIX(cannotProduce_uncompressedBlock): return "This mode cannot generate an uncompressed block"; + case PREFIX(stabilityCondition_notRespected): return "pledged buffer stability condition is not respected"; case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; case PREFIX(dictionary_wrong): return "Dictionary mismatch"; case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples"; case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; case PREFIX(srcSize_wrong): return "Src size is incorrect"; case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer"; + case PREFIX(noForwardProgress_destFull): return "Operation made no progress over multiple calls, due to output buffer being full"; + case PREFIX(noForwardProgress_inputEmpty): return "Operation made no progress over multiple calls, due to input being empty"; /* following error codes are not stable and may be removed or changed in a future version */ case PREFIX(frameIndex_tooLarge): return "Frame index is too large"; case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking"; case PREFIX(dstBuffer_wrong): return "Destination buffer is wrong"; case PREFIX(srcBuffer_wrong): return "Source buffer is wrong"; + case PREFIX(sequenceProducer_failed): return "Block-level external sequence producer returned an error code"; + case PREFIX(externalSequences_invalid): return "External sequences are not valid"; case PREFIX(maxCode): default: return notErrorCode; } @@ -3159,7 +3622,7 @@ const char* ERR_getErrorString(ERR_enum code) /**** start inlining common/fse_decompress.c ****/ /* ****************************************************************** * FSE : Finite State Entropy decoder - * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -3181,8 +3644,8 @@ const char* ERR_getErrorString(ERR_enum code) #define FSE_STATIC_LINKING_ONLY /**** skipping file: fse.h ****/ /**** skipping file: error_private.h ****/ -#define ZSTD_DEPS_NEED_MALLOC /**** skipping file: zstd_deps.h ****/ +/**** skipping file: bits.h ****/ /* ************************************************************** @@ -3214,19 +3677,6 @@ const char* ERR_getErrorString(ERR_enum code) #define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) #define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) - -/* Function templates */ -FSE_DTable* FSE_createDTable (unsigned tableLog) -{ - if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; - return (FSE_DTable*)ZSTD_malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) ); -} - -void FSE_freeDTable (FSE_DTable* dt) -{ - ZSTD_free(dt); -} - static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize) { void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */ @@ -3255,7 +3705,7 @@ static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCo symbolNext[s] = 1; } else { if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0; - symbolNext[s] = normalizedCounter[s]; + symbolNext[s] = (U16)normalizedCounter[s]; } } } ZSTD_memcpy(dt, &DTableH, sizeof(DTableH)); } @@ -3270,8 +3720,7 @@ static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCo * all symbols have counts <= 8. We ensure we have 8 bytes at the end of * our buffer to handle the over-write. */ - { - U64 const add = 0x0101010101010101ull; + { U64 const add = 0x0101010101010101ull; size_t pos = 0; U64 sv = 0; U32 s; @@ -3282,14 +3731,13 @@ static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCo for (i = 8; i < n; i += 8) { MEM_write64(spread + pos + i, sv); } - pos += n; - } - } + pos += (size_t)n; + } } /* Now we spread those positions across the table. - * The benefit of doing it in two stages is that we avoid the the + * The benefit of doing it in two stages is that we avoid the * variable size inner loop, which caused lots of branch misses. * Now we can run through all the positions without any branch misses. - * We unroll the loop twice, since that is what emperically worked best. + * We unroll the loop twice, since that is what empirically worked best. */ { size_t position = 0; @@ -3325,7 +3773,7 @@ static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCo for (u=0; utableLog = 0; - DTableH->fastMode = 0; - - cell->newState = 0; - cell->symbol = symbolValue; - cell->nbBits = 0; - - return 0; -} - - -size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits) -{ - void* ptr = dt; - FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; - void* dPtr = dt + 1; - FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr; - const unsigned tableSize = 1 << nbBits; - const unsigned tableMask = tableSize - 1; - const unsigned maxSV1 = tableMask+1; - unsigned s; - - /* Sanity checks */ - if (nbBits < 1) return ERROR(GENERIC); /* min size */ - - /* Build Decoding Table */ - DTableH->tableLog = (U16)nbBits; - DTableH->fastMode = 1; - for (s=0; sfastMode; - - /* select fast mode (static) */ - if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); - return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); + assert(op >= ostart); + return (size_t)(op-ostart); } +typedef struct { + short ncount[FSE_MAX_SYMBOL_VALUE + 1]; +} FSE_DecompressWksp; -size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) -{ - return FSE_decompress_wksp_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, /* bmi2 */ 0); -} FORCE_INLINE_TEMPLATE size_t FSE_decompress_wksp_body( void* dst, size_t dstCapacity, @@ -3477,24 +3870,34 @@ FORCE_INLINE_TEMPLATE size_t FSE_decompress_wksp_body( { const BYTE* const istart = (const BYTE*)cSrc; const BYTE* ip = istart; - short counting[FSE_MAX_SYMBOL_VALUE+1]; unsigned tableLog; unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; - FSE_DTable* const dtable = (FSE_DTable*)workSpace; + FSE_DecompressWksp* const wksp = (FSE_DecompressWksp*)workSpace; + size_t const dtablePos = sizeof(FSE_DecompressWksp) / sizeof(FSE_DTable); + FSE_DTable* const dtable = (FSE_DTable*)workSpace + dtablePos; + + FSE_STATIC_ASSERT((FSE_MAX_SYMBOL_VALUE + 1) % 2 == 0); + if (wkspSize < sizeof(*wksp)) return ERROR(GENERIC); + + /* correct offset to dtable depends on this property */ + FSE_STATIC_ASSERT(sizeof(FSE_DecompressWksp) % sizeof(FSE_DTable) == 0); /* normal FSE decoding mode */ - size_t const NCountLength = FSE_readNCount_bmi2(counting, &maxSymbolValue, &tableLog, istart, cSrcSize, bmi2); - if (FSE_isError(NCountLength)) return NCountLength; - if (tableLog > maxLog) return ERROR(tableLog_tooLarge); - assert(NCountLength <= cSrcSize); - ip += NCountLength; - cSrcSize -= NCountLength; + { size_t const NCountLength = + FSE_readNCount_bmi2(wksp->ncount, &maxSymbolValue, &tableLog, istart, cSrcSize, bmi2); + if (FSE_isError(NCountLength)) return NCountLength; + if (tableLog > maxLog) return ERROR(tableLog_tooLarge); + assert(NCountLength <= cSrcSize); + ip += NCountLength; + cSrcSize -= NCountLength; + } if (FSE_DECOMPRESS_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(tableLog_tooLarge); - workSpace = dtable + FSE_DTABLE_SIZE_U32(tableLog); - wkspSize -= FSE_DTABLE_SIZE(tableLog); + assert(sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog) <= wkspSize); + workSpace = (BYTE*)workSpace + sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog); + wkspSize -= sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog); - CHECK_F( FSE_buildDTable_internal(dtable, counting, maxSymbolValue, tableLog, workSpace, wkspSize) ); + CHECK_F( FSE_buildDTable_internal(dtable, wksp->ncount, maxSymbolValue, tableLog, workSpace, wkspSize) ); { const void* ptr = dtable; @@ -3514,7 +3917,7 @@ static size_t FSE_decompress_wksp_body_default(void* dst, size_t dstCapacity, co } #if DYNAMIC_BMI2 -TARGET_ATTRIBUTE("bmi2") static size_t FSE_decompress_wksp_body_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) +BMI2_TARGET_ATTRIBUTE static size_t FSE_decompress_wksp_body_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) { return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 1); } @@ -3531,24 +3934,6 @@ size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, return FSE_decompress_wksp_body_default(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize); } - -typedef FSE_DTable DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)]; - -#ifndef ZSTD_NO_UNUSED_FUNCTIONS -size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) { - U32 wksp[FSE_BUILD_DTABLE_WKSP_SIZE_U32(FSE_TABLELOG_ABSOLUTE_MAX, FSE_MAX_SYMBOL_VALUE)]; - return FSE_buildDTable_wksp(dt, normalizedCounter, maxSymbolValue, tableLog, wksp, sizeof(wksp)); -} - -size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize) -{ - /* Static analyzer seems unable to understand this table will be properly initialized later */ - U32 wksp[FSE_DECOMPRESS_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)]; - return FSE_decompress_wksp(dst, dstCapacity, cSrc, cSrcSize, FSE_MAX_TABLELOG, wksp, sizeof(wksp)); -} -#endif - - #endif /* FSE_COMMONDEFS_ONLY */ /**** ended inlining common/fse_decompress.c ****/ /**** start inlining common/threading.c ****/ @@ -3588,15 +3973,10 @@ size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cS /**** skipping file: debug.h ****/ -#if defined (__cplusplus) -extern "C" { -#endif - #if defined(ZSTD_MULTITHREAD) && defined(_WIN32) /** - * Windows minimalist Pthread Wrapper, based on : - * http://www.cse.wustl.edu/~schmidt/win32-cv-1.html + * Windows minimalist Pthread Wrapper */ #ifdef WINVER # undef WINVER @@ -3634,22 +4014,17 @@ extern "C" { #define ZSTD_pthread_cond_broadcast(a) WakeAllConditionVariable((a)) /* ZSTD_pthread_create() and ZSTD_pthread_join() */ -typedef struct { - HANDLE handle; - void* (*start_routine)(void*); - void* arg; -} ZSTD_pthread_t; +typedef HANDLE ZSTD_pthread_t; int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused, void* (*start_routine) (void*), void* arg); -int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr); +int ZSTD_pthread_join(ZSTD_pthread_t thread); /** * add here more wrappers as required */ - #elif defined(ZSTD_MULTITHREAD) /* posix assumed ; need a better detection method */ /* === POSIX Systems === */ # include @@ -3671,7 +4046,7 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr); #define ZSTD_pthread_t pthread_t #define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d)) -#define ZSTD_pthread_join(a, b) pthread_join((a),(b)) +#define ZSTD_pthread_join(a) pthread_join((a),NULL) #else /* DEBUGLEVEL >= 1 */ @@ -3696,7 +4071,7 @@ int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond); #define ZSTD_pthread_t pthread_t #define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d)) -#define ZSTD_pthread_join(a, b) pthread_join((a),(b)) +#define ZSTD_pthread_join(a) pthread_join((a),NULL) #endif @@ -3720,9 +4095,6 @@ typedef int ZSTD_pthread_cond_t; #endif /* ZSTD_MULTITHREAD */ -#if defined (__cplusplus) -} -#endif #endif /* THREADING_H_938743 */ /**** ended inlining threading.h ****/ @@ -3733,8 +4105,7 @@ int g_ZSTD_threading_useless_symbol; #if defined(ZSTD_MULTITHREAD) && defined(_WIN32) /** - * Windows minimalist Pthread Wrapper, based on : - * http://www.cse.wustl.edu/~schmidt/win32-cv-1.html + * Windows minimalist Pthread Wrapper */ @@ -3745,37 +4116,94 @@ int g_ZSTD_threading_useless_symbol; /* === Implementation === */ +typedef struct { + void* (*start_routine)(void*); + void* arg; + int initialized; + ZSTD_pthread_cond_t initialized_cond; + ZSTD_pthread_mutex_t initialized_mutex; +} ZSTD_thread_params_t; + static unsigned __stdcall worker(void *arg) { - ZSTD_pthread_t* const thread = (ZSTD_pthread_t*) arg; - thread->arg = thread->start_routine(thread->arg); + void* (*start_routine)(void*); + void* thread_arg; + + /* Initialized thread_arg and start_routine and signal main thread that we don't need it + * to wait any longer. + */ + { + ZSTD_thread_params_t* thread_param = (ZSTD_thread_params_t*)arg; + thread_arg = thread_param->arg; + start_routine = thread_param->start_routine; + + /* Signal main thread that we are running and do not depend on its memory anymore */ + ZSTD_pthread_mutex_lock(&thread_param->initialized_mutex); + thread_param->initialized = 1; + ZSTD_pthread_cond_signal(&thread_param->initialized_cond); + ZSTD_pthread_mutex_unlock(&thread_param->initialized_mutex); + } + + start_routine(thread_arg); + return 0; } int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused, void* (*start_routine) (void*), void* arg) { + ZSTD_thread_params_t thread_param; (void)unused; - thread->arg = arg; - thread->start_routine = start_routine; - thread->handle = (HANDLE) _beginthreadex(NULL, 0, worker, thread, 0, NULL); - if (!thread->handle) + if (thread==NULL) return -1; + *thread = NULL; + + thread_param.start_routine = start_routine; + thread_param.arg = arg; + thread_param.initialized = 0; + + /* Setup thread initialization synchronization */ + if(ZSTD_pthread_cond_init(&thread_param.initialized_cond, NULL)) { + /* Should never happen on Windows */ + return -1; + } + if(ZSTD_pthread_mutex_init(&thread_param.initialized_mutex, NULL)) { + /* Should never happen on Windows */ + ZSTD_pthread_cond_destroy(&thread_param.initialized_cond); + return -1; + } + + /* Spawn thread */ + *thread = (HANDLE)_beginthreadex(NULL, 0, worker, &thread_param, 0, NULL); + if (*thread==NULL) { + ZSTD_pthread_mutex_destroy(&thread_param.initialized_mutex); + ZSTD_pthread_cond_destroy(&thread_param.initialized_cond); return errno; - else - return 0; + } + + /* Wait for thread to be initialized */ + ZSTD_pthread_mutex_lock(&thread_param.initialized_mutex); + while(!thread_param.initialized) { + ZSTD_pthread_cond_wait(&thread_param.initialized_cond, &thread_param.initialized_mutex); + } + ZSTD_pthread_mutex_unlock(&thread_param.initialized_mutex); + ZSTD_pthread_mutex_destroy(&thread_param.initialized_mutex); + ZSTD_pthread_cond_destroy(&thread_param.initialized_cond); + + return 0; } -int ZSTD_pthread_join(ZSTD_pthread_t thread, void **value_ptr) +int ZSTD_pthread_join(ZSTD_pthread_t thread) { DWORD result; - if (!thread.handle) return 0; + if (!thread) return 0; + + result = WaitForSingleObject(thread, INFINITE); + CloseHandle(thread); - result = WaitForSingleObject(thread.handle, INFINITE); switch (result) { case WAIT_OBJECT_0: - if (value_ptr) *value_ptr = thread.arg; return 0; case WAIT_ABANDONED: return EINVAL; @@ -3793,6 +4221,7 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread, void **value_ptr) int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr) { + assert(mutex != NULL); *mutex = (pthread_mutex_t*)ZSTD_malloc(sizeof(pthread_mutex_t)); if (!*mutex) return 1; @@ -3801,6 +4230,7 @@ int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t con int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex) { + assert(mutex != NULL); if (!*mutex) return 0; { @@ -3812,6 +4242,7 @@ int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex) int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr) { + assert(cond != NULL); *cond = (pthread_cond_t*)ZSTD_malloc(sizeof(pthread_cond_t)); if (!*cond) return 1; @@ -3820,6 +4251,7 @@ int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond) { + assert(cond != NULL); if (!*cond) return 0; { @@ -3833,7 +4265,7 @@ int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond) /**** ended inlining common/threading.c ****/ /**** start inlining common/pool.c ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -3844,11 +4276,9 @@ int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond) /* ====== Dependencies ======= */ -/**** skipping file: zstd_deps.h ****/ -/**** skipping file: debug.h ****/ -/**** start inlining zstd_internal.h ****/ +/**** start inlining ../common/allocations.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -3857,8672 +4287,10744 @@ int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond) * You may select, at your option, one of the above-listed licenses. */ -#ifndef ZSTD_CCOMMON_H_MODULE -#define ZSTD_CCOMMON_H_MODULE +/* This file provides custom allocation primitives + */ -/* this module contains definitions which must be identical - * across compression, decompression and dictBuilder. - * It also contains a few functions useful to at least 2 of them - * and which benefit from being inlined */ +#define ZSTD_DEPS_NEED_MALLOC +/**** skipping file: zstd_deps.h ****/ -/*-************************************* -* Dependencies -***************************************/ -#if !defined(ZSTD_NO_INTRINSICS) && defined(__ARM_NEON) -#include -#endif /**** skipping file: compiler.h ****/ -/**** skipping file: mem.h ****/ -/**** skipping file: debug.h ****/ -/**** skipping file: error_private.h ****/ #define ZSTD_STATIC_LINKING_ONLY -/**** start inlining ../zstd.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ -#if defined (__cplusplus) -extern "C" { -#endif +/**** *NOT* inlining ../zstd.h ****/ +#include "zstd.h" /* ZSTD_customMem */ -#ifndef ZSTD_H_235446 -#define ZSTD_H_235446 +#ifndef ZSTD_ALLOCATIONS_H +#define ZSTD_ALLOCATIONS_H -/* ====== Dependency ======*/ -#include /* INT_MAX */ -#include /* size_t */ +/* custom memory allocation functions */ +MEM_STATIC void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem) +{ + if (customMem.customAlloc) + return customMem.customAlloc(customMem.opaque, size); + return ZSTD_malloc(size); +} -/* ===== ZSTDLIB_API : control library symbols visibility ===== */ -#ifndef ZSTDLIB_VISIBILITY -# if defined(__GNUC__) && (__GNUC__ >= 4) -# define ZSTDLIB_VISIBILITY __attribute__ ((visibility ("default"))) -# else -# define ZSTDLIB_VISIBILITY -# endif -#endif -#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) -# define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBILITY -#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) -# define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ -#else -# define ZSTDLIB_API ZSTDLIB_VISIBILITY -#endif - - -/******************************************************************************* - Introduction - - zstd, short for Zstandard, is a fast lossless compression algorithm, targeting - real-time compression scenarios at zlib-level and better compression ratios. - The zstd compression library provides in-memory compression and decompression - functions. - - The library supports regular compression levels from 1 up to ZSTD_maxCLevel(), - which is currently 22. Levels >= 20, labeled `--ultra`, should be used with - caution, as they require more memory. The library also offers negative - compression levels, which extend the range of speed vs. ratio preferences. - The lower the level, the faster the speed (at the cost of compression). - - Compression can be done in: - - a single step (described as Simple API) - - a single step, reusing a context (described as Explicit context) - - unbounded multiple steps (described as Streaming compression) - - The compression ratio achievable on small data can be highly improved using - a dictionary. Dictionary compression can be performed in: - - a single step (described as Simple dictionary API) - - a single step, reusing a dictionary (described as Bulk-processing - dictionary API) - - Advanced experimental functions can be accessed using - `#define ZSTD_STATIC_LINKING_ONLY` before including zstd.h. - - Advanced experimental APIs should never be used with a dynamically-linked - library. They are not "stable"; their definitions or signatures may change in - the future. Only static linking is allowed. -*******************************************************************************/ - -/*------ Version ------*/ -#define ZSTD_VERSION_MAJOR 1 -#define ZSTD_VERSION_MINOR 4 -#define ZSTD_VERSION_RELEASE 9 -#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) - -/*! ZSTD_versionNumber() : - * Return runtime library version, the value is (MAJOR*100*100 + MINOR*100 + RELEASE). */ -ZSTDLIB_API unsigned ZSTD_versionNumber(void); - -#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE -#define ZSTD_QUOTE(str) #str -#define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str) -#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION) - -/*! ZSTD_versionString() : - * Return runtime library version, like "1.4.5". Requires v1.3.0+. */ -ZSTDLIB_API const char* ZSTD_versionString(void); - -/* ************************************* - * Default constant - ***************************************/ -#ifndef ZSTD_CLEVEL_DEFAULT -# define ZSTD_CLEVEL_DEFAULT 3 -#endif - -/* ************************************* - * Constants - ***************************************/ - -/* All magic numbers are supposed read/written to/from files/memory using little-endian convention */ -#define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */ -#define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* valid since v0.7.0 */ -#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50 /* all 16 values, from 0x184D2A50 to 0x184D2A5F, signal the beginning of a skippable frame */ -#define ZSTD_MAGIC_SKIPPABLE_MASK 0xFFFFFFF0 - -#define ZSTD_BLOCKSIZELOG_MAX 17 -#define ZSTD_BLOCKSIZE_MAX (1<= `ZSTD_compressBound(srcSize)`. - * @return : compressed size written into `dst` (<= `dstCapacity), - * or an error code if it fails (which can be tested using ZSTD_isError()). */ -ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - int compressionLevel); - -/*! ZSTD_decompress() : - * `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames. - * `dstCapacity` is an upper bound of originalSize to regenerate. - * If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data. - * @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), - * or an errorCode if it fails (which can be tested using ZSTD_isError()). */ -ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, - const void* src, size_t compressedSize); - -/*! ZSTD_getFrameContentSize() : requires v1.3.0+ - * `src` should point to the start of a ZSTD encoded frame. - * `srcSize` must be at least as large as the frame header. - * hint : any size >= `ZSTD_frameHeaderSize_max` is large enough. - * @return : - decompressed size of `src` frame content, if known - * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined - * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) - * note 1 : a 0 return value means the frame is valid but "empty". - * note 2 : decompressed size is an optional field, it may not be present, typically in streaming mode. - * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. - * In which case, it's necessary to use streaming mode to decompress data. - * Optionally, application can rely on some implicit limit, - * as ZSTD_decompress() only needs an upper bound of decompressed size. - * (For example, data could be necessarily cut into blocks <= 16 KB). - * note 3 : decompressed size is always present when compression is completed using single-pass functions, - * such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict(). - * note 4 : decompressed size can be very large (64-bits value), - * potentially larger than what local system can handle as a single memory segment. - * In which case, it's necessary to use streaming mode to decompress data. - * note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified. - * Always ensure return value fits within application's authorized limits. - * Each application can set its own limits. - * note 6 : This function replaces ZSTD_getDecompressedSize() */ -#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1) -#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2) -ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); - -/*! ZSTD_getDecompressedSize() : - * NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize(). - * Both functions work the same way, but ZSTD_getDecompressedSize() blends - * "empty", "unknown" and "error" results to the same return value (0), - * while ZSTD_getFrameContentSize() gives them separate return values. - * @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */ -ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); - -/*! ZSTD_findFrameCompressedSize() : - * `src` should point to the start of a ZSTD frame or skippable frame. - * `srcSize` must be >= first frame size - * @return : the compressed size of the first frame starting at `src`, - * suitable to pass as `srcSize` to `ZSTD_decompress` or similar, - * or an error code if input is invalid */ -ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize); - - -/*====== Helper functions ======*/ -#define ZSTD_COMPRESSBOUND(srcSize) ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */ -ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */ -ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ -ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */ -ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed */ -ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */ - - -/*************************************** -* Explicit context -***************************************/ -/*= Compression context - * When compressing many times, - * it is recommended to allocate a context just once, - * and re-use it for each successive compression operation. - * This will make workload friendlier for system's memory. - * Note : re-using context is just a speed / resource optimization. - * It doesn't change the compression ratio, which remains identical. - * Note 2 : In multi-threaded environments, - * use one different context per thread for parallel execution. - */ -typedef struct ZSTD_CCtx_s ZSTD_CCtx; -ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); -ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); - -/*! ZSTD_compressCCtx() : - * Same as ZSTD_compress(), using an explicit ZSTD_CCtx. - * Important : in order to behave similarly to `ZSTD_compress()`, - * this function compresses at requested compression level, - * __ignoring any other parameter__ . - * If any advanced parameter was set using the advanced API, - * they will all be reset. Only `compressionLevel` remains. - */ -ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - int compressionLevel); - -/*= Decompression context - * When decompressing many times, - * it is recommended to allocate a context only once, - * and re-use it for each successive compression operation. - * This will make workload friendlier for system's memory. - * Use one context per thread for parallel execution. */ -typedef struct ZSTD_DCtx_s ZSTD_DCtx; -ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void); -ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); - -/*! ZSTD_decompressDCtx() : - * Same as ZSTD_decompress(), - * requires an allocated ZSTD_DCtx. - * Compatible with sticky parameters. - */ -ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize); - +MEM_STATIC void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem) +{ + if (customMem.customAlloc) { + /* calloc implemented as malloc+memset; + * not as efficient as calloc, but next best guess for custom malloc */ + void* const ptr = customMem.customAlloc(customMem.opaque, size); + ZSTD_memset(ptr, 0, size); + return ptr; + } + return ZSTD_calloc(1, size); +} -/*************************************** -* Advanced compression API -***************************************/ +MEM_STATIC void ZSTD_customFree(void* ptr, ZSTD_customMem customMem) +{ + if (ptr!=NULL) { + if (customMem.customFree) + customMem.customFree(customMem.opaque, ptr); + else + ZSTD_free(ptr); + } +} -/* API design : - * Parameters are pushed one by one into an existing context, - * using ZSTD_CCtx_set*() functions. - * Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame. - * "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` ! - * __They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx()__ . - * - * It's possible to reset all parameters to "default" using ZSTD_CCtx_reset(). +#endif /* ZSTD_ALLOCATIONS_H */ +/**** ended inlining ../common/allocations.h ****/ +/**** skipping file: zstd_deps.h ****/ +/**** skipping file: debug.h ****/ +/**** start inlining pool.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. * - * This API supercedes all other "advanced" API entry points in the experimental section. - * In the future, we expect to remove from experimental API entry points which are redundant with this API. + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. */ +#ifndef POOL_H +#define POOL_H -/* Compression strategies, listed from fastest to strongest */ -typedef enum { ZSTD_fast=1, - ZSTD_dfast=2, - ZSTD_greedy=3, - ZSTD_lazy=4, - ZSTD_lazy2=5, - ZSTD_btlazy2=6, - ZSTD_btopt=7, - ZSTD_btultra=8, - ZSTD_btultra2=9 - /* note : new strategies _might_ be added in the future. - Only the order (from fast to strong) is guaranteed */ -} ZSTD_strategy; - - -typedef enum { - - /* compression parameters - * Note: When compressing with a ZSTD_CDict these parameters are superseded - * by the parameters used to construct the ZSTD_CDict. - * See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict). */ - ZSTD_c_compressionLevel=100, /* Set compression parameters according to pre-defined cLevel table. - * Note that exact compression parameters are dynamically determined, - * depending on both compression level and srcSize (when known). - * Default level is ZSTD_CLEVEL_DEFAULT==3. - * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT. - * Note 1 : it's possible to pass a negative compression level. - * Note 2 : setting a level does not automatically set all other compression parameters - * to default. Setting this will however eventually dynamically impact the compression - * parameters which have not been manually set. The manually set - * ones will 'stick'. */ - /* Advanced compression parameters : - * It's possible to pin down compression parameters to some specific values. - * In which case, these values are no longer dynamically selected by the compressor */ - ZSTD_c_windowLog=101, /* Maximum allowed back-reference distance, expressed as power of 2. - * This will set a memory budget for streaming decompression, - * with larger values requiring more memory - * and typically compressing more. - * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX. - * Special: value 0 means "use default windowLog". - * Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT - * requires explicitly allowing such size at streaming decompression stage. */ - ZSTD_c_hashLog=102, /* Size of the initial probe table, as a power of 2. - * Resulting memory usage is (1 << (hashLog+2)). - * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX. - * Larger tables improve compression ratio of strategies <= dFast, - * and improve speed of strategies > dFast. - * Special: value 0 means "use default hashLog". */ - ZSTD_c_chainLog=103, /* Size of the multi-probe search table, as a power of 2. - * Resulting memory usage is (1 << (chainLog+2)). - * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX. - * Larger tables result in better and slower compression. - * This parameter is useless for "fast" strategy. - * It's still useful when using "dfast" strategy, - * in which case it defines a secondary probe table. - * Special: value 0 means "use default chainLog". */ - ZSTD_c_searchLog=104, /* Number of search attempts, as a power of 2. - * More attempts result in better and slower compression. - * This parameter is useless for "fast" and "dFast" strategies. - * Special: value 0 means "use default searchLog". */ - ZSTD_c_minMatch=105, /* Minimum size of searched matches. - * Note that Zstandard can still find matches of smaller size, - * it just tweaks its search algorithm to look for this size and larger. - * Larger values increase compression and decompression speed, but decrease ratio. - * Must be clamped between ZSTD_MINMATCH_MIN and ZSTD_MINMATCH_MAX. - * Note that currently, for all strategies < btopt, effective minimum is 4. - * , for all strategies > fast, effective maximum is 6. - * Special: value 0 means "use default minMatchLength". */ - ZSTD_c_targetLength=106, /* Impact of this field depends on strategy. - * For strategies btopt, btultra & btultra2: - * Length of Match considered "good enough" to stop search. - * Larger values make compression stronger, and slower. - * For strategy fast: - * Distance between match sampling. - * Larger values make compression faster, and weaker. - * Special: value 0 means "use default targetLength". */ - ZSTD_c_strategy=107, /* See ZSTD_strategy enum definition. - * The higher the value of selected strategy, the more complex it is, - * resulting in stronger and slower compression. - * Special: value 0 means "use default strategy". */ - - /* LDM mode parameters */ - ZSTD_c_enableLongDistanceMatching=160, /* Enable long distance matching. - * This parameter is designed to improve compression ratio - * for large inputs, by finding large matches at long distance. - * It increases memory usage and window size. - * Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB - * except when expressly set to a different value. - * Note: will be enabled by default if ZSTD_c_windowLog >= 128 MB and - * compression strategy >= ZSTD_btopt (== compression level 16+) */ - ZSTD_c_ldmHashLog=161, /* Size of the table for long distance matching, as a power of 2. - * Larger values increase memory usage and compression ratio, - * but decrease compression speed. - * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX - * default: windowlog - 7. - * Special: value 0 means "automatically determine hashlog". */ - ZSTD_c_ldmMinMatch=162, /* Minimum match size for long distance matcher. - * Larger/too small values usually decrease compression ratio. - * Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX. - * Special: value 0 means "use default value" (default: 64). */ - ZSTD_c_ldmBucketSizeLog=163, /* Log size of each bucket in the LDM hash table for collision resolution. - * Larger values improve collision resolution but decrease compression speed. - * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX. - * Special: value 0 means "use default value" (default: 3). */ - ZSTD_c_ldmHashRateLog=164, /* Frequency of inserting/looking up entries into the LDM hash table. - * Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN). - * Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage. - * Larger values improve compression speed. - * Deviating far from default value will likely result in a compression ratio decrease. - * Special: value 0 means "automatically determine hashRateLog". */ - - /* frame parameters */ - ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1) - * Content size must be known at the beginning of compression. - * This is automatically the case when using ZSTD_compress2(), - * For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */ - ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */ - ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */ - - /* multi-threading parameters */ - /* These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD). - * Otherwise, trying to set any other value than default (0) will be a no-op and return an error. - * In a situation where it's unknown if the linked library supports multi-threading or not, - * setting ZSTD_c_nbWorkers to any value >= 1 and consulting the return value provides a quick way to check this property. - */ - ZSTD_c_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel. - * When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() : - * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller, - * while compression is performed in parallel, within worker thread(s). - * (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end : - * in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call). - * More workers improve speed, but also increase memory usage. - * Default value is `0`, aka "single-threaded mode" : no worker is spawned, - * compression is performed inside Caller's thread, and all invocations are blocking */ - ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1. - * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads. - * 0 means default, which is dynamically determined based on compression parameters. - * Job size must be a minimum of overlap size, or 1 MB, whichever is largest. - * The minimum size is automatically and transparently enforced. */ - ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size. - * The overlap size is an amount of data reloaded from previous job at the beginning of a new job. - * It helps preserve compression ratio, while each job is compressed in parallel. - * This value is enforced only when nbWorkers >= 1. - * Larger values increase compression ratio, but decrease speed. - * Possible values range from 0 to 9 : - * - 0 means "default" : value will be determined by the library, depending on strategy - * - 1 means "no overlap" - * - 9 means "full overlap", using a full window size. - * Each intermediate rank increases/decreases load size by a factor 2 : - * 9: full window; 8: w/2; 7: w/4; 6: w/8; 5:w/16; 4: w/32; 3:w/64; 2:w/128; 1:no overlap; 0:default - * default value varies between 6 and 9, depending on strategy */ - - /* note : additional experimental parameters are also available - * within the experimental section of the API. - * At the time of this writing, they include : - * ZSTD_c_rsyncable - * ZSTD_c_format - * ZSTD_c_forceMaxWindow - * ZSTD_c_forceAttachDict - * ZSTD_c_literalCompressionMode - * ZSTD_c_targetCBlockSize - * ZSTD_c_srcSizeHint - * ZSTD_c_enableDedicatedDictSearch - * ZSTD_c_stableInBuffer - * ZSTD_c_stableOutBuffer - * ZSTD_c_blockDelimiters - * ZSTD_c_validateSequences - * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. - * note : never ever use experimentalParam? names directly; - * also, the enums values themselves are unstable and can still change. - */ - ZSTD_c_experimentalParam1=500, - ZSTD_c_experimentalParam2=10, - ZSTD_c_experimentalParam3=1000, - ZSTD_c_experimentalParam4=1001, - ZSTD_c_experimentalParam5=1002, - ZSTD_c_experimentalParam6=1003, - ZSTD_c_experimentalParam7=1004, - ZSTD_c_experimentalParam8=1005, - ZSTD_c_experimentalParam9=1006, - ZSTD_c_experimentalParam10=1007, - ZSTD_c_experimentalParam11=1008, - ZSTD_c_experimentalParam12=1009 -} ZSTD_cParameter; - -typedef struct { - size_t error; - int lowerBound; - int upperBound; -} ZSTD_bounds; - -/*! ZSTD_cParam_getBounds() : - * All parameters must belong to an interval with lower and upper bounds, - * otherwise they will either trigger an error or be automatically clamped. - * @return : a structure, ZSTD_bounds, which contains - * - an error status field, which must be tested using ZSTD_isError() - * - lower and upper bounds, both inclusive - */ -ZSTDLIB_API ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter cParam); - -/*! ZSTD_CCtx_setParameter() : - * Set one compression parameter, selected by enum ZSTD_cParameter. - * All parameters have valid bounds. Bounds can be queried using ZSTD_cParam_getBounds(). - * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). - * Setting a parameter is generally only possible during frame initialization (before starting compression). - * Exception : when using multi-threading mode (nbWorkers >= 1), - * the following parameters can be updated _during_ compression (within same frame): - * => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy. - * new parameters will be active for next job only (after a flush()). - * @return : an error code (which can be tested using ZSTD_isError()). - */ -ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value); - -/*! ZSTD_CCtx_setPledgedSrcSize() : - * Total input data size to be compressed as a single frame. - * Value will be written in frame header, unless if explicitly forbidden using ZSTD_c_contentSizeFlag. - * This value will also be controlled at end of frame, and trigger an error if not respected. - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame. - * In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN. - * ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame. - * Note 2 : pledgedSrcSize is only valid once, for the next frame. - * It's discarded at the end of the frame, and replaced by ZSTD_CONTENTSIZE_UNKNOWN. - * Note 3 : Whenever all input data is provided and consumed in a single round, - * for example with ZSTD_compress2(), - * or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end), - * this value is automatically overridden by srcSize instead. - */ -ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize); -typedef enum { - ZSTD_reset_session_only = 1, - ZSTD_reset_parameters = 2, - ZSTD_reset_session_and_parameters = 3 -} ZSTD_ResetDirective; +/**** skipping file: zstd_deps.h ****/ +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_customMem */ +/**** skipping file: ../zstd.h ****/ -/*! ZSTD_CCtx_reset() : - * There are 2 different things that can be reset, independently or jointly : - * - The session : will stop compressing current frame, and make CCtx ready to start a new one. - * Useful after an error, or to interrupt any ongoing compression. - * Any internal data not yet flushed is cancelled. - * Compression parameters and dictionary remain unchanged. - * They will be used to compress next frame. - * Resetting session never fails. - * - The parameters : changes all parameters back to "default". - * This removes any reference to any dictionary too. - * Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing) - * otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError()) - * - Both : similar to resetting the session, followed by resetting parameters. - */ -ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset); - -/*! ZSTD_compress2() : - * Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API. - * ZSTD_compress2() always starts a new frame. - * Should cctx hold data from a previously unfinished frame, everything about it is forgotten. - * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() - * - The function is always blocking, returns when compression is completed. - * Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`. - * @return : compressed size written into `dst` (<= `dstCapacity), - * or an error code if it fails (which can be tested using ZSTD_isError()). - */ -ZSTDLIB_API size_t ZSTD_compress2( ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize); +typedef struct POOL_ctx_s POOL_ctx; +/*! POOL_create() : + * Create a thread pool with at most `numThreads` threads. + * `numThreads` must be at least 1. + * The maximum number of queued jobs before blocking is `queueSize`. + * @return : POOL_ctx pointer on success, else NULL. +*/ +POOL_ctx* POOL_create(size_t numThreads, size_t queueSize); -/*************************************** -* Advanced decompression API -***************************************/ +POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, + ZSTD_customMem customMem); -/* The advanced API pushes parameters one by one into an existing DCtx context. - * Parameters are sticky, and remain valid for all following frames - * using the same DCtx context. - * It's possible to reset parameters to default values using ZSTD_DCtx_reset(). - * Note : This API is compatible with existing ZSTD_decompressDCtx() and ZSTD_decompressStream(). - * Therefore, no new decompression function is necessary. +/*! POOL_free() : + * Free a thread pool returned by POOL_create(). */ +void POOL_free(POOL_ctx* ctx); -typedef enum { - - ZSTD_d_windowLogMax=100, /* Select a size limit (in power of 2) beyond which - * the streaming API will refuse to allocate memory buffer - * in order to protect the host from unreasonable memory requirements. - * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. - * By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT). - * Special: value 0 means "use default maximum windowLog". */ - - /* note : additional experimental parameters are also available - * within the experimental section of the API. - * At the time of this writing, they include : - * ZSTD_d_format - * ZSTD_d_stableOutBuffer - * ZSTD_d_forceIgnoreChecksum - * ZSTD_d_refMultipleDDicts - * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. - * note : never ever use experimentalParam? names directly - */ - ZSTD_d_experimentalParam1=1000, - ZSTD_d_experimentalParam2=1001, - ZSTD_d_experimentalParam3=1002, - ZSTD_d_experimentalParam4=1003 - -} ZSTD_dParameter; -/*! ZSTD_dParam_getBounds() : - * All parameters must belong to an interval with lower and upper bounds, - * otherwise they will either trigger an error or be automatically clamped. - * @return : a structure, ZSTD_bounds, which contains - * - an error status field, which must be tested using ZSTD_isError() - * - both lower and upper bounds, inclusive +/*! POOL_joinJobs() : + * Waits for all queued jobs to finish executing. */ -ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam); +void POOL_joinJobs(POOL_ctx* ctx); -/*! ZSTD_DCtx_setParameter() : - * Set one compression parameter, selected by enum ZSTD_dParameter. - * All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds(). - * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). - * Setting a parameter is only possible during frame initialization (before starting decompression). - * @return : 0, or an error code (which can be tested using ZSTD_isError()). +/*! POOL_resize() : + * Expands or shrinks pool's number of threads. + * This is more efficient than releasing + creating a new context, + * since it tries to preserve and reuse existing threads. + * `numThreads` must be at least 1. + * @return : 0 when resize was successful, + * !0 (typically 1) if there is an error. + * note : only numThreads can be resized, queueSize remains unchanged. */ -ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value); +int POOL_resize(POOL_ctx* ctx, size_t numThreads); -/*! ZSTD_DCtx_reset() : - * Return a DCtx to clean state. - * Session and parameters can be reset jointly or separately. - * Parameters can only be reset when no active frame is being decompressed. - * @return : 0, or an error code, which can be tested with ZSTD_isError() +/*! POOL_sizeof() : + * @return threadpool memory usage + * note : compatible with NULL (returns 0 in this case) */ -ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset); - - -/**************************** -* Streaming -****************************/ - -typedef struct ZSTD_inBuffer_s { - const void* src; /**< start of input buffer */ - size_t size; /**< size of input buffer */ - size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */ -} ZSTD_inBuffer; - -typedef struct ZSTD_outBuffer_s { - void* dst; /**< start of output buffer */ - size_t size; /**< size of output buffer */ - size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */ -} ZSTD_outBuffer; - - - -/*-*********************************************************************** -* Streaming compression - HowTo -* -* A ZSTD_CStream object is required to track streaming operation. -* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. -* ZSTD_CStream objects can be reused multiple times on consecutive compression operations. -* It is recommended to re-use ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory. -* -* For parallel execution, use one separate ZSTD_CStream per thread. -* -* note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing. -* -* Parameters are sticky : when starting a new compression on the same context, -* it will re-use the same sticky parameters as previous compression session. -* When in doubt, it's recommended to fully initialize the context before usage. -* Use ZSTD_CCtx_reset() to reset the context and ZSTD_CCtx_setParameter(), -* ZSTD_CCtx_setPledgedSrcSize(), or ZSTD_CCtx_loadDictionary() and friends to -* set more specific parameters, the pledged source size, or load a dictionary. -* -* Use ZSTD_compressStream2() with ZSTD_e_continue as many times as necessary to -* consume input stream. The function will automatically update both `pos` -* fields within `input` and `output`. -* Note that the function may not consume the entire input, for example, because -* the output buffer is already full, in which case `input.pos < input.size`. -* The caller must check if input has been entirely consumed. -* If not, the caller must make some room to receive more compressed data, -* and then present again remaining input data. -* note: ZSTD_e_continue is guaranteed to make some forward progress when called, -* but doesn't guarantee maximal forward progress. This is especially relevant -* when compressing with multiple threads. The call won't block if it can -* consume some input, but if it can't it will wait for some, but not all, -* output to be flushed. -* @return : provides a minimum amount of data remaining to be flushed from internal buffers -* or an error code, which can be tested using ZSTD_isError(). -* -* At any moment, it's possible to flush whatever data might remain stuck within internal buffer, -* using ZSTD_compressStream2() with ZSTD_e_flush. `output->pos` will be updated. -* Note that, if `output->size` is too small, a single invocation with ZSTD_e_flush might not be enough (return code > 0). -* In which case, make some room to receive more compressed data, and call again ZSTD_compressStream2() with ZSTD_e_flush. -* You must continue calling ZSTD_compressStream2() with ZSTD_e_flush until it returns 0, at which point you can change the -* operation. -* note: ZSTD_e_flush will flush as much output as possible, meaning when compressing with multiple threads, it will -* block until the flush is complete or the output buffer is full. -* @return : 0 if internal buffers are entirely flushed, -* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), -* or an error code, which can be tested using ZSTD_isError(). -* -* Calling ZSTD_compressStream2() with ZSTD_e_end instructs to finish a frame. -* It will perform a flush and write frame epilogue. -* The epilogue is required for decoders to consider a frame completed. -* flush operation is the same, and follows same rules as calling ZSTD_compressStream2() with ZSTD_e_flush. -* You must continue calling ZSTD_compressStream2() with ZSTD_e_end until it returns 0, at which point you are free to -* start a new frame. -* note: ZSTD_e_end will flush as much output as possible, meaning when compressing with multiple threads, it will -* block until the flush is complete or the output buffer is full. -* @return : 0 if frame fully completed and fully flushed, -* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), -* or an error code, which can be tested using ZSTD_isError(). -* -* *******************************************************************/ - -typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are now effectively same object (>= v1.3.0) */ - /* Continue to distinguish them for compatibility with older versions <= v1.2.0 */ -/*===== ZSTD_CStream management functions =====*/ -ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void); -ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); - -/*===== Streaming compression functions =====*/ -typedef enum { - ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */ - ZSTD_e_flush=1, /* flush any data provided so far, - * it creates (at least) one new block, that can be decoded immediately on reception; - * frame will continue: any future data can still reference previously compressed data, improving compression. - * note : multithreaded compression will block to flush as much output as possible. */ - ZSTD_e_end=2 /* flush any remaining data _and_ close current frame. - * note that frame is only closed after compressed data is fully flushed (return value == 0). - * After that point, any additional data starts a new frame. - * note : each frame is independent (does not reference any content from previous frame). - : note : multithreaded compression will block to flush as much output as possible. */ -} ZSTD_EndDirective; - -/*! ZSTD_compressStream2() : - * Behaves about the same as ZSTD_compressStream, with additional control on end directive. - * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() - * - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode) - * - output->pos must be <= dstCapacity, input->pos must be <= srcSize - * - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit. - * - endOp must be a valid directive - * - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller. - * - When nbWorkers>=1, function is non-blocking : it copies a portion of input, distributes jobs to internal worker threads, flush to output whatever is available, - * and then immediately returns, just indicating that there is some data remaining to be flushed. - * The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte. - * - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking. - * - @return provides a minimum amount of data remaining to be flushed from internal buffers - * or an error code, which can be tested using ZSTD_isError(). - * if @return != 0, flush is not fully completed, there is still some data left within internal buffers. - * This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers. - * For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed. - * - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0), - * only ZSTD_e_end or ZSTD_e_flush operations are allowed. - * Before starting a new compression job, or changing compression parameters, - * it is required to fully flush internal buffers. - */ -ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, - ZSTD_outBuffer* output, - ZSTD_inBuffer* input, - ZSTD_EndDirective endOp); - - -/* These buffer sizes are softly recommended. - * They are not required : ZSTD_compressStream*() happily accepts any buffer size, for both input and output. - * Respecting the recommended size just makes it a bit easier for ZSTD_compressStream*(), - * reducing the amount of memory shuffling and buffering, resulting in minor performance savings. - * - * However, note that these recommendations are from the perspective of a C caller program. - * If the streaming interface is invoked from some other language, - * especially managed ones such as Java or Go, through a foreign function interface such as jni or cgo, - * a major performance rule is to reduce crossing such interface to an absolute minimum. - * It's not rare that performance ends being spent more into the interface, rather than compression itself. - * In which cases, prefer using large buffers, as large as practical, - * for both input and output, to reduce the nb of roundtrips. - */ -ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ -ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block. */ - +size_t POOL_sizeof(const POOL_ctx* ctx); -/* ***************************************************************************** - * This following is a legacy streaming API. - * It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2(). - * It is redundant, but remains fully supported. - * Advanced parameters and dictionary compression can only be used through the - * new API. - ******************************************************************************/ - -/*! - * Equivalent to: - * - * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) - * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); +/*! POOL_function : + * The function type that can be added to a thread pool. */ -ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); -/*! - * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue). - * NOTE: The return value is different. ZSTD_compressStream() returns a hint for - * the next read size (if non-zero and not an error). ZSTD_compressStream2() - * returns the minimum nb of bytes left to flush (if non-zero and not an error). +typedef void (*POOL_function)(void*); + +/*! POOL_add() : + * Add the job `function(opaque)` to the thread pool. `ctx` must be valid. + * Possibly blocks until there is room in the queue. + * Note : The function may be executed asynchronously, + * therefore, `opaque` must live until function has been completed. */ -ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); -/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */ -ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); -/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */ -ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); +void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque); -/*-*************************************************************************** -* Streaming decompression - HowTo -* -* A ZSTD_DStream object is required to track streaming operations. -* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. -* ZSTD_DStream objects can be re-used multiple times. -* -* Use ZSTD_initDStream() to start a new decompression operation. -* @return : recommended first input size -* Alternatively, use advanced API to set specific properties. -* -* Use ZSTD_decompressStream() repetitively to consume your input. -* The function will update both `pos` fields. -* If `input.pos < input.size`, some input has not been consumed. -* It's up to the caller to present again remaining data. -* The function tries to flush all data decoded immediately, respecting output buffer size. -* If `output.pos < output.size`, decoder has flushed everything it could. -* But if `output.pos == output.size`, there might be some data left within internal buffers., -* In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer. -* Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX. -* @return : 0 when a frame is completely decoded and fully flushed, -* or an error code, which can be tested using ZSTD_isError(), -* or any other value > 0, which means there is still some decoding or flushing to do to complete current frame : -* the return value is a suggested next input size (just a hint for better latency) -* that will never request more than the remaining frame size. -* *******************************************************************************/ - -typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */ - /* For compatibility with versions <= v1.2.0, prefer differentiating them. */ -/*===== ZSTD_DStream management functions =====*/ -ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void); -ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); - -/*===== Streaming decompression functions =====*/ - -/* This function is redundant with the advanced API and equivalent to: - * - * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); - * ZSTD_DCtx_refDDict(zds, NULL); - */ -ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); - -ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); - -ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ -ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */ - - -/************************** -* Simple dictionary API -***************************/ -/*! ZSTD_compress_usingDict() : - * Compression at an explicit compression level using a Dictionary. - * A dictionary can be any arbitrary data segment (also called a prefix), - * or a buffer with specified information (see dictBuilder/zdict.h). - * Note : This function loads the dictionary, resulting in significant startup delay. - * It's intended for a dictionary used only once. - * Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */ -ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize, - int compressionLevel); - -/*! ZSTD_decompress_usingDict() : - * Decompression using a known Dictionary. - * Dictionary must be identical to the one used during compression. - * Note : This function loads the dictionary, resulting in significant startup delay. - * It's intended for a dictionary used only once. - * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */ -ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize); +/*! POOL_tryAdd() : + * Add the job `function(opaque)` to thread pool _if_ a queue slot is available. + * Returns immediately even if not (does not block). + * @return : 1 if successful, 0 if not. + */ +int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque); +#endif +/**** ended inlining pool.h ****/ -/*********************************** - * Bulk processing dictionary API - **********************************/ -typedef struct ZSTD_CDict_s ZSTD_CDict; - -/*! ZSTD_createCDict() : - * When compressing multiple messages or blocks using the same dictionary, - * it's recommended to digest the dictionary only once, since it's a costly operation. - * ZSTD_createCDict() will create a state from digesting a dictionary. - * The resulting state can be used for future compression operations with very limited startup cost. - * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. - * @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict. - * Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content. - * Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer, - * in which case the only thing that it transports is the @compressionLevel. - * This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively, - * expecting a ZSTD_CDict parameter with any data, including those without a known dictionary. */ -ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, - int compressionLevel); - -/*! ZSTD_freeCDict() : - * Function frees memory allocated by ZSTD_createCDict(). */ -ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); +/* ====== Compiler specifics ====== */ +#if defined(_MSC_VER) +# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ +#endif -/*! ZSTD_compress_usingCDict() : - * Compression using a digested Dictionary. - * Recommended when same dictionary is used multiple times. - * Note : compression level is _decided at dictionary creation time_, - * and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */ -ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTD_CDict* cdict); +#ifdef ZSTD_MULTITHREAD -typedef struct ZSTD_DDict_s ZSTD_DDict; +/**** skipping file: threading.h ****/ -/*! ZSTD_createDDict() : - * Create a digested dictionary, ready to start decompression operation without startup delay. - * dictBuffer can be released after DDict creation, as its content is copied inside DDict. */ -ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize); +/* A job is a function and an opaque argument */ +typedef struct POOL_job_s { + POOL_function function; + void *opaque; +} POOL_job; -/*! ZSTD_freeDDict() : - * Function frees memory allocated with ZSTD_createDDict() */ -ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict); +struct POOL_ctx_s { + ZSTD_customMem customMem; + /* Keep track of the threads */ + ZSTD_pthread_t* threads; + size_t threadCapacity; + size_t threadLimit; -/*! ZSTD_decompress_usingDDict() : - * Decompression using a digested Dictionary. - * Recommended when same dictionary is used multiple times. */ -ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTD_DDict* ddict); + /* The queue is a circular buffer */ + POOL_job *queue; + size_t queueHead; + size_t queueTail; + size_t queueSize; + /* The number of threads working on jobs */ + size_t numThreadsBusy; + /* Indicates if the queue is empty */ + int queueEmpty; -/******************************** - * Dictionary helper functions - *******************************/ + /* The mutex protects the queue */ + ZSTD_pthread_mutex_t queueMutex; + /* Condition variable for pushers to wait on when the queue is full */ + ZSTD_pthread_cond_t queuePushCond; + /* Condition variables for poppers to wait on when the queue is empty */ + ZSTD_pthread_cond_t queuePopCond; + /* Indicates if the queue is shutting down */ + int shutdown; +}; -/*! ZSTD_getDictID_fromDict() : - * Provides the dictID stored within dictionary. - * if @return == 0, the dictionary is not conformant with Zstandard specification. - * It can still be loaded, but as a content-only dictionary. */ -ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize); +/* POOL_thread() : + * Work thread for the thread pool. + * Waits for jobs and executes them. + * @returns : NULL on failure else non-null. + */ +static void* POOL_thread(void* opaque) { + POOL_ctx* const ctx = (POOL_ctx*)opaque; + if (!ctx) { return NULL; } + for (;;) { + /* Lock the mutex and wait for a non-empty queue or until shutdown */ + ZSTD_pthread_mutex_lock(&ctx->queueMutex); -/*! ZSTD_getDictID_fromDDict() : - * Provides the dictID of the dictionary loaded into `ddict`. - * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. - * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ -ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict); + while ( ctx->queueEmpty + || (ctx->numThreadsBusy >= ctx->threadLimit) ) { + if (ctx->shutdown) { + /* even if !queueEmpty, (possible if numThreadsBusy >= threadLimit), + * a few threads will be shutdown while !queueEmpty, + * but enough threads will remain active to finish the queue */ + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + return opaque; + } + ZSTD_pthread_cond_wait(&ctx->queuePopCond, &ctx->queueMutex); + } + /* Pop a job off the queue */ + { POOL_job const job = ctx->queue[ctx->queueHead]; + ctx->queueHead = (ctx->queueHead + 1) % ctx->queueSize; + ctx->numThreadsBusy++; + ctx->queueEmpty = (ctx->queueHead == ctx->queueTail); + /* Unlock the mutex, signal a pusher, and run the job */ + ZSTD_pthread_cond_signal(&ctx->queuePushCond); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); -/*! ZSTD_getDictID_fromFrame() : - * Provides the dictID required to decompressed the frame stored within `src`. - * If @return == 0, the dictID could not be decoded. - * This could for one of the following reasons : - * - The frame does not require a dictionary to be decoded (most common case). - * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information. - * Note : this use case also happens when using a non-conformant dictionary. - * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). - * - This is not a Zstandard frame. - * When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */ -ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); + job.function(job.opaque); + /* If the intended queue size was 0, signal after finishing job */ + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + ctx->numThreadsBusy--; + ZSTD_pthread_cond_signal(&ctx->queuePushCond); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + } + } /* for (;;) */ + assert(0); /* Unreachable */ +} -/******************************************************************************* - * Advanced dictionary and prefix API - * - * This API allows dictionaries to be used with ZSTD_compress2(), - * ZSTD_compressStream2(), and ZSTD_decompress(). Dictionaries are sticky, and - * only reset with the context is reset with ZSTD_reset_parameters or - * ZSTD_reset_session_and_parameters. Prefixes are single-use. - ******************************************************************************/ +/* ZSTD_createThreadPool() : public access point */ +POOL_ctx* ZSTD_createThreadPool(size_t numThreads) { + return POOL_create (numThreads, 0); +} +POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) { + return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem); +} -/*! ZSTD_CCtx_loadDictionary() : - * Create an internal CDict from `dict` buffer. - * Decompression will have to use same dictionary. - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary, - * meaning "return to no-dictionary mode". - * Note 1 : Dictionary is sticky, it will be used for all future compressed frames. - * To return to "no-dictionary" situation, load a NULL dictionary (or reset parameters). - * Note 2 : Loading a dictionary involves building tables. - * It's also a CPU consuming operation, with non-negligible impact on latency. - * Tables are dependent on compression parameters, and for this reason, - * compression parameters can no longer be changed after loading a dictionary. - * Note 3 :`dict` content will be copied internally. - * Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead. - * In such a case, dictionary buffer must outlive its users. - * Note 4 : Use ZSTD_CCtx_loadDictionary_advanced() - * to precisely select how dictionary content must be interpreted. */ -ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); - -/*! ZSTD_CCtx_refCDict() : - * Reference a prepared dictionary, to be used for all next compressed frames. - * Note that compression parameters are enforced from within CDict, - * and supersede any compression parameter previously set within CCtx. - * The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs. - * The ignored parameters will be used again if the CCtx is returned to no-dictionary mode. - * The dictionary will remain valid for future compressed frames using same CCtx. - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Special : Referencing a NULL CDict means "return to no-dictionary mode". - * Note 1 : Currently, only one dictionary can be managed. - * Referencing a new dictionary effectively "discards" any previous one. - * Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */ -ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); - -/*! ZSTD_CCtx_refPrefix() : - * Reference a prefix (single-usage dictionary) for next compressed frame. - * A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end). - * Decompression will need same prefix to properly regenerate data. - * Compressing with a prefix is similar in outcome as performing a diff and compressing it, - * but performs much faster, especially during decompression (compression speed is tunable with compression level). - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary - * Note 1 : Prefix buffer is referenced. It **must** outlive compression. - * Its content must remain unmodified during compression. - * Note 2 : If the intention is to diff some large src data blob with some prior version of itself, - * ensure that the window size is large enough to contain the entire source. - * See ZSTD_c_windowLog. - * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters. - * It's a CPU consuming operation, with non-negligible impact on latency. - * If there is a need to use the same prefix multiple times, consider loadDictionary instead. - * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent). - * Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */ -ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, - const void* prefix, size_t prefixSize); - -/*! ZSTD_DCtx_loadDictionary() : - * Create an internal DDict from dict buffer, - * to be used to decompress next frames. - * The dictionary remains valid for all future frames, until explicitly invalidated. - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary, - * meaning "return to no-dictionary mode". - * Note 1 : Loading a dictionary involves building tables, - * which has a non-negligible impact on CPU usage and latency. - * It's recommended to "load once, use many times", to amortize the cost - * Note 2 :`dict` content will be copied internally, so `dict` can be released after loading. - * Use ZSTD_DCtx_loadDictionary_byReference() to reference dictionary content instead. - * Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to take control of - * how dictionary content is loaded and interpreted. - */ -ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); - -/*! ZSTD_DCtx_refDDict() : - * Reference a prepared dictionary, to be used to decompress next frames. - * The dictionary remains active for decompression of future frames using same DCtx. - * - * If called with ZSTD_d_refMultipleDDicts enabled, repeated calls of this function - * will store the DDict references in a table, and the DDict used for decompression - * will be determined at decompression time, as per the dict ID in the frame. - * The memory for the table is allocated on the first call to refDDict, and can be - * freed with ZSTD_freeDCtx(). - * - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Note 1 : Currently, only one dictionary can be managed. - * Referencing a new dictionary effectively "discards" any previous one. - * Special: referencing a NULL DDict means "return to no-dictionary mode". - * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx. - */ -ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); - -/*! ZSTD_DCtx_refPrefix() : - * Reference a prefix (single-usage dictionary) to decompress next frame. - * This is the reverse operation of ZSTD_CCtx_refPrefix(), - * and must use the same prefix as the one used during compression. - * Prefix is **only used once**. Reference is discarded at end of frame. - * End of frame is reached when ZSTD_decompressStream() returns 0. - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary - * Note 2 : Prefix buffer is referenced. It **must** outlive decompression. - * Prefix buffer must remain unmodified up to the end of frame, - * reached when ZSTD_decompressStream() returns 0. - * Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent). - * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section) - * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost. - * A full dictionary is more costly, as it requires building tables. - */ -ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, - const void* prefix, size_t prefixSize); +POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, + ZSTD_customMem customMem) +{ + POOL_ctx* ctx; + /* Check parameters */ + if (!numThreads) { return NULL; } + /* Allocate the context and zero initialize */ + ctx = (POOL_ctx*)ZSTD_customCalloc(sizeof(POOL_ctx), customMem); + if (!ctx) { return NULL; } + /* Initialize the job queue. + * It needs one extra space since one space is wasted to differentiate + * empty and full queues. + */ + ctx->queueSize = queueSize + 1; + ctx->queue = (POOL_job*)ZSTD_customCalloc(ctx->queueSize * sizeof(POOL_job), customMem); + ctx->queueHead = 0; + ctx->queueTail = 0; + ctx->numThreadsBusy = 0; + ctx->queueEmpty = 1; + { + int error = 0; + error |= ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL); + error |= ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL); + error |= ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL); + if (error) { POOL_free(ctx); return NULL; } + } + ctx->shutdown = 0; + /* Allocate space for the thread handles */ + ctx->threads = (ZSTD_pthread_t*)ZSTD_customCalloc(numThreads * sizeof(ZSTD_pthread_t), customMem); + ctx->threadCapacity = 0; + ctx->customMem = customMem; + /* Check for errors */ + if (!ctx->threads || !ctx->queue) { POOL_free(ctx); return NULL; } + /* Initialize the threads */ + { size_t i; + for (i = 0; i < numThreads; ++i) { + if (ZSTD_pthread_create(&ctx->threads[i], NULL, &POOL_thread, ctx)) { + ctx->threadCapacity = i; + POOL_free(ctx); + return NULL; + } } + ctx->threadCapacity = numThreads; + ctx->threadLimit = numThreads; + } + return ctx; +} -/* === Memory management === */ +/*! POOL_join() : + Shutdown the queue, wake any sleeping threads, and join all of the threads. +*/ +static void POOL_join(POOL_ctx* ctx) { + /* Shut down the queue */ + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + ctx->shutdown = 1; + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + /* Wake up sleeping threads */ + ZSTD_pthread_cond_broadcast(&ctx->queuePushCond); + ZSTD_pthread_cond_broadcast(&ctx->queuePopCond); + /* Join all of the threads */ + { size_t i; + for (i = 0; i < ctx->threadCapacity; ++i) { + ZSTD_pthread_join(ctx->threads[i]); /* note : could fail */ + } } +} -/*! ZSTD_sizeof_*() : - * These functions give the _current_ memory usage of selected object. - * Note that object memory usage can evolve (increase or decrease) over time. */ -ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); -ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); -ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); -ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); -ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict); -ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); - -#endif /* ZSTD_H_235446 */ - - -/* ************************************************************************************** - * ADVANCED AND EXPERIMENTAL FUNCTIONS - **************************************************************************************** - * The definitions in the following section are considered experimental. - * They are provided for advanced scenarios. - * They should never be used with a dynamic library, as prototypes may change in the future. - * Use them only in association with static linking. - * ***************************************************************************************/ - -#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY) -#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY - -/**************************************************************************************** - * experimental API (static linking only) - **************************************************************************************** - * The following symbols and constants - * are not planned to join "stable API" status in the near future. - * They can still change in future versions. - * Some of them are planned to remain in the static_only section indefinitely. - * Some of them might be removed in the future (especially when redundant with existing stable functions) - * ***************************************************************************************/ - -#define ZSTD_FRAMEHEADERSIZE_PREFIX(format) ((format) == ZSTD_f_zstd1 ? 5 : 1) /* minimum input size required to query frame header size */ -#define ZSTD_FRAMEHEADERSIZE_MIN(format) ((format) == ZSTD_f_zstd1 ? 6 : 2) -#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */ -#define ZSTD_SKIPPABLEHEADERSIZE 8 - -/* compression parameter bounds */ -#define ZSTD_WINDOWLOG_MAX_32 30 -#define ZSTD_WINDOWLOG_MAX_64 31 -#define ZSTD_WINDOWLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64)) -#define ZSTD_WINDOWLOG_MIN 10 -#define ZSTD_HASHLOG_MAX ((ZSTD_WINDOWLOG_MAX < 30) ? ZSTD_WINDOWLOG_MAX : 30) -#define ZSTD_HASHLOG_MIN 6 -#define ZSTD_CHAINLOG_MAX_32 29 -#define ZSTD_CHAINLOG_MAX_64 30 -#define ZSTD_CHAINLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_CHAINLOG_MAX_32 : ZSTD_CHAINLOG_MAX_64)) -#define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN -#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1) -#define ZSTD_SEARCHLOG_MIN 1 -#define ZSTD_MINMATCH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */ -#define ZSTD_MINMATCH_MIN 3 /* only for ZSTD_btopt+, faster strategies are limited to 4 */ -#define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX -#define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */ -#define ZSTD_STRATEGY_MIN ZSTD_fast -#define ZSTD_STRATEGY_MAX ZSTD_btultra2 - - -#define ZSTD_OVERLAPLOG_MIN 0 -#define ZSTD_OVERLAPLOG_MAX 9 - -#define ZSTD_WINDOWLOG_LIMIT_DEFAULT 27 /* by default, the streaming decoder will refuse any frame - * requiring larger than (1<queueMutex); + ZSTD_pthread_cond_destroy(&ctx->queuePushCond); + ZSTD_pthread_cond_destroy(&ctx->queuePopCond); + ZSTD_customFree(ctx->queue, ctx->customMem); + ZSTD_customFree(ctx->threads, ctx->customMem); + ZSTD_customFree(ctx, ctx->customMem); +} -typedef struct { - unsigned int offset; /* The offset of the match. (NOT the same as the offset code) - * If offset == 0 and matchLength == 0, this sequence represents the last - * literals in the block of litLength size. - */ +/*! POOL_joinJobs() : + * Waits for all queued jobs to finish executing. + */ +void POOL_joinJobs(POOL_ctx* ctx) { + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + while(!ctx->queueEmpty || ctx->numThreadsBusy > 0) { + ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex); + } + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); +} - unsigned int litLength; /* Literal length of the sequence. */ - unsigned int matchLength; /* Match length of the sequence. */ +void ZSTD_freeThreadPool (ZSTD_threadPool* pool) { + POOL_free (pool); +} - /* Note: Users of this API may provide a sequence with matchLength == litLength == offset == 0. - * In this case, we will treat the sequence as a marker for a block boundary. - */ +size_t POOL_sizeof(const POOL_ctx* ctx) { + if (ctx==NULL) return 0; /* supports sizeof NULL */ + return sizeof(*ctx) + + ctx->queueSize * sizeof(POOL_job) + + ctx->threadCapacity * sizeof(ZSTD_pthread_t); +} - unsigned int rep; /* Represents which repeat offset is represented by the field 'offset'. - * Ranges from [0, 3]. - * - * Repeat offsets are essentially previous offsets from previous sequences sorted in - * recency order. For more detail, see doc/zstd_compression_format.md - * - * If rep == 0, then 'offset' does not contain a repeat offset. - * If rep > 0: - * If litLength != 0: - * rep == 1 --> offset == repeat_offset_1 - * rep == 2 --> offset == repeat_offset_2 - * rep == 3 --> offset == repeat_offset_3 - * If litLength == 0: - * rep == 1 --> offset == repeat_offset_2 - * rep == 2 --> offset == repeat_offset_3 - * rep == 3 --> offset == repeat_offset_1 - 1 - * - * Note: This field is optional. ZSTD_generateSequences() will calculate the value of - * 'rep', but repeat offsets do not necessarily need to be calculated from an external - * sequence provider's perspective. For example, ZSTD_compressSequences() does not - * use this 'rep' field at all (as of now). - */ -} ZSTD_Sequence; -typedef struct { - unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */ - unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */ - unsigned hashLog; /**< dispatch table : larger == faster, more memory */ - unsigned searchLog; /**< nb of searches : larger == more compression, slower */ - unsigned minMatch; /**< match length searched : larger == faster decompression, sometimes less compression */ - unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */ - ZSTD_strategy strategy; /**< see ZSTD_strategy definition above */ -} ZSTD_compressionParameters; +/* @return : 0 on success, 1 on error */ +static int POOL_resize_internal(POOL_ctx* ctx, size_t numThreads) +{ + if (numThreads <= ctx->threadCapacity) { + if (!numThreads) return 1; + ctx->threadLimit = numThreads; + return 0; + } + /* numThreads > threadCapacity */ + { ZSTD_pthread_t* const threadPool = (ZSTD_pthread_t*)ZSTD_customCalloc(numThreads * sizeof(ZSTD_pthread_t), ctx->customMem); + if (!threadPool) return 1; + /* replace existing thread pool */ + ZSTD_memcpy(threadPool, ctx->threads, ctx->threadCapacity * sizeof(ZSTD_pthread_t)); + ZSTD_customFree(ctx->threads, ctx->customMem); + ctx->threads = threadPool; + /* Initialize additional threads */ + { size_t threadId; + for (threadId = ctx->threadCapacity; threadId < numThreads; ++threadId) { + if (ZSTD_pthread_create(&threadPool[threadId], NULL, &POOL_thread, ctx)) { + ctx->threadCapacity = threadId; + return 1; + } } + } } + /* successfully expanded */ + ctx->threadCapacity = numThreads; + ctx->threadLimit = numThreads; + return 0; +} -typedef struct { - int contentSizeFlag; /**< 1: content size will be in frame header (when known) */ - int checksumFlag; /**< 1: generate a 32-bits checksum using XXH64 algorithm at end of frame, for error detection */ - int noDictIDFlag; /**< 1: no dictID will be saved into frame header (dictID is only useful for dictionary compression) */ -} ZSTD_frameParameters; +/* @return : 0 on success, 1 on error */ +int POOL_resize(POOL_ctx* ctx, size_t numThreads) +{ + int result; + if (ctx==NULL) return 1; + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + result = POOL_resize_internal(ctx, numThreads); + ZSTD_pthread_cond_broadcast(&ctx->queuePopCond); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + return result; +} -typedef struct { - ZSTD_compressionParameters cParams; - ZSTD_frameParameters fParams; -} ZSTD_parameters; +/** + * Returns 1 if the queue is full and 0 otherwise. + * + * When queueSize is 1 (pool was created with an intended queueSize of 0), + * then a queue is empty if there is a thread free _and_ no job is waiting. + */ +static int isQueueFull(POOL_ctx const* ctx) { + if (ctx->queueSize > 1) { + return ctx->queueHead == ((ctx->queueTail + 1) % ctx->queueSize); + } else { + return (ctx->numThreadsBusy == ctx->threadLimit) || + !ctx->queueEmpty; + } +} -typedef enum { - ZSTD_dct_auto = 0, /* dictionary is "full" when starting with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */ - ZSTD_dct_rawContent = 1, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */ - ZSTD_dct_fullDict = 2 /* refuses to load a dictionary if it does not respect Zstandard's specification, starting with ZSTD_MAGIC_DICTIONARY */ -} ZSTD_dictContentType_e; -typedef enum { - ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */ - ZSTD_dlm_byRef = 1 /**< Reference dictionary content -- the dictionary buffer must outlive its users. */ -} ZSTD_dictLoadMethod_e; +static void +POOL_add_internal(POOL_ctx* ctx, POOL_function function, void *opaque) +{ + POOL_job job; + job.function = function; + job.opaque = opaque; + assert(ctx != NULL); + if (ctx->shutdown) return; -typedef enum { - ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */ - ZSTD_f_zstd1_magicless = 1 /* Variant of zstd frame format, without initial 4-bytes magic number. - * Useful to save 4 bytes per generated frame. - * Decoder cannot recognise automatically this format, requiring this instruction. */ -} ZSTD_format_e; + ctx->queueEmpty = 0; + ctx->queue[ctx->queueTail] = job; + ctx->queueTail = (ctx->queueTail + 1) % ctx->queueSize; + ZSTD_pthread_cond_signal(&ctx->queuePopCond); +} -typedef enum { - /* Note: this enum controls ZSTD_d_forceIgnoreChecksum */ - ZSTD_d_validateChecksum = 0, - ZSTD_d_ignoreChecksum = 1 -} ZSTD_forceIgnoreChecksum_e; +void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque) +{ + assert(ctx != NULL); + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + /* Wait until there is space in the queue for the new job */ + while (isQueueFull(ctx) && (!ctx->shutdown)) { + ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex); + } + POOL_add_internal(ctx, function, opaque); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); +} -typedef enum { - /* Note: this enum controls ZSTD_d_refMultipleDDicts */ - ZSTD_rmd_refSingleDDict = 0, - ZSTD_rmd_refMultipleDDicts = 1 -} ZSTD_refMultipleDDicts_e; -typedef enum { - /* Note: this enum and the behavior it controls are effectively internal - * implementation details of the compressor. They are expected to continue - * to evolve and should be considered only in the context of extremely - * advanced performance tuning. - * - * Zstd currently supports the use of a CDict in three ways: - * - * - The contents of the CDict can be copied into the working context. This - * means that the compression can search both the dictionary and input - * while operating on a single set of internal tables. This makes - * the compression faster per-byte of input. However, the initial copy of - * the CDict's tables incurs a fixed cost at the beginning of the - * compression. For small compressions (< 8 KB), that copy can dominate - * the cost of the compression. - * - * - The CDict's tables can be used in-place. In this model, compression is - * slower per input byte, because the compressor has to search two sets of - * tables. However, this model incurs no start-up cost (as long as the - * working context's tables can be reused). For small inputs, this can be - * faster than copying the CDict's tables. - * - * - The CDict's tables are not used at all, and instead we use the working - * context alone to reload the dictionary and use params based on the source - * size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict(). - * This method is effective when the dictionary sizes are very small relative - * to the input size, and the input size is fairly large to begin with. - * - * Zstd has a simple internal heuristic that selects which strategy to use - * at the beginning of a compression. However, if experimentation shows that - * Zstd is making poor choices, it is possible to override that choice with - * this enum. - */ - ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */ - ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */ - ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */ - ZSTD_dictForceLoad = 3 /* Always reload the dictionary */ -} ZSTD_dictAttachPref_e; +int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) +{ + assert(ctx != NULL); + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + if (isQueueFull(ctx)) { + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + return 0; + } + POOL_add_internal(ctx, function, opaque); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + return 1; +} -typedef enum { - ZSTD_lcm_auto = 0, /**< Automatically determine the compression mode based on the compression level. - * Negative compression levels will be uncompressed, and positive compression - * levels will be compressed. */ - ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be - * emitted if Huffman compression is not profitable. */ - ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */ -} ZSTD_literalCompressionMode_e; +#else /* ZSTD_MULTITHREAD not defined */ -/*************************************** -* Frame size functions -***************************************/ +/* ========================== */ +/* No multi-threading support */ +/* ========================== */ -/*! ZSTD_findDecompressedSize() : - * `src` should point to the start of a series of ZSTD encoded and/or skippable frames - * `srcSize` must be the _exact_ size of this series - * (i.e. there should be a frame boundary at `src + srcSize`) - * @return : - decompressed size of all data in all successive frames - * - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN - * - if an error occurred: ZSTD_CONTENTSIZE_ERROR - * - * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. - * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. - * In which case, it's necessary to use streaming mode to decompress data. - * note 2 : decompressed size is always present when compression is done with ZSTD_compress() - * note 3 : decompressed size can be very large (64-bits value), - * potentially larger than what local system can handle as a single memory segment. - * In which case, it's necessary to use streaming mode to decompress data. - * note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified. - * Always ensure result fits within application's authorized limits. - * Each application can set its own limits. - * note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to - * read each contained frame header. This is fast as most of the data is skipped, - * however it does mean that all frame data must be present and valid. */ -ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); - -/*! ZSTD_decompressBound() : - * `src` should point to the start of a series of ZSTD encoded and/or skippable frames - * `srcSize` must be the _exact_ size of this series - * (i.e. there should be a frame boundary at `src + srcSize`) - * @return : - upper-bound for the decompressed size of all data in all successive frames - * - if an error occurred: ZSTD_CONTENTSIZE_ERROR - * - * note 1 : an error can occur if `src` contains an invalid or incorrectly formatted frame. - * note 2 : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`. - * in this case, `ZSTD_findDecompressedSize` and `ZSTD_decompressBound` return the same value. - * note 3 : when the decompressed size field isn't available, the upper-bound for that frame is calculated by: - * upper-bound = # blocks * min(128 KB, Window_Size) - */ -ZSTDLIB_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize); - -/*! ZSTD_frameHeaderSize() : - * srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX. - * @return : size of the Frame Header, - * or an error code (if srcSize is too small) */ -ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); -typedef enum { - ZSTD_sf_noBlockDelimiters = 0, /* Representation of ZSTD_Sequence has no block delimiters, sequences only */ - ZSTD_sf_explicitBlockDelimiters = 1 /* Representation of ZSTD_Sequence contains explicit block delimiters */ -} ZSTD_sequenceFormat_e; +/* We don't need any data, but if it is empty, malloc() might return NULL. */ +struct POOL_ctx_s { + int dummy; +}; +static POOL_ctx g_poolCtx; -/*! ZSTD_generateSequences() : - * Generate sequences using ZSTD_compress2, given a source buffer. - * - * Each block will end with a dummy sequence - * with offset == 0, matchLength == 0, and litLength == length of last literals. - * litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0) - * simply acts as a block delimiter. - * - * zc can be used to insert custom compression params. - * This function invokes ZSTD_compress2 - * - * The output of this function can be fed into ZSTD_compressSequences() with CCtx - * setting of ZSTD_c_blockDelimiters as ZSTD_sf_explicitBlockDelimiters - * @return : number of sequences generated - */ +POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) { + return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem); +} -ZSTDLIB_API size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, - size_t outSeqsSize, const void* src, size_t srcSize); +POOL_ctx* +POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem) +{ + (void)numThreads; + (void)queueSize; + (void)customMem; + return &g_poolCtx; +} -/*! ZSTD_mergeBlockDelimiters() : - * Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals - * by merging them into into the literals of the next sequence. - * - * As such, the final generated result has no explicit representation of block boundaries, - * and the final last literals segment is not represented in the sequences. - * - * The output of this function can be fed into ZSTD_compressSequences() with CCtx - * setting of ZSTD_c_blockDelimiters as ZSTD_sf_noBlockDelimiters - * @return : number of sequences left after merging - */ -ZSTDLIB_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize); +void POOL_free(POOL_ctx* ctx) { + assert(!ctx || ctx == &g_poolCtx); + (void)ctx; +} -/*! ZSTD_compressSequences() : - * Compress an array of ZSTD_Sequence, generated from the original source buffer, into dst. - * If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.) - * The entire source is compressed into a single frame. - * - * The compression behavior changes based on cctx params. In particular: - * If ZSTD_c_blockDelimiters == ZSTD_sf_noBlockDelimiters, the array of ZSTD_Sequence is expected to contain - * no block delimiters (defined in ZSTD_Sequence). Block boundaries are roughly determined based on - * the block size derived from the cctx, and sequences may be split. This is the default setting. - * - * If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain - * block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided. - * - * If ZSTD_c_validateSequences == 0, this function will blindly accept the sequences provided. Invalid sequences cause undefined - * behavior. If ZSTD_c_validateSequences == 1, then if sequence is invalid (see doc/zstd_compression_format.md for - * specifics regarding offset/matchlength requirements) then the function will bail out and return an error. - * - * In addition to the two adjustable experimental params, there are other important cctx params. - * - ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN. - * - ZSTD_c_compressionLevel accordingly adjusts the strength of the entropy coder, as it would in typical compression. - * - ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset - * is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md - * - * Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused. - * Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly, - * and cannot emit an RLE block that disagrees with the repcode history - * @return : final compressed size or a ZSTD error. - */ -ZSTDLIB_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstSize, - const ZSTD_Sequence* inSeqs, size_t inSeqsSize, - const void* src, size_t srcSize); +void POOL_joinJobs(POOL_ctx* ctx){ + assert(!ctx || ctx == &g_poolCtx); + (void)ctx; +} +int POOL_resize(POOL_ctx* ctx, size_t numThreads) { + (void)ctx; (void)numThreads; + return 0; +} -/*! ZSTD_writeSkippableFrame() : - * Generates a zstd skippable frame containing data given by src, and writes it to dst buffer. - * - * Skippable frames begin with a a 4-byte magic number. There are 16 possible choices of magic number, - * ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15. - * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so - * the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant. - * - * Returns an error if destination buffer is not large enough, if the source size is not representable - * with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid). - * - * @return : number of bytes written or a ZSTD error. - */ -ZSTDLIB_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity, - const void* src, size_t srcSize, unsigned magicVariant); +void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque) { + (void)ctx; + function(opaque); +} +int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) { + (void)ctx; + function(opaque); + return 1; +} -/*************************************** -* Memory management -***************************************/ - -/*! ZSTD_estimate*() : - * These functions make it possible to estimate memory usage - * of a future {D,C}Ctx, before its creation. - * - * ZSTD_estimateCCtxSize() will provide a memory budget large enough - * for any compression level up to selected one. - * Note : Unlike ZSTD_estimateCStreamSize*(), this estimate - * does not include space for a window buffer. - * Therefore, the estimation is only guaranteed for single-shot compressions, not streaming. - * The estimate will assume the input may be arbitrarily large, - * which is the worst case. - * - * When srcSize can be bound by a known and rather "small" value, - * this fact can be used to provide a tighter estimation - * because the CCtx compression context will need less memory. - * This tighter estimation can be provided by more advanced functions - * ZSTD_estimateCCtxSize_usingCParams(), which can be used in tandem with ZSTD_getCParams(), - * and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter(). - * Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits. - * - * Note 2 : only single-threaded compression is supported. - * ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1. - */ -ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel); -ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); -ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params); -ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void); - -/*! ZSTD_estimateCStreamSize() : - * ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one. - * It will also consider src size to be arbitrarily "large", which is worst case. - * If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation. - * ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. - * ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1. - * Note : CStream size estimation is only correct for single-threaded compression. - * ZSTD_DStream memory budget depends on window Size. - * This information can be passed manually, using ZSTD_estimateDStreamSize, - * or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame(); - * Note : if streaming is init with function ZSTD_init?Stream_usingDict(), - * an internal ?Dict will be created, which additional size is not estimated here. - * In this case, get total size by adding ZSTD_estimate?DictSize */ -ZSTDLIB_API size_t ZSTD_estimateCStreamSize(int compressionLevel); -ZSTDLIB_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams); -ZSTDLIB_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params); -ZSTDLIB_API size_t ZSTD_estimateDStreamSize(size_t windowSize); -ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize); - -/*! ZSTD_estimate?DictSize() : - * ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict(). - * ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, like ZSTD_createCDict_advanced(). - * Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller. - */ -ZSTDLIB_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel); -ZSTDLIB_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod); -ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod); - -/*! ZSTD_initStatic*() : - * Initialize an object using a pre-allocated fixed-size buffer. - * workspace: The memory area to emplace the object into. - * Provided pointer *must be 8-bytes aligned*. - * Buffer must outlive object. - * workspaceSize: Use ZSTD_estimate*Size() to determine - * how large workspace must be to support target scenario. - * @return : pointer to object (same address as workspace, just different type), - * or NULL if error (size too small, incorrect alignment, etc.) - * Note : zstd will never resize nor malloc() when using a static buffer. - * If the object requires more memory than available, - * zstd will just error out (typically ZSTD_error_memory_allocation). - * Note 2 : there is no corresponding "free" function. - * Since workspace is allocated externally, it must be freed externally too. - * Note 3 : cParams : use ZSTD_getCParams() to convert a compression level - * into its associated cParams. - * Limitation 1 : currently not compatible with internal dictionary creation, triggered by - * ZSTD_CCtx_loadDictionary(), ZSTD_initCStream_usingDict() or ZSTD_initDStream_usingDict(). - * Limitation 2 : static cctx currently not compatible with multi-threading. - * Limitation 3 : static dctx is incompatible with legacy support. - */ -ZSTDLIB_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize); -ZSTDLIB_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */ - -ZSTDLIB_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize); -ZSTDLIB_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */ - -ZSTDLIB_API const ZSTD_CDict* ZSTD_initStaticCDict( - void* workspace, size_t workspaceSize, - const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, - ZSTD_dictContentType_e dictContentType, - ZSTD_compressionParameters cParams); - -ZSTDLIB_API const ZSTD_DDict* ZSTD_initStaticDDict( - void* workspace, size_t workspaceSize, - const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, - ZSTD_dictContentType_e dictContentType); - - -/*! Custom memory allocation : - * These prototypes make it possible to pass your own allocation/free functions. - * ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below. - * All allocation/free operations will be completed using these custom variants instead of regular ones. - */ -typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size); -typedef void (*ZSTD_freeFunction) (void* opaque, void* address); -typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; -static -#ifdef __GNUC__ -__attribute__((__unused__)) -#endif -ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */ - -ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); -ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); -ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); -ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); - -ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, - ZSTD_dictContentType_e dictContentType, - ZSTD_compressionParameters cParams, - ZSTD_customMem customMem); - -/* ! Thread pool : - * These prototypes make it possible to share a thread pool among multiple compression contexts. - * This can limit resources for applications with multiple threads where each one uses - * a threaded compression mode (via ZSTD_c_nbWorkers parameter). - * ZSTD_createThreadPool creates a new thread pool with a given number of threads. - * Note that the lifetime of such pool must exist while being used. - * ZSTD_CCtx_refThreadPool assigns a thread pool to a context (use NULL argument value - * to use an internal thread pool). - * ZSTD_freeThreadPool frees a thread pool. - */ -typedef struct POOL_ctx_s ZSTD_threadPool; -ZSTDLIB_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads); -ZSTDLIB_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool); -ZSTDLIB_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool); - +size_t POOL_sizeof(const POOL_ctx* ctx) { + if (ctx==NULL) return 0; /* supports sizeof NULL */ + assert(ctx == &g_poolCtx); + return sizeof(*ctx); +} +#endif /* ZSTD_MULTITHREAD */ +/**** ended inlining common/pool.c ****/ +/**** start inlining common/zstd_common.c ****/ /* - * This API is temporary and is expected to change or disappear in the future! - */ -ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced2( - const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, - ZSTD_dictContentType_e dictContentType, - const ZSTD_CCtx_params* cctxParams, - ZSTD_customMem customMem); - -ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced( - const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, - ZSTD_dictContentType_e dictContentType, - ZSTD_customMem customMem); - - -/*************************************** -* Advanced compression functions -***************************************/ - -/*! ZSTD_createCDict_byReference() : - * Create a digested dictionary for compression - * Dictionary content is just referenced, not duplicated. - * As a consequence, `dictBuffer` **must** outlive CDict, - * and its content must remain unmodified throughout the lifetime of CDict. - * note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */ -ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); - -/*! ZSTD_getDictID_fromCDict() : - * Provides the dictID of the dictionary loaded into `cdict`. - * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. - * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ -ZSTDLIB_API unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict); - -/*! ZSTD_getCParams() : - * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize. - * `estimatedSrcSize` value is optional, select 0 if not known */ -ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); - -/*! ZSTD_getParams() : - * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`. - * All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */ -ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); - -/*! ZSTD_checkCParams() : - * Ensure param values remain within authorized range. - * @return 0 on success, or an error code (can be checked with ZSTD_isError()) */ -ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); - -/*! ZSTD_adjustCParams() : - * optimize params for a given `srcSize` and `dictSize`. - * `srcSize` can be unknown, in which case use ZSTD_CONTENTSIZE_UNKNOWN. - * `dictSize` must be `0` when there is no dictionary. - * cPar can be invalid : all parameters will be clamped within valid range in the @return struct. - * This function never fails (wide contract) */ -ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); - -/*! ZSTD_compress_advanced() : - * Note : this function is now DEPRECATED. - * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters. - * This prototype will be marked as deprecated and generate compilation warning on reaching v1.5.x */ -ZSTDLIB_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize, - ZSTD_parameters params); - -/*! ZSTD_compress_usingCDict_advanced() : - * Note : this function is now REDUNDANT. - * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters. - * This prototype will be marked as deprecated and generate compilation warning in some future version */ -ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTD_CDict* cdict, - ZSTD_frameParameters fParams); - - -/*! ZSTD_CCtx_loadDictionary_byReference() : - * Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx. - * It saves some memory, but also requires that `dict` outlives its usage within `cctx` */ -ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); - -/*! ZSTD_CCtx_loadDictionary_advanced() : - * Same as ZSTD_CCtx_loadDictionary(), but gives finer control over - * how to load the dictionary (by copy ? by reference ?) - * and how to interpret it (automatic ? force raw mode ? full mode only ?) */ -ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); - -/*! ZSTD_CCtx_refPrefix_advanced() : - * Same as ZSTD_CCtx_refPrefix(), but gives finer control over - * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ -ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); - -/* === experimental parameters === */ -/* these parameters can be used with ZSTD_setParameter() - * they are not guaranteed to remain supported in the future */ - - /* Enables rsyncable mode, - * which makes compressed files more rsync friendly - * by adding periodic synchronization points to the compressed data. - * The target average block size is ZSTD_c_jobSize / 2. - * It's possible to modify the job size to increase or decrease - * the granularity of the synchronization point. - * Once the jobSize is smaller than the window size, - * it will result in compression ratio degradation. - * NOTE 1: rsyncable mode only works when multithreading is enabled. - * NOTE 2: rsyncable performs poorly in combination with long range mode, - * since it will decrease the effectiveness of synchronization points, - * though mileage may vary. - * NOTE 3: Rsyncable mode limits maximum compression speed to ~400 MB/s. - * If the selected compression level is already running significantly slower, - * the overall speed won't be significantly impacted. - */ - #define ZSTD_c_rsyncable ZSTD_c_experimentalParam1 - -/* Select a compression format. - * The value must be of type ZSTD_format_e. - * See ZSTD_format_e enum definition for details */ -#define ZSTD_c_format ZSTD_c_experimentalParam2 - -/* Force back-reference distances to remain < windowSize, - * even when referencing into Dictionary content (default:0) */ -#define ZSTD_c_forceMaxWindow ZSTD_c_experimentalParam3 - -/* Controls whether the contents of a CDict - * are used in place, or copied into the working context. - * Accepts values from the ZSTD_dictAttachPref_e enum. - * See the comments on that enum for an explanation of the feature. */ -#define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4 - -/* Controls how the literals are compressed (default is auto). - * The value must be of type ZSTD_literalCompressionMode_e. - * See ZSTD_literalCompressionMode_t enum definition for details. - */ -#define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5 - -/* Tries to fit compressed block size to be around targetCBlockSize. - * No target when targetCBlockSize == 0. - * There is no guarantee on compressed block size (default:0) */ -#define ZSTD_c_targetCBlockSize ZSTD_c_experimentalParam6 - -/* User's best guess of source size. - * Hint is not valid when srcSizeHint == 0. - * There is no guarantee that hint is close to actual source size, - * but compression ratio may regress significantly if guess considerably underestimates */ -#define ZSTD_c_srcSizeHint ZSTD_c_experimentalParam7 - -/* Controls whether the new and experimental "dedicated dictionary search - * structure" can be used. This feature is still rough around the edges, be - * prepared for surprising behavior! - * - * How to use it: - * - * When using a CDict, whether to use this feature or not is controlled at - * CDict creation, and it must be set in a CCtxParams set passed into that - * construction (via ZSTD_createCDict_advanced2()). A compression will then - * use the feature or not based on how the CDict was constructed; the value of - * this param, set in the CCtx, will have no effect. - * - * However, when a dictionary buffer is passed into a CCtx, such as via - * ZSTD_CCtx_loadDictionary(), this param can be set on the CCtx to control - * whether the CDict that is created internally can use the feature or not. - * - * What it does: - * - * Normally, the internal data structures of the CDict are analogous to what - * would be stored in a CCtx after compressing the contents of a dictionary. - * To an approximation, a compression using a dictionary can then use those - * data structures to simply continue what is effectively a streaming - * compression where the simulated compression of the dictionary left off. - * Which is to say, the search structures in the CDict are normally the same - * format as in the CCtx. - * - * It is possible to do better, since the CDict is not like a CCtx: the search - * structures are written once during CDict creation, and then are only read - * after that, while the search structures in the CCtx are both read and - * written as the compression goes along. This means we can choose a search - * structure for the dictionary that is read-optimized. - * - * This feature enables the use of that different structure. - * - * Note that some of the members of the ZSTD_compressionParameters struct have - * different semantics and constraints in the dedicated search structure. It is - * highly recommended that you simply set a compression level in the CCtxParams - * you pass into the CDict creation call, and avoid messing with the cParams - * directly. - * - * Effects: - * - * This will only have any effect when the selected ZSTD_strategy - * implementation supports this feature. Currently, that's limited to - * ZSTD_greedy, ZSTD_lazy, and ZSTD_lazy2. - * - * Note that this means that the CDict tables can no longer be copied into the - * CCtx, so the dict attachment mode ZSTD_dictForceCopy will no longer be - * useable. The dictionary can only be attached or reloaded. - * - * In general, you should expect compression to be faster--sometimes very much - * so--and CDict creation to be slightly slower. Eventually, we will probably - * make this mode the default. - */ -#define ZSTD_c_enableDedicatedDictSearch ZSTD_c_experimentalParam8 - -/* ZSTD_c_stableInBuffer - * Experimental parameter. - * Default is 0 == disabled. Set to 1 to enable. - * - * Tells the compressor that the ZSTD_inBuffer will ALWAYS be the same - * between calls, except for the modifications that zstd makes to pos (the - * caller must not modify pos). This is checked by the compressor, and - * compression will fail if it ever changes. This means the only flush - * mode that makes sense is ZSTD_e_end, so zstd will error if ZSTD_e_end - * is not used. The data in the ZSTD_inBuffer in the range [src, src + pos) - * MUST not be modified during compression or you will get data corruption. - * - * When this flag is enabled zstd won't allocate an input window buffer, - * because the user guarantees it can reference the ZSTD_inBuffer until - * the frame is complete. But, it will still allocate an output buffer - * large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also - * avoid the memcpy() from the input buffer to the input window buffer. - * - * NOTE: ZSTD_compressStream2() will error if ZSTD_e_end is not used. - * That means this flag cannot be used with ZSTD_compressStream(). - * - * NOTE: So long as the ZSTD_inBuffer always points to valid memory, using - * this flag is ALWAYS memory safe, and will never access out-of-bounds - * memory. However, compression WILL fail if you violate the preconditions. - * - * WARNING: The data in the ZSTD_inBuffer in the range [dst, dst + pos) MUST - * not be modified during compression or you will get data corruption. This - * is because zstd needs to reference data in the ZSTD_inBuffer to find - * matches. Normally zstd maintains its own window buffer for this purpose, - * but passing this flag tells zstd to use the user provided buffer. - */ -#define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9 - -/* ZSTD_c_stableOutBuffer - * Experimental parameter. - * Default is 0 == disabled. Set to 1 to enable. - * - * Tells he compressor that the ZSTD_outBuffer will not be resized between - * calls. Specifically: (out.size - out.pos) will never grow. This gives the - * compressor the freedom to say: If the compressed data doesn't fit in the - * output buffer then return ZSTD_error_dstSizeTooSmall. This allows us to - * always decompress directly into the output buffer, instead of decompressing - * into an internal buffer and copying to the output buffer. - * - * When this flag is enabled zstd won't allocate an output buffer, because - * it can write directly to the ZSTD_outBuffer. It will still allocate the - * input window buffer (see ZSTD_c_stableInBuffer). - * - * Zstd will check that (out.size - out.pos) never grows and return an error - * if it does. While not strictly necessary, this should prevent surprises. - */ -#define ZSTD_c_stableOutBuffer ZSTD_c_experimentalParam10 - -/* ZSTD_c_blockDelimiters - * Default is 0 == ZSTD_sf_noBlockDelimiters. - * - * For use with sequence compression API: ZSTD_compressSequences(). - * - * Designates whether or not the given array of ZSTD_Sequence contains block delimiters - * and last literals, which are defined as sequences with offset == 0 and matchLength == 0. - * See the definition of ZSTD_Sequence for more specifics. - */ -#define ZSTD_c_blockDelimiters ZSTD_c_experimentalParam11 - -/* ZSTD_c_validateSequences - * Default is 0 == disabled. Set to 1 to enable sequence validation. - * - * For use with sequence compression API: ZSTD_compressSequences(). - * Designates whether or not we validate sequences provided to ZSTD_compressSequences() - * during function execution. - * - * Without validation, providing a sequence that does not conform to the zstd spec will cause - * undefined behavior, and may produce a corrupted block. - * - * With validation enabled, a if sequence is invalid (see doc/zstd_compression_format.md for - * specifics regarding offset/matchlength requirements) then the function will bail out and - * return an error. - * - */ -#define ZSTD_c_validateSequences ZSTD_c_experimentalParam12 - -/*! ZSTD_CCtx_getParameter() : - * Get the requested compression parameter value, selected by enum ZSTD_cParameter, - * and store it into int* value. - * @return : 0, or an error code (which can be tested with ZSTD_isError()). - */ -ZSTDLIB_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value); - - -/*! ZSTD_CCtx_params : - * Quick howto : - * - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure - * - ZSTD_CCtxParams_setParameter() : Push parameters one by one into - * an existing ZSTD_CCtx_params structure. - * This is similar to - * ZSTD_CCtx_setParameter(). - * - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to - * an existing CCtx. - * These parameters will be applied to - * all subsequent frames. - * - ZSTD_compressStream2() : Do compression using the CCtx. - * - ZSTD_freeCCtxParams() : Free the memory. + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. * - * This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams() - * for static allocation of CCtx for single-threaded compression. - */ -ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void); -ZSTDLIB_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); - -/*! ZSTD_CCtxParams_reset() : - * Reset params to default values. - */ -ZSTDLIB_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params); - -/*! ZSTD_CCtxParams_init() : - * Initializes the compression parameters of cctxParams according to - * compression level. All other parameters are reset to their default values. - */ -ZSTDLIB_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel); - -/*! ZSTD_CCtxParams_init_advanced() : - * Initializes the compression and frame parameters of cctxParams according to - * params. All other parameters are reset to their default values. - */ -ZSTDLIB_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params); - -/*! ZSTD_CCtxParams_setParameter() : - * Similar to ZSTD_CCtx_setParameter. - * Set one compression parameter, selected by enum ZSTD_cParameter. - * Parameters must be applied to a ZSTD_CCtx using - * ZSTD_CCtx_setParametersUsingCCtxParams(). - * @result : a code representing success or failure (which can be tested with - * ZSTD_isError()). - */ -ZSTDLIB_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value); - -/*! ZSTD_CCtxParams_getParameter() : - * Similar to ZSTD_CCtx_getParameter. - * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter. - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - */ -ZSTDLIB_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value); - -/*! ZSTD_CCtx_setParametersUsingCCtxParams() : - * Apply a set of ZSTD_CCtx_params to the compression context. - * This can be done even after compression is started, - * if nbWorkers==0, this will have no impact until a new compression is started. - * if nbWorkers>=1, new parameters will be picked up at next job, - * with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated). + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. */ -ZSTDLIB_API size_t ZSTD_CCtx_setParametersUsingCCtxParams( - ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params); -/*! ZSTD_compressStream2_simpleArgs() : - * Same as ZSTD_compressStream2(), - * but using only integral types as arguments. - * This variant might be helpful for binders from dynamic languages - * which have troubles handling structures containing memory pointers. - */ -ZSTDLIB_API size_t ZSTD_compressStream2_simpleArgs ( - ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, size_t* dstPos, - const void* src, size_t srcSize, size_t* srcPos, - ZSTD_EndDirective endOp); -/*************************************** -* Advanced decompression functions +/*-************************************* +* Dependencies ***************************************/ - -/*! ZSTD_isFrame() : - * Tells if the content of `buffer` starts with a valid Frame Identifier. - * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. - * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. - * Note 3 : Skippable Frame Identifiers are considered valid. */ -ZSTDLIB_API unsigned ZSTD_isFrame(const void* buffer, size_t size); - -/*! ZSTD_createDDict_byReference() : - * Create a digested dictionary, ready to start decompression operation without startup delay. - * Dictionary content is referenced, and therefore stays in dictBuffer. - * It is important that dictBuffer outlives DDict, - * it must remain read accessible throughout the lifetime of DDict */ -ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize); - -/*! ZSTD_DCtx_loadDictionary_byReference() : - * Same as ZSTD_DCtx_loadDictionary(), - * but references `dict` content instead of copying it into `dctx`. - * This saves memory if `dict` remains around., - * However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression. */ -ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); - -/*! ZSTD_DCtx_loadDictionary_advanced() : - * Same as ZSTD_DCtx_loadDictionary(), - * but gives direct control over - * how to load the dictionary (by copy ? by reference ?) - * and how to interpret it (automatic ? force raw mode ? full mode only ?). */ -ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); - -/*! ZSTD_DCtx_refPrefix_advanced() : - * Same as ZSTD_DCtx_refPrefix(), but gives finer control over - * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ -ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); - -/*! ZSTD_DCtx_setMaxWindowSize() : - * Refuses allocating internal buffers for frames requiring a window size larger than provided limit. - * This protects a decoder context from reserving too much memory for itself (potential attack scenario). - * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. - * By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) - * @return : 0, or an error code (which can be tested using ZSTD_isError()). - */ -ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize); - -/*! ZSTD_DCtx_getParameter() : - * Get the requested decompression parameter value, selected by enum ZSTD_dParameter, - * and store it into int* value. - * @return : 0, or an error code (which can be tested with ZSTD_isError()). - */ -ZSTDLIB_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value); - -/* ZSTD_d_format - * experimental parameter, - * allowing selection between ZSTD_format_e input compression formats - */ -#define ZSTD_d_format ZSTD_d_experimentalParam1 -/* ZSTD_d_stableOutBuffer - * Experimental parameter. - * Default is 0 == disabled. Set to 1 to enable. - * - * Tells the decompressor that the ZSTD_outBuffer will ALWAYS be the same - * between calls, except for the modifications that zstd makes to pos (the - * caller must not modify pos). This is checked by the decompressor, and - * decompression will fail if it ever changes. Therefore the ZSTD_outBuffer - * MUST be large enough to fit the entire decompressed frame. This will be - * checked when the frame content size is known. The data in the ZSTD_outBuffer - * in the range [dst, dst + pos) MUST not be modified during decompression - * or you will get data corruption. - * - * When this flags is enabled zstd won't allocate an output buffer, because - * it can write directly to the ZSTD_outBuffer, but it will still allocate - * an input buffer large enough to fit any compressed block. This will also - * avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer. - * If you need to avoid the input buffer allocation use the buffer-less - * streaming API. - * - * NOTE: So long as the ZSTD_outBuffer always points to valid memory, using - * this flag is ALWAYS memory safe, and will never access out-of-bounds - * memory. However, decompression WILL fail if you violate the preconditions. - * - * WARNING: The data in the ZSTD_outBuffer in the range [dst, dst + pos) MUST - * not be modified during decompression or you will get data corruption. This - * is because zstd needs to reference data in the ZSTD_outBuffer to regenerate - * matches. Normally zstd maintains its own buffer for this purpose, but passing - * this flag tells zstd to use the user provided buffer. - */ -#define ZSTD_d_stableOutBuffer ZSTD_d_experimentalParam2 - -/* ZSTD_d_forceIgnoreChecksum - * Experimental parameter. - * Default is 0 == disabled. Set to 1 to enable - * - * Tells the decompressor to skip checksum validation during decompression, regardless - * of whether checksumming was specified during compression. This offers some - * slight performance benefits, and may be useful for debugging. - * Param has values of type ZSTD_forceIgnoreChecksum_e - */ -#define ZSTD_d_forceIgnoreChecksum ZSTD_d_experimentalParam3 - -/* ZSTD_d_refMultipleDDicts - * Experimental parameter. - * Default is 0 == disabled. Set to 1 to enable - * - * If enabled and dctx is allocated on the heap, then additional memory will be allocated - * to store references to multiple ZSTD_DDict. That is, multiple calls of ZSTD_refDDict() - * using a given ZSTD_DCtx, rather than overwriting the previous DDict reference, will instead - * store all references. At decompression time, the appropriate dictID is selected - * from the set of DDicts based on the dictID in the frame. - * - * Usage is simply calling ZSTD_refDDict() on multiple dict buffers. - * - * Param has values of byte ZSTD_refMultipleDDicts_e - * - * WARNING: Enabling this parameter and calling ZSTD_DCtx_refDDict(), will trigger memory - * allocation for the hash table. ZSTD_freeDCtx() also frees this memory. - * Memory is allocated as per ZSTD_DCtx::customMem. - * - * Although this function allocates memory for the table, the user is still responsible for - * memory management of the underlying ZSTD_DDict* themselves. - */ -#define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4 - - -/*! ZSTD_DCtx_setFormat() : - * Instruct the decoder context about what kind of data to decode next. - * This instruction is mandatory to decode data without a fully-formed header, - * such ZSTD_f_zstd1_magicless for example. - * @return : 0, or an error code (which can be tested using ZSTD_isError()). */ -ZSTDLIB_API size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); - -/*! ZSTD_decompressStream_simpleArgs() : - * Same as ZSTD_decompressStream(), - * but using only integral types as arguments. - * This can be helpful for binders from dynamic languages - * which have troubles handling structures containing memory pointers. - */ -ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs ( - ZSTD_DCtx* dctx, - void* dst, size_t dstCapacity, size_t* dstPos, - const void* src, size_t srcSize, size_t* srcPos); - - -/******************************************************************** -* Advanced streaming functions -* Warning : most of these functions are now redundant with the Advanced API. -* Once Advanced API reaches "stable" status, -* redundant functions will be deprecated, and then at some point removed. -********************************************************************/ - -/*===== Advanced Streaming compression functions =====*/ - -/*! ZSTD_initCStream_srcSize() : - * This function is deprecated, and equivalent to: - * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) - * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); - * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); - * - * pledgedSrcSize must be correct. If it is not known at init time, use - * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, - * "0" also disables frame content size field. It may be enabled in the future. - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x - */ -ZSTDLIB_API size_t -ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, - int compressionLevel, - unsigned long long pledgedSrcSize); - -/*! ZSTD_initCStream_usingDict() : - * This function is deprecated, and is equivalent to: - * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); - * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); - * - * Creates of an internal CDict (incompatible with static CCtx), except if - * dict == NULL or dictSize < 8, in which case no dict is used. - * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if - * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy. - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x - */ -ZSTDLIB_API size_t -ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, - const void* dict, size_t dictSize, - int compressionLevel); - -/*! ZSTD_initCStream_advanced() : - * This function is deprecated, and is approximately equivalent to: - * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * // Pseudocode: Set each zstd parameter and leave the rest as-is. - * for ((param, value) : params) { - * ZSTD_CCtx_setParameter(zcs, param, value); - * } - * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); - * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); - * - * dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy. - * pledgedSrcSize must be correct. - * If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x - */ -ZSTDLIB_API size_t -ZSTD_initCStream_advanced(ZSTD_CStream* zcs, - const void* dict, size_t dictSize, - ZSTD_parameters params, - unsigned long long pledgedSrcSize); - -/*! ZSTD_initCStream_usingCDict() : - * This function is deprecated, and equivalent to: - * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * ZSTD_CCtx_refCDict(zcs, cdict); - * - * note : cdict will just be referenced, and must outlive compression session - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x - */ -ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); - -/*! ZSTD_initCStream_usingCDict_advanced() : - * This function is DEPRECATED, and is approximately equivalent to: - * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * // Pseudocode: Set each zstd frame parameter and leave the rest as-is. - * for ((fParam, value) : fParams) { - * ZSTD_CCtx_setParameter(zcs, fParam, value); - * } - * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); - * ZSTD_CCtx_refCDict(zcs, cdict); - * - * same as ZSTD_initCStream_usingCDict(), with control over frame parameters. - * pledgedSrcSize must be correct. If srcSize is not known at init time, use - * value ZSTD_CONTENTSIZE_UNKNOWN. - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x - */ -ZSTDLIB_API size_t -ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, - const ZSTD_CDict* cdict, - ZSTD_frameParameters fParams, - unsigned long long pledgedSrcSize); - -/*! ZSTD_resetCStream() : - * This function is deprecated, and is equivalent to: - * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); - * - * start a new frame, using same parameters from previous frame. - * This is typically useful to skip dictionary loading stage, since it will re-use it in-place. - * Note that zcs must be init at least once before using ZSTD_resetCStream(). - * If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN. - * If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end. - * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs, - * but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead. - * @return : 0, or an error code (which can be tested using ZSTD_isError()) - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x - */ -ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); - - -typedef struct { - unsigned long long ingested; /* nb input bytes read and buffered */ - unsigned long long consumed; /* nb input bytes actually compressed */ - unsigned long long produced; /* nb of compressed bytes generated and buffered */ - unsigned long long flushed; /* nb of compressed bytes flushed : not provided; can be tracked from caller side */ - unsigned currentJobID; /* MT only : latest started job nb */ - unsigned nbActiveWorkers; /* MT only : nb of workers actively compressing at probe time */ -} ZSTD_frameProgression; - -/* ZSTD_getFrameProgression() : - * tells how much data has been ingested (read from input) - * consumed (input actually compressed) and produced (output) for current frame. - * Note : (ingested - consumed) is amount of input data buffered internally, not yet compressed. - * Aggregates progression inside active worker threads. - */ -ZSTDLIB_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx); - -/*! ZSTD_toFlushNow() : - * Tell how many bytes are ready to be flushed immediately. - * Useful for multithreading scenarios (nbWorkers >= 1). - * Probe the oldest active job, defined as oldest job not yet entirely flushed, - * and check its output buffer. - * @return : amount of data stored in oldest job and ready to be flushed immediately. - * if @return == 0, it means either : - * + there is no active job (could be checked with ZSTD_frameProgression()), or - * + oldest job is still actively compressing data, - * but everything it has produced has also been flushed so far, - * therefore flush speed is limited by production speed of oldest job - * irrespective of the speed of concurrent (and newer) jobs. - */ -ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx); - - -/*===== Advanced Streaming decompression functions =====*/ - -/*! - * This function is deprecated, and is equivalent to: - * - * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); - * ZSTD_DCtx_loadDictionary(zds, dict, dictSize); - * - * note: no dictionary will be used if dict == NULL or dictSize < 8 - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x - */ -ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); - -/*! - * This function is deprecated, and is equivalent to: - * - * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); - * ZSTD_DCtx_refDDict(zds, ddict); - * - * note : ddict is referenced, it must outlive decompression session - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x - */ -ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); - -/*! - * This function is deprecated, and is equivalent to: - * - * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); +#define ZSTD_DEPS_NEED_MALLOC +/**** skipping file: error_private.h ****/ +/**** start inlining zstd_internal.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. * - * re-use decompression parameters from previous init; saves dictionary loading - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. */ -ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); - - -/********************************************************************* -* Buffer-less and synchronous inner streaming functions -* -* This is an advanced API, giving full control over buffer management, for users which need direct control over memory. -* But it's also a complex one, with several restrictions, documented below. -* Prefer normal streaming API for an easier experience. -********************************************************************* */ - -/** - Buffer-less streaming compression (synchronous mode) - - A ZSTD_CCtx object is required to track streaming operations. - Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource. - ZSTD_CCtx object can be re-used multiple times within successive compression operations. - - Start by initializing a context. - Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression, - or ZSTD_compressBegin_advanced(), for finer parameter control. - It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx() - - Then, consume your input using ZSTD_compressContinue(). - There are some important considerations to keep in mind when using this advanced function : - - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffers only. - - Interface is synchronous : input is consumed entirely and produces 1+ compressed blocks. - - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario. - Worst case evaluation is provided by ZSTD_compressBound(). - ZSTD_compressContinue() doesn't guarantee recover after a failed compression. - - ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog). - It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks) - - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps. - In which case, it will "discard" the relevant memory section from its history. - - Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum. - It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame. - Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders. - - `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress again. -*/ - -/*===== Buffer-less streaming compression functions =====*/ -ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); -ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); -ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */ -ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */ -ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */ -ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */ - -ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); - - -/** - Buffer-less streaming decompression (synchronous mode) - - A ZSTD_DCtx object is required to track streaming operations. - Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. - A ZSTD_DCtx object can be re-used multiple times. - - First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader(). - Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough. - Data fragment must be large enough to ensure successful decoding. - `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough. - @result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled. - >0 : `srcSize` is too small, please provide at least @result bytes on next attempt. - errorCode, which can be tested using ZSTD_isError(). - - It fills a ZSTD_frameHeader structure with important information to correctly decode the frame, - such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`). - Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information. - As a consequence, check that values remain within valid application range. - For example, do not allocate memory blindly, check that `windowSize` is within expectation. - Each application can set its own limits, depending on local restrictions. - For extended interoperability, it is recommended to support `windowSize` of at least 8 MB. - - ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize` bytes. - ZSTD_decompressContinue() is very sensitive to contiguity, - if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place, - or that previous contiguous segment is large enough to properly handle maximum back-reference distance. - There are multiple ways to guarantee this condition. - - The most memory efficient way is to use a round buffer of sufficient size. - Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(), - which can @return an error code if required value is too large for current system (in 32-bits mode). - In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one, - up to the moment there is not enough room left in the buffer to guarantee decoding another full block, - which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`. - At which point, decoding can resume from the beginning of the buffer. - Note that already decoded data stored in the buffer should be flushed before being overwritten. - - There are alternatives possible, for example using two or more buffers of size `windowSize` each, though they consume more memory. - - Finally, if you control the compression process, you can also ignore all buffer size rules, - as long as the encoder and decoder progress in "lock-step", - aka use exactly the same buffer sizes, break contiguity at the same place, etc. - - Once buffers are setup, start decompression, with ZSTD_decompressBegin(). - If decompression requires a dictionary, use ZSTD_decompressBegin_usingDict() or ZSTD_decompressBegin_usingDDict(). - - Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively. - ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue(). - ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail. - - @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). - It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item. - It can also be an error code, which can be tested with ZSTD_isError(). - - A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. - Context can then be reset to start a new decompression. - - Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType(). - This information is not required to properly decode a frame. - - == Special case : skippable frames == - - Skippable frames allow integration of user-defined data into a flow of concatenated frames. - Skippable frames will be ignored (skipped) by decompressor. - The format of skippable frames is as follows : - a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F - b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits - c) Frame Content - any content (User Data) of length equal to Frame Size - For skippable frames ZSTD_getFrameHeader() returns zfhPtr->frameType==ZSTD_skippableFrame. - For skippable frames ZSTD_decompressContinue() always returns 0 : it only skips the content. -*/ - -/*===== Buffer-less streaming decompression functions =====*/ -typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e; -typedef struct { - unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */ - unsigned long long windowSize; /* can be very large, up to <= frameContentSize */ - unsigned blockSizeMax; - ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */ - unsigned headerSize; - unsigned dictID; - unsigned checksumFlag; -} ZSTD_frameHeader; - -/*! ZSTD_getFrameHeader() : - * decode Frame Header, or requires larger `srcSize`. - * @return : 0, `zfhPtr` is correctly filled, - * >0, `srcSize` is too small, value is wanted `srcSize` amount, - * or an error code, which can be tested using ZSTD_isError() */ -ZSTDLIB_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */ -/*! ZSTD_getFrameHeader_advanced() : - * same as ZSTD_getFrameHeader(), - * with added capability to select a format (like ZSTD_f_zstd1_magicless) */ -ZSTDLIB_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format); -ZSTDLIB_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */ - -ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); -ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); -ZSTDLIB_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); - -ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); -ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); - -/* misc */ -ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); -typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; -ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); - - - - -/* ============================ */ -/** Block level API */ -/* ============================ */ - -/*! - Block functions produce and decode raw zstd blocks, without frame metadata. - Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes). - But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes. - - A few rules to respect : - - Compressing and decompressing require a context structure - + Use ZSTD_createCCtx() and ZSTD_createDCtx() - - It is necessary to init context before starting - + compression : any ZSTD_compressBegin*() variant, including with dictionary - + decompression : any ZSTD_decompressBegin*() variant, including with dictionary - + copyCCtx() and copyDCtx() can be used too - - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB - + If input is larger than a block size, it's necessary to split input data into multiple blocks - + For inputs larger than a single block, consider using regular ZSTD_compress() instead. - Frame metadata is not that costly, and quickly becomes negligible as source size grows larger than a block. - - When a block is considered not compressible enough, ZSTD_compressBlock() result will be 0 (zero) ! - ===> In which case, nothing is produced into `dst` ! - + User __must__ test for such outcome and deal directly with uncompressed data - + A block cannot be declared incompressible if ZSTD_compressBlock() return value was != 0. - Doing so would mess up with statistics history, leading to potential data corruption. - + ZSTD_decompressBlock() _doesn't accept uncompressed data as input_ !! - + In case of multiple successive blocks, should some of them be uncompressed, - decoder must be informed of their existence in order to follow proper history. - Use ZSTD_insertBlock() for such a case. -*/ - -/*===== Raw zstd block functions =====*/ -ZSTDLIB_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx); -ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -ZSTDLIB_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */ +#ifndef ZSTD_CCOMMON_H_MODULE +#define ZSTD_CCOMMON_H_MODULE -#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */ +/* this module contains definitions which must be identical + * across compression, decompression and dictBuilder. + * It also contains a few functions useful to at least 2 of them + * and which benefit from being inlined */ -#if defined (__cplusplus) -} -#endif -/**** ended inlining ../zstd.h ****/ -#define FSE_STATIC_LINKING_ONLY -/**** skipping file: fse.h ****/ -#define HUF_STATIC_LINKING_ONLY -/**** skipping file: huf.h ****/ -#ifndef XXH_STATIC_LINKING_ONLY -# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ -#endif -/**** start inlining xxhash.h ****/ +/*-************************************* +* Dependencies +***************************************/ +/**** skipping file: compiler.h ****/ +/**** start inlining cpu.h ****/ /* - * xxHash - Extremely Fast Hash algorithm - * Header File - * Copyright (c) 2012-2021, Yann Collet, Facebook, Inc. - * - * You can contact the author at : - * - xxHash source repository : https://github.com/Cyan4973/xxHash + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the * LICENSE file in the root directory of this source tree) and the GPLv2 (found * in the COPYING file in the root directory of this source tree). * You may select, at your option, one of the above-listed licenses. -*/ - -/* Notice extracted from xxHash homepage : - -xxHash is an extremely fast Hash algorithm, running at RAM speed limits. -It also successfully passes all tests from the SMHasher suite. - -Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) - -Name Speed Q.Score Author -xxHash 5.4 GB/s 10 -CrapWow 3.2 GB/s 2 Andrew -MumurHash 3a 2.7 GB/s 10 Austin Appleby -SpookyHash 2.0 GB/s 10 Bob Jenkins -SBox 1.4 GB/s 9 Bret Mulvey -Lookup3 1.2 GB/s 9 Bob Jenkins -SuperFastHash 1.2 GB/s 1 Paul Hsieh -CityHash64 1.05 GB/s 10 Pike & Alakuijala -FNV 0.55 GB/s 5 Fowler, Noll, Vo -CRC32 0.43 GB/s 9 -MD5-32 0.33 GB/s 10 Ronald L. Rivest -SHA1-32 0.28 GB/s 10 - -Q.Score is a measure of quality of the hash function. -It depends on successfully passing SMHasher test set. -10 is a perfect score. - -A 64-bits version, named XXH64, is available since r35. -It offers much better speed, but for 64-bits applications only. -Name Speed on 64 bits Speed on 32 bits -XXH64 13.8 GB/s 1.9 GB/s -XXH32 6.8 GB/s 6.0 GB/s -*/ - -#if defined (__cplusplus) -extern "C" { -#endif - -#ifndef XXHASH_H_5627135585666179 -#define XXHASH_H_5627135585666179 1 - - -/* **************************** -* Definitions -******************************/ -/**** skipping file: zstd_deps.h ****/ -typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; - - -/* **************************** -* API modifier -******************************/ -/** XXH_PRIVATE_API -* This is useful if you want to include xxhash functions in `static` mode -* in order to inline them, and remove their symbol from the public list. -* Methodology : -* #define XXH_PRIVATE_API -* #include "xxhash.h" -* `xxhash.c` is automatically included. -* It's not useful to compile and link it as a separate module anymore. -*/ -#ifdef XXH_PRIVATE_API -# ifndef XXH_STATIC_LINKING_ONLY -# define XXH_STATIC_LINKING_ONLY -# endif -# if defined(__GNUC__) -# define XXH_PUBLIC_API static __inline __attribute__((unused)) -# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define XXH_PUBLIC_API static inline -# elif defined(_MSC_VER) -# define XXH_PUBLIC_API static __inline -# else -# define XXH_PUBLIC_API static /* this version may generate warnings for unused static functions; disable the relevant warning */ -# endif -#else -# define XXH_PUBLIC_API /* do nothing */ -#endif /* XXH_PRIVATE_API */ + */ -/*!XXH_NAMESPACE, aka Namespace Emulation : +#ifndef ZSTD_COMMON_CPU_H +#define ZSTD_COMMON_CPU_H -If you want to include _and expose_ xxHash functions from within your own library, -but also want to avoid symbol collisions with another library which also includes xxHash, +/** + * Implementation taken from folly/CpuId.h + * https://github.com/facebook/folly/blob/master/folly/CpuId.h + */ -you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library -with the value of XXH_NAMESPACE (so avoid to keep it NULL and avoid numeric values). +/**** skipping file: mem.h ****/ -Note that no change is required within the calling program as long as it includes `xxhash.h` : -regular symbol name will be automatically translated by this header. -*/ -#ifdef XXH_NAMESPACE -# define XXH_CAT(A,B) A##B -# define XXH_NAME2(A,B) XXH_CAT(A,B) -# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) -# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) -# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) -# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) -# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) -# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) -# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) -# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) -# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) -# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) -# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) -# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) -# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) -# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) -# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) -# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) -# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) -# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) -# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) +#ifdef _MSC_VER +#include #endif +typedef struct { + U32 f1c; + U32 f1d; + U32 f7b; + U32 f7c; +} ZSTD_cpuid_t; -/* ************************************* -* Version -***************************************/ -#define XXH_VERSION_MAJOR 0 -#define XXH_VERSION_MINOR 6 -#define XXH_VERSION_RELEASE 2 -#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) -XXH_PUBLIC_API unsigned XXH_versionNumber (void); - - -/* **************************** -* Simple Hash Functions -******************************/ -typedef unsigned int XXH32_hash_t; -typedef unsigned long long XXH64_hash_t; - -XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed); -XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed); - -/*! -XXH32() : - Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input". - The memory between input & input+length must be valid (allocated and read-accessible). - "seed" can be used to alter the result predictably. - Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s -XXH64() : - Calculate the 64-bits hash of sequence of length "len" stored at memory address "input". - "seed" can be used to alter the result predictably. - This function runs 2x faster on 64-bits systems, but slower on 32-bits systems (see benchmark). -*/ - - -/* **************************** -* Streaming Hash Functions -******************************/ -typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */ -typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ - -/*! State allocation, compatible with dynamic libraries */ - -XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); -XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); - -XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void); -XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); - - -/* hash streaming */ - -XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned int seed); -XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); -XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); - -XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed); -XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); -XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr); - -/* -These functions generate the xxHash of an input provided in multiple segments. -Note that, for small input, they are slower than single-call functions, due to state management. -For small input, prefer `XXH32()` and `XXH64()` . - -XXH state must first be allocated, using XXH*_createState() . - -Start a new hash by initializing state with a seed, using XXH*_reset(). - -Then, feed the hash state by calling XXH*_update() as many times as necessary. -Obviously, input must be allocated and read accessible. -The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. - -Finally, a hash value can be produced anytime, by using XXH*_digest(). -This function returns the nn-bits hash as an int or long long. - -It's still possible to continue inserting input into the hash state after a digest, -and generate some new hashes later on, by calling again XXH*_digest(). - -When done, free XXH state space if it was allocated dynamically. -*/ - - -/* ************************** -* Utils -****************************/ -#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* ! C99 */ -# define restrict /* disable restrict */ +MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) { + U32 f1c = 0; + U32 f1d = 0; + U32 f7b = 0; + U32 f7c = 0; +#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) +#if !defined(_M_X64) || !defined(__clang__) || __clang_major__ >= 16 + int reg[4]; + __cpuid((int*)reg, 0); + { + int const n = reg[0]; + if (n >= 1) { + __cpuid((int*)reg, 1); + f1c = (U32)reg[2]; + f1d = (U32)reg[3]; + } + if (n >= 7) { + __cpuidex((int*)reg, 7, 0); + f7b = (U32)reg[1]; + f7c = (U32)reg[2]; + } + } +#else + /* Clang compiler has a bug (fixed in https://reviews.llvm.org/D101338) in + * which the `__cpuid` intrinsic does not save and restore `rbx` as it needs + * to due to being a reserved register. So in that case, do the `cpuid` + * ourselves. Clang supports inline assembly anyway. + */ + U32 n; + __asm__( + "pushq %%rbx\n\t" + "cpuid\n\t" + "popq %%rbx\n\t" + : "=a"(n) + : "a"(0) + : "rcx", "rdx"); + if (n >= 1) { + U32 f1a; + __asm__( + "pushq %%rbx\n\t" + "cpuid\n\t" + "popq %%rbx\n\t" + : "=a"(f1a), "=c"(f1c), "=d"(f1d) + : "a"(1) + :); + } + if (n >= 7) { + __asm__( + "pushq %%rbx\n\t" + "cpuid\n\t" + "movq %%rbx, %%rax\n\t" + "popq %%rbx" + : "=a"(f7b), "=c"(f7c) + : "a"(7), "c"(0) + : "rdx"); + } #endif +#elif defined(__i386__) && defined(__PIC__) && !defined(__clang__) && defined(__GNUC__) + /* The following block like the normal cpuid branch below, but gcc + * reserves ebx for use of its pic register so we must specially + * handle the save and restore to avoid clobbering the register + */ + U32 n; + __asm__( + "pushl %%ebx\n\t" + "cpuid\n\t" + "popl %%ebx\n\t" + : "=a"(n) + : "a"(0) + : "ecx", "edx"); + if (n >= 1) { + U32 f1a; + __asm__( + "pushl %%ebx\n\t" + "cpuid\n\t" + "popl %%ebx\n\t" + : "=a"(f1a), "=c"(f1c), "=d"(f1d) + : "a"(1)); + } + if (n >= 7) { + __asm__( + "pushl %%ebx\n\t" + "cpuid\n\t" + "movl %%ebx, %%eax\n\t" + "popl %%ebx" + : "=a"(f7b), "=c"(f7c) + : "a"(7), "c"(0) + : "edx"); + } +#elif defined(__x86_64__) || defined(_M_X64) || defined(__i386__) + U32 n; + __asm__("cpuid" : "=a"(n) : "a"(0) : "ebx", "ecx", "edx"); + if (n >= 1) { + U32 f1a; + __asm__("cpuid" : "=a"(f1a), "=c"(f1c), "=d"(f1d) : "a"(1) : "ebx"); + } + if (n >= 7) { + U32 f7a; + __asm__("cpuid" + : "=a"(f7a), "=b"(f7b), "=c"(f7c) + : "a"(7), "c"(0) + : "edx"); + } +#endif + { + ZSTD_cpuid_t cpuid; + cpuid.f1c = f1c; + cpuid.f1d = f1d; + cpuid.f7b = f7b; + cpuid.f7c = f7c; + return cpuid; + } +} -XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dst_state, const XXH32_state_t* restrict src_state); -XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dst_state, const XXH64_state_t* restrict src_state); - - -/* ************************** -* Canonical representation -****************************/ -/* Default result type for XXH functions are primitive unsigned 32 and 64 bits. -* The canonical representation uses human-readable write convention, aka big-endian (large digits first). -* These functions allow transformation of hash result into and from its canonical format. -* This way, hash values can be written into a file / memory, and remain comparable on different systems and programs. -*/ -typedef struct { unsigned char digest[4]; } XXH32_canonical_t; -typedef struct { unsigned char digest[8]; } XXH64_canonical_t; - -XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); -XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash); - -XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); -XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); +#define X(name, r, bit) \ + MEM_STATIC int ZSTD_cpuid_##name(ZSTD_cpuid_t const cpuid) { \ + return ((cpuid.r) & (1U << bit)) != 0; \ + } -#endif /* XXHASH_H_5627135585666179 */ +/* cpuid(1): Processor Info and Feature Bits. */ +#define C(name, bit) X(name, f1c, bit) + C(sse3, 0) + C(pclmuldq, 1) + C(dtes64, 2) + C(monitor, 3) + C(dscpl, 4) + C(vmx, 5) + C(smx, 6) + C(eist, 7) + C(tm2, 8) + C(ssse3, 9) + C(cnxtid, 10) + C(fma, 12) + C(cx16, 13) + C(xtpr, 14) + C(pdcm, 15) + C(pcid, 17) + C(dca, 18) + C(sse41, 19) + C(sse42, 20) + C(x2apic, 21) + C(movbe, 22) + C(popcnt, 23) + C(tscdeadline, 24) + C(aes, 25) + C(xsave, 26) + C(osxsave, 27) + C(avx, 28) + C(f16c, 29) + C(rdrand, 30) +#undef C +#define D(name, bit) X(name, f1d, bit) + D(fpu, 0) + D(vme, 1) + D(de, 2) + D(pse, 3) + D(tsc, 4) + D(msr, 5) + D(pae, 6) + D(mce, 7) + D(cx8, 8) + D(apic, 9) + D(sep, 11) + D(mtrr, 12) + D(pge, 13) + D(mca, 14) + D(cmov, 15) + D(pat, 16) + D(pse36, 17) + D(psn, 18) + D(clfsh, 19) + D(ds, 21) + D(acpi, 22) + D(mmx, 23) + D(fxsr, 24) + D(sse, 25) + D(sse2, 26) + D(ss, 27) + D(htt, 28) + D(tm, 29) + D(pbe, 31) +#undef D +/* cpuid(7): Extended Features. */ +#define B(name, bit) X(name, f7b, bit) + B(bmi1, 3) + B(hle, 4) + B(avx2, 5) + B(smep, 7) + B(bmi2, 8) + B(erms, 9) + B(invpcid, 10) + B(rtm, 11) + B(mpx, 14) + B(avx512f, 16) + B(avx512dq, 17) + B(rdseed, 18) + B(adx, 19) + B(smap, 20) + B(avx512ifma, 21) + B(pcommit, 22) + B(clflushopt, 23) + B(clwb, 24) + B(avx512pf, 26) + B(avx512er, 27) + B(avx512cd, 28) + B(sha, 29) + B(avx512bw, 30) + B(avx512vl, 31) +#undef B +#define C(name, bit) X(name, f7c, bit) + C(prefetchwt1, 0) + C(avx512vbmi, 1) +#undef C +#undef X -/* ================================================================================================ - This section contains definitions which are not guaranteed to remain stable. - They may change in future versions, becoming incompatible with a different version of the library. - They shall only be used with static linking. - Never use these definitions in association with dynamic linking ! -=================================================================================================== */ -#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXH_STATIC_H_3543687687345) -#define XXH_STATIC_H_3543687687345 - -/* These definitions are only meant to allow allocation of XXH state - statically, on stack, or in a struct for example. - Do not use members directly. */ - - struct XXH32_state_s { - unsigned total_len_32; - unsigned large_len; - unsigned v1; - unsigned v2; - unsigned v3; - unsigned v4; - unsigned mem32[4]; /* buffer defined as U32 for alignment */ - unsigned memsize; - unsigned reserved; /* never read nor write, will be removed in a future version */ - }; /* typedef'd to XXH32_state_t */ - - struct XXH64_state_s { - unsigned long long total_len; - unsigned long long v1; - unsigned long long v2; - unsigned long long v3; - unsigned long long v4; - unsigned long long mem64[4]; /* buffer defined as U64 for alignment */ - unsigned memsize; - unsigned reserved[2]; /* never read nor write, will be removed in a future version */ - }; /* typedef'd to XXH64_state_t */ - - -# ifdef XXH_PRIVATE_API -/**** start inlining xxhash.c ****/ +#endif /* ZSTD_COMMON_CPU_H */ +/**** ended inlining cpu.h ****/ +/**** skipping file: mem.h ****/ +/**** skipping file: debug.h ****/ +/**** skipping file: error_private.h ****/ +#define ZSTD_STATIC_LINKING_ONLY +/**** skipping file: ../zstd.h ****/ +#define FSE_STATIC_LINKING_ONLY +/**** skipping file: fse.h ****/ +/**** skipping file: huf.h ****/ +#ifndef XXH_STATIC_LINKING_ONLY +# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ +#endif +/**** start inlining xxhash.h ****/ /* - * xxHash - Fast Hash algorithm - * Copyright (c) 2012-2021, Yann Collet, Facebook, Inc. - * - * You can contact the author at : - * - xxHash homepage: http://www.xxhash.com - * - xxHash source repository : https://github.com/Cyan4973/xxHash + * xxHash - Extremely Fast Hash algorithm + * Header File + * Copyright (c) Yann Collet - Meta Platforms, Inc * * This source code is licensed under both the BSD-style license (found in the * LICENSE file in the root directory of this source tree) and the GPLv2 (found * in the COPYING file in the root directory of this source tree). * You may select, at your option, one of the above-listed licenses. -*/ + */ +/* Local adaptations for Zstandard */ -/* ************************************* -* Tuning parameters -***************************************/ -/*!XXH_FORCE_MEMORY_ACCESS : - * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. - * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. - * The below switch allow to select different access method for improved performance. - * Method 0 (default) : use `memcpy()`. Safe and portable. - * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). - * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. - * Method 2 : direct access. This method doesn't depend on compiler but violate C standard. - * It can generate buggy code on targets which do not support unaligned memory accesses. - * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) - * See http://stackoverflow.com/a/32095106/646947 for details. - * Prefer these methods in priority order (0 > 1 > 2) - */ -#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ -# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) -# define XXH_FORCE_MEMORY_ACCESS 2 -# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \ - (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) || \ - defined(__ICCARM__) -# define XXH_FORCE_MEMORY_ACCESS 1 -# endif +#ifndef XXH_NO_XXH3 +# define XXH_NO_XXH3 +#endif + +#ifndef XXH_NAMESPACE +# define XXH_NAMESPACE ZSTD_ #endif -/*!XXH_ACCEPT_NULL_INPUT_POINTER : - * If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. - * When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. - * By default, this option is disabled. To enable it, uncomment below define : +/*! + * @mainpage xxHash + * + * xxHash is an extremely fast non-cryptographic hash algorithm, working at RAM speed + * limits. + * + * It is proposed in four flavors, in three families: + * 1. @ref XXH32_family + * - Classic 32-bit hash function. Simple, compact, and runs on almost all + * 32-bit and 64-bit systems. + * 2. @ref XXH64_family + * - Classic 64-bit adaptation of XXH32. Just as simple, and runs well on most + * 64-bit systems (but _not_ 32-bit systems). + * 3. @ref XXH3_family + * - Modern 64-bit and 128-bit hash function family which features improved + * strength and performance across the board, especially on smaller data. + * It benefits greatly from SIMD and 64-bit without requiring it. + * + * Benchmarks + * --- + * The reference system uses an Intel i7-9700K CPU, and runs Ubuntu x64 20.04. + * The open source benchmark program is compiled with clang v10.0 using -O3 flag. + * + * | Hash Name | ISA ext | Width | Large Data Speed | Small Data Velocity | + * | -------------------- | ------- | ----: | ---------------: | ------------------: | + * | XXH3_64bits() | @b AVX2 | 64 | 59.4 GB/s | 133.1 | + * | MeowHash | AES-NI | 128 | 58.2 GB/s | 52.5 | + * | XXH3_128bits() | @b AVX2 | 128 | 57.9 GB/s | 118.1 | + * | CLHash | PCLMUL | 64 | 37.1 GB/s | 58.1 | + * | XXH3_64bits() | @b SSE2 | 64 | 31.5 GB/s | 133.1 | + * | XXH3_128bits() | @b SSE2 | 128 | 29.6 GB/s | 118.1 | + * | RAM sequential read | | N/A | 28.0 GB/s | N/A | + * | ahash | AES-NI | 64 | 22.5 GB/s | 107.2 | + * | City64 | | 64 | 22.0 GB/s | 76.6 | + * | T1ha2 | | 64 | 22.0 GB/s | 99.0 | + * | City128 | | 128 | 21.7 GB/s | 57.7 | + * | FarmHash | AES-NI | 64 | 21.3 GB/s | 71.9 | + * | XXH64() | | 64 | 19.4 GB/s | 71.0 | + * | SpookyHash | | 64 | 19.3 GB/s | 53.2 | + * | Mum | | 64 | 18.0 GB/s | 67.0 | + * | CRC32C | SSE4.2 | 32 | 13.0 GB/s | 57.9 | + * | XXH32() | | 32 | 9.7 GB/s | 71.9 | + * | City32 | | 32 | 9.1 GB/s | 66.0 | + * | Blake3* | @b AVX2 | 256 | 4.4 GB/s | 8.1 | + * | Murmur3 | | 32 | 3.9 GB/s | 56.1 | + * | SipHash* | | 64 | 3.0 GB/s | 43.2 | + * | Blake3* | @b SSE2 | 256 | 2.4 GB/s | 8.1 | + * | HighwayHash | | 64 | 1.4 GB/s | 6.0 | + * | FNV64 | | 64 | 1.2 GB/s | 62.7 | + * | Blake2* | | 256 | 1.1 GB/s | 5.1 | + * | SHA1* | | 160 | 0.8 GB/s | 5.6 | + * | MD5* | | 128 | 0.6 GB/s | 7.8 | + * @note + * - Hashes which require a specific ISA extension are noted. SSE2 is also noted, + * even though it is mandatory on x64. + * - Hashes with an asterisk are cryptographic. Note that MD5 is non-cryptographic + * by modern standards. + * - Small data velocity is a rough average of algorithm's efficiency for small + * data. For more accurate information, see the wiki. + * - More benchmarks and strength tests are found on the wiki: + * https://github.com/Cyan4973/xxHash/wiki + * + * Usage + * ------ + * All xxHash variants use a similar API. Changing the algorithm is a trivial + * substitution. + * + * @pre + * For functions which take an input and length parameter, the following + * requirements are assumed: + * - The range from [`input`, `input + length`) is valid, readable memory. + * - The only exception is if the `length` is `0`, `input` may be `NULL`. + * - For C++, the objects must have the *TriviallyCopyable* property, as the + * functions access bytes directly as if it was an array of `unsigned char`. + * + * @anchor single_shot_example + * **Single Shot** + * + * These functions are stateless functions which hash a contiguous block of memory, + * immediately returning the result. They are the easiest and usually the fastest + * option. + * + * XXH32(), XXH64(), XXH3_64bits(), XXH3_128bits() + * + * @code{.c} + * #include + * #include "xxhash.h" + * + * // Example for a function which hashes a null terminated string with XXH32(). + * XXH32_hash_t hash_string(const char* string, XXH32_hash_t seed) + * { + * // NULL pointers are only valid if the length is zero + * size_t length = (string == NULL) ? 0 : strlen(string); + * return XXH32(string, length, seed); + * } + * @endcode + * + * + * @anchor streaming_example + * **Streaming** + * + * These groups of functions allow incremental hashing of unknown size, even + * more than what would fit in a size_t. + * + * XXH32_reset(), XXH64_reset(), XXH3_64bits_reset(), XXH3_128bits_reset() + * + * @code{.c} + * #include + * #include + * #include "xxhash.h" + * // Example for a function which hashes a FILE incrementally with XXH3_64bits(). + * XXH64_hash_t hashFile(FILE* f) + * { + * // Allocate a state struct. Do not just use malloc() or new. + * XXH3_state_t* state = XXH3_createState(); + * assert(state != NULL && "Out of memory!"); + * // Reset the state to start a new hashing session. + * XXH3_64bits_reset(state); + * char buffer[4096]; + * size_t count; + * // Read the file in chunks + * while ((count = fread(buffer, 1, sizeof(buffer), f)) != 0) { + * // Run update() as many times as necessary to process the data + * XXH3_64bits_update(state, buffer, count); + * } + * // Retrieve the finalized hash. This will not change the state. + * XXH64_hash_t result = XXH3_64bits_digest(state); + * // Free the state. Do not use free(). + * XXH3_freeState(state); + * return result; + * } + * @endcode + * + * Streaming functions generate the xxHash value from an incremental input. + * This method is slower than single-call functions, due to state management. + * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized. + * + * An XXH state must first be allocated using `XXH*_createState()`. + * + * Start a new hash by initializing the state with a seed using `XXH*_reset()`. + * + * Then, feed the hash state by calling `XXH*_update()` as many times as necessary. + * + * The function returns an error code, with 0 meaning OK, and any other value + * meaning there is an error. + * + * Finally, a hash value can be produced anytime, by using `XXH*_digest()`. + * This function returns the nn-bits hash as an int or long long. + * + * It's still possible to continue inserting input into the hash state after a + * digest, and generate new hash values later on by invoking `XXH*_digest()`. + * + * When done, release the state using `XXH*_freeState()`. + * + * + * @anchor canonical_representation_example + * **Canonical Representation** + * + * The default return values from XXH functions are unsigned 32, 64 and 128 bit + * integers. + * This the simplest and fastest format for further post-processing. + * + * However, this leaves open the question of what is the order on the byte level, + * since little and big endian conventions will store the same number differently. + * + * The canonical representation settles this issue by mandating big-endian + * convention, the same convention as human-readable numbers (large digits first). + * + * When writing hash values to storage, sending them over a network, or printing + * them, it's highly recommended to use the canonical representation to ensure + * portability across a wider range of systems, present and future. + * + * The following functions allow transformation of hash values to and from + * canonical format. + * + * XXH32_canonicalFromHash(), XXH32_hashFromCanonical(), + * XXH64_canonicalFromHash(), XXH64_hashFromCanonical(), + * XXH128_canonicalFromHash(), XXH128_hashFromCanonical(), + * + * @code{.c} + * #include + * #include "xxhash.h" + * + * // Example for a function which prints XXH32_hash_t in human readable format + * void printXxh32(XXH32_hash_t hash) + * { + * XXH32_canonical_t cano; + * XXH32_canonicalFromHash(&cano, hash); + * size_t i; + * for(i = 0; i < sizeof(cano.digest); ++i) { + * printf("%02x", cano.digest[i]); + * } + * printf("\n"); + * } + * + * // Example for a function which converts XXH32_canonical_t to XXH32_hash_t + * XXH32_hash_t convertCanonicalToXxh32(XXH32_canonical_t cano) + * { + * XXH32_hash_t hash = XXH32_hashFromCanonical(&cano); + * return hash; + * } + * @endcode + * + * + * @file xxhash.h + * xxHash prototypes and implementation */ -/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */ -/*!XXH_FORCE_NATIVE_FORMAT : - * By default, xxHash library provides endian-independent Hash values, based on little-endian convention. - * Results are therefore identical for little-endian and big-endian CPU. - * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. - * Should endian-independence be of no importance for your application, you may set the #define below to 1, - * to improve speed for Big-endian CPU. - * This option has no impact on Little_Endian CPU. +/* **************************** + * INLINE mode + ******************************/ +/*! + * @defgroup public Public API + * Contains details on the public xxHash functions. + * @{ */ -#ifndef XXH_FORCE_NATIVE_FORMAT /* can be defined externally */ -# define XXH_FORCE_NATIVE_FORMAT 0 -#endif +#ifdef XXH_DOXYGEN +/*! + * @brief Gives access to internal state declaration, required for static allocation. + * + * Incompatible with dynamic linking, due to risks of ABI changes. + * + * Usage: + * @code{.c} + * #define XXH_STATIC_LINKING_ONLY + * #include "xxhash.h" + * @endcode + */ +# define XXH_STATIC_LINKING_ONLY +/* Do not undef XXH_STATIC_LINKING_ONLY for Doxygen */ -/*!XXH_FORCE_ALIGN_CHECK : - * This is a minor performance trick, only useful with lots of very small keys. - * It means : check for aligned/unaligned input. - * The check costs one initial branch per hash; set to 0 when the input data - * is guaranteed to be aligned. +/*! + * @brief Gives access to internal definitions. + * + * Usage: + * @code{.c} + * #define XXH_STATIC_LINKING_ONLY + * #define XXH_IMPLEMENTATION + * #include "xxhash.h" + * @endcode */ -#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ -# if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) -# define XXH_FORCE_ALIGN_CHECK 0 +# define XXH_IMPLEMENTATION +/* Do not undef XXH_IMPLEMENTATION for Doxygen */ + +/*! + * @brief Exposes the implementation and marks all functions as `inline`. + * + * Use these build macros to inline xxhash into the target unit. + * Inlining improves performance on small inputs, especially when the length is + * expressed as a compile-time constant: + * + * https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html + * + * It also keeps xxHash symbols private to the unit, so they are not exported. + * + * Usage: + * @code{.c} + * #define XXH_INLINE_ALL + * #include "xxhash.h" + * @endcode + * Do not compile and link xxhash.o as a separate object, as it is not useful. + */ +# define XXH_INLINE_ALL +# undef XXH_INLINE_ALL +/*! + * @brief Exposes the implementation without marking functions as inline. + */ +# define XXH_PRIVATE_API +# undef XXH_PRIVATE_API +/*! + * @brief Emulate a namespace by transparently prefixing all symbols. + * + * If you want to include _and expose_ xxHash functions from within your own + * library, but also want to avoid symbol collisions with other libraries which + * may also include xxHash, you can use @ref XXH_NAMESPACE to automatically prefix + * any public symbol from xxhash library with the value of @ref XXH_NAMESPACE + * (therefore, avoid empty or numeric values). + * + * Note that no change is required within the calling program as long as it + * includes `xxhash.h`: Regular symbol names will be automatically translated + * by this header. + */ +# define XXH_NAMESPACE /* YOUR NAME HERE */ +# undef XXH_NAMESPACE +#endif + +#if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) \ + && !defined(XXH_INLINE_ALL_31684351384) + /* this section should be traversed only once */ +# define XXH_INLINE_ALL_31684351384 + /* give access to the advanced API, required to compile implementations */ +# undef XXH_STATIC_LINKING_ONLY /* avoid macro redef */ +# define XXH_STATIC_LINKING_ONLY + /* make all functions private */ +# undef XXH_PUBLIC_API +# if defined(__GNUC__) +# define XXH_PUBLIC_API static __inline __attribute__((unused)) +# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define XXH_PUBLIC_API static inline +# elif defined(_MSC_VER) +# define XXH_PUBLIC_API static __inline # else -# define XXH_FORCE_ALIGN_CHECK 1 + /* note: this version may generate warnings for unused static functions */ +# define XXH_PUBLIC_API static # endif -#endif + /* + * This part deals with the special case where a unit wants to inline xxHash, + * but "xxhash.h" has previously been included without XXH_INLINE_ALL, + * such as part of some previously included *.h header file. + * Without further action, the new include would just be ignored, + * and functions would effectively _not_ be inlined (silent failure). + * The following macros solve this situation by prefixing all inlined names, + * avoiding naming collision with previous inclusions. + */ + /* Before that, we unconditionally #undef all symbols, + * in case they were already defined with XXH_NAMESPACE. + * They will then be redefined for XXH_INLINE_ALL + */ +# undef XXH_versionNumber + /* XXH32 */ +# undef XXH32 +# undef XXH32_createState +# undef XXH32_freeState +# undef XXH32_reset +# undef XXH32_update +# undef XXH32_digest +# undef XXH32_copyState +# undef XXH32_canonicalFromHash +# undef XXH32_hashFromCanonical + /* XXH64 */ +# undef XXH64 +# undef XXH64_createState +# undef XXH64_freeState +# undef XXH64_reset +# undef XXH64_update +# undef XXH64_digest +# undef XXH64_copyState +# undef XXH64_canonicalFromHash +# undef XXH64_hashFromCanonical + /* XXH3_64bits */ +# undef XXH3_64bits +# undef XXH3_64bits_withSecret +# undef XXH3_64bits_withSeed +# undef XXH3_64bits_withSecretandSeed +# undef XXH3_createState +# undef XXH3_freeState +# undef XXH3_copyState +# undef XXH3_64bits_reset +# undef XXH3_64bits_reset_withSeed +# undef XXH3_64bits_reset_withSecret +# undef XXH3_64bits_update +# undef XXH3_64bits_digest +# undef XXH3_generateSecret + /* XXH3_128bits */ +# undef XXH128 +# undef XXH3_128bits +# undef XXH3_128bits_withSeed +# undef XXH3_128bits_withSecret +# undef XXH3_128bits_reset +# undef XXH3_128bits_reset_withSeed +# undef XXH3_128bits_reset_withSecret +# undef XXH3_128bits_reset_withSecretandSeed +# undef XXH3_128bits_update +# undef XXH3_128bits_digest +# undef XXH128_isEqual +# undef XXH128_cmp +# undef XXH128_canonicalFromHash +# undef XXH128_hashFromCanonical + /* Finally, free the namespace itself */ +# undef XXH_NAMESPACE + + /* employ the namespace for XXH_INLINE_ALL */ +# define XXH_NAMESPACE XXH_INLINE_ + /* + * Some identifiers (enums, type names) are not symbols, + * but they must nonetheless be renamed to avoid redeclaration. + * Alternative solution: do not redeclare them. + * However, this requires some #ifdefs, and has a more dispersed impact. + * Meanwhile, renaming can be achieved in a single place. + */ +# define XXH_IPREF(Id) XXH_NAMESPACE ## Id +# define XXH_OK XXH_IPREF(XXH_OK) +# define XXH_ERROR XXH_IPREF(XXH_ERROR) +# define XXH_errorcode XXH_IPREF(XXH_errorcode) +# define XXH32_canonical_t XXH_IPREF(XXH32_canonical_t) +# define XXH64_canonical_t XXH_IPREF(XXH64_canonical_t) +# define XXH128_canonical_t XXH_IPREF(XXH128_canonical_t) +# define XXH32_state_s XXH_IPREF(XXH32_state_s) +# define XXH32_state_t XXH_IPREF(XXH32_state_t) +# define XXH64_state_s XXH_IPREF(XXH64_state_s) +# define XXH64_state_t XXH_IPREF(XXH64_state_t) +# define XXH3_state_s XXH_IPREF(XXH3_state_s) +# define XXH3_state_t XXH_IPREF(XXH3_state_t) +# define XXH128_hash_t XXH_IPREF(XXH128_hash_t) + /* Ensure the header is parsed again, even if it was previously included */ +# undef XXHASH_H_5627135585666179 +# undef XXHASH_H_STATIC_13879238742 +#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */ + +/* **************************************************************** + * Stable API + *****************************************************************/ +#ifndef XXHASH_H_5627135585666179 +#define XXHASH_H_5627135585666179 1 -/* ************************************* -* Includes & Memory related functions -***************************************/ -/* Modify the local functions below should you wish to use some other memory routines */ -/* for ZSTD_malloc(), ZSTD_free() */ -#define ZSTD_DEPS_NEED_MALLOC -/**** skipping file: zstd_deps.h ****/ -static void* XXH_malloc(size_t s) { return ZSTD_malloc(s); } -static void XXH_free (void* p) { ZSTD_free(p); } -static void* XXH_memcpy(void* dest, const void* src, size_t size) { return ZSTD_memcpy(dest,src,size); } +/*! @brief Marks a global symbol. */ +#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) +# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) +# ifdef XXH_EXPORT +# define XXH_PUBLIC_API __declspec(dllexport) +# elif XXH_IMPORT +# define XXH_PUBLIC_API __declspec(dllimport) +# endif +# else +# define XXH_PUBLIC_API /* do nothing */ +# endif +#endif -#ifndef XXH_STATIC_LINKING_ONLY -# define XXH_STATIC_LINKING_ONLY +#ifdef XXH_NAMESPACE +# define XXH_CAT(A,B) A##B +# define XXH_NAME2(A,B) XXH_CAT(A,B) +# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) +/* XXH32 */ +# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) +# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) +# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) +# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) +# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) +# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) +# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) +# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) +# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) +/* XXH64 */ +# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) +# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) +# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) +# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) +# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) +# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) +# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) +# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) +# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) +/* XXH3_64bits */ +# define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits) +# define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret) +# define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed) +# define XXH3_64bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecretandSeed) +# define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState) +# define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState) +# define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState) +# define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset) +# define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed) +# define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret) +# define XXH3_64bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecretandSeed) +# define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update) +# define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest) +# define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret) +# define XXH3_generateSecret_fromSeed XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret_fromSeed) +/* XXH3_128bits */ +# define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128) +# define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits) +# define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed) +# define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret) +# define XXH3_128bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecretandSeed) +# define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset) +# define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed) +# define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret) +# define XXH3_128bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecretandSeed) +# define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update) +# define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest) +# define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual) +# define XXH128_cmp XXH_NAME2(XXH_NAMESPACE, XXH128_cmp) +# define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash) +# define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical) #endif -/**** skipping file: xxhash.h ****/ /* ************************************* -* Compiler Specific Options +* Compiler specifics ***************************************/ -/**** skipping file: compiler.h ****/ +/* specific declaration modes for Windows */ +#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) +# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) +# ifdef XXH_EXPORT +# define XXH_PUBLIC_API __declspec(dllexport) +# elif XXH_IMPORT +# define XXH_PUBLIC_API __declspec(dllimport) +# endif +# else +# define XXH_PUBLIC_API /* do nothing */ +# endif +#endif + +#if defined (__GNUC__) +# define XXH_CONSTF __attribute__((const)) +# define XXH_PUREF __attribute__((pure)) +# define XXH_MALLOCF __attribute__((malloc)) +#else +# define XXH_CONSTF /* disable */ +# define XXH_PUREF +# define XXH_MALLOCF +#endif /* ************************************* -* Basic Types +* Version ***************************************/ -/**** skipping file: mem.h ****/ +#define XXH_VERSION_MAJOR 0 +#define XXH_VERSION_MINOR 8 +#define XXH_VERSION_RELEASE 2 +/*! @brief Version number, encoded as two digits each */ +#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) -#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) +#if defined (__cplusplus) +extern "C" { +#endif +/*! + * @brief Obtains the xxHash version. + * + * This is mostly useful when xxHash is compiled as a shared library, + * since the returned value comes from the library, as opposed to header file. + * + * @return @ref XXH_VERSION_NUMBER of the invoked library. + */ +XXH_PUBLIC_API XXH_CONSTF unsigned XXH_versionNumber (void); -/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ -static U32 XXH_read32(const void* memPtr) { return *(const U32*) memPtr; } -static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; } +#if defined (__cplusplus) +} +#endif + +/* **************************** +* Common basic types +******************************/ +#include /* size_t */ +/*! + * @brief Exit code for the streaming API. + */ +typedef enum { + XXH_OK = 0, /*!< OK */ + XXH_ERROR /*!< Error */ +} XXH_errorcode; -#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) -/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ -/* currently only defined for gcc and icc */ -typedef union { U32 u32; U64 u64; } __attribute__((packed)) unalign; +/*-********************************************************************** +* 32-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* Don't show include */ +/*! + * @brief An unsigned 32-bit integer. + * + * Not necessarily defined to `uint32_t` but functionally equivalent. + */ +typedef uint32_t XXH32_hash_t; -static U32 XXH_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } -static U64 XXH_read64(const void* ptr) { return ((const unalign*)ptr)->u64; } +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# ifdef _AIX +# include +# else +# include +# endif + typedef uint32_t XXH32_hash_t; #else +# include +# if UINT_MAX == 0xFFFFFFFFUL + typedef unsigned int XXH32_hash_t; +# elif ULONG_MAX == 0xFFFFFFFFUL + typedef unsigned long XXH32_hash_t; +# else +# error "unsupported platform: need a 32-bit type" +# endif +#endif + +#if defined (__cplusplus) +extern "C" { +#endif + +/*! + * @} + * + * @defgroup XXH32_family XXH32 family + * @ingroup public + * Contains functions used in the classic 32-bit xxHash algorithm. + * + * @note + * XXH32 is useful for older platforms, with no or poor 64-bit performance. + * Note that the @ref XXH3_family provides competitive speed for both 32-bit + * and 64-bit systems, and offers true 64/128 bit hash results. + * + * @see @ref XXH64_family, @ref XXH3_family : Other xxHash families + * @see @ref XXH32_impl for implementation details + * @{ + */ -/* portable and safe solution. Generally efficient. - * see : http://stackoverflow.com/a/32095106/646947 +/*! + * @brief Calculates the 32-bit hash of @p input using xxHash32. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 32-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 32-bit xxHash32 value. + * + * @see @ref single_shot_example "Single Shot Example" for an example. */ +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32 (const void* input, size_t length, XXH32_hash_t seed); -static U32 XXH_read32(const void* memPtr) -{ - U32 val; - ZSTD_memcpy(&val, memPtr, sizeof(val)); - return val; -} +#ifndef XXH_NO_STREAM +/*! + * @typedef struct XXH32_state_s XXH32_state_t + * @brief The opaque state struct for the XXH32 streaming API. + * + * @see XXH32_state_s for details. + */ +typedef struct XXH32_state_s XXH32_state_t; -static U64 XXH_read64(const void* memPtr) -{ - U64 val; - ZSTD_memcpy(&val, memPtr, sizeof(val)); - return val; -} +/*! + * @brief Allocates an @ref XXH32_state_t. + * + * @return An allocated pointer of @ref XXH32_state_t on success. + * @return `NULL` on failure. + * + * @note Must be freed with XXH32_freeState(). + */ +XXH_PUBLIC_API XXH_MALLOCF XXH32_state_t* XXH32_createState(void); +/*! + * @brief Frees an @ref XXH32_state_t. + * + * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref XXH32_createState(). + * + * @return @ref XXH_OK. + * + * @note @p statePtr must be allocated with XXH32_createState(). + * + */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); +/*! + * @brief Copies one @ref XXH32_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state); -#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ +/*! + * @brief Resets an @ref XXH32_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 32-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note This function resets and seeds a state. Call it before @ref XXH32_update(). + */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, XXH32_hash_t seed); +/*! + * @brief Consumes a block of @p input to an @ref XXH32_state_t. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note Call this to incrementally consume blocks of data. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); -/* **************************************** -* Compiler-specific Functions and Macros -******************************************/ -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +/*! + * @brief Returns the calculated hash value from an @ref XXH32_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated 32-bit xxHash32 value from that state. + * + * @note + * Calling XXH32_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + */ +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ -/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */ -#if defined(_MSC_VER) -# define XXH_rotl32(x,r) _rotl(x,r) -# define XXH_rotl64(x,r) _rotl64(x,r) -#else -#if defined(__ICCARM__) -# include -# define XXH_rotl32(x,r) __ROR(x,(32 - r)) +/******* Canonical representation *******/ + +/*! + * @brief Canonical (big endian) representation of @ref XXH32_hash_t. + */ +typedef struct { + unsigned char digest[4]; /*!< Hash bytes, big endian */ +} XXH32_canonical_t; + +/*! + * @brief Converts an @ref XXH32_hash_t to a big endian @ref XXH32_canonical_t. + * + * @param dst The @ref XXH32_canonical_t pointer to be stored to. + * @param hash The @ref XXH32_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); + +/*! + * @brief Converts an @ref XXH32_canonical_t to a native @ref XXH32_hash_t. + * + * @param src The @ref XXH32_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); + + +/*! @cond Doxygen ignores this part */ +#ifdef __has_attribute +# define XXH_HAS_ATTRIBUTE(x) __has_attribute(x) #else -# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) +# define XXH_HAS_ATTRIBUTE(x) 0 #endif -# define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r))) +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +/* + * C23 __STDC_VERSION__ number hasn't been specified yet. For now + * leave as `201711L` (C17 + 1). + * TODO: Update to correct value when its been specified. + */ +#define XXH_C23_VN 201711L +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +/* C-language Attributes are added in C23. */ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN) && defined(__has_c_attribute) +# define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) +#else +# define XXH_HAS_C_ATTRIBUTE(x) 0 #endif +/*! @endcond */ -#if defined(_MSC_VER) /* Visual Studio */ -# define XXH_swap32 _byteswap_ulong -# define XXH_swap64 _byteswap_uint64 -#elif GCC_VERSION >= 403 -# define XXH_swap32 __builtin_bswap32 -# define XXH_swap64 __builtin_bswap64 +/*! @cond Doxygen ignores this part */ +#if defined(__cplusplus) && defined(__has_cpp_attribute) +# define XXH_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) #else -static U32 XXH_swap32 (U32 x) -{ - return ((x << 24) & 0xff000000 ) | - ((x << 8) & 0x00ff0000 ) | - ((x >> 8) & 0x0000ff00 ) | - ((x >> 24) & 0x000000ff ); -} -static U64 XXH_swap64 (U64 x) -{ - return ((x << 56) & 0xff00000000000000ULL) | - ((x << 40) & 0x00ff000000000000ULL) | - ((x << 24) & 0x0000ff0000000000ULL) | - ((x << 8) & 0x000000ff00000000ULL) | - ((x >> 8) & 0x00000000ff000000ULL) | - ((x >> 24) & 0x0000000000ff0000ULL) | - ((x >> 40) & 0x000000000000ff00ULL) | - ((x >> 56) & 0x00000000000000ffULL); -} +# define XXH_HAS_CPP_ATTRIBUTE(x) 0 #endif +/*! @endcond */ +/*! @cond Doxygen ignores this part */ +/* + * Define XXH_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute + * introduced in CPP17 and C23. + * CPP17 : https://en.cppreference.com/w/cpp/language/attributes/fallthrough + * C23 : https://en.cppreference.com/w/c/language/attributes/fallthrough + */ +#if XXH_HAS_C_ATTRIBUTE(fallthrough) || XXH_HAS_CPP_ATTRIBUTE(fallthrough) +# define XXH_FALLTHROUGH [[fallthrough]] +#elif XXH_HAS_ATTRIBUTE(__fallthrough__) +# define XXH_FALLTHROUGH __attribute__ ((__fallthrough__)) +#else +# define XXH_FALLTHROUGH /* fallthrough */ +#endif +/*! @endcond */ -/* ************************************* -* Architecture Macros -***************************************/ -typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; +/*! @cond Doxygen ignores this part */ +/* + * Define XXH_NOESCAPE for annotated pointers in public API. + * https://clang.llvm.org/docs/AttributeReference.html#noescape + * As of writing this, only supported by clang. + */ +#if XXH_HAS_ATTRIBUTE(noescape) +# define XXH_NOESCAPE __attribute__((noescape)) +#else +# define XXH_NOESCAPE +#endif +/*! @endcond */ -/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */ -#ifndef XXH_CPU_LITTLE_ENDIAN - static const int g_one = 1; -# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&g_one)) +#if defined (__cplusplus) +} /* end of extern "C" */ #endif +/*! + * @} + * @ingroup public + * @{ + */ -/* *************************** -* Memory reads -*****************************/ -typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; +#ifndef XXH_NO_LONG_LONG +/*-********************************************************************** +* 64-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* don't include */ +/*! + * @brief An unsigned 64-bit integer. + * + * Not necessarily defined to `uint64_t` but functionally equivalent. + */ +typedef uint64_t XXH64_hash_t; +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# ifdef _AIX +# include +# else +# include +# endif + typedef uint64_t XXH64_hash_t; +#else +# include +# if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL + /* LP64 ABI says uint64_t is unsigned long */ + typedef unsigned long XXH64_hash_t; +# else + /* the following type must have a width of 64-bit */ + typedef unsigned long long XXH64_hash_t; +# endif +#endif -FORCE_INLINE_TEMPLATE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) -{ - if (align==XXH_unaligned) - return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); - else - return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr); -} +#if defined (__cplusplus) +extern "C" { +#endif +/*! + * @} + * + * @defgroup XXH64_family XXH64 family + * @ingroup public + * @{ + * Contains functions used in the classic 64-bit xxHash algorithm. + * + * @note + * XXH3 provides competitive speed for both 32-bit and 64-bit systems, + * and offers true 64/128 bit hash results. + * It provides better speed for systems with vector processing capabilities. + */ -FORCE_INLINE_TEMPLATE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) -{ - return XXH_readLE32_align(ptr, endian, XXH_unaligned); -} +/*! + * @brief Calculates the 64-bit hash of @p input using xxHash64. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 64-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit xxHash64 value. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed); -static U32 XXH_readBE32(const void* ptr) -{ - return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); -} +/******* Streaming *******/ +#ifndef XXH_NO_STREAM +/*! + * @brief The opaque state struct for the XXH64 streaming API. + * + * @see XXH64_state_s for details. + */ +typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ -FORCE_INLINE_TEMPLATE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) -{ - if (align==XXH_unaligned) - return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); - else - return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr); -} +/*! + * @brief Allocates an @ref XXH64_state_t. + * + * @return An allocated pointer of @ref XXH64_state_t on success. + * @return `NULL` on failure. + * + * @note Must be freed with XXH64_freeState(). + */ +XXH_PUBLIC_API XXH_MALLOCF XXH64_state_t* XXH64_createState(void); -FORCE_INLINE_TEMPLATE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) -{ - return XXH_readLE64_align(ptr, endian, XXH_unaligned); -} +/*! + * @brief Frees an @ref XXH64_state_t. + * + * @param statePtr A pointer to an @ref XXH64_state_t allocated with @ref XXH64_createState(). + * + * @return @ref XXH_OK. + * + * @note @p statePtr must be allocated with XXH64_createState(). + */ +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); -static U64 XXH_readBE64(const void* ptr) -{ - return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); -} +/*! + * @brief Copies one @ref XXH64_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dst_state, const XXH64_state_t* src_state); +/*! + * @brief Resets an @ref XXH64_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note This function resets and seeds a state. Call it before @ref XXH64_update(). + */ +XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed); -/* ************************************* -* Macros -***************************************/ -#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ +/*! + * @brief Consumes a block of @p input to an @ref XXH64_state_t. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note Call this to incrementally consume blocks of data. + */ +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH_NOESCAPE XXH64_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); +/*! + * @brief Returns the calculated hash value from an @ref XXH64_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated 64-bit xxHash64 value from that state. + * + * @note + * Calling XXH64_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_digest (XXH_NOESCAPE const XXH64_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ +/******* Canonical representation *******/ -/* ************************************* -* Constants -***************************************/ -static const U32 PRIME32_1 = 2654435761U; -static const U32 PRIME32_2 = 2246822519U; -static const U32 PRIME32_3 = 3266489917U; -static const U32 PRIME32_4 = 668265263U; -static const U32 PRIME32_5 = 374761393U; - -static const U64 PRIME64_1 = 11400714785074694791ULL; -static const U64 PRIME64_2 = 14029467366897019727ULL; -static const U64 PRIME64_3 = 1609587929392839161ULL; -static const U64 PRIME64_4 = 9650029242287828579ULL; -static const U64 PRIME64_5 = 2870177450012600261ULL; +/*! + * @brief Canonical (big endian) representation of @ref XXH64_hash_t. + */ +typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t; -XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } +/*! + * @brief Converts an @ref XXH64_hash_t to a big endian @ref XXH64_canonical_t. + * + * @param dst The @ref XXH64_canonical_t pointer to be stored to. + * @param hash The @ref XXH64_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash); +/*! + * @brief Converts an @ref XXH64_canonical_t to a native @ref XXH64_hash_t. + * + * @param src The @ref XXH64_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src); -/* ************************** -* Utils -****************************/ -XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dstState, const XXH32_state_t* restrict srcState) -{ - ZSTD_memcpy(dstState, srcState, sizeof(*dstState)); -} +#ifndef XXH_NO_XXH3 -XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dstState, const XXH64_state_t* restrict srcState) -{ - ZSTD_memcpy(dstState, srcState, sizeof(*dstState)); -} +/*! + * @} + * ************************************************************************ + * @defgroup XXH3_family XXH3 family + * @ingroup public + * @{ + * + * XXH3 is a more recent hash algorithm featuring: + * - Improved speed for both small and large inputs + * - True 64-bit and 128-bit outputs + * - SIMD acceleration + * - Improved 32-bit viability + * + * Speed analysis methodology is explained here: + * + * https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html + * + * Compared to XXH64, expect XXH3 to run approximately + * ~2x faster on large inputs and >3x faster on small ones, + * exact differences vary depending on platform. + * + * XXH3's speed benefits greatly from SIMD and 64-bit arithmetic, + * but does not require it. + * Most 32-bit and 64-bit targets that can run XXH32 smoothly can run XXH3 + * at competitive speeds, even without vector support. Further details are + * explained in the implementation. + * + * XXH3 has a fast scalar implementation, but it also includes accelerated SIMD + * implementations for many common platforms: + * - AVX512 + * - AVX2 + * - SSE2 + * - ARM NEON + * - WebAssembly SIMD128 + * - POWER8 VSX + * - s390x ZVector + * This can be controlled via the @ref XXH_VECTOR macro, but it automatically + * selects the best version according to predefined macros. For the x86 family, an + * automatic runtime dispatcher is included separately in @ref xxh_x86dispatch.c. + * + * XXH3 implementation is portable: + * it has a generic C90 formulation that can be compiled on any platform, + * all implementations generate exactly the same hash value on all platforms. + * Starting from v0.8.0, it's also labelled "stable", meaning that + * any future version will also generate the same hash value. + * + * XXH3 offers 2 variants, _64bits and _128bits. + * + * When only 64 bits are needed, prefer invoking the _64bits variant, as it + * reduces the amount of mixing, resulting in faster speed on small inputs. + * It's also generally simpler to manipulate a scalar return type than a struct. + * + * The API supports one-shot hashing, streaming mode, and custom secrets. + */ +/*-********************************************************************** +* XXH3 64-bit variant +************************************************************************/ +/*! + * @brief Calculates 64-bit unseeded variant of XXH3 hash of @p input. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit XXH3 hash value. + * + * @note + * This is equivalent to @ref XXH3_64bits_withSeed() with a seed of `0`, however + * it may have slightly better performance due to constant propagation of the + * defaults. + * + * @see + * XXH3_64bits_withSeed(), XXH3_64bits_withSecret(): other seeding variants + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length); -/* *************************** -* Simple Hash Functions -*****************************/ +/*! + * @brief Calculates 64-bit seeded variant of XXH3 hash of @p input. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit XXH3 hash value. + * + * @note + * seed == 0 produces the same results as @ref XXH3_64bits(). + * + * This variant generates a custom secret on the fly based on default secret + * altered using the @p seed value. + * + * While this operation is decently fast, note that it's not completely free. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed); -static U32 XXH32_round(U32 seed, U32 input) -{ - seed += input * PRIME32_2; - seed = XXH_rotl32(seed, 13); - seed *= PRIME32_1; - return seed; -} +/*! + * The bare minimum size for a custom secret. + * + * @see + * XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(), + * XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret(). + */ +#define XXH3_SECRET_SIZE_MIN 136 -FORCE_INLINE_TEMPLATE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* bEnd = p + len; - U32 h32; -#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align) +/*! + * @brief Calculates 64-bit variant of XXH3 with a custom "secret". + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @return The calculated 64-bit XXH3 hash value. + * + * @pre + * The memory between @p data and @p data + @p len must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p data may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * It's possible to provide any blob of bytes as a "secret" to generate the hash. + * This makes it more difficult for an external actor to prepare an intentional collision. + * The main condition is that @p secretSize *must* be large enough (>= @ref XXH3_SECRET_SIZE_MIN). + * However, the quality of the secret impacts the dispersion of the hash algorithm. + * Therefore, the secret _must_ look like a bunch of random bytes. + * Avoid "trivial" or structured data such as repeated sequences or a text document. + * Whenever in doubt about the "randomness" of the blob of bytes, + * consider employing @ref XXH3_generateSecret() instead (see below). + * It will generate a proper high entropy secret derived from the blob of bytes. + * Another advantage of using XXH3_generateSecret() is that + * it guarantees that all bits within the initial blob of bytes + * will impact every bit of the output. + * This is not necessarily the case when using the blob of bytes directly + * because, when hashing _small_ inputs, only a portion of the secret is employed. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize); + + +/******* Streaming *******/ +#ifndef XXH_NO_STREAM +/* + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + */ -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (p==NULL) { - len=0; - bEnd=p=(const BYTE*)(size_t)16; - } -#endif +/*! + * @brief The opaque state struct for the XXH3 streaming API. + * + * @see XXH3_state_s for details. + */ +typedef struct XXH3_state_s XXH3_state_t; +XXH_PUBLIC_API XXH_MALLOCF XXH3_state_t* XXH3_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr); - if (len>=16) { - const BYTE* const limit = bEnd - 16; - U32 v1 = seed + PRIME32_1 + PRIME32_2; - U32 v2 = seed + PRIME32_2; - U32 v3 = seed + 0; - U32 v4 = seed - PRIME32_1; +/*! + * @brief Copies one @ref XXH3_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state); - do { - v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4; - v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4; - v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4; - v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4; - } while (p<=limit); +/*! + * @brief Resets an @ref XXH3_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret with default parameters. + * - Call this function before @ref XXH3_64bits_update(). + * - Digest will be equivalent to `XXH3_64bits()`. + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr); - h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); - } else { - h32 = seed + PRIME32_5; - } +/*! + * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret from `seed`. + * - Call this function before @ref XXH3_64bits_update(). + * - Digest will be equivalent to `XXH3_64bits_withSeed()`. + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed); - h32 += (U32) len; +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * `secret` is referenced, it _must outlive_ the hash streaming session. + * + * Similar to one-shot API, `secretSize` must be >= @ref XXH3_SECRET_SIZE_MIN, + * and the quality of produced hash values depends on secret's entropy + * (secret's content should look like a bunch of random bytes). + * When in doubt about the randomness of a candidate `secret`, + * consider employing `XXH3_generateSecret()` instead (see below). + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize); - while (p+4<=bEnd) { - h32 += XXH_get32bits(p) * PRIME32_3; - h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; - p+=4; - } +/*! + * @brief Consumes a block of @p input to an @ref XXH3_state_t. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note Call this to incrementally consume blocks of data. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); - while (p> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; +/* note : canonical representation of XXH3 is the same as XXH64 + * since they both produce XXH64_hash_t values */ - return h32; -} +/*-********************************************************************** +* XXH3 128-bit variant +************************************************************************/ -XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed) -{ -#if 0 - /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ - XXH32_CREATESTATE_STATIC(state); - XXH32_reset(state, seed); - XXH32_update(state, input, len); - return XXH32_digest(state); -#else - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; +/*! + * @brief The return value from 128-bit hashes. + * + * Stored in little endian order, although the fields themselves are in native + * endianness. + */ +typedef struct { + XXH64_hash_t low64; /*!< `value & 0xFFFFFFFFFFFFFFFF` */ + XXH64_hash_t high64; /*!< `value >> 64` */ +} XXH128_hash_t; - if (XXH_FORCE_ALIGN_CHECK) { - if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); - else - return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); - } } +/*! + * @brief Calculates 128-bit unseeded variant of XXH3 of @p data. + * + * @param data The block of data to be hashed, at least @p length bytes in size. + * @param len The length of @p data, in bytes. + * + * @return The calculated 128-bit variant of XXH3 value. + * + * The 128-bit variant of XXH3 has more strength, but it has a bit of overhead + * for shorter inputs. + * + * This is equivalent to @ref XXH3_128bits_withSeed() with a seed of `0`, however + * it may have slightly better performance due to constant propagation of the + * defaults. + * + * @see XXH3_128bits_withSeed(), XXH3_128bits_withSecret(): other seeding variants + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* data, size_t len); +/*! @brief Calculates 128-bit seeded variant of XXH3 hash of @p data. + * + * @param data The block of data to be hashed, at least @p length bytes in size. + * @param len The length of @p data, in bytes. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @return The calculated 128-bit variant of XXH3 value. + * + * @note + * seed == 0 produces the same results as @ref XXH3_64bits(). + * + * This variant generates a custom secret on the fly based on default secret + * altered using the @p seed value. + * + * While this operation is decently fast, note that it's not completely free. + * + * @see XXH3_128bits(), XXH3_128bits_withSecret(): other seeding variants + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSeed(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed); +/*! + * @brief Calculates 128-bit variant of XXH3 with a custom "secret". + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @return The calculated 128-bit variant of XXH3 value. + * + * It's possible to provide any blob of bytes as a "secret" to generate the hash. + * This makes it more difficult for an external actor to prepare an intentional collision. + * The main condition is that @p secretSize *must* be large enough (>= @ref XXH3_SECRET_SIZE_MIN). + * However, the quality of the secret impacts the dispersion of the hash algorithm. + * Therefore, the secret _must_ look like a bunch of random bytes. + * Avoid "trivial" or structured data such as repeated sequences or a text document. + * Whenever in doubt about the "randomness" of the blob of bytes, + * consider employing @ref XXH3_generateSecret() instead (see below). + * It will generate a proper high entropy secret derived from the blob of bytes. + * Another advantage of using XXH3_generateSecret() is that + * it guarantees that all bits within the initial blob of bytes + * will impact every bit of the output. + * This is not necessarily the case when using the blob of bytes directly + * because, when hashing _small_ inputs, only a portion of the secret is employed. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize); + +/******* Streaming *******/ +#ifndef XXH_NO_STREAM +/* + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + * + * XXH3_128bits uses the same XXH3_state_t as XXH3_64bits(). + * Use already declared XXH3_createState() and XXH3_freeState(). + * + * All reset and streaming functions have same meaning as their 64-bit counterpart. + */ - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); - else - return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); -#endif -} +/*! + * @brief Resets an @ref XXH3_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret with default parameters. + * - Call it before @ref XXH3_128bits_update(). + * - Digest will be equivalent to `XXH3_128bits()`. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr); +/*! + * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret from `seed`. + * - Call it before @ref XXH3_128bits_update(). + * - Digest will be equivalent to `XXH3_128bits_withSeed()`. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed); +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * `secret` is referenced, it _must outlive_ the hash streaming session. + * Similar to one-shot API, `secretSize` must be >= @ref XXH3_SECRET_SIZE_MIN, + * and the quality of produced hash values depends on secret's entropy + * (secret's content should look like a bunch of random bytes). + * When in doubt about the randomness of a candidate `secret`, + * consider employing `XXH3_generateSecret()` instead (see below). + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize); -static U64 XXH64_round(U64 acc, U64 input) -{ - acc += input * PRIME64_2; - acc = XXH_rotl64(acc, 31); - acc *= PRIME64_1; - return acc; -} +/*! + * @brief Consumes a block of @p input to an @ref XXH3_state_t. + * + * Call this to incrementally consume blocks of data. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); -static U64 XXH64_mergeRound(U64 acc, U64 val) -{ - val = XXH64_round(0, val); - acc ^= val; - acc = acc * PRIME64_1 + PRIME64_4; - return acc; -} +/*! + * @brief Returns the calculated XXH3 128-bit hash value from an @ref XXH3_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated XXH3 128-bit hash value from that state. + * + * @note + * Calling XXH3_128bits_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ -FORCE_INLINE_TEMPLATE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - U64 h64; -#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align) +/* Following helper functions make it possible to compare XXH128_hast_t values. + * Since XXH128_hash_t is a structure, this capability is not offered by the language. + * Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */ -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (p==NULL) { - len=0; - bEnd=p=(const BYTE*)(size_t)32; - } -#endif +/*! + * @brief Check equality of two XXH128_hash_t values + * + * @param h1 The 128-bit hash value. + * @param h2 Another 128-bit hash value. + * + * @return `1` if `h1` and `h2` are equal. + * @return `0` if they are not. + */ +XXH_PUBLIC_API XXH_PUREF int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2); - if (len>=32) { - const BYTE* const limit = bEnd - 32; - U64 v1 = seed + PRIME64_1 + PRIME64_2; - U64 v2 = seed + PRIME64_2; - U64 v3 = seed + 0; - U64 v4 = seed - PRIME64_1; +/*! + * @brief Compares two @ref XXH128_hash_t + * + * This comparator is compatible with stdlib's `qsort()`/`bsearch()`. + * + * @param h128_1 Left-hand side value + * @param h128_2 Right-hand side value + * + * @return >0 if @p h128_1 > @p h128_2 + * @return =0 if @p h128_1 == @p h128_2 + * @return <0 if @p h128_1 < @p h128_2 + */ +XXH_PUBLIC_API XXH_PUREF int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2); - do { - v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8; - v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8; - v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8; - v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8; - } while (p<=limit); - h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); - h64 = XXH64_mergeRound(h64, v1); - h64 = XXH64_mergeRound(h64, v2); - h64 = XXH64_mergeRound(h64, v3); - h64 = XXH64_mergeRound(h64, v4); +/******* Canonical representation *******/ +typedef struct { unsigned char digest[sizeof(XXH128_hash_t)]; } XXH128_canonical_t; - } else { - h64 = seed + PRIME64_5; - } - h64 += (U64) len; +/*! + * @brief Converts an @ref XXH128_hash_t to a big endian @ref XXH128_canonical_t. + * + * @param dst The @ref XXH128_canonical_t pointer to be stored to. + * @param hash The @ref XXH128_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash); - while (p+8<=bEnd) { - U64 const k1 = XXH64_round(0, XXH_get64bits(p)); - h64 ^= k1; - h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; - p+=8; - } +/*! + * @brief Converts an @ref XXH128_canonical_t to a native @ref XXH128_hash_t. + * + * @param src The @ref XXH128_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src); - if (p+4<=bEnd) { - h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; - h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; - p+=4; - } - while (p> 33; - h64 *= PRIME64_2; - h64 ^= h64 >> 29; - h64 *= PRIME64_3; - h64 ^= h64 >> 32; +#if defined (__cplusplus) +} /* extern "C" */ +#endif - return h64; -} +#endif /* XXH_NO_LONG_LONG */ +/*! + * @} + */ +#endif /* XXHASH_H_5627135585666179 */ -XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed) -{ -#if 0 - /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ - XXH64_CREATESTATE_STATIC(state); - XXH64_reset(state, seed); - XXH64_update(state, input, len); - return XXH64_digest(state); -#else - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - if (XXH_FORCE_ALIGN_CHECK) { - if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); - else - return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); - } } - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); - else - return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); -#endif -} +#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) +#define XXHASH_H_STATIC_13879238742 +/* **************************************************************************** + * This section contains declarations which are not guaranteed to remain stable. + * They may change in future versions, becoming incompatible with a different + * version of the library. + * These declarations should only be used with static linking. + * Never use them in association with dynamic linking! + ***************************************************************************** */ +/* + * These definitions are only present to allow static allocation + * of XXH states, on stack or in a struct, for example. + * Never **ever** access their members directly. + */ -/* ************************************************** -* Advanced Hash Functions -****************************************************/ +/*! + * @internal + * @brief Structure for XXH32 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH32_state_t. + * Do not access the members of this struct directly. + * @see XXH64_state_s, XXH3_state_s + */ +struct XXH32_state_s { + XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */ + XXH32_hash_t large_len; /*!< Whether the hash is >= 16 (handles @ref total_len_32 overflow) */ + XXH32_hash_t v[4]; /*!< Accumulator lanes */ + XXH32_hash_t mem32[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[16]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem32 */ + XXH32_hash_t reserved; /*!< Reserved field. Do not read nor write to it. */ +}; /* typedef'd to XXH32_state_t */ -XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) -{ - return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); -} -XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) -{ - XXH_free(statePtr); - return XXH_OK; -} -XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) -{ - return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); -} -XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) -{ - XXH_free(statePtr); - return XXH_OK; -} +#ifndef XXH_NO_LONG_LONG /* defined when there is no 64-bit support */ +/*! + * @internal + * @brief Structure for XXH64 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH64_state_t. + * Do not access the members of this struct directly. + * @see XXH32_state_s, XXH3_state_s + */ +struct XXH64_state_s { + XXH64_hash_t total_len; /*!< Total length hashed. This is always 64-bit. */ + XXH64_hash_t v[4]; /*!< Accumulator lanes */ + XXH64_hash_t mem64[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[32]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem64 */ + XXH32_hash_t reserved32; /*!< Reserved field, needed for padding anyways*/ + XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it. */ +}; /* typedef'd to XXH64_state_t */ + +#ifndef XXH_NO_XXH3 + +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */ +# include +# define XXH_ALIGN(n) alignas(n) +#elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */ +/* In C++ alignas() is a keyword */ +# define XXH_ALIGN(n) alignas(n) +#elif defined(__GNUC__) +# define XXH_ALIGN(n) __attribute__ ((aligned(n))) +#elif defined(_MSC_VER) +# define XXH_ALIGN(n) __declspec(align(n)) +#else +# define XXH_ALIGN(n) /* disabled */ +#endif -/*** Hash feed ***/ +/* Old GCC versions only accept the attribute after the type in structures. */ +#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) /* C11+ */ \ + && ! (defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \ + && defined(__GNUC__) +# define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align) +#else +# define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type +#endif -XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed) -{ - XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ - ZSTD_memset(&state, 0, sizeof(state)-4); /* do not write into reserved, for future removal */ - state.v1 = seed + PRIME32_1 + PRIME32_2; - state.v2 = seed + PRIME32_2; - state.v3 = seed + 0; - state.v4 = seed - PRIME32_1; - ZSTD_memcpy(statePtr, &state, sizeof(state)); - return XXH_OK; -} +/*! + * @brief The size of the internal XXH3 buffer. + * + * This is the optimal update size for incremental hashing. + * + * @see XXH3_64b_update(), XXH3_128b_update(). + */ +#define XXH3_INTERNALBUFFER_SIZE 256 +/*! + * @internal + * @brief Default size of the secret buffer (and @ref XXH3_kSecret). + * + * This is the size used in @ref XXH3_kSecret and the seeded functions. + * + * Not to be confused with @ref XXH3_SECRET_SIZE_MIN. + */ +#define XXH3_SECRET_DEFAULT_SIZE 192 -XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed) -{ - XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ - ZSTD_memset(&state, 0, sizeof(state)-8); /* do not write into reserved, for future removal */ - state.v1 = seed + PRIME64_1 + PRIME64_2; - state.v2 = seed + PRIME64_2; - state.v3 = seed + 0; - state.v4 = seed - PRIME64_1; - ZSTD_memcpy(statePtr, &state, sizeof(state)); - return XXH_OK; -} +/*! + * @internal + * @brief Structure for XXH3 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. + * Otherwise it is an opaque type. + * Never use this definition in combination with dynamic library. + * This allows fields to safely be changed in the future. + * + * @note ** This structure has a strict alignment requirement of 64 bytes!! ** + * Do not allocate this with `malloc()` or `new`, + * it will not be sufficiently aligned. + * Use @ref XXH3_createState() and @ref XXH3_freeState(), or stack allocation. + * + * Typedef'd to @ref XXH3_state_t. + * Do never access the members of this struct directly. + * + * @see XXH3_INITSTATE() for stack initialization. + * @see XXH3_createState(), XXH3_freeState(). + * @see XXH32_state_s, XXH64_state_s + */ +struct XXH3_state_s { + XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]); + /*!< The 8 accumulators. See @ref XXH32_state_s::v and @ref XXH64_state_s::v */ + XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]); + /*!< Used to store a custom secret generated from a seed. */ + XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]); + /*!< The internal buffer. @see XXH32_state_s::mem32 */ + XXH32_hash_t bufferedSize; + /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */ + XXH32_hash_t useSeed; + /*!< Reserved field. Needed for padding on 64-bit. */ + size_t nbStripesSoFar; + /*!< Number or stripes processed. */ + XXH64_hash_t totalLen; + /*!< Total length hashed. 64-bit even on 32-bit targets. */ + size_t nbStripesPerBlock; + /*!< Number of stripes per block. */ + size_t secretLimit; + /*!< Size of @ref customSecret or @ref extSecret */ + XXH64_hash_t seed; + /*!< Seed for _withSeed variants. Must be zero otherwise, @see XXH3_INITSTATE() */ + XXH64_hash_t reserved64; + /*!< Reserved field. */ + const unsigned char* extSecret; + /*!< Reference to an external secret for the _withSecret variants, NULL + * for other variants. */ + /* note: there may be some padding at the end due to alignment on 64 bytes */ +}; /* typedef'd to XXH3_state_t */ + +#undef XXH_ALIGN_MEMBER +/*! + * @brief Initializes a stack-allocated `XXH3_state_s`. + * + * When the @ref XXH3_state_t structure is merely emplaced on stack, + * it should be initialized with XXH3_INITSTATE() or a memset() + * in case its first reset uses XXH3_NNbits_reset_withSeed(). + * This init can be omitted if the first reset uses default or _withSecret mode. + * This operation isn't necessary when the state is created with XXH3_createState(). + * Note that this doesn't prepare the state for a streaming operation, + * it's still necessary to use XXH3_NNbits_reset*() afterwards. + */ +#define XXH3_INITSTATE(XXH3_state_ptr) \ + do { \ + XXH3_state_t* tmp_xxh3_state_ptr = (XXH3_state_ptr); \ + tmp_xxh3_state_ptr->seed = 0; \ + tmp_xxh3_state_ptr->extSecret = NULL; \ + } while(0) -FORCE_INLINE_TEMPLATE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (input==NULL) return XXH_ERROR; +#if defined (__cplusplus) +extern "C" { #endif - state->total_len_32 += (unsigned)len; - state->large_len |= (len>=16) | (state->total_len_32>=16); - - if (state->memsize + len < 16) { /* fill in tmp buffer */ - XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); - state->memsize += (unsigned)len; - return XXH_OK; - } - - if (state->memsize) { /* some data left from previous update */ - XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize); - { const U32* p32 = state->mem32; - state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++; - state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++; - state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++; - state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); p32++; - } - p += 16-state->memsize; - state->memsize = 0; - } +/*! + * @brief Calculates the 128-bit hash of @p data using XXH3. + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param seed The 64-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p data and @p data + @p len must be valid, + * readable, contiguous memory. However, if @p len is `0`, @p data may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 128-bit XXH3 value. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed); - if (p <= bEnd-16) { - const BYTE* const limit = bEnd - 16; - U32 v1 = state->v1; - U32 v2 = state->v2; - U32 v3 = state->v3; - U32 v4 = state->v4; - do { - v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4; - v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4; - v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4; - v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4; - } while (p<=limit); +/* === Experimental API === */ +/* Symbols defined below must be considered tied to a specific library version. */ - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; - } +/*! + * @brief Derive a high-entropy secret from any user-defined content, named customSeed. + * + * @param secretBuffer A writable buffer for derived high-entropy secret data. + * @param secretSize Size of secretBuffer, in bytes. Must be >= XXH3_SECRET_DEFAULT_SIZE. + * @param customSeed A user-defined content. + * @param customSeedSize Size of customSeed, in bytes. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * The generated secret can be used in combination with `*_withSecret()` functions. + * The `_withSecret()` variants are useful to provide a higher level of protection + * than 64-bit seed, as it becomes much more difficult for an external actor to + * guess how to impact the calculation logic. + * + * The function accepts as input a custom seed of any length and any content, + * and derives from it a high-entropy secret of length @p secretSize into an + * already allocated buffer @p secretBuffer. + * + * The generated secret can then be used with any `*_withSecret()` variant. + * The functions @ref XXH3_128bits_withSecret(), @ref XXH3_64bits_withSecret(), + * @ref XXH3_128bits_reset_withSecret() and @ref XXH3_64bits_reset_withSecret() + * are part of this list. They all accept a `secret` parameter + * which must be large enough for implementation reasons (>= @ref XXH3_SECRET_SIZE_MIN) + * _and_ feature very high entropy (consist of random-looking bytes). + * These conditions can be a high bar to meet, so @ref XXH3_generateSecret() can + * be employed to ensure proper quality. + * + * @p customSeed can be anything. It can have any size, even small ones, + * and its content can be anything, even "poor entropy" sources such as a bunch + * of zeroes. The resulting `secret` will nonetheless provide all required qualities. + * + * @pre + * - @p secretSize must be >= @ref XXH3_SECRET_SIZE_MIN + * - When @p customSeedSize > 0, supplying NULL as customSeed is undefined behavior. + * + * Example code: + * @code{.c} + * #include + * #include + * #include + * #define XXH_STATIC_LINKING_ONLY // expose unstable API + * #include "xxhash.h" + * // Hashes argv[2] using the entropy from argv[1]. + * int main(int argc, char* argv[]) + * { + * char secret[XXH3_SECRET_SIZE_MIN]; + * if (argv != 3) { return 1; } + * XXH3_generateSecret(secret, sizeof(secret), argv[1], strlen(argv[1])); + * XXH64_hash_t h = XXH3_64bits_withSecret( + * argv[2], strlen(argv[2]), + * secret, sizeof(secret) + * ); + * printf("%016llx\n", (unsigned long long) h); + * } + * @endcode + */ +XXH_PUBLIC_API XXH_errorcode XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize); - if (p < bEnd) { - XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); - state->memsize = (unsigned)(bEnd-p); - } +/*! + * @brief Generate the same secret as the _withSeed() variants. + * + * @param secretBuffer A writable buffer of @ref XXH3_SECRET_SIZE_MIN bytes + * @param seed The 64-bit seed to alter the hash result predictably. + * + * The generated secret can be used in combination with + *`*_withSecret()` and `_withSecretandSeed()` variants. + * + * Example C++ `std::string` hash class: + * @code{.cpp} + * #include + * #define XXH_STATIC_LINKING_ONLY // expose unstable API + * #include "xxhash.h" + * // Slow, seeds each time + * class HashSlow { + * XXH64_hash_t seed; + * public: + * HashSlow(XXH64_hash_t s) : seed{s} {} + * size_t operator()(const std::string& x) const { + * return size_t{XXH3_64bits_withSeed(x.c_str(), x.length(), seed)}; + * } + * }; + * // Fast, caches the seeded secret for future uses. + * class HashFast { + * unsigned char secret[XXH3_SECRET_SIZE_MIN]; + * public: + * HashFast(XXH64_hash_t s) { + * XXH3_generateSecret_fromSeed(secret, seed); + * } + * size_t operator()(const std::string& x) const { + * return size_t{ + * XXH3_64bits_withSecret(x.c_str(), x.length(), secret, sizeof(secret)) + * }; + * } + * }; + * @endcode + */ +XXH_PUBLIC_API void XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer, XXH64_hash_t seed); - return XXH_OK; -} +/*! + * @brief Calculates 64/128-bit seeded variant of XXH3 hash of @p data. + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * These variants generate hash values using either + * @p seed for "short" keys (< @ref XXH3_MIDSIZE_MAX = 240 bytes) + * or @p secret for "large" keys (>= @ref XXH3_MIDSIZE_MAX). + * + * This generally benefits speed, compared to `_withSeed()` or `_withSecret()`. + * `_withSeed()` has to generate the secret on the fly for "large" keys. + * It's fast, but can be perceptible for "not so large" keys (< 1 KB). + * `_withSecret()` has to generate the masks on the fly for "small" keys, + * which requires more instructions than _withSeed() variants. + * Therefore, _withSecretandSeed variant combines the best of both worlds. + * + * When @p secret has been generated by XXH3_generateSecret_fromSeed(), + * this variant produces *exactly* the same results as `_withSeed()` variant, + * hence offering only a pure speed benefit on "large" input, + * by skipping the need to regenerate the secret for every large input. + * + * Another usage scenario is to hash the secret to a 64-bit hash value, + * for example with XXH3_64bits(), which then becomes the seed, + * and then employ both the seed and the secret in _withSecretandSeed(). + * On top of speed, an added benefit is that each bit in the secret + * has a 50% chance to swap each bit in the output, via its impact to the seed. + * + * This is not guaranteed when using the secret directly in "small data" scenarios, + * because only portions of the secret are employed for small data. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t +XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* data, size_t len, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed); +/*! + * @brief Calculates 128-bit seeded variant of XXH3 hash of @p data. + * + * @param input The block of data to be hashed, at least @p len bytes in size. + * @param length The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed64 The 64-bit seed to alter the hash result predictably. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @see XXH3_64bits_withSecretandSeed() + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t +XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed64); +#ifndef XXH_NO_STREAM +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed64 The 64-bit seed to alter the hash result predictably. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @see XXH3_64bits_withSecretandSeed() + */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed64); +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed64 The 64-bit seed to alter the hash result predictably. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @see XXH3_64bits_withSecretandSeed() + */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed64); +#endif /* !XXH_NO_STREAM */ -XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; +#if defined (__cplusplus) +} /* extern "C" */ +#endif - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_update_endian(state_in, input, len, XXH_littleEndian); - else - return XXH32_update_endian(state_in, input, len, XXH_bigEndian); -} +#endif /* !XXH_NO_XXH3 */ +#endif /* XXH_NO_LONG_LONG */ +#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) +# define XXH_IMPLEMENTATION +#endif +#endif /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */ -FORCE_INLINE_TEMPLATE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian) -{ - const BYTE * p = (const BYTE*)state->mem32; - const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize; - U32 h32; - if (state->large_len) { - h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); - } else { - h32 = state->v3 /* == seed */ + PRIME32_5; - } +/* ======================================================================== */ +/* ======================================================================== */ +/* ======================================================================== */ - h32 += state->total_len_32; - while (p+4<=bEnd) { - h32 += XXH_readLE32(p, endian) * PRIME32_3; - h32 = XXH_rotl32(h32, 17) * PRIME32_4; - p+=4; - } +/*-********************************************************************** + * xxHash implementation + *-********************************************************************** + * xxHash's implementation used to be hosted inside xxhash.c. + * + * However, inlining requires implementation to be visible to the compiler, + * hence be included alongside the header. + * Previously, implementation was hosted inside xxhash.c, + * which was then #included when inlining was activated. + * This construction created issues with a few build and install systems, + * as it required xxhash.c to be stored in /include directory. + * + * xxHash implementation is now directly integrated within xxhash.h. + * As a consequence, xxhash.c is no longer needed in /include. + * + * xxhash.c is still available and is still useful. + * In a "normal" setup, when xxhash is not inlined, + * xxhash.h only exposes the prototypes and public symbols, + * while xxhash.c can be built into an object file xxhash.o + * which can then be linked into the final binary. + ************************************************************************/ - while (p> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; +/* ************************************* +* Tuning parameters +***************************************/ - return h32; -} +/*! + * @defgroup tuning Tuning parameters + * @{ + * + * Various macros to control xxHash's behavior. + */ +#ifdef XXH_DOXYGEN +/*! + * @brief Define this to disable 64-bit code. + * + * Useful if only using the @ref XXH32_family and you have a strict C90 compiler. + */ +# define XXH_NO_LONG_LONG +# undef XXH_NO_LONG_LONG /* don't actually */ +/*! + * @brief Controls how unaligned memory is accessed. + * + * By default, access to unaligned memory is controlled by `memcpy()`, which is + * safe and portable. + * + * Unfortunately, on some target/compiler combinations, the generated assembly + * is sub-optimal. + * + * The below switch allow selection of a different access method + * in the search for improved performance. + * + * @par Possible options: + * + * - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy` + * @par + * Use `memcpy()`. Safe and portable. Note that most modern compilers will + * eliminate the function call and treat it as an unaligned access. + * + * - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((aligned(1)))` + * @par + * Depends on compiler extensions and is therefore not portable. + * This method is safe _if_ your compiler supports it, + * and *generally* as fast or faster than `memcpy`. + * + * - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast + * @par + * Casts directly and dereferences. This method doesn't depend on the + * compiler, but it violates the C standard as it directly dereferences an + * unaligned pointer. It can generate buggy code on targets which do not + * support unaligned memory accesses, but in some circumstances, it's the + * only known way to get the most performance. + * + * - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift + * @par + * Also portable. This can generate the best code on old compilers which don't + * inline small `memcpy()` calls, and it might also be faster on big-endian + * systems which lack a native byteswap instruction. However, some compilers + * will emit literal byteshifts even if the target supports unaligned access. + * + * + * @warning + * Methods 1 and 2 rely on implementation-defined behavior. Use these with + * care, as what works on one compiler/platform/optimization level may cause + * another to read garbage data or even crash. + * + * See https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details. + * + * Prefer these methods in priority order (0 > 3 > 1 > 2) + */ +# define XXH_FORCE_MEMORY_ACCESS 0 +/*! + * @def XXH_SIZE_OPT + * @brief Controls how much xxHash optimizes for size. + * + * xxHash, when compiled, tends to result in a rather large binary size. This + * is mostly due to heavy usage to forced inlining and constant folding of the + * @ref XXH3_family to increase performance. + * + * However, some developers prefer size over speed. This option can + * significantly reduce the size of the generated code. When using the `-Os` + * or `-Oz` options on GCC or Clang, this is defined to 1 by default, + * otherwise it is defined to 0. + * + * Most of these size optimizations can be controlled manually. + * + * This is a number from 0-2. + * - `XXH_SIZE_OPT` == 0: Default. xxHash makes no size optimizations. Speed + * comes first. + * - `XXH_SIZE_OPT` == 1: Default for `-Os` and `-Oz`. xxHash is more + * conservative and disables hacks that increase code size. It implies the + * options @ref XXH_NO_INLINE_HINTS == 1, @ref XXH_FORCE_ALIGN_CHECK == 0, + * and @ref XXH3_NEON_LANES == 8 if they are not already defined. + * - `XXH_SIZE_OPT` == 2: xxHash tries to make itself as small as possible. + * Performance may cry. For example, the single shot functions just use the + * streaming API. + */ +# define XXH_SIZE_OPT 0 -XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; +/*! + * @def XXH_FORCE_ALIGN_CHECK + * @brief If defined to non-zero, adds a special path for aligned inputs (XXH32() + * and XXH64() only). + * + * This is an important performance trick for architectures without decent + * unaligned memory access performance. + * + * It checks for input alignment, and when conditions are met, uses a "fast + * path" employing direct 32-bit/64-bit reads, resulting in _dramatically + * faster_ read speed. + * + * The check costs one initial branch per hash, which is generally negligible, + * but not zero. + * + * Moreover, it's not useful to generate an additional code path if memory + * access uses the same instruction for both aligned and unaligned + * addresses (e.g. x86 and aarch64). + * + * In these cases, the alignment check can be removed by setting this macro to 0. + * Then the code will always use unaligned memory access. + * Align check is automatically disabled on x86, x64, ARM64, and some ARM chips + * which are platforms known to offer good unaligned memory accesses performance. + * + * It is also disabled by default when @ref XXH_SIZE_OPT >= 1. + * + * This option does not affect XXH3 (only XXH32 and XXH64). + */ +# define XXH_FORCE_ALIGN_CHECK 0 - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_digest_endian(state_in, XXH_littleEndian); - else - return XXH32_digest_endian(state_in, XXH_bigEndian); -} +/*! + * @def XXH_NO_INLINE_HINTS + * @brief When non-zero, sets all functions to `static`. + * + * By default, xxHash tries to force the compiler to inline almost all internal + * functions. + * + * This can usually improve performance due to reduced jumping and improved + * constant folding, but significantly increases the size of the binary which + * might not be favorable. + * + * Additionally, sometimes the forced inlining can be detrimental to performance, + * depending on the architecture. + * + * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the + * compiler full control on whether to inline or not. + * + * When not optimizing (-O0), using `-fno-inline` with GCC or Clang, or if + * @ref XXH_SIZE_OPT >= 1, this will automatically be defined. + */ +# define XXH_NO_INLINE_HINTS 0 +/*! + * @def XXH3_INLINE_SECRET + * @brief Determines whether to inline the XXH3 withSecret code. + * + * When the secret size is known, the compiler can improve the performance + * of XXH3_64bits_withSecret() and XXH3_128bits_withSecret(). + * + * However, if the secret size is not known, it doesn't have any benefit. This + * happens when xxHash is compiled into a global symbol. Therefore, if + * @ref XXH_INLINE_ALL is *not* defined, this will be defined to 0. + * + * Additionally, this defaults to 0 on GCC 12+, which has an issue with function pointers + * that are *sometimes* force inline on -Og, and it is impossible to automatically + * detect this optimization level. + */ +# define XXH3_INLINE_SECRET 0 +/*! + * @def XXH32_ENDJMP + * @brief Whether to use a jump for `XXH32_finalize`. + * + * For performance, `XXH32_finalize` uses multiple branches in the finalizer. + * This is generally preferable for performance, + * but depending on exact architecture, a jmp may be preferable. + * + * This setting is only possibly making a difference for very small inputs. + */ +# define XXH32_ENDJMP 0 -/* **** XXH64 **** */ +/*! + * @internal + * @brief Redefines old internal names. + * + * For compatibility with code that uses xxHash's internals before the names + * were changed to improve namespacing. There is no other reason to use this. + */ +# define XXH_OLD_NAMES +# undef XXH_OLD_NAMES /* don't actually use, it is ugly. */ -FORCE_INLINE_TEMPLATE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; +/*! + * @def XXH_NO_STREAM + * @brief Disables the streaming API. + * + * When xxHash is not inlined and the streaming functions are not used, disabling + * the streaming functions can improve code size significantly, especially with + * the @ref XXH3_family which tends to make constant folded copies of itself. + */ +# define XXH_NO_STREAM +# undef XXH_NO_STREAM /* don't actually */ +#endif /* XXH_DOXYGEN */ +/*! + * @} + */ -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (input==NULL) return XXH_ERROR; +#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ + /* prefer __packed__ structures (method 1) for GCC + * < ARMv7 with unaligned access (e.g. Raspbian armhf) still uses byte shifting, so we use memcpy + * which for some reason does unaligned loads. */ +# if defined(__GNUC__) && !(defined(__ARM_ARCH) && __ARM_ARCH < 7 && defined(__ARM_FEATURE_UNALIGNED)) +# define XXH_FORCE_MEMORY_ACCESS 1 +# endif #endif - state->total_len += len; +#ifndef XXH_SIZE_OPT + /* default to 1 for -Os or -Oz */ +# if (defined(__GNUC__) || defined(__clang__)) && defined(__OPTIMIZE_SIZE__) +# define XXH_SIZE_OPT 1 +# else +# define XXH_SIZE_OPT 0 +# endif +#endif - if (state->memsize + len < 32) { /* fill in tmp buffer */ - if (input != NULL) { - XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); - } - state->memsize += (U32)len; - return XXH_OK; - } +#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ + /* don't check on sizeopt, x86, aarch64, or arm when unaligned access is available */ +# if XXH_SIZE_OPT >= 1 || \ + defined(__i386) || defined(__x86_64__) || defined(__aarch64__) || defined(__ARM_FEATURE_UNALIGNED) \ + || defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) || defined(_M_ARM) /* visual */ +# define XXH_FORCE_ALIGN_CHECK 0 +# else +# define XXH_FORCE_ALIGN_CHECK 1 +# endif +#endif - if (state->memsize) { /* tmp buffer is full */ - XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize); - state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian)); - state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian)); - state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian)); - state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian)); - p += 32-state->memsize; - state->memsize = 0; - } +#ifndef XXH_NO_INLINE_HINTS +# if XXH_SIZE_OPT >= 1 || defined(__NO_INLINE__) /* -O0, -fno-inline */ +# define XXH_NO_INLINE_HINTS 1 +# else +# define XXH_NO_INLINE_HINTS 0 +# endif +#endif - if (p+32 <= bEnd) { - const BYTE* const limit = bEnd - 32; - U64 v1 = state->v1; - U64 v2 = state->v2; - U64 v3 = state->v3; - U64 v4 = state->v4; +#ifndef XXH3_INLINE_SECRET +# if (defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 12) \ + || !defined(XXH_INLINE_ALL) +# define XXH3_INLINE_SECRET 0 +# else +# define XXH3_INLINE_SECRET 1 +# endif +#endif - do { - v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8; - v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8; - v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8; - v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8; - } while (p<=limit); +#ifndef XXH32_ENDJMP +/* generally preferable for performance */ +# define XXH32_ENDJMP 0 +#endif - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; - } +/*! + * @defgroup impl Implementation + * @{ + */ - if (p < bEnd) { - XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); - state->memsize = (unsigned)(bEnd-p); - } +/* ************************************* +* Includes & Memory related functions +***************************************/ +#include /* memcmp, memcpy */ +#include /* ULLONG_MAX */ - return XXH_OK; -} +#if defined(XXH_NO_STREAM) +/* nothing */ +#elif defined(XXH_NO_STDLIB) -XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; +/* When requesting to disable any mention of stdlib, + * the library loses the ability to invoked malloc / free. + * In practice, it means that functions like `XXH*_createState()` + * will always fail, and return NULL. + * This flag is useful in situations where + * xxhash.h is integrated into some kernel, embedded or limited environment + * without access to dynamic allocation. + */ - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_update_endian(state_in, input, len, XXH_littleEndian); - else - return XXH64_update_endian(state_in, input, len, XXH_bigEndian); -} +#if defined (__cplusplus) +extern "C" { +#endif +static XXH_CONSTF void* XXH_malloc(size_t s) { (void)s; return NULL; } +static void XXH_free(void* p) { (void)p; } +#if defined (__cplusplus) +} /* extern "C" */ +#endif -FORCE_INLINE_TEMPLATE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian) -{ - const BYTE * p = (const BYTE*)state->mem64; - const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize; - U64 h64; +#else - if (state->total_len >= 32) { - U64 const v1 = state->v1; - U64 const v2 = state->v2; - U64 const v3 = state->v3; - U64 const v4 = state->v4; +/* + * Modify the local functions below should you wish to use + * different memory routines for malloc() and free() + */ +#include - h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); - h64 = XXH64_mergeRound(h64, v1); - h64 = XXH64_mergeRound(h64, v2); - h64 = XXH64_mergeRound(h64, v3); - h64 = XXH64_mergeRound(h64, v4); - } else { - h64 = state->v3 + PRIME64_5; - } - - h64 += (U64) state->total_len; - - while (p+8<=bEnd) { - U64 const k1 = XXH64_round(0, XXH_readLE64(p, endian)); - h64 ^= k1; - h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; - p+=8; - } - - if (p+4<=bEnd) { - h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1; - h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; - p+=4; - } - - while (p> 33; - h64 *= PRIME64_2; - h64 ^= h64 >> 29; - h64 *= PRIME64_3; - h64 ^= h64 >> 32; +/*! + * @internal + * @brief Modify this function to use a different routine than free(). + */ +static void XXH_free(void* p) { free(p); } - return h64; -} +#if defined (__cplusplus) +} /* extern "C" */ +#endif +#endif /* XXH_NO_STDLIB */ -XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in) +#if defined (__cplusplus) +extern "C" { +#endif +/*! + * @internal + * @brief Modify this function to use a different routine than memcpy(). + */ +static void* XXH_memcpy(void* dest, const void* src, size_t size) { - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_digest_endian(state_in, XXH_littleEndian); - else - return XXH64_digest_endian(state_in, XXH_bigEndian); + return memcpy(dest,src,size); } +#if defined (__cplusplus) +} /* extern "C" */ +#endif -/* ************************** -* Canonical representation -****************************/ +/* ************************************* +* Compiler Specific Options +***************************************/ +#ifdef _MSC_VER /* Visual Studio warning fix */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif -/*! Default XXH result types are basic unsigned 32 and 64 bits. -* The canonical representation follows human-readable write convention, aka big-endian (large digits first). -* These functions allow transformation of hash result into and from its canonical format. -* This way, hash values can be written into a file or buffer, and remain comparable across different systems and programs. -*/ +#if XXH_NO_INLINE_HINTS /* disable inlining hints */ +# if defined(__GNUC__) || defined(__clang__) +# define XXH_FORCE_INLINE static __attribute__((unused)) +# else +# define XXH_FORCE_INLINE static +# endif +# define XXH_NO_INLINE static +/* enable inlining hints */ +#elif defined(__GNUC__) || defined(__clang__) +# define XXH_FORCE_INLINE static __inline__ __attribute__((always_inline, unused)) +# define XXH_NO_INLINE static __attribute__((noinline)) +#elif defined(_MSC_VER) /* Visual Studio */ +# define XXH_FORCE_INLINE static __forceinline +# define XXH_NO_INLINE static __declspec(noinline) +#elif defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* C99 */ +# define XXH_FORCE_INLINE static inline +# define XXH_NO_INLINE static +#else +# define XXH_FORCE_INLINE static +# define XXH_NO_INLINE static +#endif -XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) -{ - XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); - if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); - ZSTD_memcpy(dst, &hash, sizeof(*dst)); -} +#if XXH3_INLINE_SECRET +# define XXH3_WITH_SECRET_INLINE XXH_FORCE_INLINE +#else +# define XXH3_WITH_SECRET_INLINE XXH_NO_INLINE +#endif -XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash) -{ - XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); - if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); - ZSTD_memcpy(dst, &hash, sizeof(*dst)); -} -XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) -{ - return XXH_readBE32(src); -} +/* ************************************* +* Debug +***************************************/ +/*! + * @ingroup tuning + * @def XXH_DEBUGLEVEL + * @brief Sets the debugging level. + * + * XXH_DEBUGLEVEL is expected to be defined externally, typically via the + * compiler's command line options. The value must be a number. + */ +#ifndef XXH_DEBUGLEVEL +# ifdef DEBUGLEVEL /* backwards compat */ +# define XXH_DEBUGLEVEL DEBUGLEVEL +# else +# define XXH_DEBUGLEVEL 0 +# endif +#endif -XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src) -{ - return XXH_readBE64(src); -} -/**** ended inlining xxhash.c ****/ +#if (XXH_DEBUGLEVEL>=1) +# include /* note: can still be disabled with NDEBUG */ +# define XXH_ASSERT(c) assert(c) +#else +# if defined(__INTEL_COMPILER) +# define XXH_ASSERT(c) XXH_ASSUME((unsigned char) (c)) +# else +# define XXH_ASSERT(c) XXH_ASSUME(c) # endif +#endif -#endif /* XXH_STATIC_LINKING_ONLY && XXH_STATIC_H_3543687687345 */ +/* note: use after variable declarations */ +#ifndef XXH_STATIC_ASSERT +# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */ +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { _Static_assert((c),m); } while(0) +# elif defined(__cplusplus) && (__cplusplus >= 201103L) /* C++11 */ +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0) +# else +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { struct xxh_sa { char x[(c) ? 1 : -1]; }; } while(0) +# endif +# define XXH_STATIC_ASSERT(c) XXH_STATIC_ASSERT_WITH_MESSAGE((c),#c) +#endif +/*! + * @internal + * @def XXH_COMPILER_GUARD(var) + * @brief Used to prevent unwanted optimizations for @p var. + * + * It uses an empty GCC inline assembly statement with a register constraint + * which forces @p var into a general purpose register (eg eax, ebx, ecx + * on x86) and marks it as modified. + * + * This is used in a few places to avoid unwanted autovectorization (e.g. + * XXH32_round()). All vectorization we want is explicit via intrinsics, + * and _usually_ isn't wanted elsewhere. + * + * We also use it to prevent unwanted constant folding for AArch64 in + * XXH3_initCustomSecret_scalar(). + */ +#if defined(__GNUC__) || defined(__clang__) +# define XXH_COMPILER_GUARD(var) __asm__("" : "+r" (var)) +#else +# define XXH_COMPILER_GUARD(var) ((void)0) +#endif -#if defined (__cplusplus) -} +/* Specifically for NEON vectors which use the "w" constraint, on + * Clang. */ +#if defined(__clang__) && defined(__ARM_ARCH) && !defined(__wasm__) +# define XXH_COMPILER_GUARD_CLANG_NEON(var) __asm__("" : "+w" (var)) +#else +# define XXH_COMPILER_GUARD_CLANG_NEON(var) ((void)0) #endif -/**** ended inlining xxhash.h ****/ -#if defined (__cplusplus) -extern "C" { +/* ************************************* +* Basic Types +***************************************/ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# ifdef _AIX +# include +# else +# include +# endif + typedef uint8_t xxh_u8; +#else + typedef unsigned char xxh_u8; #endif +typedef XXH32_hash_t xxh_u32; -/* ---- static assert (debug) --- */ -#define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) -#define ZSTD_isError ERR_isError /* for inlining */ -#define FSE_isError ERR_isError -#define HUF_isError ERR_isError +#ifdef XXH_OLD_NAMES +# warning "XXH_OLD_NAMES is planned to be removed starting v0.9. If the program depends on it, consider moving away from it by employing newer type names directly" +# define BYTE xxh_u8 +# define U8 xxh_u8 +# define U32 xxh_u32 +#endif +#if defined (__cplusplus) +extern "C" { +#endif -/*-************************************* -* shared macros -***************************************/ -#undef MIN -#undef MAX -#define MIN(a,b) ((a)<(b) ? (a) : (b)) -#define MAX(a,b) ((a)>(b) ? (a) : (b)) +/* *** Memory access *** */ -/** - * Ignore: this is an internal helper. +/*! + * @internal + * @fn xxh_u32 XXH_read32(const void* ptr) + * @brief Reads an unaligned 32-bit integer from @p ptr in native endianness. * - * This is a helper function to help force C99-correctness during compilation. - * Under strict compilation modes, variadic macro arguments can't be empty. - * However, variadic function arguments can be. Using a function therefore lets - * us statically check that at least one (string) argument was passed, - * independent of the compilation flags. + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit native endian integer from the bytes at @p ptr. */ -static INLINE_KEYWORD UNUSED_ATTR -void _force_has_format_string(const char *format, ...) { - (void)format; -} -/** - * Ignore: this is an internal helper. +/*! + * @internal + * @fn xxh_u32 XXH_readLE32(const void* ptr) + * @brief Reads an unaligned 32-bit little endian integer from @p ptr. * - * We want to force this function invocation to be syntactically correct, but - * we don't want to force runtime evaluation of its arguments. + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit little endian integer from the bytes at @p ptr. */ -#define _FORCE_HAS_FORMAT_STRING(...) \ - if (0) { \ - _force_has_format_string(__VA_ARGS__); \ - } -/** - * Return the specified error if the condition evaluates to true. +/*! + * @internal + * @fn xxh_u32 XXH_readBE32(const void* ptr) + * @brief Reads an unaligned 32-bit big endian integer from @p ptr. * - * In debug modes, prints additional information. - * In order to do that (particularly, printing the conditional that failed), - * this can't just wrap RETURN_ERROR(). + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit big endian integer from the bytes at @p ptr. */ -#define RETURN_ERROR_IF(cond, err, ...) \ - if (cond) { \ - RAWLOG(3, "%s:%d: ERROR!: check %s failed, returning %s", \ - __FILE__, __LINE__, ZSTD_QUOTE(cond), ZSTD_QUOTE(ERROR(err))); \ - _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ - RAWLOG(3, ": " __VA_ARGS__); \ - RAWLOG(3, "\n"); \ - return ERROR(err); \ - } -/** - * Unconditionally return the specified error. +/*! + * @internal + * @fn xxh_u32 XXH_readLE32_align(const void* ptr, XXH_alignment align) + * @brief Like @ref XXH_readLE32(), but has an option for aligned reads. * - * In debug modes, prints additional information. + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * Note that when @ref XXH_FORCE_ALIGN_CHECK == 0, the @p align parameter is + * always @ref XXH_alignment::XXH_unaligned. + * + * @param ptr The pointer to read from. + * @param align Whether @p ptr is aligned. + * @pre + * If @p align == @ref XXH_alignment::XXH_aligned, @p ptr must be 4 byte + * aligned. + * @return The 32-bit little endian integer from the bytes at @p ptr. */ -#define RETURN_ERROR(err, ...) \ - do { \ - RAWLOG(3, "%s:%d: ERROR!: unconditional check failed, returning %s", \ - __FILE__, __LINE__, ZSTD_QUOTE(ERROR(err))); \ - _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ - RAWLOG(3, ": " __VA_ARGS__); \ - RAWLOG(3, "\n"); \ - return ERROR(err); \ - } while(0); -/** - * If the provided expression evaluates to an error code, returns that error code. - * - * In debug modes, prints additional information. +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE32 and XXH_readBE32. */ -#define FORWARD_IF_ERROR(err, ...) \ - do { \ - size_t const err_code = (err); \ - if (ERR_isError(err_code)) { \ - RAWLOG(3, "%s:%d: ERROR!: forwarding error in %s: %s", \ - __FILE__, __LINE__, ZSTD_QUOTE(err), ERR_getErrorName(err_code)); \ - _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ - RAWLOG(3, ": " __VA_ARGS__); \ - RAWLOG(3, "\n"); \ - return err_code; \ - } \ - } while(0); +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) +/* + * Force direct memory access. Only works on CPU which support unaligned memory + * access in hardware. + */ +static xxh_u32 XXH_read32(const void* memPtr) { return *(const xxh_u32*) memPtr; } -/*-************************************* -* Common constants -***************************************/ -#define ZSTD_OPT_NUM (1<<12) +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) -#define ZSTD_REP_NUM 3 /* number of repcodes */ -#define ZSTD_REP_MOVE (ZSTD_REP_NUM-1) -static UNUSED_ATTR const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 }; +/* + * __attribute__((aligned(1))) is supported by gcc and clang. Originally the + * documentation claimed that it only increased the alignment, but actually it + * can decrease it on gcc, clang, and icc: + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502, + * https://gcc.godbolt.org/z/xYez1j67Y. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; } __attribute__((packed)) unalign; +#endif +static xxh_u32 XXH_read32(const void* ptr) +{ + typedef __attribute__((aligned(1))) xxh_u32 xxh_unalign32; + return *((const xxh_unalign32*)ptr); +} -#define KB *(1 <<10) -#define MB *(1 <<20) -#define GB *(1U<<30) +#else -#define BIT7 128 -#define BIT6 64 -#define BIT5 32 -#define BIT4 16 -#define BIT1 2 -#define BIT0 1 +/* + * Portable and safe solution. Generally efficient. + * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + */ +static xxh_u32 XXH_read32(const void* memPtr) +{ + xxh_u32 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; +} -#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10 -static UNUSED_ATTR const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 }; -static UNUSED_ATTR const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 }; +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ -#define ZSTD_FRAMEIDSIZE 4 /* magic number size */ -#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ -static UNUSED_ATTR const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; -typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e; +/* *** Endianness *** */ -#define ZSTD_FRAMECHECKSUMSIZE 4 +/*! + * @ingroup tuning + * @def XXH_CPU_LITTLE_ENDIAN + * @brief Whether the target is little endian. + * + * Defined to 1 if the target is little endian, or 0 if it is big endian. + * It can be defined externally, for example on the compiler command line. + * + * If it is not defined, + * a runtime check (which is usually constant folded) is used instead. + * + * @note + * This is not necessarily defined to an integer constant. + * + * @see XXH_isLittleEndian() for the runtime check. + */ +#ifndef XXH_CPU_LITTLE_ENDIAN +/* + * Try to detect endianness automatically, to avoid the nonstandard behavior + * in `XXH_isLittleEndian()` + */ +# if defined(_WIN32) /* Windows is always little endian */ \ + || defined(__LITTLE_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 1 +# elif defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 0 +# else +/*! + * @internal + * @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN. + * + * Most compilers will constant fold this. + */ +static int XXH_isLittleEndian(void) +{ + /* + * Portable and well-defined behavior. + * Don't use static: it is detrimental to performance. + */ + const union { xxh_u32 u; xxh_u8 c[4]; } one = { 1 }; + return one.c[0]; +} +# define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian() +# endif +#endif -#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */ -#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */ -#define HufLog 12 -typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e; -#define LONGNBSEQ 0x7F00 -#define MINMATCH 3 +/* **************************************** +* Compiler-specific Functions and Macros +******************************************/ +#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -#define Litbits 8 -#define MaxLit ((1<= XXH_C23_VN) + * # include + * # ifdef unreachable + * # define XXH_UNREACHABLE() unreachable() + * # endif + * #endif + * ``` + * + * Note C++23 also has std::unreachable() which can be detected + * as follows: + * ``` + * #if defined(__cpp_lib_unreachable) && (__cpp_lib_unreachable >= 202202L) + * # include + * # define XXH_UNREACHABLE() std::unreachable() + * #endif + * ``` + * NB: `__cpp_lib_unreachable` is defined in the `` header. + * We don't use that as including `` in `extern "C"` blocks + * doesn't work on GCC12 + */ + +#if XXH_HAS_BUILTIN(__builtin_unreachable) +# define XXH_UNREACHABLE() __builtin_unreachable() -static UNUSED_ATTR const S16 OF_defaultNorm[DefaultMaxOff+1] = { - 1, 1, 1, 1, 1, 1, 2, 2, - 2, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - -1,-1,-1,-1,-1 -}; -#define OF_DEFAULTNORMLOG 5 /* for static allocation */ -static UNUSED_ATTR const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG; +#elif defined(_MSC_VER) +# define XXH_UNREACHABLE() __assume(0) +#else +# define XXH_UNREACHABLE() +#endif -/*-******************************************* -* Shared functions to include for inlining -*********************************************/ -static void ZSTD_copy8(void* dst, const void* src) { -#if !defined(ZSTD_NO_INTRINSICS) && defined(__ARM_NEON) - vst1_u8((uint8_t*)dst, vld1_u8((const uint8_t*)src)); +#if XXH_HAS_BUILTIN(__builtin_assume) +# define XXH_ASSUME(c) __builtin_assume(c) #else - ZSTD_memcpy(dst, src, 8); +# define XXH_ASSUME(c) if (!(c)) { XXH_UNREACHABLE(); } #endif -} -#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; } -static void ZSTD_copy16(void* dst, const void* src) { -#if !defined(ZSTD_NO_INTRINSICS) && defined(__ARM_NEON) - vst1q_u8((uint8_t*)dst, vld1q_u8((const uint8_t*)src)); +/*! + * @internal + * @def XXH_rotl32(x,r) + * @brief 32-bit rotate left. + * + * @param x The 32-bit integer to be rotated. + * @param r The number of bits to rotate. + * @pre + * @p r > 0 && @p r < 32 + * @note + * @p x and @p r may be evaluated multiple times. + * @return The rotated result. + */ +#if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) \ + && XXH_HAS_BUILTIN(__builtin_rotateleft64) +# define XXH_rotl32 __builtin_rotateleft32 +# define XXH_rotl64 __builtin_rotateleft64 +/* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */ +#elif defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +# define XXH_rotl64(x,r) _rotl64(x,r) #else - ZSTD_memcpy(dst, src, 16); +# define XXH_rotl32(x,r) (((x) << (r)) | ((x) >> (32 - (r)))) +# define XXH_rotl64(x,r) (((x) << (r)) | ((x) >> (64 - (r)))) #endif + +/*! + * @internal + * @fn xxh_u32 XXH_swap32(xxh_u32 x) + * @brief A 32-bit byteswap. + * + * @param x The 32-bit integer to byteswap. + * @return @p x, byteswapped. + */ +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap32 _byteswap_ulong +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +#else +static xxh_u32 XXH_swap32 (xxh_u32 x) +{ + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); } -#define COPY16(d,s) { ZSTD_copy16(d,s); d+=16; s+=16; } +#endif -#define WILDCOPY_OVERLENGTH 32 -#define WILDCOPY_VECLEN 16 +/* *************************** +* Memory reads +*****************************/ + +/*! + * @internal + * @brief Enum to indicate whether a pointer is aligned. + */ typedef enum { - ZSTD_no_overlap, - ZSTD_overlap_src_before_dst - /* ZSTD_overlap_dst_before_src, */ -} ZSTD_overlap_e; + XXH_aligned, /*!< Aligned */ + XXH_unaligned /*!< Possibly unaligned */ +} XXH_alignment; -/*! ZSTD_wildcopy() : - * Custom version of ZSTD_memcpy(), can over read/write up to WILDCOPY_OVERLENGTH bytes (if length==0) - * @param ovtype controls the overlap detection - * - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart. - * - ZSTD_overlap_src_before_dst: The src and dst may overlap, but they MUST be at least 8 bytes apart. - * The src buffer must be before the dst buffer. +/* + * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. + * + * This is ideal for older compilers which don't inline memcpy. */ -MEM_STATIC FORCE_INLINE_ATTR -void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e const ovtype) +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* memPtr) { - ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src; - const BYTE* ip = (const BYTE*)src; - BYTE* op = (BYTE*)dst; - BYTE* const oend = op + length; + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u32)bytePtr[1] << 8) + | ((xxh_u32)bytePtr[2] << 16) + | ((xxh_u32)bytePtr[3] << 24); +} - assert(diff >= 8 || (ovtype == ZSTD_no_overlap && diff <= -WILDCOPY_VECLEN)); +XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[3] + | ((xxh_u32)bytePtr[2] << 8) + | ((xxh_u32)bytePtr[1] << 16) + | ((xxh_u32)bytePtr[0] << 24); +} - if (ovtype == ZSTD_overlap_src_before_dst && diff < WILDCOPY_VECLEN) { - /* Handle short offset copies. */ - do { - COPY8(op, ip) - } while (op < oend); - } else { - assert(diff >= WILDCOPY_VECLEN || diff <= -WILDCOPY_VECLEN); - /* Separate out the first COPY16() call because the copy length is - * almost certain to be short, so the branches have different - * probabilities. Since it is almost certain to be short, only do - * one COPY16() in the first call. Then, do two calls per loop since - * at that point it is more likely to have a high trip count. - */ -#ifdef __aarch64__ - do { - COPY16(op, ip); - } - while (op < oend); #else - ZSTD_copy16(op, ip); - if (16 >= length) return; - op += 16; - ip += 16; - do { - COPY16(op, ip); - COPY16(op, ip); - } - while (op < oend); -#endif - } +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); } -MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +static xxh_u32 XXH_readBE32(const void* ptr) { - size_t const length = MIN(dstCapacity, srcSize); - if (length > 0) { - ZSTD_memcpy(dst, src, length); - } - return length; + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); } +#endif -/* define "workspace is too large" as this number of times larger than needed */ -#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 +XXH_FORCE_INLINE xxh_u32 +XXH_readLE32_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) { + return XXH_readLE32(ptr); + } else { + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32*)ptr : XXH_swap32(*(const xxh_u32*)ptr); + } +} -/* when workspace is continuously too large - * during at least this number of times, - * context's memory usage is considered wasteful, - * because it's sized to handle a worst case scenario which rarely happens. - * In which case, resize it down to free some memory */ -#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 -/* Controls whether the input/output buffer is buffered or stable. */ -typedef enum { - ZSTD_bm_buffered = 0, /* Buffer the input/output */ - ZSTD_bm_stable = 1 /* ZSTD_inBuffer/ZSTD_outBuffer is stable */ -} ZSTD_bufferMode_e; +/* ************************************* +* Misc +***************************************/ +/*! @ingroup public */ +XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } -/*-******************************************* -* Private declarations -*********************************************/ -typedef struct seqDef_s { - U32 offset; /* Offset code of the sequence */ - U16 litLength; - U16 matchLength; -} seqDef; +/* ******************************************************************* +* 32-bit hash functions +*********************************************************************/ +/*! + * @} + * @defgroup XXH32_impl XXH32 implementation + * @ingroup impl + * + * Details on the XXH32 implementation. + * @{ + */ + /* #define instead of static const, to be used as initializers */ +#define XXH_PRIME32_1 0x9E3779B1U /*!< 0b10011110001101110111100110110001 */ +#define XXH_PRIME32_2 0x85EBCA77U /*!< 0b10000101111010111100101001110111 */ +#define XXH_PRIME32_3 0xC2B2AE3DU /*!< 0b11000010101100101010111000111101 */ +#define XXH_PRIME32_4 0x27D4EB2FU /*!< 0b00100111110101001110101100101111 */ +#define XXH_PRIME32_5 0x165667B1U /*!< 0b00010110010101100110011110110001 */ -typedef struct { - seqDef* sequencesStart; - seqDef* sequences; /* ptr to end of sequences */ - BYTE* litStart; - BYTE* lit; /* ptr to end of literals */ - BYTE* llCode; - BYTE* mlCode; - BYTE* ofCode; - size_t maxNbSeq; - size_t maxNbLit; +#ifdef XXH_OLD_NAMES +# define PRIME32_1 XXH_PRIME32_1 +# define PRIME32_2 XXH_PRIME32_2 +# define PRIME32_3 XXH_PRIME32_3 +# define PRIME32_4 XXH_PRIME32_4 +# define PRIME32_5 XXH_PRIME32_5 +#endif - /* longLengthPos and longLengthID to allow us to represent either a single litLength or matchLength - * in the seqStore that has a value larger than U16 (if it exists). To do so, we increment - * the existing value of the litLength or matchLength by 0x10000. +/*! + * @internal + * @brief Normal stripe processing routine. + * + * This shuffles the bits so that any bit from @p input impacts several bits in + * @p acc. + * + * @param acc The accumulator lane. + * @param input The stripe of input to mix. + * @return The mixed accumulator lane. + */ +static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input) +{ + acc += input * XXH_PRIME32_2; + acc = XXH_rotl32(acc, 13); + acc *= XXH_PRIME32_1; +#if (defined(__SSE4_1__) || defined(__aarch64__) || defined(__wasm_simd128__)) && !defined(XXH_ENABLE_AUTOVECTORIZE) + /* + * UGLY HACK: + * A compiler fence is the only thing that prevents GCC and Clang from + * autovectorizing the XXH32 loop (pragmas and attributes don't work for some + * reason) without globally disabling SSE4.1. + * + * The reason we want to avoid vectorization is because despite working on + * 4 integers at a time, there are multiple factors slowing XXH32 down on + * SSE4: + * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on + * newer chips!) making it slightly slower to multiply four integers at + * once compared to four integers independently. Even when pmulld was + * fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE + * just to multiply unless doing a long operation. + * + * - Four instructions are required to rotate, + * movqda tmp, v // not required with VEX encoding + * pslld tmp, 13 // tmp <<= 13 + * psrld v, 19 // x >>= 19 + * por v, tmp // x |= tmp + * compared to one for scalar: + * roll v, 13 // reliably fast across the board + * shldl v, v, 13 // Sandy Bridge and later prefer this for some reason + * + * - Instruction level parallelism is actually more beneficial here because + * the SIMD actually serializes this operation: While v1 is rotating, v2 + * can load data, while v3 can multiply. SSE forces them to operate + * together. + * + * This is also enabled on AArch64, as Clang is *very aggressive* in vectorizing + * the loop. NEON is only faster on the A53, and with the newer cores, it is less + * than half the speed. + * + * Additionally, this is used on WASM SIMD128 because it JITs to the same + * SIMD instructions and has the same issue. */ - U32 longLengthID; /* 0 == no longLength; 1 == Represent the long literal; 2 == Represent the long match; */ - U32 longLengthPos; /* Index of the sequence to apply long length modification to */ -} seqStore_t; + XXH_COMPILER_GUARD(acc); +#endif + return acc; +} -typedef struct { - U32 litLength; - U32 matchLength; -} ZSTD_sequenceLength; +/*! + * @internal + * @brief Mixes all bits to finalize the hash. + * + * The final mix ensures that all input bits have a chance to impact any bit in + * the output digest, resulting in an unbiased distribution. + * + * @param hash The hash to avalanche. + * @return The avalanched hash. + */ +static xxh_u32 XXH32_avalanche(xxh_u32 hash) +{ + hash ^= hash >> 15; + hash *= XXH_PRIME32_2; + hash ^= hash >> 13; + hash *= XXH_PRIME32_3; + hash ^= hash >> 16; + return hash; +} -/** - * Returns the ZSTD_sequenceLength for the given sequences. It handles the decoding of long sequences - * indicated by longLengthPos and longLengthID, and adds MINMATCH back to matchLength. +#define XXH_get32bits(p) XXH_readLE32_align(p, align) + +/*! + * @internal + * @brief Processes the last 0-15 bytes of @p ptr. + * + * There may be up to 15 bytes remaining to consume from the input. + * This final stage will digest them to ensure that all input bytes are present + * in the final mix. + * + * @param hash The hash to finalize. + * @param ptr The pointer to the remaining input. + * @param len The remaining length, modulo 16. + * @param align Whether @p ptr is aligned. + * @return The finalized hash. + * @see XXH64_finalize(). + */ +static XXH_PUREF xxh_u32 +XXH32_finalize(xxh_u32 hash, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ +#define XXH_PROCESS1 do { \ + hash += (*ptr++) * XXH_PRIME32_5; \ + hash = XXH_rotl32(hash, 11) * XXH_PRIME32_1; \ +} while (0) + +#define XXH_PROCESS4 do { \ + hash += XXH_get32bits(ptr) * XXH_PRIME32_3; \ + ptr += 4; \ + hash = XXH_rotl32(hash, 17) * XXH_PRIME32_4; \ +} while (0) + + if (ptr==NULL) XXH_ASSERT(len == 0); + + /* Compact rerolled version; generally faster */ + if (!XXH32_ENDJMP) { + len &= 15; + while (len >= 4) { + XXH_PROCESS4; + len -= 4; + } + while (len > 0) { + XXH_PROCESS1; + --len; + } + return XXH32_avalanche(hash); + } else { + switch(len&15) /* or switch(bEnd - p) */ { + case 12: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 8: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 4: XXH_PROCESS4; + return XXH32_avalanche(hash); + + case 13: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 9: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 5: XXH_PROCESS4; + XXH_PROCESS1; + return XXH32_avalanche(hash); + + case 14: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 10: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 6: XXH_PROCESS4; + XXH_PROCESS1; + XXH_PROCESS1; + return XXH32_avalanche(hash); + + case 15: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 11: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 7: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 3: XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 2: XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 1: XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 0: return XXH32_avalanche(hash); + } + XXH_ASSERT(0); + return hash; /* reaching this point is deemed impossible */ + } +} + +#ifdef XXH_OLD_NAMES +# define PROCESS1 XXH_PROCESS1 +# define PROCESS4 XXH_PROCESS4 +#else +# undef XXH_PROCESS1 +# undef XXH_PROCESS4 +#endif + +/*! + * @internal + * @brief The implementation for @ref XXH32(). + * + * @param input , len , seed Directly passed from @ref XXH32(). + * @param align Whether @p input is aligned. + * @return The calculated hash. */ -MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore, seqDef const* seq) +XXH_FORCE_INLINE XXH_PUREF xxh_u32 +XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align) { - ZSTD_sequenceLength seqLen; - seqLen.litLength = seq->litLength; - seqLen.matchLength = seq->matchLength + MINMATCH; - if (seqStore->longLengthPos == (U32)(seq - seqStore->sequencesStart)) { - if (seqStore->longLengthID == 1) { - seqLen.litLength += 0xFFFF; - } - if (seqStore->longLengthID == 2) { - seqLen.matchLength += 0xFFFF; - } + xxh_u32 h32; + + if (input==NULL) XXH_ASSERT(len == 0); + + if (len>=16) { + const xxh_u8* const bEnd = input + len; + const xxh_u8* const limit = bEnd - 15; + xxh_u32 v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + xxh_u32 v2 = seed + XXH_PRIME32_2; + xxh_u32 v3 = seed + 0; + xxh_u32 v4 = seed - XXH_PRIME32_1; + + do { + v1 = XXH32_round(v1, XXH_get32bits(input)); input += 4; + v2 = XXH32_round(v2, XXH_get32bits(input)); input += 4; + v3 = XXH32_round(v3, XXH_get32bits(input)); input += 4; + v4 = XXH32_round(v4, XXH_get32bits(input)); input += 4; + } while (input < limit); + + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + } else { + h32 = seed + XXH_PRIME32_5; } - return seqLen; + + h32 += (xxh_u32)len; + + return XXH32_finalize(h32, input, len&15, align); } -/** - * Contains the compressed frame size and an upper-bound for the decompressed frame size. - * Note: before using `compressedSize`, check for errors using ZSTD_isError(). - * similarly, before using `decompressedBound`, check for errors using: - * `decompressedBound != ZSTD_CONTENTSIZE_ERROR` - */ -typedef struct { - size_t compressedSize; - unsigned long long decompressedBound; -} ZSTD_frameSizeInfo; /* decompress & legacy */ +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed) +{ +#if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH32_state_t state; + XXH32_reset(&state, seed); + XXH32_update(&state, (const xxh_u8*)input, len); + return XXH32_digest(&state); +#else + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); + } } -const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBuilder */ -void ZSTD_seqToCodes(const seqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */ + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); +#endif +} -/* custom memory allocation functions */ -void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem); -void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem); -void ZSTD_customFree(void* ptr, ZSTD_customMem customMem); -MEM_STATIC U32 ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus */ +/******* Hash streaming *******/ +#ifndef XXH_NO_STREAM +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) { - assert(val != 0); - { -# if defined(_MSC_VER) /* Visual */ -# if STATIC_BMI2 == 1 - return _lzcnt_u32(val)^31; -# else - unsigned long r=0; - return _BitScanReverse(&r, val) ? (unsigned)r : 0; -# endif -# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */ - return __builtin_clz (val) ^ 31; -# elif defined(__ICCARM__) /* IAR Intrinsic */ - return 31 - __CLZ(val); -# else /* Software version */ - static const U32 DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; - U32 v = val; - v |= v >> 1; - v |= v >> 2; - v |= v >> 4; - v |= v >> 8; - v |= v >> 16; - return DeBruijnClz[(v * 0x07C4ACDDU) >> 27]; -# endif - } + return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); +} +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; } +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState) +{ + XXH_memcpy(dstState, srcState, sizeof(*dstState)); +} -/* ZSTD_invalidateRepCodes() : - * ensures next compression will not use repcodes from previous block. - * Note : only works with regular variant; - * do not use with extDict variant ! */ -void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx); /* zstdmt, adaptive_compression (shouldn't get this definition from here) */ +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed) +{ + XXH_ASSERT(statePtr != NULL); + memset(statePtr, 0, sizeof(*statePtr)); + statePtr->v[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + statePtr->v[1] = seed + XXH_PRIME32_2; + statePtr->v[2] = seed + 0; + statePtr->v[3] = seed - XXH_PRIME32_1; + return XXH_OK; +} -typedef struct { - blockType_e blockType; - U32 lastBlock; - U32 origSize; -} blockProperties_t; /* declared here for decompress and fullbench */ +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode +XXH32_update(XXH32_state_t* state, const void* input, size_t len) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } -/*! ZSTD_getcBlockSize() : - * Provides the size of compressed block from block header `src` */ -/* Used by: decompress, fullbench (does not get its definition from here) */ -size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, - blockProperties_t* bpPtr); + { const xxh_u8* p = (const xxh_u8*)input; + const xxh_u8* const bEnd = p + len; -/*! ZSTD_decodeSeqHeaders() : - * decode sequence header from src */ -/* Used by: decompress, fullbench (does not get its definition from here) */ -size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, - const void* src, size_t srcSize); + state->total_len_32 += (XXH32_hash_t)len; + state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16)); + if (state->memsize + len < 16) { /* fill in tmp buffer */ + XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, len); + state->memsize += (XXH32_hash_t)len; + return XXH_OK; + } -#if defined (__cplusplus) + if (state->memsize) { /* some data left from previous update */ + XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, 16-state->memsize); + { const xxh_u32* p32 = state->mem32; + state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p32)); p32++; + state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p32)); p32++; + state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p32)); p32++; + state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p32)); + } + p += 16-state->memsize; + state->memsize = 0; + } + + if (p <= bEnd-16) { + const xxh_u8* const limit = bEnd - 16; + + do { + state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p)); p+=4; + state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p)); p+=4; + state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p)); p+=4; + state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p)); p+=4; + } while (p<=limit); + + } + + if (p < bEnd) { + XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + } + + return XXH_OK; } -#endif -#endif /* ZSTD_CCOMMON_H_MODULE */ -/**** ended inlining zstd_internal.h ****/ -/**** start inlining pool.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ -#ifndef POOL_H -#define POOL_H +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t* state) +{ + xxh_u32 h32; -#if defined (__cplusplus) -extern "C" { -#endif + if (state->large_len) { + h32 = XXH_rotl32(state->v[0], 1) + + XXH_rotl32(state->v[1], 7) + + XXH_rotl32(state->v[2], 12) + + XXH_rotl32(state->v[3], 18); + } else { + h32 = state->v[2] /* == seed */ + XXH_PRIME32_5; + } + h32 += state->total_len_32; -/**** skipping file: zstd_deps.h ****/ -#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_customMem */ -/**** skipping file: ../zstd.h ****/ + return XXH32_finalize(h32, (const xxh_u8*)state->mem32, state->memsize, XXH_aligned); +} +#endif /* !XXH_NO_STREAM */ -typedef struct POOL_ctx_s POOL_ctx; +/******* Canonical representation *******/ -/*! POOL_create() : - * Create a thread pool with at most `numThreads` threads. - * `numThreads` must be at least 1. - * The maximum number of queued jobs before blocking is `queueSize`. - * @return : POOL_ctx pointer on success, else NULL. -*/ -POOL_ctx* POOL_create(size_t numThreads, size_t queueSize); +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); +} +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) +{ + return XXH_readBE32(src); +} -POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, - ZSTD_customMem customMem); -/*! POOL_free() : - * Free a thread pool returned by POOL_create(). - */ -void POOL_free(POOL_ctx* ctx); +#ifndef XXH_NO_LONG_LONG -/*! POOL_resize() : - * Expands or shrinks pool's number of threads. - * This is more efficient than releasing + creating a new context, - * since it tries to preserve and re-use existing threads. - * `numThreads` must be at least 1. - * @return : 0 when resize was successful, - * !0 (typically 1) if there is an error. - * note : only numThreads can be resized, queueSize remains unchanged. +/* ******************************************************************* +* 64-bit hash functions +*********************************************************************/ +/*! + * @} + * @ingroup impl + * @{ */ -int POOL_resize(POOL_ctx* ctx, size_t numThreads); +/******* Memory access *******/ -/*! POOL_sizeof() : - * @return threadpool memory usage - * note : compatible with NULL (returns 0 in this case) - */ -size_t POOL_sizeof(POOL_ctx* ctx); +typedef XXH64_hash_t xxh_u64; -/*! POOL_function : - * The function type that can be added to a thread pool. +#ifdef XXH_OLD_NAMES +# define U64 xxh_u64 +#endif + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE64 and XXH_readBE64. */ -typedef void (*POOL_function)(void*); +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) -/*! POOL_add() : - * Add the job `function(opaque)` to the thread pool. `ctx` must be valid. - * Possibly blocks until there is room in the queue. - * Note : The function may be executed asynchronously, - * therefore, `opaque` must live until function has been completed. +/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + return *(const xxh_u64*) memPtr; +} + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* + * __attribute__((aligned(1))) is supported by gcc and clang. Originally the + * documentation claimed that it only increased the alignment, but actually it + * can decrease it on gcc, clang, and icc: + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502, + * https://gcc.godbolt.org/z/xYez1j67Y. */ -void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque); +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) unalign64; +#endif +static xxh_u64 XXH_read64(const void* ptr) +{ + typedef __attribute__((aligned(1))) xxh_u64 xxh_unalign64; + return *((const xxh_unalign64*)ptr); +} +#else -/*! POOL_tryAdd() : - * Add the job `function(opaque)` to thread pool _if_ a worker is available. - * Returns immediately even if not (does not block). - * @return : 1 if successful, 0 if not. +/* + * Portable and safe solution. Generally efficient. + * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html */ -int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque); +static xxh_u64 XXH_read64(const void* memPtr) +{ + xxh_u64 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; +} +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ -#if defined (__cplusplus) +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap64 _byteswap_uint64 +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap64 __builtin_bswap64 +#else +static xxh_u64 XXH_swap64(xxh_u64 x) +{ + return ((x << 56) & 0xff00000000000000ULL) | + ((x << 40) & 0x00ff000000000000ULL) | + ((x << 24) & 0x0000ff0000000000ULL) | + ((x << 8) & 0x000000ff00000000ULL) | + ((x >> 8) & 0x00000000ff000000ULL) | + ((x >> 24) & 0x0000000000ff0000ULL) | + ((x >> 40) & 0x000000000000ff00ULL) | + ((x >> 56) & 0x00000000000000ffULL); } #endif -#endif -/**** ended inlining pool.h ****/ -/* ====== Compiler specifics ====== */ -#if defined(_MSC_VER) -# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ +/* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u64)bytePtr[1] << 8) + | ((xxh_u64)bytePtr[2] << 16) + | ((xxh_u64)bytePtr[3] << 24) + | ((xxh_u64)bytePtr[4] << 32) + | ((xxh_u64)bytePtr[5] << 40) + | ((xxh_u64)bytePtr[6] << 48) + | ((xxh_u64)bytePtr[7] << 56); +} + +XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[7] + | ((xxh_u64)bytePtr[6] << 8) + | ((xxh_u64)bytePtr[5] << 16) + | ((xxh_u64)bytePtr[4] << 24) + | ((xxh_u64)bytePtr[3] << 32) + | ((xxh_u64)bytePtr[2] << 40) + | ((xxh_u64)bytePtr[1] << 48) + | ((xxh_u64)bytePtr[0] << 56); +} + +#else +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); +} + +static xxh_u64 XXH_readBE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); +} #endif +XXH_FORCE_INLINE xxh_u64 +XXH_readLE64_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) + return XXH_readLE64(ptr); + else + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64*)ptr : XXH_swap64(*(const xxh_u64*)ptr); +} + -#ifdef ZSTD_MULTITHREAD +/******* xxh64 *******/ +/*! + * @} + * @defgroup XXH64_impl XXH64 implementation + * @ingroup impl + * + * Details on the XXH64 implementation. + * @{ + */ +/* #define rather that static const, to be used as initializers */ +#define XXH_PRIME64_1 0x9E3779B185EBCA87ULL /*!< 0b1001111000110111011110011011000110000101111010111100101010000111 */ +#define XXH_PRIME64_2 0xC2B2AE3D27D4EB4FULL /*!< 0b1100001010110010101011100011110100100111110101001110101101001111 */ +#define XXH_PRIME64_3 0x165667B19E3779F9ULL /*!< 0b0001011001010110011001111011000110011110001101110111100111111001 */ +#define XXH_PRIME64_4 0x85EBCA77C2B2AE63ULL /*!< 0b1000010111101011110010100111011111000010101100101010111001100011 */ +#define XXH_PRIME64_5 0x27D4EB2F165667C5ULL /*!< 0b0010011111010100111010110010111100010110010101100110011111000101 */ -/**** skipping file: threading.h ****/ +#ifdef XXH_OLD_NAMES +# define PRIME64_1 XXH_PRIME64_1 +# define PRIME64_2 XXH_PRIME64_2 +# define PRIME64_3 XXH_PRIME64_3 +# define PRIME64_4 XXH_PRIME64_4 +# define PRIME64_5 XXH_PRIME64_5 +#endif -/* A job is a function and an opaque argument */ -typedef struct POOL_job_s { - POOL_function function; - void *opaque; -} POOL_job; +/*! @copydoc XXH32_round */ +static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input) +{ + acc += input * XXH_PRIME64_2; + acc = XXH_rotl64(acc, 31); + acc *= XXH_PRIME64_1; +#if (defined(__AVX512F__)) && !defined(XXH_ENABLE_AUTOVECTORIZE) + /* + * DISABLE AUTOVECTORIZATION: + * A compiler fence is used to prevent GCC and Clang from + * autovectorizing the XXH64 loop (pragmas and attributes don't work for some + * reason) without globally disabling AVX512. + * + * Autovectorization of XXH64 tends to be detrimental, + * though the exact outcome may change depending on exact cpu and compiler version. + * For information, it has been reported as detrimental for Skylake-X, + * but possibly beneficial for Zen4. + * + * The default is to disable auto-vectorization, + * but you can select to enable it instead using `XXH_ENABLE_AUTOVECTORIZE` build variable. + */ + XXH_COMPILER_GUARD(acc); +#endif + return acc; +} -struct POOL_ctx_s { - ZSTD_customMem customMem; - /* Keep track of the threads */ - ZSTD_pthread_t* threads; - size_t threadCapacity; - size_t threadLimit; +static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val) +{ + val = XXH64_round(0, val); + acc ^= val; + acc = acc * XXH_PRIME64_1 + XXH_PRIME64_4; + return acc; +} - /* The queue is a circular buffer */ - POOL_job *queue; - size_t queueHead; - size_t queueTail; - size_t queueSize; +/*! @copydoc XXH32_avalanche */ +static xxh_u64 XXH64_avalanche(xxh_u64 hash) +{ + hash ^= hash >> 33; + hash *= XXH_PRIME64_2; + hash ^= hash >> 29; + hash *= XXH_PRIME64_3; + hash ^= hash >> 32; + return hash; +} - /* The number of threads working on jobs */ - size_t numThreadsBusy; - /* Indicates if the queue is empty */ - int queueEmpty; - /* The mutex protects the queue */ - ZSTD_pthread_mutex_t queueMutex; - /* Condition variable for pushers to wait on when the queue is full */ - ZSTD_pthread_cond_t queuePushCond; - /* Condition variables for poppers to wait on when the queue is empty */ - ZSTD_pthread_cond_t queuePopCond; - /* Indicates if the queue is shutting down */ - int shutdown; -}; +#define XXH_get64bits(p) XXH_readLE64_align(p, align) -/* POOL_thread() : - * Work thread for the thread pool. - * Waits for jobs and executes them. - * @returns : NULL on failure else non-null. +/*! + * @internal + * @brief Processes the last 0-31 bytes of @p ptr. + * + * There may be up to 31 bytes remaining to consume from the input. + * This final stage will digest them to ensure that all input bytes are present + * in the final mix. + * + * @param hash The hash to finalize. + * @param ptr The pointer to the remaining input. + * @param len The remaining length, modulo 32. + * @param align Whether @p ptr is aligned. + * @return The finalized hash + * @see XXH32_finalize(). + */ +static XXH_PUREF xxh_u64 +XXH64_finalize(xxh_u64 hash, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ + if (ptr==NULL) XXH_ASSERT(len == 0); + len &= 31; + while (len >= 8) { + xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr)); + ptr += 8; + hash ^= k1; + hash = XXH_rotl64(hash,27) * XXH_PRIME64_1 + XXH_PRIME64_4; + len -= 8; + } + if (len >= 4) { + hash ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1; + ptr += 4; + hash = XXH_rotl64(hash, 23) * XXH_PRIME64_2 + XXH_PRIME64_3; + len -= 4; + } + while (len > 0) { + hash ^= (*ptr++) * XXH_PRIME64_5; + hash = XXH_rotl64(hash, 11) * XXH_PRIME64_1; + --len; + } + return XXH64_avalanche(hash); +} + +#ifdef XXH_OLD_NAMES +# define PROCESS1_64 XXH_PROCESS1_64 +# define PROCESS4_64 XXH_PROCESS4_64 +# define PROCESS8_64 XXH_PROCESS8_64 +#else +# undef XXH_PROCESS1_64 +# undef XXH_PROCESS4_64 +# undef XXH_PROCESS8_64 +#endif + +/*! + * @internal + * @brief The implementation for @ref XXH64(). + * + * @param input , len , seed Directly passed from @ref XXH64(). + * @param align Whether @p input is aligned. + * @return The calculated hash. */ -static void* POOL_thread(void* opaque) { - POOL_ctx* const ctx = (POOL_ctx*)opaque; - if (!ctx) { return NULL; } - for (;;) { - /* Lock the mutex and wait for a non-empty queue or until shutdown */ - ZSTD_pthread_mutex_lock(&ctx->queueMutex); +XXH_FORCE_INLINE XXH_PUREF xxh_u64 +XXH64_endian_align(const xxh_u8* input, size_t len, xxh_u64 seed, XXH_alignment align) +{ + xxh_u64 h64; + if (input==NULL) XXH_ASSERT(len == 0); - while ( ctx->queueEmpty - || (ctx->numThreadsBusy >= ctx->threadLimit) ) { - if (ctx->shutdown) { - /* even if !queueEmpty, (possible if numThreadsBusy >= threadLimit), - * a few threads will be shutdown while !queueEmpty, - * but enough threads will remain active to finish the queue */ - ZSTD_pthread_mutex_unlock(&ctx->queueMutex); - return opaque; - } - ZSTD_pthread_cond_wait(&ctx->queuePopCond, &ctx->queueMutex); - } - /* Pop a job off the queue */ - { POOL_job const job = ctx->queue[ctx->queueHead]; - ctx->queueHead = (ctx->queueHead + 1) % ctx->queueSize; - ctx->numThreadsBusy++; - ctx->queueEmpty = ctx->queueHead == ctx->queueTail; - /* Unlock the mutex, signal a pusher, and run the job */ - ZSTD_pthread_cond_signal(&ctx->queuePushCond); - ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + if (len>=32) { + const xxh_u8* const bEnd = input + len; + const xxh_u8* const limit = bEnd - 31; + xxh_u64 v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + xxh_u64 v2 = seed + XXH_PRIME64_2; + xxh_u64 v3 = seed + 0; + xxh_u64 v4 = seed - XXH_PRIME64_1; - job.function(job.opaque); + do { + v1 = XXH64_round(v1, XXH_get64bits(input)); input+=8; + v2 = XXH64_round(v2, XXH_get64bits(input)); input+=8; + v3 = XXH64_round(v3, XXH_get64bits(input)); input+=8; + v4 = XXH64_round(v4, XXH_get64bits(input)); input+=8; + } while (inputqueueMutex); - ctx->numThreadsBusy--; - if (ctx->queueSize == 1) { - ZSTD_pthread_cond_signal(&ctx->queuePushCond); - } - ZSTD_pthread_mutex_unlock(&ctx->queueMutex); - } - } /* for (;;) */ - assert(0); /* Unreachable */ -} + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + h64 = XXH64_mergeRound(h64, v1); + h64 = XXH64_mergeRound(h64, v2); + h64 = XXH64_mergeRound(h64, v3); + h64 = XXH64_mergeRound(h64, v4); -POOL_ctx* ZSTD_createThreadPool(size_t numThreads) { - return POOL_create (numThreads, 0); + } else { + h64 = seed + XXH_PRIME64_5; + } + + h64 += (xxh_u64) len; + + return XXH64_finalize(h64, input, len, align); } -POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) { - return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem); + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64 (XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) +{ +#if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH64_state_t state; + XXH64_reset(&state, seed); + XXH64_update(&state, (const xxh_u8*)input, len); + return XXH64_digest(&state); +#else + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ + return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); + } } + + return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); + +#endif } -POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, - ZSTD_customMem customMem) { - POOL_ctx* ctx; - /* Check parameters */ - if (!numThreads) { return NULL; } - /* Allocate the context and zero initialize */ - ctx = (POOL_ctx*)ZSTD_customCalloc(sizeof(POOL_ctx), customMem); - if (!ctx) { return NULL; } - /* Initialize the job queue. - * It needs one extra space since one space is wasted to differentiate - * empty and full queues. - */ - ctx->queueSize = queueSize + 1; - ctx->queue = (POOL_job*)ZSTD_customMalloc(ctx->queueSize * sizeof(POOL_job), customMem); - ctx->queueHead = 0; - ctx->queueTail = 0; - ctx->numThreadsBusy = 0; - ctx->queueEmpty = 1; - { - int error = 0; - error |= ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL); - error |= ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL); - error |= ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL); - if (error) { POOL_free(ctx); return NULL; } - } - ctx->shutdown = 0; - /* Allocate space for the thread handles */ - ctx->threads = (ZSTD_pthread_t*)ZSTD_customMalloc(numThreads * sizeof(ZSTD_pthread_t), customMem); - ctx->threadCapacity = 0; - ctx->customMem = customMem; - /* Check for errors */ - if (!ctx->threads || !ctx->queue) { POOL_free(ctx); return NULL; } - /* Initialize the threads */ - { size_t i; - for (i = 0; i < numThreads; ++i) { - if (ZSTD_pthread_create(&ctx->threads[i], NULL, &POOL_thread, ctx)) { - ctx->threadCapacity = i; - POOL_free(ctx); - return NULL; - } } - ctx->threadCapacity = numThreads; - ctx->threadLimit = numThreads; - } - return ctx; -} - -/*! POOL_join() : - Shutdown the queue, wake any sleeping threads, and join all of the threads. -*/ -static void POOL_join(POOL_ctx* ctx) { - /* Shut down the queue */ - ZSTD_pthread_mutex_lock(&ctx->queueMutex); - ctx->shutdown = 1; - ZSTD_pthread_mutex_unlock(&ctx->queueMutex); - /* Wake up sleeping threads */ - ZSTD_pthread_cond_broadcast(&ctx->queuePushCond); - ZSTD_pthread_cond_broadcast(&ctx->queuePopCond); - /* Join all of the threads */ - { size_t i; - for (i = 0; i < ctx->threadCapacity; ++i) { - ZSTD_pthread_join(ctx->threads[i], NULL); /* note : could fail */ - } } +/******* Hash Streaming *******/ +#ifndef XXH_NO_STREAM +/*! @ingroup XXH64_family*/ +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) +{ + return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); } - -void POOL_free(POOL_ctx *ctx) { - if (!ctx) { return; } - POOL_join(ctx); - ZSTD_pthread_mutex_destroy(&ctx->queueMutex); - ZSTD_pthread_cond_destroy(&ctx->queuePushCond); - ZSTD_pthread_cond_destroy(&ctx->queuePopCond); - ZSTD_customFree(ctx->queue, ctx->customMem); - ZSTD_customFree(ctx->threads, ctx->customMem); - ZSTD_customFree(ctx, ctx->customMem); +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; } -void ZSTD_freeThreadPool (ZSTD_threadPool* pool) { - POOL_free (pool); +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dstState, const XXH64_state_t* srcState) +{ + XXH_memcpy(dstState, srcState, sizeof(*dstState)); } -size_t POOL_sizeof(POOL_ctx *ctx) { - if (ctx==NULL) return 0; /* supports sizeof NULL */ - return sizeof(*ctx) - + ctx->queueSize * sizeof(POOL_job) - + ctx->threadCapacity * sizeof(ZSTD_pthread_t); +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed) +{ + XXH_ASSERT(statePtr != NULL); + memset(statePtr, 0, sizeof(*statePtr)); + statePtr->v[0] = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + statePtr->v[1] = seed + XXH_PRIME64_2; + statePtr->v[2] = seed + 0; + statePtr->v[3] = seed - XXH_PRIME64_1; + return XXH_OK; } - -/* @return : 0 on success, 1 on error */ -static int POOL_resize_internal(POOL_ctx* ctx, size_t numThreads) +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode +XXH64_update (XXH_NOESCAPE XXH64_state_t* state, XXH_NOESCAPE const void* input, size_t len) { - if (numThreads <= ctx->threadCapacity) { - if (!numThreads) return 1; - ctx->threadLimit = numThreads; - return 0; + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; } - /* numThreads > threadCapacity */ - { ZSTD_pthread_t* const threadPool = (ZSTD_pthread_t*)ZSTD_customMalloc(numThreads * sizeof(ZSTD_pthread_t), ctx->customMem); - if (!threadPool) return 1; - /* replace existing thread pool */ - ZSTD_memcpy(threadPool, ctx->threads, ctx->threadCapacity * sizeof(*threadPool)); - ZSTD_customFree(ctx->threads, ctx->customMem); - ctx->threads = threadPool; - /* Initialize additional threads */ - { size_t threadId; - for (threadId = ctx->threadCapacity; threadId < numThreads; ++threadId) { - if (ZSTD_pthread_create(&threadPool[threadId], NULL, &POOL_thread, ctx)) { - ctx->threadCapacity = threadId; - return 1; - } } - } } - /* successfully expanded */ - ctx->threadCapacity = numThreads; - ctx->threadLimit = numThreads; - return 0; -} -/* @return : 0 on success, 1 on error */ -int POOL_resize(POOL_ctx* ctx, size_t numThreads) -{ - int result; - if (ctx==NULL) return 1; - ZSTD_pthread_mutex_lock(&ctx->queueMutex); - result = POOL_resize_internal(ctx, numThreads); - ZSTD_pthread_cond_broadcast(&ctx->queuePopCond); - ZSTD_pthread_mutex_unlock(&ctx->queueMutex); - return result; -} + { const xxh_u8* p = (const xxh_u8*)input; + const xxh_u8* const bEnd = p + len; -/** - * Returns 1 if the queue is full and 0 otherwise. - * - * When queueSize is 1 (pool was created with an intended queueSize of 0), - * then a queue is empty if there is a thread free _and_ no job is waiting. - */ -static int isQueueFull(POOL_ctx const* ctx) { - if (ctx->queueSize > 1) { - return ctx->queueHead == ((ctx->queueTail + 1) % ctx->queueSize); - } else { - return (ctx->numThreadsBusy == ctx->threadLimit) || - !ctx->queueEmpty; - } -} + state->total_len += len; + if (state->memsize + len < 32) { /* fill in tmp buffer */ + XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, len); + state->memsize += (xxh_u32)len; + return XXH_OK; + } -static void POOL_add_internal(POOL_ctx* ctx, POOL_function function, void *opaque) -{ - POOL_job const job = {function, opaque}; - assert(ctx != NULL); - if (ctx->shutdown) return; + if (state->memsize) { /* tmp buffer is full */ + XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, 32-state->memsize); + state->v[0] = XXH64_round(state->v[0], XXH_readLE64(state->mem64+0)); + state->v[1] = XXH64_round(state->v[1], XXH_readLE64(state->mem64+1)); + state->v[2] = XXH64_round(state->v[2], XXH_readLE64(state->mem64+2)); + state->v[3] = XXH64_round(state->v[3], XXH_readLE64(state->mem64+3)); + p += 32 - state->memsize; + state->memsize = 0; + } - ctx->queueEmpty = 0; - ctx->queue[ctx->queueTail] = job; - ctx->queueTail = (ctx->queueTail + 1) % ctx->queueSize; - ZSTD_pthread_cond_signal(&ctx->queuePopCond); -} + if (p+32 <= bEnd) { + const xxh_u8* const limit = bEnd - 32; -void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque) -{ - assert(ctx != NULL); - ZSTD_pthread_mutex_lock(&ctx->queueMutex); - /* Wait until there is space in the queue for the new job */ - while (isQueueFull(ctx) && (!ctx->shutdown)) { - ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex); - } - POOL_add_internal(ctx, function, opaque); - ZSTD_pthread_mutex_unlock(&ctx->queueMutex); -} + do { + state->v[0] = XXH64_round(state->v[0], XXH_readLE64(p)); p+=8; + state->v[1] = XXH64_round(state->v[1], XXH_readLE64(p)); p+=8; + state->v[2] = XXH64_round(state->v[2], XXH_readLE64(p)); p+=8; + state->v[3] = XXH64_round(state->v[3], XXH_readLE64(p)); p+=8; + } while (p<=limit); + } -int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) -{ - assert(ctx != NULL); - ZSTD_pthread_mutex_lock(&ctx->queueMutex); - if (isQueueFull(ctx)) { - ZSTD_pthread_mutex_unlock(&ctx->queueMutex); - return 0; + if (p < bEnd) { + XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } } - POOL_add_internal(ctx, function, opaque); - ZSTD_pthread_mutex_unlock(&ctx->queueMutex); - return 1; -} + return XXH_OK; +} -#else /* ZSTD_MULTITHREAD not defined */ - -/* ========================== */ -/* No multi-threading support */ -/* ========================== */ +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_digest(XXH_NOESCAPE const XXH64_state_t* state) +{ + xxh_u64 h64; -/* We don't need any data, but if it is empty, malloc() might return NULL. */ -struct POOL_ctx_s { - int dummy; -}; -static POOL_ctx g_poolCtx; + if (state->total_len >= 32) { + h64 = XXH_rotl64(state->v[0], 1) + XXH_rotl64(state->v[1], 7) + XXH_rotl64(state->v[2], 12) + XXH_rotl64(state->v[3], 18); + h64 = XXH64_mergeRound(h64, state->v[0]); + h64 = XXH64_mergeRound(h64, state->v[1]); + h64 = XXH64_mergeRound(h64, state->v[2]); + h64 = XXH64_mergeRound(h64, state->v[3]); + } else { + h64 = state->v[2] /*seed*/ + XXH_PRIME64_5; + } -POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) { - return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem); -} + h64 += (xxh_u64) state->total_len; -POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem) { - (void)numThreads; - (void)queueSize; - (void)customMem; - return &g_poolCtx; + return XXH64_finalize(h64, (const xxh_u8*)state->mem64, (size_t)state->total_len, XXH_aligned); } +#endif /* !XXH_NO_STREAM */ -void POOL_free(POOL_ctx* ctx) { - assert(!ctx || ctx == &g_poolCtx); - (void)ctx; -} +/******* Canonical representation *******/ -int POOL_resize(POOL_ctx* ctx, size_t numThreads) { - (void)ctx; (void)numThreads; - return 0; +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); } -void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque) { - (void)ctx; - function(opaque); +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src) +{ + return XXH_readBE64(src); } -int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) { - (void)ctx; - function(opaque); - return 1; +#if defined (__cplusplus) } +#endif -size_t POOL_sizeof(POOL_ctx* ctx) { - if (ctx==NULL) return 0; /* supports sizeof NULL */ - assert(ctx == &g_poolCtx); - return sizeof(*ctx); -} +#ifndef XXH_NO_XXH3 -#endif /* ZSTD_MULTITHREAD */ -/**** ended inlining common/pool.c ****/ -/**** start inlining common/zstd_common.c ****/ +/* ********************************************************************* +* XXH3 +* New generation hash designed for speed on small keys and vectorization +************************************************************************ */ +/*! + * @} + * @defgroup XXH3_impl XXH3 implementation + * @ingroup impl + * @{ + */ + +/* === Compiler specifics === */ + +#if ((defined(sun) || defined(__sun)) && __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested with GCC 5.5 */ +# define XXH_RESTRICT /* disable */ +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */ +# define XXH_RESTRICT restrict +#elif (defined (__GNUC__) && ((__GNUC__ > 3) || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1))) \ + || (defined (__clang__)) \ + || (defined (_MSC_VER) && (_MSC_VER >= 1400)) \ + || (defined (__INTEL_COMPILER) && (__INTEL_COMPILER >= 1300)) /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. + * There are a LOT more compilers that recognize __restrict but this + * covers the major ones. */ +# define XXH_RESTRICT __restrict +#else +# define XXH_RESTRICT /* disable */ +#endif +#if (defined(__GNUC__) && (__GNUC__ >= 3)) \ + || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) \ + || defined(__clang__) +# define XXH_likely(x) __builtin_expect(x, 1) +# define XXH_unlikely(x) __builtin_expect(x, 0) +#else +# define XXH_likely(x) (x) +# define XXH_unlikely(x) (x) +#endif +#ifndef XXH_HAS_INCLUDE +# ifdef __has_include +/* + * Not defined as XXH_HAS_INCLUDE(x) (function-like) because + * this causes segfaults in Apple Clang 4.2 (on Mac OS X 10.7 Lion) + */ +# define XXH_HAS_INCLUDE __has_include +# else +# define XXH_HAS_INCLUDE(x) 0 +# endif +#endif -/*-************************************* -* Dependencies -***************************************/ -#define ZSTD_DEPS_NEED_MALLOC -/**** skipping file: zstd_deps.h ****/ -/**** skipping file: error_private.h ****/ -/**** skipping file: zstd_internal.h ****/ - - -/*-**************************************** -* Version -******************************************/ -unsigned ZSTD_versionNumber(void) { return ZSTD_VERSION_NUMBER; } - -const char* ZSTD_versionString(void) { return ZSTD_VERSION_STRING; } +#if defined(__GNUC__) || defined(__clang__) +# if defined(__ARM_FEATURE_SVE) +# include +# endif +# if defined(__ARM_NEON__) || defined(__ARM_NEON) \ + || (defined(_M_ARM) && _M_ARM >= 7) \ + || defined(_M_ARM64) || defined(_M_ARM64EC) \ + || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE()) /* WASM SIMD128 via SIMDe */ +# define inline __inline__ /* circumvent a clang bug */ +# include +# undef inline +# elif defined(__AVX2__) +# include +# elif defined(__SSE2__) +# include +# endif +#endif +#if defined(_MSC_VER) +# include +#endif -/*-**************************************** -* ZSTD Error Management -******************************************/ -#undef ZSTD_isError /* defined within zstd_internal.h */ -/*! ZSTD_isError() : - * tells if a return value is an error code - * symbol is required for external callers */ -unsigned ZSTD_isError(size_t code) { return ERR_isError(code); } +/* + * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while + * remaining a true 64-bit/128-bit hash function. + * + * This is done by prioritizing a subset of 64-bit operations that can be + * emulated without too many steps on the average 32-bit machine. + * + * For example, these two lines seem similar, and run equally fast on 64-bit: + * + * xxh_u64 x; + * x ^= (x >> 47); // good + * x ^= (x >> 13); // bad + * + * However, to a 32-bit machine, there is a major difference. + * + * x ^= (x >> 47) looks like this: + * + * x.lo ^= (x.hi >> (47 - 32)); + * + * while x ^= (x >> 13) looks like this: + * + * // note: funnel shifts are not usually cheap. + * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13)); + * x.hi ^= (x.hi >> 13); + * + * The first one is significantly faster than the second, simply because the + * shift is larger than 32. This means: + * - All the bits we need are in the upper 32 bits, so we can ignore the lower + * 32 bits in the shift. + * - The shift result will always fit in the lower 32 bits, and therefore, + * we can ignore the upper 32 bits in the xor. + * + * Thanks to this optimization, XXH3 only requires these features to be efficient: + * + * - Usable unaligned access + * - A 32-bit or 64-bit ALU + * - If 32-bit, a decent ADC instruction + * - A 32 or 64-bit multiply with a 64-bit result + * - For the 128-bit variant, a decent byteswap helps short inputs. + * + * The first two are already required by XXH32, and almost all 32-bit and 64-bit + * platforms which can run XXH32 can run XXH3 efficiently. + * + * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one + * notable exception. + * + * First of all, Thumb-1 lacks support for the UMULL instruction which + * performs the important long multiply. This means numerous __aeabi_lmul + * calls. + * + * Second of all, the 8 functional registers are just not enough. + * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need + * Lo registers, and this shuffling results in thousands more MOVs than A32. + * + * A32 and T32 don't have this limitation. They can access all 14 registers, + * do a 32->64 multiply with UMULL, and the flexible operand allowing free + * shifts is helpful, too. + * + * Therefore, we do a quick sanity check. + * + * If compiling Thumb-1 for a target which supports ARM instructions, we will + * emit a warning, as it is not a "sane" platform to compile for. + * + * Usually, if this happens, it is because of an accident and you probably need + * to specify -march, as you likely meant to compile for a newer architecture. + * + * Credit: large sections of the vectorial and asm source code paths + * have been contributed by @easyaspi314 + */ +#if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM) +# warning "XXH3 is highly inefficient without ARM or Thumb-2." +#endif -/*! ZSTD_getErrorName() : - * provides error code string from function result (useful for debugging) */ -const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); } +/* ========================================== + * Vectorization detection + * ========================================== */ -/*! ZSTD_getError() : - * convert a `size_t` function result into a proper ZSTD_errorCode enum */ -ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); } +#ifdef XXH_DOXYGEN +/*! + * @ingroup tuning + * @brief Overrides the vectorization implementation chosen for XXH3. + * + * Can be defined to 0 to disable SIMD or any of the values mentioned in + * @ref XXH_VECTOR_TYPE. + * + * If this is not defined, it uses predefined macros to determine the best + * implementation. + */ +# define XXH_VECTOR XXH_SCALAR +/*! + * @ingroup tuning + * @brief Possible values for @ref XXH_VECTOR. + * + * Note that these are actually implemented as macros. + * + * If this is not defined, it is detected automatically. + * internal macro XXH_X86DISPATCH overrides this. + */ +enum XXH_VECTOR_TYPE /* fake enum */ { + XXH_SCALAR = 0, /*!< Portable scalar version */ + XXH_SSE2 = 1, /*!< + * SSE2 for Pentium 4, Opteron, all x86_64. + * + * @note SSE2 is also guaranteed on Windows 10, macOS, and + * Android x86. + */ + XXH_AVX2 = 2, /*!< AVX2 for Haswell and Bulldozer */ + XXH_AVX512 = 3, /*!< AVX512 for Skylake and Icelake */ + XXH_NEON = 4, /*!< + * NEON for most ARMv7-A, all AArch64, and WASM SIMD128 + * via the SIMDeverywhere polyfill provided with the + * Emscripten SDK. + */ + XXH_VSX = 5, /*!< VSX and ZVector for POWER8/z13 (64-bit) */ + XXH_SVE = 6, /*!< SVE for some ARMv8-A and ARMv9-A */ +}; +/*! + * @ingroup tuning + * @brief Selects the minimum alignment for XXH3's accumulators. + * + * When using SIMD, this should match the alignment required for said vector + * type, so, for example, 32 for AVX2. + * + * Default: Auto detected. + */ +# define XXH_ACC_ALIGN 8 +#endif + +/* Actual definition */ +#ifndef XXH_DOXYGEN +# define XXH_SCALAR 0 +# define XXH_SSE2 1 +# define XXH_AVX2 2 +# define XXH_AVX512 3 +# define XXH_NEON 4 +# define XXH_VSX 5 +# define XXH_SVE 6 +#endif + +#ifndef XXH_VECTOR /* can be defined on command line */ +# if defined(__ARM_FEATURE_SVE) +# define XXH_VECTOR XXH_SVE +# elif ( \ + defined(__ARM_NEON__) || defined(__ARM_NEON) /* gcc */ \ + || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC) /* msvc */ \ + || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE()) /* wasm simd128 via SIMDe */ \ + ) && ( \ + defined(_WIN32) || defined(__LITTLE_ENDIAN__) /* little endian only */ \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) \ + ) +# define XXH_VECTOR XXH_NEON +# elif defined(__AVX512F__) +# define XXH_VECTOR XXH_AVX512 +# elif defined(__AVX2__) +# define XXH_VECTOR XXH_AVX2 +# elif defined(__SSE2__) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2)) +# define XXH_VECTOR XXH_SSE2 +# elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \ + || (defined(__s390x__) && defined(__VEC__)) \ + && defined(__GNUC__) /* TODO: IBM XL */ +# define XXH_VECTOR XXH_VSX +# else +# define XXH_VECTOR XXH_SCALAR +# endif +#endif -/*! ZSTD_getErrorString() : - * provides error code string from enum */ -const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); } +/* __ARM_FEATURE_SVE is only supported by GCC & Clang. */ +#if (XXH_VECTOR == XXH_SVE) && !defined(__ARM_FEATURE_SVE) +# ifdef _MSC_VER +# pragma warning(once : 4606) +# else +# warning "__ARM_FEATURE_SVE isn't supported. Use SCALAR instead." +# endif +# undef XXH_VECTOR +# define XXH_VECTOR XXH_SCALAR +#endif +/* + * Controls the alignment of the accumulator, + * for compatibility with aligned vector loads, which are usually faster. + */ +#ifndef XXH_ACC_ALIGN +# if defined(XXH_X86DISPATCH) +# define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */ +# elif XXH_VECTOR == XXH_SCALAR /* scalar */ +# define XXH_ACC_ALIGN 8 +# elif XXH_VECTOR == XXH_SSE2 /* sse2 */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX2 /* avx2 */ +# define XXH_ACC_ALIGN 32 +# elif XXH_VECTOR == XXH_NEON /* neon */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_VSX /* vsx */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX512 /* avx512 */ +# define XXH_ACC_ALIGN 64 +# elif XXH_VECTOR == XXH_SVE /* sve */ +# define XXH_ACC_ALIGN 64 +# endif +#endif +#if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 \ + || XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512 +# define XXH_SEC_ALIGN XXH_ACC_ALIGN +#elif XXH_VECTOR == XXH_SVE +# define XXH_SEC_ALIGN XXH_ACC_ALIGN +#else +# define XXH_SEC_ALIGN 8 +#endif -/*=************************************************************** -* Custom allocator -****************************************************************/ -void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem) +#if defined(__GNUC__) || defined(__clang__) +# define XXH_ALIASING __attribute__((may_alias)) +#else +# define XXH_ALIASING /* nothing */ +#endif + +/* + * UGLY HACK: + * GCC usually generates the best code with -O3 for xxHash. + * + * However, when targeting AVX2, it is overzealous in its unrolling resulting + * in code roughly 3/4 the speed of Clang. + * + * There are other issues, such as GCC splitting _mm256_loadu_si256 into + * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which + * only applies to Sandy and Ivy Bridge... which don't even support AVX2. + * + * That is why when compiling the AVX2 version, it is recommended to use either + * -O2 -mavx2 -march=haswell + * or + * -O2 -mavx2 -mno-avx256-split-unaligned-load + * for decent performance, or to use Clang instead. + * + * Fortunately, we can control the first one with a pragma that forces GCC into + * -O2, but the other one we can't control without "failed to inline always + * inline function due to target mismatch" warnings. + */ +#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ + && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__OPTIMIZE__) && XXH_SIZE_OPT <= 0 /* respect -O0 and -Os */ +# pragma GCC push_options +# pragma GCC optimize("-O2") +#endif + +#if defined (__cplusplus) +extern "C" { +#endif + +#if XXH_VECTOR == XXH_NEON + +/* + * UGLY HACK: While AArch64 GCC on Linux does not seem to care, on macOS, GCC -O3 + * optimizes out the entire hashLong loop because of the aliasing violation. + * + * However, GCC is also inefficient at load-store optimization with vld1q/vst1q, + * so the only option is to mark it as aliasing. + */ +typedef uint64x2_t xxh_aliasing_uint64x2_t XXH_ALIASING; + +/*! + * @internal + * @brief `vld1q_u64` but faster and alignment-safe. + * + * On AArch64, unaligned access is always safe, but on ARMv7-a, it is only + * *conditionally* safe (`vld1` has an alignment bit like `movdq[ua]` in x86). + * + * GCC for AArch64 sees `vld1q_u8` as an intrinsic instead of a load, so it + * prohibits load-store optimizations. Therefore, a direct dereference is used. + * + * Otherwise, `vld1q_u8` is used with `vreinterpretq_u8_u64` to do a safe + * unaligned load. + */ +#if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) +XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr) /* silence -Wcast-align */ { - if (customMem.customAlloc) - return customMem.customAlloc(customMem.opaque, size); - return ZSTD_malloc(size); + return *(xxh_aliasing_uint64x2_t const *)ptr; } - -void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem) +#else +XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr) { - if (customMem.customAlloc) { - /* calloc implemented as malloc+memset; - * not as efficient as calloc, but next best guess for custom malloc */ - void* const ptr = customMem.customAlloc(customMem.opaque, size); - ZSTD_memset(ptr, 0, size); - return ptr; - } - return ZSTD_calloc(1, size); + return vreinterpretq_u64_u8(vld1q_u8((uint8_t const*)ptr)); } +#endif -void ZSTD_customFree(void* ptr, ZSTD_customMem customMem) +/*! + * @internal + * @brief `vmlal_u32` on low and high halves of a vector. + * + * This is a workaround for AArch64 GCC < 11 which implemented arm_neon.h with + * inline assembly and were therefore incapable of merging the `vget_{low, high}_u32` + * with `vmlal_u32`. + */ +#if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 11 +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) { - if (ptr!=NULL) { - if (customMem.customFree) - customMem.customFree(customMem.opaque, ptr); - else - ZSTD_free(ptr); - } + /* Inline assembly is the only way */ + __asm__("umlal %0.2d, %1.2s, %2.2s" : "+w" (acc) : "w" (lhs), "w" (rhs)); + return acc; } -/**** ended inlining common/zstd_common.c ****/ +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + /* This intrinsic works as expected */ + return vmlal_high_u32(acc, lhs, rhs); +} +#else +/* Portable intrinsic versions */ +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + return vmlal_u32(acc, vget_low_u32(lhs), vget_low_u32(rhs)); +} +/*! @copydoc XXH_vmlal_low_u32 + * Assume the compiler converts this to vmlal_high_u32 on aarch64 */ +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + return vmlal_u32(acc, vget_high_u32(lhs), vget_high_u32(rhs)); +} +#endif -/**** start inlining compress/fse_compress.c ****/ -/* ****************************************************************** - * FSE : Finite State Entropy encoder - * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc. +/*! + * @ingroup tuning + * @brief Controls the NEON to scalar ratio for XXH3 * - * You can contact the author at : - * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy - * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * This can be set to 2, 4, 6, or 8. * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. -****************************************************************** */ - -/* ************************************************************** -* Includes -****************************************************************/ -/**** skipping file: ../common/compiler.h ****/ -/**** skipping file: ../common/mem.h ****/ -/**** skipping file: ../common/debug.h ****/ -/**** start inlining hist.h ****/ -/* ****************************************************************** - * hist : Histogram functions - * part of Finite State Entropy project - * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc. + * ARM Cortex CPUs are _very_ sensitive to how their pipelines are used. * - * You can contact the author at : - * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy - * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * For example, the Cortex-A73 can dispatch 3 micro-ops per cycle, but only 2 of those + * can be NEON. If you are only using NEON instructions, you are only using 2/3 of the CPU + * bandwidth. * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. -****************************************************************** */ - -/* --- dependencies --- */ -/**** skipping file: ../common/zstd_deps.h ****/ - + * This is even more noticeable on the more advanced cores like the Cortex-A76 which + * can dispatch 8 micro-ops per cycle, but still only 2 NEON micro-ops at once. + * + * Therefore, to make the most out of the pipeline, it is beneficial to run 6 NEON lanes + * and 2 scalar lanes, which is chosen by default. + * + * This does not apply to Apple processors or 32-bit processors, which run better with + * full NEON. These will default to 8. Additionally, size-optimized builds run 8 lanes. + * + * This change benefits CPUs with large micro-op buffers without negatively affecting + * most other CPUs: + * + * | Chipset | Dispatch type | NEON only | 6:2 hybrid | Diff. | + * |:----------------------|:--------------------|----------:|-----------:|------:| + * | Snapdragon 730 (A76) | 2 NEON/8 micro-ops | 8.8 GB/s | 10.1 GB/s | ~16% | + * | Snapdragon 835 (A73) | 2 NEON/3 micro-ops | 5.1 GB/s | 5.3 GB/s | ~5% | + * | Marvell PXA1928 (A53) | In-order dual-issue | 1.9 GB/s | 1.9 GB/s | 0% | + * | Apple M1 | 4 NEON/8 micro-ops | 37.3 GB/s | 36.1 GB/s | ~-3% | + * + * It also seems to fix some bad codegen on GCC, making it almost as fast as clang. + * + * When using WASM SIMD128, if this is 2 or 6, SIMDe will scalarize 2 of the lanes meaning + * it effectively becomes worse 4. + * + * @see XXH3_accumulate_512_neon() + */ +# ifndef XXH3_NEON_LANES +# if (defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) || defined(_M_ARM64EC)) \ + && !defined(__APPLE__) && XXH_SIZE_OPT <= 0 +# define XXH3_NEON_LANES 6 +# else +# define XXH3_NEON_LANES XXH_ACC_NB +# endif +# endif +#endif /* XXH_VECTOR == XXH_NEON */ -/* --- simple histogram functions --- */ +#if defined (__cplusplus) +} /* extern "C" */ +#endif -/*! HIST_count(): - * Provides the precise count of each byte within a table 'count'. - * 'count' is a table of unsigned int, of minimum size (*maxSymbolValuePtr+1). - * Updates *maxSymbolValuePtr with actual largest symbol value detected. - * @return : count of the most frequent symbol (which isn't identified). - * or an error code, which can be tested using HIST_isError(). - * note : if return == srcSize, there is only one symbol. +/* + * VSX and Z Vector helpers. + * + * This is very messy, and any pull requests to clean this up are welcome. + * + * There are a lot of problems with supporting VSX and s390x, due to + * inconsistent intrinsics, spotty coverage, and multiple endiannesses. */ -size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr, - const void* src, size_t srcSize); +#if XXH_VECTOR == XXH_VSX +/* Annoyingly, these headers _may_ define three macros: `bool`, `vector`, + * and `pixel`. This is a problem for obvious reasons. + * + * These keywords are unnecessary; the spec literally says they are + * equivalent to `__bool`, `__vector`, and `__pixel` and may be undef'd + * after including the header. + * + * We use pragma push_macro/pop_macro to keep the namespace clean. */ +# pragma push_macro("bool") +# pragma push_macro("vector") +# pragma push_macro("pixel") +/* silence potential macro redefined warnings */ +# undef bool +# undef vector +# undef pixel -unsigned HIST_isError(size_t code); /**< tells if a return value is an error code */ +# if defined(__s390x__) +# include +# else +# include +# endif +/* Restore the original macro values, if applicable. */ +# pragma pop_macro("pixel") +# pragma pop_macro("vector") +# pragma pop_macro("bool") -/* --- advanced histogram functions --- */ +typedef __vector unsigned long long xxh_u64x2; +typedef __vector unsigned char xxh_u8x16; +typedef __vector unsigned xxh_u32x4; -#define HIST_WKSP_SIZE_U32 1024 -#define HIST_WKSP_SIZE (HIST_WKSP_SIZE_U32 * sizeof(unsigned)) -/** HIST_count_wksp() : - * Same as HIST_count(), but using an externally provided scratch buffer. - * Benefit is this function will use very little stack space. - * `workSpace` is a writable buffer which must be 4-bytes aligned, - * `workSpaceSize` must be >= HIST_WKSP_SIZE +/* + * UGLY HACK: Similar to aarch64 macOS GCC, s390x GCC has the same aliasing issue. */ -size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr, - const void* src, size_t srcSize, - void* workSpace, size_t workSpaceSize); +typedef xxh_u64x2 xxh_aliasing_u64x2 XXH_ALIASING; -/** HIST_countFast() : - * same as HIST_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr. - * This function is unsafe, and will segfault if any value within `src` is `> *maxSymbolValuePtr` - */ -size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr, - const void* src, size_t srcSize); +# ifndef XXH_VSX_BE +# if defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_VSX_BE 1 +# elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__ +# warning "-maltivec=be is not recommended. Please use native endianness." +# define XXH_VSX_BE 1 +# else +# define XXH_VSX_BE 0 +# endif +# endif /* !defined(XXH_VSX_BE) */ -/** HIST_countFast_wksp() : - * Same as HIST_countFast(), but using an externally provided scratch buffer. - * `workSpace` is a writable buffer which must be 4-bytes aligned, - * `workSpaceSize` must be >= HIST_WKSP_SIZE +# if XXH_VSX_BE +# if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__)) +# define XXH_vec_revb vec_revb +# else +#if defined (__cplusplus) +extern "C" { +#endif +/*! + * A polyfill for POWER9's vec_revb(). */ -size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, - const void* src, size_t srcSize, - void* workSpace, size_t workSpaceSize); +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val) +{ + xxh_u8x16 const vByteSwap = { 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00, + 0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 }; + return vec_perm(val, val, vByteSwap); +} +#if defined (__cplusplus) +} /* extern "C" */ +#endif +# endif +# endif /* XXH_VSX_BE */ -/*! HIST_count_simple() : - * Same as HIST_countFast(), this function is unsafe, - * and will segfault if any value within `src` is `> *maxSymbolValuePtr`. - * It is also a bit slower for large inputs. - * However, it does not need any additional memory (not even on stack). - * @return : count of the most frequent symbol. - * Note this function doesn't produce any error (i.e. it must succeed). +#if defined (__cplusplus) +extern "C" { +#endif +/*! + * Performs an unaligned vector load and byte swaps it on big endian. */ -unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, - const void* src, size_t srcSize); -/**** ended inlining hist.h ****/ -/**** skipping file: ../common/bitstream.h ****/ -#define FSE_STATIC_LINKING_ONLY -/**** skipping file: ../common/fse.h ****/ -/**** skipping file: ../common/error_private.h ****/ -#define ZSTD_DEPS_NEED_MALLOC -#define ZSTD_DEPS_NEED_MATH64 -/**** skipping file: ../common/zstd_deps.h ****/ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr) +{ + xxh_u64x2 ret; + XXH_memcpy(&ret, ptr, sizeof(xxh_u64x2)); +# if XXH_VSX_BE + ret = XXH_vec_revb(ret); +# endif + return ret; +} +/* + * vec_mulo and vec_mule are very problematic intrinsics on PowerPC + * + * These intrinsics weren't added until GCC 8, despite existing for a while, + * and they are endian dependent. Also, their meaning swap depending on version. + * */ +# if defined(__s390x__) + /* s390x is always big endian, no issue on this platform */ +# define XXH_vec_mulo vec_mulo +# define XXH_vec_mule vec_mule +# elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) && !defined(__ibmxl__) +/* Clang has a better way to control this, we can just use the builtin which doesn't swap. */ + /* The IBM XL Compiler (which defined __clang__) only implements the vec_* operations */ +# define XXH_vec_mulo __builtin_altivec_vmulouw +# define XXH_vec_mule __builtin_altivec_vmuleuw +# else +/* gcc needs inline assembly */ +/* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +# endif /* XXH_vec_mulo, XXH_vec_mule */ -/* ************************************************************** -* Error Management -****************************************************************/ -#define FSE_isError ERR_isError +#if defined (__cplusplus) +} /* extern "C" */ +#endif +#endif /* XXH_VECTOR == XXH_VSX */ -/* ************************************************************** -* Templates -****************************************************************/ +#if XXH_VECTOR == XXH_SVE +#define ACCRND(acc, offset) \ +do { \ + svuint64_t input_vec = svld1_u64(mask, xinput + offset); \ + svuint64_t secret_vec = svld1_u64(mask, xsecret + offset); \ + svuint64_t mixed = sveor_u64_x(mask, secret_vec, input_vec); \ + svuint64_t swapped = svtbl_u64(input_vec, kSwap); \ + svuint64_t mixed_lo = svextw_u64_x(mask, mixed); \ + svuint64_t mixed_hi = svlsr_n_u64_x(mask, mixed, 32); \ + svuint64_t mul = svmad_u64_x(mask, mixed_lo, mixed_hi, swapped); \ + acc = svadd_u64_x(mask, acc, mul); \ +} while (0) +#endif /* XXH_VECTOR == XXH_SVE */ + +/* prefetch + * can be disabled, by declaring XXH_NO_PREFETCH build macro */ +#if defined(XXH_NO_PREFETCH) +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +#else +# if XXH_SIZE_OPT >= 1 +# define XXH_PREFETCH(ptr) (void)(ptr) +# elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */ +# include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ +# define XXH_PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) +# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) +# define XXH_PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) +# else +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +# endif +#endif /* XXH_NO_PREFETCH */ + +#if defined (__cplusplus) +extern "C" { +#endif +/* ========================================== + * XXH3 default settings + * ========================================== */ + +#define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */ + +#if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN) +# error "default keyset is not large enough" +#endif + +/*! Pseudorandom secret taken directly from FARSH. */ +XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = { + 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c, + 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f, + 0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21, + 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c, + 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3, + 0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8, + 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d, + 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64, + 0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb, + 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e, + 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce, + 0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e, +}; + +static const xxh_u64 PRIME_MX1 = 0x165667919E3779F9ULL; /*!< 0b0001011001010110011001111001000110011110001101110111100111111001 */ +static const xxh_u64 PRIME_MX2 = 0x9FB21C651E98DF25ULL; /*!< 0b1001111110110010000111000110010100011110100110001101111100100101 */ + +#ifdef XXH_OLD_NAMES +# define kSecret XXH3_kSecret +#endif + +#ifdef XXH_DOXYGEN +/*! + * @brief Calculates a 32-bit to 64-bit long multiply. + * + * Implemented as a macro. + * + * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it doesn't + * need to (but it shouldn't need to anyways, it is about 7 instructions to do + * a 64x64 multiply...). Since we know that this will _always_ emit `MULL`, we + * use that instead of the normal method. + * + * If you are compiling for platforms like Thumb-1 and don't have a better option, + * you may also want to write your own long multiply routine here. + * + * @param x, y Numbers to be multiplied + * @return 64-bit product of the low 32 bits of @p x and @p y. + */ +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64(xxh_u64 x, xxh_u64 y) +{ + return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF); +} +#elif defined(_MSC_VER) && defined(_M_IX86) +# define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y)) +#else /* - designed to be included - for type-specific functions (template emulation in C) - Objective is to write these functions only once, for improved maintenance -*/ + * Downcast + upcast is usually better than masking on older compilers like + * GCC 4.2 (especially 32-bit ones), all without affecting newer compilers. + * + * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both operands + * and perform a full 64x64 multiply -- entirely redundant on 32-bit. + */ +# define XXH_mult32to64(x, y) ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y)) +#endif -/* safety checks */ -#ifndef FSE_FUNCTION_EXTENSION -# error "FSE_FUNCTION_EXTENSION must be defined" +/*! + * @brief Calculates a 64->128-bit long multiply. + * + * Uses `__uint128_t` and `_umul128` if available, otherwise uses a scalar + * version. + * + * @param lhs , rhs The 64-bit integers to be multiplied + * @return The 128-bit result represented in an @ref XXH128_hash_t. + */ +static XXH128_hash_t +XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) +{ + /* + * GCC/Clang __uint128_t method. + * + * On most 64-bit targets, GCC and Clang define a __uint128_t type. + * This is usually the best way as it usually uses a native long 64-bit + * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64. + * + * Usually. + * + * Despite being a 32-bit platform, Clang (and emscripten) define this type + * despite not having the arithmetic for it. This results in a laggy + * compiler builtin call which calculates a full 128-bit multiply. + * In that case it is best to use the portable one. + * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677 + */ +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__wasm__) \ + && defined(__SIZEOF_INT128__) \ + || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + + __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs; + XXH128_hash_t r128; + r128.low64 = (xxh_u64)(product); + r128.high64 = (xxh_u64)(product >> 64); + return r128; + + /* + * MSVC for x64's _umul128 method. + * + * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct); + * + * This compiles to single operand MUL on x64. + */ +#elif (defined(_M_X64) || defined(_M_IA64)) && !defined(_M_ARM64EC) + +#ifndef _MSC_VER +# pragma intrinsic(_umul128) #endif -#ifndef FSE_FUNCTION_TYPE -# error "FSE_FUNCTION_TYPE must be defined" + xxh_u64 product_high; + xxh_u64 const product_low = _umul128(lhs, rhs, &product_high); + XXH128_hash_t r128; + r128.low64 = product_low; + r128.high64 = product_high; + return r128; + + /* + * MSVC for ARM64's __umulh method. + * + * This compiles to the same MUL + UMULH as GCC/Clang's __uint128_t method. + */ +#elif defined(_M_ARM64) || defined(_M_ARM64EC) + +#ifndef _MSC_VER +# pragma intrinsic(__umulh) #endif + XXH128_hash_t r128; + r128.low64 = lhs * rhs; + r128.high64 = __umulh(lhs, rhs); + return r128; -/* Function names */ -#define FSE_CAT(X,Y) X##Y -#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) -#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) +#else + /* + * Portable scalar method. Optimized for 32-bit and 64-bit ALUs. + * + * This is a fast and simple grade school multiply, which is shown below + * with base 10 arithmetic instead of base 0x100000000. + * + * 9 3 // D2 lhs = 93 + * x 7 5 // D2 rhs = 75 + * ---------- + * 1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15 + * 4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45 + * 2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21 + * + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63 + * --------- + * 2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27 + * + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67 + * --------- + * 6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975 + * + * The reasons for adding the products like this are: + * 1. It avoids manual carry tracking. Just like how + * (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX. + * This avoids a lot of complexity. + * + * 2. It hints for, and on Clang, compiles to, the powerful UMAAL + * instruction available in ARM's Digital Signal Processing extension + * in 32-bit ARMv6 and later, which is shown below: + * + * void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm) + * { + * xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm; + * *RdLo = (xxh_u32)(product & 0xFFFFFFFF); + * *RdHi = (xxh_u32)(product >> 32); + * } + * + * This instruction was designed for efficient long multiplication, and + * allows this to be calculated in only 4 instructions at speeds + * comparable to some 64-bit ALUs. + * + * 3. It isn't terrible on other platforms. Usually this will be a couple + * of 32-bit ADD/ADCs. + */ + /* First calculate all of the cross products. */ + xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF); + xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF); + xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32); + xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32, rhs >> 32); -/* Function templates */ + /* Now add the products together. These will never overflow. */ + xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi; + xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32) + hi_hi; + xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF); -/* FSE_buildCTable_wksp() : - * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). - * wkspSize should be sized to handle worst case situation, which is `1<>1 : 1) ; - FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT); - U32 const step = FSE_TABLESTEP(tableSize); + XXH128_hash_t product = XXH_mult64to128(lhs, rhs); + return product.low64 ^ product.high64; +} - U32* cumul = (U32*)workSpace; - FSE_FUNCTION_TYPE* tableSymbol = (FSE_FUNCTION_TYPE*)(cumul + (maxSymbolValue + 2)); +/*! Seems to produce slightly better code on GCC for some reason. */ +XXH_FORCE_INLINE XXH_CONSTF xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift) +{ + XXH_ASSERT(0 <= shift && shift < 64); + return v64 ^ (v64 >> shift); +} - U32 highThreshold = tableSize-1; +/* + * This is a fast avalanche stage, + * suitable when input bits are already partially mixed + */ +static XXH64_hash_t XXH3_avalanche(xxh_u64 h64) +{ + h64 = XXH_xorshift64(h64, 37); + h64 *= PRIME_MX1; + h64 = XXH_xorshift64(h64, 32); + return h64; +} - if ((size_t)workSpace & 3) return ERROR(GENERIC); /* Must be 4 byte aligned */ - if (FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) > wkspSize) return ERROR(tableLog_tooLarge); - /* CTable header */ - tableU16[-2] = (U16) tableLog; - tableU16[-1] = (U16) maxSymbolValue; - assert(tableLog < 16); /* required for threshold strategy to work */ +/* + * This is a stronger avalanche, + * inspired by Pelle Evensen's rrmxmx + * preferable when input has not been previously mixed + */ +static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len) +{ + /* this mix is inspired by Pelle Evensen's rrmxmx */ + h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24); + h64 *= PRIME_MX2; + h64 ^= (h64 >> 35) + len ; + h64 *= PRIME_MX2; + return XXH_xorshift64(h64, 28); +} - /* For explanations on how to distribute symbol values over the table : - * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */ - #ifdef __clang_analyzer__ - ZSTD_memset(tableSymbol, 0, sizeof(*tableSymbol) * tableSize); /* useless initialization, just to keep scan-build happy */ - #endif +/* ========================================== + * Short keys + * ========================================== + * One of the shortcomings of XXH32 and XXH64 was that their performance was + * sub-optimal on short lengths. It used an iterative algorithm which strongly + * favored lengths that were a multiple of 4 or 8. + * + * Instead of iterating over individual inputs, we use a set of single shot + * functions which piece together a range of lengths and operate in constant time. + * + * Additionally, the number of multiplies has been significantly reduced. This + * reduces latency, especially when emulating 64-bit multiplies on 32-bit. + * + * Depending on the platform, this may or may not be faster than XXH32, but it + * is almost guaranteed to be faster than XXH64. + */ - /* symbol start positions */ - { U32 u; - cumul[0] = 0; - for (u=1; u <= maxSymbolValue+1; u++) { - if (normalizedCounter[u-1]==-1) { /* Low proba symbol */ - cumul[u] = cumul[u-1] + 1; - tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u-1); - } else { - cumul[u] = cumul[u-1] + normalizedCounter[u-1]; - } } - cumul[maxSymbolValue+1] = tableSize+1; +/* + * At very short lengths, there isn't enough input to fully hide secrets, or use + * the entire secret. + * + * There is also only a limited amount of mixing we can do before significantly + * impacting performance. + * + * Therefore, we use different sections of the secret and always mix two secret + * samples with an XOR. This should have no effect on performance on the + * seedless or withSeed variants because everything _should_ be constant folded + * by modern compilers. + * + * The XOR mixing hides individual parts of the secret and increases entropy. + * + * This adds an extra layer of strength for custom secrets. + */ +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combined = { input[0], 0x01, input[0], input[0] } + * len = 2: combined = { input[1], 0x02, input[0], input[1] } + * len = 3: combined = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u64 const bitflip = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const keyed = (xxh_u64)combined ^ bitflip; + return XXH64_avalanche(keyed); } +} - /* Spread symbols */ - { U32 position = 0; - U32 symbol; - for (symbol=0; symbol<=maxSymbolValue; symbol++) { - int nbOccurrences; - int const freq = normalizedCounter[symbol]; - for (nbOccurrences=0; nbOccurrences highThreshold) - position = (position + step) & tableMask; /* Low proba area */ - } } +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input1 = XXH_readLE32(input); + xxh_u32 const input2 = XXH_readLE32(input + len - 4); + xxh_u64 const bitflip = (XXH_readLE64(secret+8) ^ XXH_readLE64(secret+16)) - seed; + xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32); + xxh_u64 const keyed = input64 ^ bitflip; + return XXH3_rrmxmx(keyed, len); + } +} - assert(position==0); /* Must have initialized all positions */ +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflip1 = (XXH_readLE64(secret+24) ^ XXH_readLE64(secret+32)) + seed; + xxh_u64 const bitflip2 = (XXH_readLE64(secret+40) ^ XXH_readLE64(secret+48)) - seed; + xxh_u64 const input_lo = XXH_readLE64(input) ^ bitflip1; + xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2; + xxh_u64 const acc = len + + XXH_swap64(input_lo) + input_hi + + XXH3_mul128_fold64(input_lo, input_hi); + return XXH3_avalanche(acc); } +} - /* Build table */ - { U32 u; for (u=0; u 8)) return XXH3_len_9to16_64b(input, len, secret, seed); + if (XXH_likely(len >= 4)) return XXH3_len_4to8_64b(input, len, secret, seed); + if (len) return XXH3_len_1to3_64b(input, len, secret, seed); + return XXH64_avalanche(seed ^ (XXH_readLE64(secret+56) ^ XXH_readLE64(secret+64))); + } +} - /* Build Symbol Transformation Table */ - { unsigned total = 0; - unsigned s; - for (s=0; s<=maxSymbolValue; s++) { - switch (normalizedCounter[s]) - { - case 0: - /* filling nonetheless, for compatibility with FSE_getMaxNbBits() */ - symbolTT[s].deltaNbBits = ((tableLog+1) << 16) - (1<= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); -#if 0 /* debug : symbol costs */ - DEBUGLOG(5, "\n --- table statistics : "); - { U32 symbol; - for (symbol=0; symbol<=maxSymbolValue; symbol++) { - DEBUGLOG(5, "%3u: w=%3i, maxBits=%u, fracBits=%.2f", - symbol, normalizedCounter[symbol], - FSE_getMaxNbBits(symbolTT, symbol), - (double)FSE_bitCost(symbolTT, tableLog, symbol, 8) / 256); + { xxh_u64 acc = len * XXH_PRIME64_1; +#if XXH_SIZE_OPT >= 1 + /* Smaller and cleaner, but slightly slower. */ + unsigned int i = (unsigned int)(len - 1) / 32; + do { + acc += XXH3_mix16B(input+16 * i, secret+32*i, seed); + acc += XXH3_mix16B(input+len-16*(i+1), secret+32*i+16, seed); + } while (i-- != 0); +#else + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc += XXH3_mix16B(input+48, secret+96, seed); + acc += XXH3_mix16B(input+len-64, secret+112, seed); + } + acc += XXH3_mix16B(input+32, secret+64, seed); + acc += XXH3_mix16B(input+len-48, secret+80, seed); + } + acc += XXH3_mix16B(input+16, secret+32, seed); + acc += XXH3_mix16B(input+len-32, secret+48, seed); } + acc += XXH3_mix16B(input+0, secret+0, seed); + acc += XXH3_mix16B(input+len-16, secret+16, seed); +#endif + return XXH3_avalanche(acc); } +} + +/*! + * @brief Maximum size of "short" key in bytes. + */ +#define XXH3_MIDSIZE_MAX 240 + +XXH_NO_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + #define XXH3_MIDSIZE_STARTOFFSET 3 + #define XXH3_MIDSIZE_LASTOFFSET 17 + + { xxh_u64 acc = len * XXH_PRIME64_1; + xxh_u64 acc_end; + unsigned int const nbRounds = (unsigned int)len / 16; + unsigned int i; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + for (i=0; i<8; i++) { + acc += XXH3_mix16B(input+(16*i), secret+(16*i), seed); + } + /* last bytes */ + acc_end = XXH3_mix16B(input + len - 16, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed); + XXH_ASSERT(nbRounds >= 8); + acc = XXH3_avalanche(acc); +#if defined(__clang__) /* Clang */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86. + * In everywhere else, it uses scalar code. + * + * For 64->128-bit multiplies, even if the NEON was 100% optimal, it + * would still be slower than UMAAL (see XXH_mult64to128). + * + * Unfortunately, Clang doesn't handle the long multiplies properly and + * converts them to the nonexistent "vmulq_u64" intrinsic, which is then + * scalarized into an ugly mess of VMOV.32 instructions. + * + * This mess is difficult to avoid without turning autovectorization + * off completely, but they are usually relatively minor and/or not + * worth it to fix. + * + * This loop is the easiest to fix, as unlike XXH32, this pragma + * _actually works_ because it is a loop vectorization instead of an + * SLP vectorization. + */ + #pragma clang loop vectorize(disable) #endif + for (i=8 ; i < nbRounds; i++) { + /* + * Prevents clang for unrolling the acc loop and interleaving with this one. + */ + XXH_COMPILER_GUARD(acc); + acc_end += XXH3_mix16B(input+(16*i), secret+(16*(i-8)) + XXH3_MIDSIZE_STARTOFFSET, seed); + } + return XXH3_avalanche(acc + acc_end); + } +} - return 0; + +/* ======= Long Keys ======= */ + +#define XXH_STRIPE_LEN 64 +#define XXH_SECRET_CONSUME_RATE 8 /* nb of secret bytes consumed at each accumulation */ +#define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64)) + +#ifdef XXH_OLD_NAMES +# define STRIPE_LEN XXH_STRIPE_LEN +# define ACC_NB XXH_ACC_NB +#endif + +#ifndef XXH_PREFETCH_DIST +# ifdef __clang__ +# define XXH_PREFETCH_DIST 320 +# else +# if (XXH_VECTOR == XXH_AVX512) +# define XXH_PREFETCH_DIST 512 +# else +# define XXH_PREFETCH_DIST 384 +# endif +# endif /* __clang__ */ +#endif /* XXH_PREFETCH_DIST */ + +/* + * These macros are to generate an XXH3_accumulate() function. + * The two arguments select the name suffix and target attribute. + * + * The name of this symbol is XXH3_accumulate_() and it calls + * XXH3_accumulate_512_(). + * + * It may be useful to hand implement this function if the compiler fails to + * optimize the inline function. + */ +#define XXH3_ACCUMULATE_TEMPLATE(name) \ +void \ +XXH3_accumulate_##name(xxh_u64* XXH_RESTRICT acc, \ + const xxh_u8* XXH_RESTRICT input, \ + const xxh_u8* XXH_RESTRICT secret, \ + size_t nbStripes) \ +{ \ + size_t n; \ + for (n = 0; n < nbStripes; n++ ) { \ + const xxh_u8* const in = input + n*XXH_STRIPE_LEN; \ + XXH_PREFETCH(in + XXH_PREFETCH_DIST); \ + XXH3_accumulate_512_##name( \ + acc, \ + in, \ + secret + n*XXH_SECRET_CONSUME_RATE); \ + } \ } -#ifndef ZSTD_NO_UNUSED_FUNCTIONS -size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) + +XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64) { - FSE_FUNCTION_TYPE tableSymbol[FSE_MAX_TABLESIZE]; /* memset() is not necessary, even if static analyzer complain about it */ - return FSE_buildCTable_wksp(ct, normalizedCounter, maxSymbolValue, tableLog, tableSymbol, sizeof(tableSymbol)); + if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64); + XXH_memcpy(dst, &v64, sizeof(v64)); } + +/* Several intrinsic functions below are supposed to accept __int64 as argument, + * as documented in https://software.intel.com/sites/landingpage/IntrinsicsGuide/ . + * However, several environments do not define __int64 type, + * requiring a workaround. + */ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) + typedef int64_t xxh_i64; +#else + /* the following type must have a width of 64-bit */ + typedef long long xxh_i64; #endif +/* + * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the most optimized. + * + * It is a hardened version of UMAC, based off of FARSH's implementation. + * + * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD + * implementations, and it is ridiculously fast. + * + * We harden it by mixing the original input to the accumulators as well as the product. + * + * This means that in the (relatively likely) case of a multiply by zero, the + * original input is preserved. + * + * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve + * cross-pollination, as otherwise the upper and lower halves would be + * essentially independent. + * + * This doesn't matter on 64-bit hashes since they all get merged together in + * the end, so we skip the extra step. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ -#ifndef FSE_COMMONDEFS_ONLY +#if (XXH_VECTOR == XXH_AVX512) \ + || (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0) +#ifndef XXH_TARGET_AVX512 +# define XXH_TARGET_AVX512 /* disable attribute target */ +#endif -/*-************************************************************** -* FSE NCount encoding -****************************************************************/ -size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog) +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_accumulate_512_avx512(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) { - size_t const maxHeaderSize = (((maxSymbolValue+1) * tableLog) >> 3) + 3; - return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */ -} + __m512i* const xacc = (__m512i *) acc; + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); -static size_t -FSE_writeNCount_generic (void* header, size_t headerBufferSize, - const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, - unsigned writeIsSafe) -{ - BYTE* const ostart = (BYTE*) header; - BYTE* out = ostart; - BYTE* const oend = ostart + headerBufferSize; - int nbBits; - const int tableSize = 1 << tableLog; - int remaining; - int threshold; - U32 bitStream = 0; - int bitCount = 0; - unsigned symbol = 0; - unsigned const alphabetSize = maxSymbolValue + 1; - int previousIs0 = 0; + { + /* data_vec = input[0]; */ + __m512i const data_vec = _mm512_loadu_si512 (input); + /* key_vec = secret[0]; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + /* data_key = data_vec ^ key_vec; */ + __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m512i const data_key_lo = _mm512_srli_epi64 (data_key, 32); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m512i const product = _mm512_mul_epu32 (data_key, data_key_lo); + /* xacc[0] += swap(data_vec); */ + __m512i const data_swap = _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2)); + __m512i const sum = _mm512_add_epi64(*xacc, data_swap); + /* xacc[0] += product; */ + *xacc = _mm512_add_epi64(product, sum); + } +} +XXH_FORCE_INLINE XXH_TARGET_AVX512 XXH3_ACCUMULATE_TEMPLATE(avx512) - /* Table Size */ - bitStream += (tableLog-FSE_MIN_TABLELOG) << bitCount; - bitCount += 4; +/* + * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing. + * + * Multiplication isn't perfect, as explained by Google in HighwayHash: + * + * // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to + * // varying degrees. In descending order of goodness, bytes + * // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32. + * // As expected, the upper and lower bytes are much worse. + * + * Source: https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291 + * + * Since our algorithm uses a pseudorandom secret to add some variance into the + * mix, we don't need to (or want to) mix as often or as much as HighwayHash does. + * + * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid + * extraction. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ - /* Init */ - remaining = tableSize+1; /* +1 for extra accuracy */ - threshold = tableSize; - nbBits = tableLog+1; +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_scrambleAcc_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + { __m512i* const xacc = (__m512i*) acc; + const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1); - while ((symbol < alphabetSize) && (remaining>1)) { /* stops at 1 */ - if (previousIs0) { - unsigned start = symbol; - while ((symbol < alphabetSize) && !normalizedCounter[symbol]) symbol++; - if (symbol == alphabetSize) break; /* incorrect distribution */ - while (symbol >= start+24) { - start+=24; - bitStream += 0xFFFFU << bitCount; - if ((!writeIsSafe) && (out > oend-2)) - return ERROR(dstSize_tooSmall); /* Buffer overflow */ - out[0] = (BYTE) bitStream; - out[1] = (BYTE)(bitStream>>8); - out+=2; - bitStream>>=16; - } - while (symbol >= start+3) { - start+=3; - bitStream += 3 << bitCount; - bitCount += 2; - } - bitStream += (symbol-start) << bitCount; - bitCount += 2; - if (bitCount>16) { - if ((!writeIsSafe) && (out > oend - 2)) - return ERROR(dstSize_tooSmall); /* Buffer overflow */ - out[0] = (BYTE)bitStream; - out[1] = (BYTE)(bitStream>>8); - out += 2; - bitStream >>= 16; - bitCount -= 16; - } } - { int count = normalizedCounter[symbol++]; - int const max = (2*threshold-1) - remaining; - remaining -= count < 0 ? -count : count; - count++; /* +1 for extra accuracy */ - if (count>=threshold) - count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */ - bitStream += count << bitCount; - bitCount += nbBits; - bitCount -= (count>=1; } - } - if (bitCount>16) { - if ((!writeIsSafe) && (out > oend - 2)) - return ERROR(dstSize_tooSmall); /* Buffer overflow */ - out[0] = (BYTE)bitStream; - out[1] = (BYTE)(bitStream>>8); - out += 2; - bitStream >>= 16; - bitCount -= 16; - } } + /* xacc[0] ^= (xacc[0] >> 47) */ + __m512i const acc_vec = *xacc; + __m512i const shifted = _mm512_srli_epi64 (acc_vec, 47); + /* xacc[0] ^= secret; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + __m512i const data_key = _mm512_ternarylogic_epi32(key_vec, acc_vec, shifted, 0x96 /* key_vec ^ acc_vec ^ shifted */); - if (remaining != 1) - return ERROR(GENERIC); /* incorrect normalized distribution */ - assert(symbol <= alphabetSize); + /* xacc[0] *= XXH_PRIME32_1; */ + __m512i const data_key_hi = _mm512_srli_epi64 (data_key, 32); + __m512i const prod_lo = _mm512_mul_epu32 (data_key, prime32); + __m512i const prod_hi = _mm512_mul_epu32 (data_key_hi, prime32); + *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32)); + } +} - /* flush remaining bitStream */ - if ((!writeIsSafe) && (out > oend - 2)) - return ERROR(dstSize_tooSmall); /* Buffer overflow */ - out[0] = (BYTE)bitStream; - out[1] = (BYTE)(bitStream>>8); - out+= (bitCount+7) /8; +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_initCustomSecret_avx512(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64); + XXH_ASSERT(((size_t)customSecret & 63) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i); + __m512i const seed_pos = _mm512_set1_epi64((xxh_i64)seed64); + __m512i const seed = _mm512_mask_sub_epi64(seed_pos, 0xAA, _mm512_set1_epi8(0), seed_pos); - return (out-ostart); + const __m512i* const src = (const __m512i*) ((const void*) XXH3_kSecret); + __m512i* const dest = ( __m512i*) customSecret; + int i; + XXH_ASSERT(((size_t)src & 63) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 63) == 0); + for (i=0; i < nbRounds; ++i) { + dest[i] = _mm512_add_epi64(_mm512_load_si512(src + i), seed); + } } } +#endif -size_t FSE_writeNCount (void* buffer, size_t bufferSize, - const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) +#if (XXH_VECTOR == XXH_AVX2) \ + || (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0) + +#ifndef XXH_TARGET_AVX2 +# define XXH_TARGET_AVX2 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_accumulate_512_avx2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { __m256i* const xacc = (__m256i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xinput = (const __m256i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* data_vec = xinput[i]; */ + __m256i const data_vec = _mm256_loadu_si256 (xinput+i); + /* key_vec = xsecret[i]; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m256i const data_key_lo = _mm256_srli_epi64 (data_key, 32); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m256i const product = _mm256_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2)); + __m256i const sum = _mm256_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm256_add_epi64(product, sum); + } } +} +XXH_FORCE_INLINE XXH_TARGET_AVX2 XXH3_ACCUMULATE_TEMPLATE(avx2) + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_scrambleAcc_avx2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) { - if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported */ - if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported */ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { __m256i* const xacc = (__m256i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1); - if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog)) - return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0); + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m256i const acc_vec = xacc[i]; + __m256i const shifted = _mm256_srli_epi64 (acc_vec, 47); + __m256i const data_vec = _mm256_xor_si256 (acc_vec, shifted); + /* xacc[i] ^= xsecret; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); - return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1 /* write in buffer is safe */); + /* xacc[i] *= XXH_PRIME32_1; */ + __m256i const data_key_hi = _mm256_srli_epi64 (data_key, 32); + __m256i const prod_lo = _mm256_mul_epu32 (data_key, prime32); + __m256i const prod_hi = _mm256_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32)); + } + } } - -/*-************************************************************** -* FSE Compression Code -****************************************************************/ - -FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog) +XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) { - size_t size; - if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; - size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32); - return (FSE_CTable*)ZSTD_malloc(size); -} + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0); + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64); + (void)(&XXH_writeLE64); + XXH_PREFETCH(customSecret); + { __m256i const seed = _mm256_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64, (xxh_i64)(0U - seed64), (xxh_i64)seed64); -void FSE_freeCTable (FSE_CTable* ct) { ZSTD_free(ct); } + const __m256i* const src = (const __m256i*) ((const void*) XXH3_kSecret); + __m256i* dest = ( __m256i*) customSecret; -/* provides the minimum logSize to safely represent a distribution */ -static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue) -{ - U32 minBitsSrc = BIT_highbit32((U32)(srcSize)) + 1; - U32 minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2; - U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols; - assert(srcSize > 1); /* Not supported, RLE should be used instead */ - return minBits; +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dest); +# endif + XXH_ASSERT(((size_t)src & 31) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 31) == 0); + + /* GCC -O2 need unroll loop manually */ + dest[0] = _mm256_add_epi64(_mm256_load_si256(src+0), seed); + dest[1] = _mm256_add_epi64(_mm256_load_si256(src+1), seed); + dest[2] = _mm256_add_epi64(_mm256_load_si256(src+2), seed); + dest[3] = _mm256_add_epi64(_mm256_load_si256(src+3), seed); + dest[4] = _mm256_add_epi64(_mm256_load_si256(src+4), seed); + dest[5] = _mm256_add_epi64(_mm256_load_si256(src+5), seed); + } +} + +#endif + +/* x86dispatch always generates SSE2 */ +#if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH) + +#ifndef XXH_TARGET_SSE2 +# define XXH_TARGET_SSE2 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_accumulate_512_sse2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* SSE2 is just a half-scale version of the AVX2 version. */ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { __m128i* const xacc = (__m128i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xinput = (const __m128i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* data_vec = xinput[i]; */ + __m128i const data_vec = _mm_loadu_si128 (xinput+i); + /* key_vec = xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m128i const data_key_lo = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m128i const product = _mm_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2)); + __m128i const sum = _mm_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm_add_epi64(product, sum); + } } } +XXH_FORCE_INLINE XXH_TARGET_SSE2 XXH3_ACCUMULATE_TEMPLATE(sse2) -unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus) +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_scrambleAcc_sse2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) { - U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - minus; - U32 tableLog = maxTableLog; - U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue); - assert(srcSize > 1); /* Not supported, RLE should be used instead */ - if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; - if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */ - if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */ - if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG; - if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG; - return tableLog; + XXH_ASSERT((((size_t)acc) & 15) == 0); + { __m128i* const xacc = (__m128i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m128i const acc_vec = xacc[i]; + __m128i const shifted = _mm_srli_epi64 (acc_vec, 47); + __m128i const data_vec = _mm_xor_si128 (acc_vec, shifted); + /* xacc[i] ^= xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m128i const data_key_hi = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + __m128i const prod_lo = _mm_mul_epu32 (data_key, prime32); + __m128i const prod_hi = _mm_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32)); + } + } } -unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) +XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) { - return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2); + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i); + +# if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900 + /* MSVC 32bit mode does not support _mm_set_epi64x before 2015 */ + XXH_ALIGN(16) const xxh_i64 seed64x2[2] = { (xxh_i64)seed64, (xxh_i64)(0U - seed64) }; + __m128i const seed = _mm_load_si128((__m128i const*)seed64x2); +# else + __m128i const seed = _mm_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64); +# endif + int i; + + const void* const src16 = XXH3_kSecret; + __m128i* dst16 = (__m128i*) customSecret; +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dst16); +# endif + XXH_ASSERT(((size_t)src16 & 15) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dst16 & 15) == 0); + + for (i=0; i < nbRounds; ++i) { + dst16[i] = _mm_add_epi64(_mm_load_si128((const __m128i *)src16+i), seed); + } } } -/* Secondary normalization method. - To be used when primary method fails. */ +#endif -static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue, short lowProbCount) -{ - short const NOT_YET_ASSIGNED = -2; - U32 s; - U32 distributed = 0; - U32 ToDistribute; +#if (XXH_VECTOR == XXH_NEON) - /* Init */ - U32 const lowThreshold = (U32)(total >> tableLog); - U32 lowOne = (U32)((total * 3) >> (tableLog + 1)); +/* forward declarations for the scalar routines */ +XXH_FORCE_INLINE void +XXH3_scalarRound(void* XXH_RESTRICT acc, void const* XXH_RESTRICT input, + void const* XXH_RESTRICT secret, size_t lane); - for (s=0; s<=maxSymbolValue; s++) { - if (count[s] == 0) { - norm[s]=0; - continue; - } - if (count[s] <= lowThreshold) { - norm[s] = lowProbCount; - distributed++; - total -= count[s]; - continue; - } - if (count[s] <= lowOne) { - norm[s] = 1; - distributed++; - total -= count[s]; - continue; - } +XXH_FORCE_INLINE void +XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT secret, size_t lane); - norm[s]=NOT_YET_ASSIGNED; - } - ToDistribute = (1 << tableLog) - distributed; +/*! + * @internal + * @brief The bulk processing loop for NEON and WASM SIMD128. + * + * The NEON code path is actually partially scalar when running on AArch64. This + * is to optimize the pipelining and can have up to 15% speedup depending on the + * CPU, and it also mitigates some GCC codegen issues. + * + * @see XXH3_NEON_LANES for configuring this and details about this optimization. + * + * NEON's 32-bit to 64-bit long multiply takes a half vector of 32-bit + * integers instead of the other platforms which mask full 64-bit vectors, + * so the setup is more complicated than just shifting right. + * + * Additionally, there is an optimization for 4 lanes at once noted below. + * + * Since, as stated, the most optimal amount of lanes for Cortexes is 6, + * there needs to be *three* versions of the accumulate operation used + * for the remaining 2 lanes. + * + * WASM's SIMD128 uses SIMDe's arm_neon.h polyfill because the intrinsics overlap + * nearly perfectly. + */ - if (ToDistribute == 0) - return 0; +XXH_FORCE_INLINE void +XXH3_accumulate_512_neon( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + XXH_STATIC_ASSERT(XXH3_NEON_LANES > 0 && XXH3_NEON_LANES <= XXH_ACC_NB && XXH3_NEON_LANES % 2 == 0); + { /* GCC for darwin arm64 does not like aliasing here */ + xxh_aliasing_uint64x2_t* const xacc = (xxh_aliasing_uint64x2_t*) acc; + /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */ + uint8_t const* xinput = (const uint8_t *) input; + uint8_t const* xsecret = (const uint8_t *) secret; - if ((total / ToDistribute) > lowOne) { - /* risk of rounding to zero */ - lowOne = (U32)((total * 3) / (ToDistribute * 2)); - for (s=0; s<=maxSymbolValue; s++) { - if ((norm[s] == NOT_YET_ASSIGNED) && (count[s] <= lowOne)) { - norm[s] = 1; - distributed++; - total -= count[s]; - continue; - } } - ToDistribute = (1 << tableLog) - distributed; + size_t i; +#ifdef __wasm_simd128__ + /* + * On WASM SIMD128, Clang emits direct address loads when XXH3_kSecret + * is constant propagated, which results in it converting it to this + * inside the loop: + * + * a = v128.load(XXH3_kSecret + 0 + $secret_offset, offset = 0) + * b = v128.load(XXH3_kSecret + 16 + $secret_offset, offset = 0) + * ... + * + * This requires a full 32-bit address immediate (and therefore a 6 byte + * instruction) as well as an add for each offset. + * + * Putting an asm guard prevents it from folding (at the cost of losing + * the alignment hint), and uses the free offset in `v128.load` instead + * of adding secret_offset each time which overall reduces code size by + * about a kilobyte and improves performance. + */ + XXH_COMPILER_GUARD(xsecret); +#endif + /* Scalar lanes use the normal scalarRound routine */ + for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { + XXH3_scalarRound(acc, input, secret, i); + } + i = 0; + /* 4 NEON lanes at a time. */ + for (; i+1 < XXH3_NEON_LANES / 2; i+=2) { + /* data_vec = xinput[i]; */ + uint64x2_t data_vec_1 = XXH_vld1q_u64(xinput + (i * 16)); + uint64x2_t data_vec_2 = XXH_vld1q_u64(xinput + ((i+1) * 16)); + /* key_vec = xsecret[i]; */ + uint64x2_t key_vec_1 = XXH_vld1q_u64(xsecret + (i * 16)); + uint64x2_t key_vec_2 = XXH_vld1q_u64(xsecret + ((i+1) * 16)); + /* data_swap = swap(data_vec) */ + uint64x2_t data_swap_1 = vextq_u64(data_vec_1, data_vec_1, 1); + uint64x2_t data_swap_2 = vextq_u64(data_vec_2, data_vec_2, 1); + /* data_key = data_vec ^ key_vec; */ + uint64x2_t data_key_1 = veorq_u64(data_vec_1, key_vec_1); + uint64x2_t data_key_2 = veorq_u64(data_vec_2, key_vec_2); + + /* + * If we reinterpret the 64x2 vectors as 32x4 vectors, we can use a + * de-interleave operation for 4 lanes in 1 step with `vuzpq_u32` to + * get one vector with the low 32 bits of each lane, and one vector + * with the high 32 bits of each lane. + * + * The intrinsic returns a double vector because the original ARMv7-a + * instruction modified both arguments in place. AArch64 and SIMD128 emit + * two instructions from this intrinsic. + * + * [ dk11L | dk11H | dk12L | dk12H ] -> [ dk11L | dk12L | dk21L | dk22L ] + * [ dk21L | dk21H | dk22L | dk22H ] -> [ dk11H | dk12H | dk21H | dk22H ] + */ + uint32x4x2_t unzipped = vuzpq_u32( + vreinterpretq_u32_u64(data_key_1), + vreinterpretq_u32_u64(data_key_2) + ); + /* data_key_lo = data_key & 0xFFFFFFFF */ + uint32x4_t data_key_lo = unzipped.val[0]; + /* data_key_hi = data_key >> 32 */ + uint32x4_t data_key_hi = unzipped.val[1]; + /* + * Then, we can split the vectors horizontally and multiply which, as for most + * widening intrinsics, have a variant that works on both high half vectors + * for free on AArch64. A similar instruction is available on SIMD128. + * + * sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi + */ + uint64x2_t sum_1 = XXH_vmlal_low_u32(data_swap_1, data_key_lo, data_key_hi); + uint64x2_t sum_2 = XXH_vmlal_high_u32(data_swap_2, data_key_lo, data_key_hi); + /* + * Clang reorders + * a += b * c; // umlal swap.2d, dkl.2s, dkh.2s + * c += a; // add acc.2d, acc.2d, swap.2d + * to + * c += a; // add acc.2d, acc.2d, swap.2d + * c += b * c; // umlal acc.2d, dkl.2s, dkh.2s + * + * While it would make sense in theory since the addition is faster, + * for reasons likely related to umlal being limited to certain NEON + * pipelines, this is worse. A compiler guard fixes this. + */ + XXH_COMPILER_GUARD_CLANG_NEON(sum_1); + XXH_COMPILER_GUARD_CLANG_NEON(sum_2); + /* xacc[i] = acc_vec + sum; */ + xacc[i] = vaddq_u64(xacc[i], sum_1); + xacc[i+1] = vaddq_u64(xacc[i+1], sum_2); + } + /* Operate on the remaining NEON lanes 2 at a time. */ + for (; i < XXH3_NEON_LANES / 2; i++) { + /* data_vec = xinput[i]; */ + uint64x2_t data_vec = XXH_vld1q_u64(xinput + (i * 16)); + /* key_vec = xsecret[i]; */ + uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16)); + /* acc_vec_2 = swap(data_vec) */ + uint64x2_t data_swap = vextq_u64(data_vec, data_vec, 1); + /* data_key = data_vec ^ key_vec; */ + uint64x2_t data_key = veorq_u64(data_vec, key_vec); + /* For two lanes, just use VMOVN and VSHRN. */ + /* data_key_lo = data_key & 0xFFFFFFFF; */ + uint32x2_t data_key_lo = vmovn_u64(data_key); + /* data_key_hi = data_key >> 32; */ + uint32x2_t data_key_hi = vshrn_n_u64(data_key, 32); + /* sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi; */ + uint64x2_t sum = vmlal_u32(data_swap, data_key_lo, data_key_hi); + /* Same Clang workaround as before */ + XXH_COMPILER_GUARD_CLANG_NEON(sum); + /* xacc[i] = acc_vec + sum; */ + xacc[i] = vaddq_u64 (xacc[i], sum); + } + } +} +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(neon) + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_neon(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { xxh_aliasing_uint64x2_t* xacc = (xxh_aliasing_uint64x2_t*) acc; + uint8_t const* xsecret = (uint8_t const*) secret; + + size_t i; + /* WASM uses operator overloads and doesn't need these. */ +#ifndef __wasm_simd128__ + /* { prime32_1, prime32_1 } */ + uint32x2_t const kPrimeLo = vdup_n_u32(XXH_PRIME32_1); + /* { 0, prime32_1, 0, prime32_1 } */ + uint32x4_t const kPrimeHi = vreinterpretq_u32_u64(vdupq_n_u64((xxh_u64)XXH_PRIME32_1 << 32)); +#endif + + /* AArch64 uses both scalar and neon at the same time */ + for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { + XXH3_scalarScrambleRound(acc, secret, i); + } + for (i=0; i < XXH3_NEON_LANES / 2; i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + uint64x2_t acc_vec = xacc[i]; + uint64x2_t shifted = vshrq_n_u64(acc_vec, 47); + uint64x2_t data_vec = veorq_u64(acc_vec, shifted); + + /* xacc[i] ^= xsecret[i]; */ + uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16)); + uint64x2_t data_key = veorq_u64(data_vec, key_vec); + /* xacc[i] *= XXH_PRIME32_1 */ +#ifdef __wasm_simd128__ + /* SIMD128 has multiply by u64x2, use it instead of expanding and scalarizing */ + xacc[i] = data_key * XXH_PRIME32_1; +#else + /* + * Expanded version with portable NEON intrinsics + * + * lo(x) * lo(y) + (hi(x) * lo(y) << 32) + * + * prod_hi = hi(data_key) * lo(prime) << 32 + * + * Since we only need 32 bits of this multiply a trick can be used, reinterpreting the vector + * as a uint32x4_t and multiplying by { 0, prime, 0, prime } to cancel out the unwanted bits + * and avoid the shift. + */ + uint32x4_t prod_hi = vmulq_u32 (vreinterpretq_u32_u64(data_key), kPrimeHi); + /* Extract low bits for vmlal_u32 */ + uint32x2_t data_key_lo = vmovn_u64(data_key); + /* xacc[i] = prod_hi + lo(data_key) * XXH_PRIME32_1; */ + xacc[i] = vmlal_u32(vreinterpretq_u64_u32(prod_hi), data_key_lo, kPrimeLo); +#endif + } } +} +#endif - if (distributed == maxSymbolValue+1) { - /* all values are pretty poor; - probably incompressible data (should have already been detected); - find max, then give all remaining points to max */ - U32 maxV = 0, maxC = 0; - for (s=0; s<=maxSymbolValue; s++) - if (count[s] > maxC) { maxV=s; maxC=count[s]; } - norm[maxV] += (short)ToDistribute; - return 0; - } +#if (XXH_VECTOR == XXH_VSX) - if (total == 0) { - /* all of the symbols were low enough for the lowOne or lowThreshold */ - for (s=0; ToDistribute > 0; s = (s+1)%(maxSymbolValue+1)) - if (norm[s] > 0) { ToDistribute--; norm[s]++; } - return 0; +XXH_FORCE_INLINE void +XXH3_accumulate_512_vsx( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* presumed aligned */ + xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc; + xxh_u8 const* const xinput = (xxh_u8 const*) input; /* no alignment restriction */ + xxh_u8 const* const xsecret = (xxh_u8 const*) secret; /* no alignment restriction */ + xxh_u64x2 const v32 = { 32, 32 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* data_vec = xinput[i]; */ + xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + 16*i); + /* key_vec = xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16*i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + /* shuffled = (data_key << 32) | (data_key >> 32); */ + xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32); + /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & 0xFFFFFFFF); */ + xxh_u64x2 const product = XXH_vec_mulo((xxh_u32x4)data_key, shuffled); + /* acc_vec = xacc[i]; */ + xxh_u64x2 acc_vec = xacc[i]; + acc_vec += product; + + /* swap high and low halves */ +#ifdef __s390x__ + acc_vec += vec_permi(data_vec, data_vec, 2); +#else + acc_vec += vec_xxpermdi(data_vec, data_vec, 2); +#endif + xacc[i] = acc_vec; } +} +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(vsx) - { U64 const vStepLog = 62 - tableLog; - U64 const mid = (1ULL << (vStepLog-1)) - 1; - U64 const rStep = ZSTD_div64((((U64)1<> vStepLog); - U32 const sEnd = (U32)(end >> vStepLog); - U32 const weight = sEnd - sStart; - if (weight < 1) - return ERROR(GENERIC); - norm[s] = (short)weight; - tmpTotal = end; - } } } +XXH_FORCE_INLINE void +XXH3_scrambleAcc_vsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); - return 0; -} + { xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc; + const xxh_u8* const xsecret = (const xxh_u8*) secret; + /* constants */ + xxh_u64x2 const v32 = { 32, 32 }; + xxh_u64x2 const v47 = { 47, 47 }; + xxh_u32x4 const prime = { XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + xxh_u64x2 const acc_vec = xacc[i]; + xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47); -size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog, - const unsigned* count, size_t total, - unsigned maxSymbolValue, unsigned useLowProbCount) -{ - /* Sanity checks */ - if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; - if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported size */ - if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported size */ - if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */ + /* xacc[i] ^= xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16*i); + xxh_u64x2 const data_key = data_vec ^ key_vec; - { static U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 }; - short const lowProbCount = useLowProbCount ? -1 : 1; - U64 const scale = 62 - tableLog; - U64 const step = ZSTD_div64((U64)1<<62, (U32)total); /* <== here, one division ! */ - U64 const vStep = 1ULL<<(scale-20); - int stillToDistribute = 1<> tableLog); + /* xacc[i] *= XXH_PRIME32_1 */ + /* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & 0xFFFFFFFF); */ + xxh_u64x2 const prod_even = XXH_vec_mule((xxh_u32x4)data_key, prime); + /* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32); */ + xxh_u64x2 const prod_odd = XXH_vec_mulo((xxh_u32x4)data_key, prime); + xacc[i] = prod_odd + (prod_even << v32); + } } +} - for (s=0; s<=maxSymbolValue; s++) { - if (count[s] == total) return 0; /* rle special case */ - if (count[s] == 0) { normalizedCounter[s]=0; continue; } - if (count[s] <= lowThreshold) { - normalizedCounter[s] = lowProbCount; - stillToDistribute--; - } else { - short proba = (short)((count[s]*step) >> scale); - if (proba<8) { - U64 restToBeat = vStep * rtbTable[proba]; - proba += (count[s]*step) - ((U64)proba< restToBeat; - } - if (proba > largestP) { largestP=proba; largest=s; } - normalizedCounter[s] = proba; - stillToDistribute -= proba; - } } - if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) { - /* corner case, need another normalization method */ - size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue, lowProbCount); - if (FSE_isError(errorCode)) return errorCode; - } - else normalizedCounter[largest] += (short)stillToDistribute; - } +#endif -#if 0 - { /* Print Table (debug) */ - U32 s; - U32 nTotal = 0; - for (s=0; s<=maxSymbolValue; s++) - RAWLOG(2, "%3i: %4i \n", s, normalizedCounter[s]); - for (s=0; s<=maxSymbolValue; s++) - nTotal += abs(normalizedCounter[s]); - if (nTotal != (1U<= 8) { + svbool_t mask = svptrue_pat_b64(SV_VL8); + svuint64_t vacc = svld1_u64(mask, xacc); + ACCRND(vacc, 0); + svst1_u64(mask, xacc, vacc); + } else if (element_count == 2) { /* sve128 */ + svbool_t mask = svptrue_pat_b64(SV_VL2); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 2); + svuint64_t acc2 = svld1_u64(mask, xacc + 4); + svuint64_t acc3 = svld1_u64(mask, xacc + 6); + ACCRND(acc0, 0); + ACCRND(acc1, 2); + ACCRND(acc2, 4); + ACCRND(acc3, 6); + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 2, acc1); + svst1_u64(mask, xacc + 4, acc2); + svst1_u64(mask, xacc + 6, acc3); + } else { + svbool_t mask = svptrue_pat_b64(SV_VL4); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 4); + ACCRND(acc0, 0); + ACCRND(acc1, 4); + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 4, acc1); + } +} + +XXH_FORCE_INLINE void +XXH3_accumulate_sve(xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, + size_t nbStripes) +{ + if (nbStripes != 0) { + uint64_t *xacc = (uint64_t *)acc; + const uint64_t *xinput = (const uint64_t *)(const void *)input; + const uint64_t *xsecret = (const uint64_t *)(const void *)secret; + svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1); + uint64_t element_count = svcntd(); + if (element_count >= 8) { + svbool_t mask = svptrue_pat_b64(SV_VL8); + svuint64_t vacc = svld1_u64(mask, xacc + 0); + do { + /* svprfd(svbool_t, void *, enum svfprop); */ + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(vacc, 0); + xinput += 8; + xsecret += 1; + nbStripes--; + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, vacc); + } else if (element_count == 2) { /* sve128 */ + svbool_t mask = svptrue_pat_b64(SV_VL2); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 2); + svuint64_t acc2 = svld1_u64(mask, xacc + 4); + svuint64_t acc3 = svld1_u64(mask, xacc + 6); + do { + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(acc0, 0); + ACCRND(acc1, 2); + ACCRND(acc2, 4); + ACCRND(acc3, 6); + xinput += 8; + xsecret += 1; + nbStripes--; + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 2, acc1); + svst1_u64(mask, xacc + 4, acc2); + svst1_u64(mask, xacc + 6, acc3); + } else { + svbool_t mask = svptrue_pat_b64(SV_VL4); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 4); + do { + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(acc0, 0); + ACCRND(acc1, 4); + xinput += 8; + xsecret += 1; + nbStripes--; + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 4, acc1); + } } +} + #endif - return tableLog; +/* scalar variants - universal */ + +#if defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__)) +/* + * In XXH3_scalarRound(), GCC and Clang have a similar codegen issue, where they + * emit an excess mask and a full 64-bit multiply-add (MADD X-form). + * + * While this might not seem like much, as AArch64 is a 64-bit architecture, only + * big Cortex designs have a full 64-bit multiplier. + * + * On the little cores, the smaller 32-bit multiplier is used, and full 64-bit + * multiplies expand to 2-3 multiplies in microcode. This has a major penalty + * of up to 4 latency cycles and 2 stall cycles in the multiply pipeline. + * + * Thankfully, AArch64 still provides the 32-bit long multiply-add (UMADDL) which does + * not have this penalty and does the mask automatically. + */ +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc) +{ + xxh_u64 ret; + /* note: %x = 64-bit register, %w = 32-bit register */ + __asm__("umaddl %x0, %w1, %w2, %x3" : "=r" (ret) : "r" (lhs), "r" (rhs), "r" (acc)); + return ret; } +#else +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc) +{ + return XXH_mult32to64((xxh_u32)lhs, (xxh_u32)rhs) + acc; +} +#endif +/*! + * @internal + * @brief Scalar round for @ref XXH3_accumulate_512_scalar(). + * + * This is extracted to its own function because the NEON path uses a combination + * of NEON and scalar. + */ +XXH_FORCE_INLINE void +XXH3_scalarRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT input, + void const* XXH_RESTRICT secret, + size_t lane) +{ + xxh_u64* xacc = (xxh_u64*) acc; + xxh_u8 const* xinput = (xxh_u8 const*) input; + xxh_u8 const* xsecret = (xxh_u8 const*) secret; + XXH_ASSERT(lane < XXH_ACC_NB); + XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN-1)) == 0); + { + xxh_u64 const data_val = XXH_readLE64(xinput + lane * 8); + xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + lane * 8); + xacc[lane ^ 1] += data_val; /* swap adjacent lanes */ + xacc[lane] = XXH_mult32to64_add64(data_key /* & 0xFFFFFFFF */, data_key >> 32, xacc[lane]); + } +} -/* fake FSE_CTable, for raw (uncompressed) input */ -size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits) +/*! + * @internal + * @brief Processes a 64 byte block of data using the scalar path. + */ +XXH_FORCE_INLINE void +XXH3_accumulate_512_scalar(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) { - const unsigned tableSize = 1 << nbBits; - const unsigned tableMask = tableSize - 1; - const unsigned maxSymbolValue = tableMask; - void* const ptr = ct; - U16* const tableU16 = ( (U16*) ptr) + 2; - void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableSize>>1); /* assumption : tableLog >= 1 */ - FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT); - unsigned s; - - /* Sanity checks */ - if (nbBits < 1) return ERROR(GENERIC); /* min size */ + size_t i; + /* ARM GCC refuses to unroll this loop, resulting in a 24% slowdown on ARMv6. */ +#if defined(__GNUC__) && !defined(__clang__) \ + && (defined(__arm__) || defined(__thumb2__)) \ + && defined(__ARM_FEATURE_UNALIGNED) /* no unaligned access just wastes bytes */ \ + && XXH_SIZE_OPT <= 0 +# pragma GCC unroll 8 +#endif + for (i=0; i < XXH_ACC_NB; i++) { + XXH3_scalarRound(acc, input, secret, i); + } +} +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(scalar) - /* header */ - tableU16[-2] = (U16) nbBits; - tableU16[-1] = (U16) maxSymbolValue; +/*! + * @internal + * @brief Scalar scramble step for @ref XXH3_scrambleAcc_scalar(). + * + * This is extracted to its own function because the NEON path uses a combination + * of NEON and scalar. + */ +XXH_FORCE_INLINE void +XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT secret, + size_t lane) +{ + xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */ + const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */ + XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN-1)) == 0); + XXH_ASSERT(lane < XXH_ACC_NB); + { + xxh_u64 const key64 = XXH_readLE64(xsecret + lane * 8); + xxh_u64 acc64 = xacc[lane]; + acc64 = XXH_xorshift64(acc64, 47); + acc64 ^= key64; + acc64 *= XXH_PRIME32_1; + xacc[lane] = acc64; + } +} - /* Build table */ - for (s=0; s FSE_MAX_TABLELOG*4+7 ) && (srcSize & 2)) { /* test bit 2 */ - FSE_encodeSymbol(&bitC, &CState2, *--ip); - FSE_encodeSymbol(&bitC, &CState1, *--ip); - FSE_FLUSHBITS(&bitC); - } +#elif (XXH_VECTOR == XXH_VSX) - /* 2 or 4 encoding per loop */ - while ( ip>istart ) { +#define XXH3_accumulate_512 XXH3_accumulate_512_vsx +#define XXH3_accumulate XXH3_accumulate_vsx +#define XXH3_scrambleAcc XXH3_scrambleAcc_vsx +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar - FSE_encodeSymbol(&bitC, &CState2, *--ip); +#elif (XXH_VECTOR == XXH_SVE) +#define XXH3_accumulate_512 XXH3_accumulate_512_sve +#define XXH3_accumulate XXH3_accumulate_sve +#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar - if (sizeof(bitC.bitContainer)*8 < FSE_MAX_TABLELOG*2+7 ) /* this test must be static */ - FSE_FLUSHBITS(&bitC); +#else /* scalar */ - FSE_encodeSymbol(&bitC, &CState1, *--ip); +#define XXH3_accumulate_512 XXH3_accumulate_512_scalar +#define XXH3_accumulate XXH3_accumulate_scalar +#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar - if (sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) { /* this test must be static */ - FSE_encodeSymbol(&bitC, &CState2, *--ip); - FSE_encodeSymbol(&bitC, &CState1, *--ip); - } +#endif - FSE_FLUSHBITS(&bitC); +#if XXH_SIZE_OPT >= 1 /* don't do SIMD for initialization */ +# undef XXH3_initCustomSecret +# define XXH3_initCustomSecret XXH3_initCustomSecret_scalar +#endif + +XXH_FORCE_INLINE void +XXH3_hashLong_internal_loop(xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + size_t const nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE; + size_t const block_len = XXH_STRIPE_LEN * nbStripesPerBlock; + size_t const nb_blocks = (len - 1) / block_len; + + size_t n; + + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + + for (n = 0; n < nb_blocks; n++) { + f_acc(acc, input + n*block_len, secret, nbStripesPerBlock); + f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN); } - FSE_flushCState(&bitC, &CState2); - FSE_flushCState(&bitC, &CState1); - return BIT_closeCStream(&bitC); + /* last partial block */ + XXH_ASSERT(len > XXH_STRIPE_LEN); + { size_t const nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN; + XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE)); + f_acc(acc, input + nb_blocks*block_len, secret, nbStripes); + + /* last stripe */ + { const xxh_u8* const p = input + len - XXH_STRIPE_LEN; +#define XXH_SECRET_LASTACC_START 7 /* not aligned on 8, last secret is different from acc & scrambler */ + XXH3_accumulate_512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START); + } } } -size_t FSE_compress_usingCTable (void* dst, size_t dstSize, - const void* src, size_t srcSize, - const FSE_CTable* ct) +XXH_FORCE_INLINE xxh_u64 +XXH3_mix2Accs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret) { - unsigned const fast = (dstSize >= FSE_BLOCKBOUND(srcSize)); - - if (fast) - return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1); - else - return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0); + return XXH3_mul128_fold64( + acc[0] ^ XXH_readLE64(secret), + acc[1] ^ XXH_readLE64(secret+8) ); } +static XXH64_hash_t +XXH3_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start) +{ + xxh_u64 result64 = start; + size_t i = 0; -size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); } + for (i = 0; i < 4; i++) { + result64 += XXH3_mix2Accs(acc+2*i, secret + 16*i); +#if defined(__clang__) /* Clang */ \ + && (defined(__arm__) || defined(__thumb__)) /* ARMv7 */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Prevent autovectorization on Clang ARMv7-a. Exact same problem as + * the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b. + * XXH3_64bits, len == 256, Snapdragon 835: + * without hack: 2063.7 MB/s + * with hack: 2560.7 MB/s + */ + XXH_COMPILER_GUARD(result64); +#endif + } -#ifndef ZSTD_NO_UNUSED_FUNCTIONS -/* FSE_compress_wksp() : - * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`). - * `wkspSize` size must be `(1<= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + return XXH3_mergeAccs(acc, (const xxh_u8*)secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)len * XXH_PRIME64_1); +} - /* Scan input and build symbol stats */ - { CHECK_V_F(maxCount, HIST_count_wksp(count, &maxSymbolValue, src, srcSize, scratchBuffer, scratchBufferSize) ); - if (maxCount == srcSize) return 1; /* only a single symbol in src : rle */ - if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */ - if (maxCount < (srcSize >> 7)) return 0; /* Heuristic : not compressible enough */ - } +/* + * It's important for performance to transmit secret's size (when it's static) + * so that the compiler can properly optimize the vectorized loop. + * This makes a big performance difference for "medium" keys (<1 KB) when using AVX instruction set. + * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE + * breaks -Og, this is XXH_NO_INLINE. + */ +XXH3_WITH_SECRET_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_64b_internal(input, len, secret, secretLen, XXH3_accumulate, XXH3_scrambleAcc); +} - tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue); - CHECK_F( FSE_normalizeCount(norm, tableLog, count, srcSize, maxSymbolValue, /* useLowProbCount */ srcSize >= 2048) ); +/* + * It's preferable for performance that XXH3_hashLong is not inlined, + * as it results in a smaller function for small data, easier to the instruction cache. + * Note that inside this no_inline function, we do inline the internal loop, + * and provide a statically defined secret size to allow optimization of vector loop. + */ +XXH_NO_INLINE XXH_PUREF XXH64_hash_t +XXH3_hashLong_64b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_accumulate, XXH3_scrambleAcc); +} - /* Write table description header */ - { CHECK_V_F(nc_err, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) ); - op += nc_err; +/* + * XXH3_hashLong_64b_withSeed(): + * Generate a custom key based on alteration of default XXH3_kSecret with the seed, + * and then use this key for long mode hashing. + * + * This operation is decently fast but nonetheless costs a little bit of time. + * Try to avoid it whenever possible (typically when seed==0). + * + * It's important for performance that XXH3_hashLong is not inlined. Not sure + * why (uop cache maybe?), but the difference is large and easily measurable. + */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed_internal(const void* input, size_t len, + XXH64_hash_t seed, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ +#if XXH_SIZE_OPT <= 0 + if (seed == 0) + return XXH3_hashLong_64b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc, f_scramble); +#endif + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed); + return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret), + f_acc, f_scramble); } +} - /* Compress */ - CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, scratchBufferSize) ); - { CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, src, srcSize, CTable) ); - if (cSize == 0) return 0; /* not enough space for compressed data */ - op += cSize; - } +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_64b_withSeed_internal(input, len, seed, + XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret); +} - /* check compressibility */ - if ( (size_t)(op-ostart) >= srcSize-1 ) return 0; - return op-ostart; +typedef XXH64_hash_t (*XXH3_hashLong64_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const xxh_u8* XXH_RESTRICT, size_t); + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_64bits_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong64_f f_hashLong) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secretLen` condition is not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + * Also, note that function signature doesn't offer room to return an error. + */ + if (len <= 16) + return XXH3_len_0to16_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hashLong(input, len, seed64, (const xxh_u8*)secret, secretLen); } -typedef struct { - FSE_CTable CTable_max[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)]; - union { - U32 hist_wksp[HIST_WKSP_SIZE_U32]; - BYTE scratchBuffer[1 << FSE_MAX_TABLELOG]; - } workspace; -} fseWkspMax_t; -size_t FSE_compress2 (void* dst, size_t dstCapacity, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog) +/* === Public entry point === */ + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length) { - fseWkspMax_t scratchBuffer; - DEBUG_STATIC_ASSERT(sizeof(scratchBuffer) >= FSE_COMPRESS_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)); /* compilation failures here means scratchBuffer is not large enough */ - if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); - return FSE_compress_wksp(dst, dstCapacity, src, srcSize, maxSymbolValue, tableLog, &scratchBuffer, sizeof(scratchBuffer)); + return XXH3_64bits_internal(input, length, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_default); } -size_t FSE_compress (void* dst, size_t dstCapacity, const void* src, size_t srcSize) +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecret(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize) { - return FSE_compress2(dst, dstCapacity, src, srcSize, FSE_MAX_SYMBOL_VALUE, FSE_DEFAULT_TABLELOG); + return XXH3_64bits_internal(input, length, 0, secret, secretSize, XXH3_hashLong_64b_withSecret); } -#endif -#endif /* FSE_COMMONDEFS_ONLY */ -/**** ended inlining compress/fse_compress.c ****/ -/**** start inlining compress/hist.c ****/ -/* ****************************************************************** - * hist : Histogram functions - * part of Finite State Entropy project - * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc. - * - * You can contact the author at : - * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy - * - Public forum : https://groups.google.com/forum/#!forum/lz4c - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. -****************************************************************** */ - -/* --- dependencies --- */ -/**** skipping file: ../common/mem.h ****/ -/**** skipping file: ../common/debug.h ****/ -/**** skipping file: ../common/error_private.h ****/ -/**** skipping file: hist.h ****/ - - -/* --- Error management --- */ -unsigned HIST_isError(size_t code) { return ERR_isError(code); } - -/*-************************************************************** - * Histogram functions - ****************************************************************/ -unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, - const void* src, size_t srcSize) +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed) { - const BYTE* ip = (const BYTE*)src; - const BYTE* const end = ip + srcSize; - unsigned maxSymbolValue = *maxSymbolValuePtr; - unsigned largestCount=0; - - ZSTD_memset(count, 0, (maxSymbolValue+1) * sizeof(*count)); - if (srcSize==0) { *maxSymbolValuePtr = 0; return 0; } - - while (ip largestCount) largestCount = count[s]; - } + return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed); +} - return largestCount; +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + if (length <= XXH3_MIDSIZE_MAX) + return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_64b_withSecret(input, length, seed, (const xxh_u8*)secret, secretSize); } -typedef enum { trustInput, checkMaxSymbolValue } HIST_checkInput_e; -/* HIST_count_parallel_wksp() : - * store histogram into 4 intermediate tables, recombined at the end. - * this design makes better use of OoO cpus, - * and is noticeably faster when some values are heavily repeated. - * But it needs some additional workspace for intermediate tables. - * `workSpace` must be a U32 table of size >= HIST_WKSP_SIZE_U32. - * @return : largest histogram frequency, - * or an error code (notably when histogram's alphabet is larger than *maxSymbolValuePtr) */ -static size_t HIST_count_parallel_wksp( - unsigned* count, unsigned* maxSymbolValuePtr, - const void* source, size_t sourceSize, - HIST_checkInput_e check, - U32* const workSpace) +/* === XXH3 streaming === */ +#ifndef XXH_NO_STREAM +/* + * Malloc's a pointer that is always aligned to align. + * + * This must be freed with `XXH_alignedFree()`. + * + * malloc typically guarantees 16 byte alignment on 64-bit systems and 8 byte + * alignment on 32-bit. This isn't enough for the 32 byte aligned loads in AVX2 + * or on 32-bit, the 16 byte aligned loads in SSE2 and NEON. + * + * This underalignment previously caused a rather obvious crash which went + * completely unnoticed due to XXH3_createState() not actually being tested. + * Credit to RedSpah for noticing this bug. + * + * The alignment is done manually: Functions like posix_memalign or _mm_malloc + * are avoided: To maintain portability, we would have to write a fallback + * like this anyways, and besides, testing for the existence of library + * functions without relying on external build tools is impossible. + * + * The method is simple: Overallocate, manually align, and store the offset + * to the original behind the returned pointer. + * + * Align must be a power of 2 and 8 <= align <= 128. + */ +static XXH_MALLOCF void* XXH_alignedMalloc(size_t s, size_t align) { - const BYTE* ip = (const BYTE*)source; - const BYTE* const iend = ip+sourceSize; - size_t const countSize = (*maxSymbolValuePtr + 1) * sizeof(*count); - unsigned max=0; - U32* const Counting1 = workSpace; - U32* const Counting2 = Counting1 + 256; - U32* const Counting3 = Counting2 + 256; - U32* const Counting4 = Counting3 + 256; + XXH_ASSERT(align <= 128 && align >= 8); /* range check */ + XXH_ASSERT((align & (align-1)) == 0); /* power of 2 */ + XXH_ASSERT(s != 0 && s < (s + align)); /* empty/overflow */ + { /* Overallocate to make room for manual realignment and an offset byte */ + xxh_u8* base = (xxh_u8*)XXH_malloc(s + align); + if (base != NULL) { + /* + * Get the offset needed to align this pointer. + * + * Even if the returned pointer is aligned, there will always be + * at least one byte to store the offset to the original pointer. + */ + size_t offset = align - ((size_t)base & (align - 1)); /* base % align */ + /* Add the offset for the now-aligned pointer */ + xxh_u8* ptr = base + offset; - /* safety checks */ - assert(*maxSymbolValuePtr <= 255); - if (!sourceSize) { - ZSTD_memset(count, 0, countSize); - *maxSymbolValuePtr = 0; - return 0; - } - ZSTD_memset(workSpace, 0, 4*256*sizeof(unsigned)); + XXH_ASSERT((size_t)ptr % align == 0); - /* by stripes of 16 bytes */ - { U32 cached = MEM_read32(ip); ip += 4; - while (ip < iend-15) { - U32 c = cached; cached = MEM_read32(ip); ip += 4; - Counting1[(BYTE) c ]++; - Counting2[(BYTE)(c>>8) ]++; - Counting3[(BYTE)(c>>16)]++; - Counting4[ c>>24 ]++; - c = cached; cached = MEM_read32(ip); ip += 4; - Counting1[(BYTE) c ]++; - Counting2[(BYTE)(c>>8) ]++; - Counting3[(BYTE)(c>>16)]++; - Counting4[ c>>24 ]++; - c = cached; cached = MEM_read32(ip); ip += 4; - Counting1[(BYTE) c ]++; - Counting2[(BYTE)(c>>8) ]++; - Counting3[(BYTE)(c>>16)]++; - Counting4[ c>>24 ]++; - c = cached; cached = MEM_read32(ip); ip += 4; - Counting1[(BYTE) c ]++; - Counting2[(BYTE)(c>>8) ]++; - Counting3[(BYTE)(c>>16)]++; - Counting4[ c>>24 ]++; + /* Store the offset immediately before the returned pointer. */ + ptr[-1] = (xxh_u8)offset; + return ptr; } - ip-=4; + return NULL; } - - /* finish last symbols */ - while (ip max) max = Counting1[s]; - } } - - { unsigned maxSymbolValue = 255; - while (!Counting1[maxSymbolValue]) maxSymbolValue--; - if (check && maxSymbolValue > *maxSymbolValuePtr) return ERROR(maxSymbolValue_tooSmall); - *maxSymbolValuePtr = maxSymbolValue; - ZSTD_memmove(count, Counting1, countSize); /* in case count & Counting1 are overlapping */ +} +/* + * Frees an aligned pointer allocated by XXH_alignedMalloc(). Don't pass + * normal malloc'd pointers, XXH_alignedMalloc has a specific data layout. + */ +static void XXH_alignedFree(void* p) +{ + if (p != NULL) { + xxh_u8* ptr = (xxh_u8*)p; + /* Get the offset byte we added in XXH_malloc. */ + xxh_u8 offset = ptr[-1]; + /* Free the original malloc'd pointer */ + xxh_u8* base = ptr - offset; + XXH_free(base); } - return (size_t)max; +} +/*! @ingroup XXH3_family */ +/*! + * @brief Allocate an @ref XXH3_state_t. + * + * @return An allocated pointer of @ref XXH3_state_t on success. + * @return `NULL` on failure. + * + * @note Must be freed with XXH3_freeState(). + */ +XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void) +{ + XXH3_state_t* const state = (XXH3_state_t*)XXH_alignedMalloc(sizeof(XXH3_state_t), 64); + if (state==NULL) return NULL; + XXH3_INITSTATE(state); + return state; } -/* HIST_countFast_wksp() : - * Same as HIST_countFast(), but using an externally provided scratch buffer. - * `workSpace` is a writable buffer which must be 4-bytes aligned, - * `workSpaceSize` must be >= HIST_WKSP_SIZE +/*! @ingroup XXH3_family */ +/*! + * @brief Frees an @ref XXH3_state_t. + * + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). + * + * @return @ref XXH_OK. + * + * @note Must be allocated with XXH3_createState(). */ -size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, - const void* source, size_t sourceSize, - void* workSpace, size_t workSpaceSize) +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr) { - if (sourceSize < 1500) /* heuristic threshold */ - return HIST_count_simple(count, maxSymbolValuePtr, source, sourceSize); - if ((size_t)workSpace & 3) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */ - if (workSpaceSize < HIST_WKSP_SIZE) return ERROR(workSpace_tooSmall); - return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, trustInput, (U32*)workSpace); + XXH_alignedFree(statePtr); + return XXH_OK; } -/* HIST_count_wksp() : - * Same as HIST_count(), but using an externally provided scratch buffer. - * `workSpace` size must be table of >= HIST_WKSP_SIZE_U32 unsigned */ -size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr, - const void* source, size_t sourceSize, - void* workSpace, size_t workSpaceSize) +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API void +XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state) { - if ((size_t)workSpace & 3) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */ - if (workSpaceSize < HIST_WKSP_SIZE) return ERROR(workSpace_tooSmall); - if (*maxSymbolValuePtr < 255) - return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, checkMaxSymbolValue, (U32*)workSpace); - *maxSymbolValuePtr = 255; - return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace, workSpaceSize); + XXH_memcpy(dst_state, src_state, sizeof(*dst_state)); } -#ifndef ZSTD_NO_UNUSED_FUNCTIONS -/* fast variant (unsafe : won't check if src contains values beyond count[] limit) */ -size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr, - const void* source, size_t sourceSize) +static void +XXH3_reset_internal(XXH3_state_t* statePtr, + XXH64_hash_t seed, + const void* secret, size_t secretSize) +{ + size_t const initStart = offsetof(XXH3_state_t, bufferedSize); + size_t const initLength = offsetof(XXH3_state_t, nbStripesPerBlock) - initStart; + XXH_ASSERT(offsetof(XXH3_state_t, nbStripesPerBlock) > initStart); + XXH_ASSERT(statePtr != NULL); + /* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */ + memset((char*)statePtr + initStart, 0, initLength); + statePtr->acc[0] = XXH_PRIME32_3; + statePtr->acc[1] = XXH_PRIME64_1; + statePtr->acc[2] = XXH_PRIME64_2; + statePtr->acc[3] = XXH_PRIME64_3; + statePtr->acc[4] = XXH_PRIME64_4; + statePtr->acc[5] = XXH_PRIME32_2; + statePtr->acc[6] = XXH_PRIME64_5; + statePtr->acc[7] = XXH_PRIME32_1; + statePtr->seed = seed; + statePtr->useSeed = (seed != 0); + statePtr->extSecret = (const unsigned char*)secret; + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + statePtr->secretLimit = secretSize - XXH_STRIPE_LEN; + statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize) { - unsigned tmpCounters[HIST_WKSP_SIZE_U32]; - return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, tmpCounters, sizeof(tmpCounters)); + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, secret, secretSize); + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + return XXH_OK; } -size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr, - const void* src, size_t srcSize) +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed) { - unsigned tmpCounters[HIST_WKSP_SIZE_U32]; - return HIST_count_wksp(count, maxSymbolValuePtr, src, srcSize, tmpCounters, sizeof(tmpCounters)); + if (statePtr == NULL) return XXH_ERROR; + if (seed==0) return XXH3_64bits_reset(statePtr); + if ((seed != statePtr->seed) || (statePtr->extSecret != NULL)) + XXH3_initCustomSecret(statePtr->customSecret, seed); + XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; } -#endif -/**** ended inlining compress/hist.c ****/ -/**** start inlining compress/huf_compress.c ****/ -/* ****************************************************************** - * Huffman encoder, part of New Generation Entropy library - * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc. - * - * You can contact the author at : - * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy - * - Public forum : https://groups.google.com/forum/#!forum/lz4c - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. -****************************************************************** */ -/* ************************************************************** -* Compiler specifics -****************************************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed64) +{ + if (statePtr == NULL) return XXH_ERROR; + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + XXH3_reset_internal(statePtr, seed64, secret, secretSize); + statePtr->useSeed = 1; /* always, even if seed64==0 */ + return XXH_OK; +} + +/*! + * @internal + * @brief Processes a large input for XXH3_update() and XXH3_digest_long(). + * + * Unlike XXH3_hashLong_internal_loop(), this can process data that overlaps a block. + * + * @param acc Pointer to the 8 accumulator lanes + * @param nbStripesSoFarPtr In/out pointer to the number of leftover stripes in the block* + * @param nbStripesPerBlock Number of stripes in a block + * @param input Input pointer + * @param nbStripes Number of stripes to process + * @param secret Secret pointer + * @param secretLimit Offset of the last block in @p secret + * @param f_acc Pointer to an XXH3_accumulate implementation + * @param f_scramble Pointer to an XXH3_scrambleAcc implementation + * @return Pointer past the end of @p input after processing + */ +XXH_FORCE_INLINE const xxh_u8 * +XXH3_consumeStripes(xxh_u64* XXH_RESTRICT acc, + size_t* XXH_RESTRICT nbStripesSoFarPtr, size_t nbStripesPerBlock, + const xxh_u8* XXH_RESTRICT input, size_t nbStripes, + const xxh_u8* XXH_RESTRICT secret, size_t secretLimit, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + const xxh_u8* initialSecret = secret + *nbStripesSoFarPtr * XXH_SECRET_CONSUME_RATE; + /* Process full blocks */ + if (nbStripes >= (nbStripesPerBlock - *nbStripesSoFarPtr)) { + /* Process the initial partial block... */ + size_t nbStripesThisIter = nbStripesPerBlock - *nbStripesSoFarPtr; + + do { + /* Accumulate and scramble */ + f_acc(acc, input, initialSecret, nbStripesThisIter); + f_scramble(acc, secret + secretLimit); + input += nbStripesThisIter * XXH_STRIPE_LEN; + nbStripes -= nbStripesThisIter; + /* Then continue the loop with the full block size */ + nbStripesThisIter = nbStripesPerBlock; + initialSecret = secret; + } while (nbStripes >= nbStripesPerBlock); + *nbStripesSoFarPtr = 0; + } + /* Process a partial block */ + if (nbStripes > 0) { + f_acc(acc, input, initialSecret, nbStripes); + input += nbStripes * XXH_STRIPE_LEN; + *nbStripesSoFarPtr += nbStripes; + } + /* Return end pointer */ + return input; +} + +#ifndef XXH3_STREAM_USE_STACK +# if XXH_SIZE_OPT <= 0 && !defined(__clang__) /* clang doesn't need additional stack space */ +# define XXH3_STREAM_USE_STACK 1 +# endif #endif +/* + * Both XXH3_64bits_update and XXH3_128bits_update use this routine. + */ +XXH_FORCE_INLINE XXH_errorcode +XXH3_update(XXH3_state_t* XXH_RESTRICT const state, + const xxh_u8* XXH_RESTRICT input, size_t len, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + XXH_ASSERT(state != NULL); + { const xxh_u8* const bEnd = input + len; + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; +#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* For some reason, gcc and MSVC seem to suffer greatly + * when operating accumulators directly into state. + * Operating into stack space seems to enable proper optimization. + * clang, on the other hand, doesn't seem to need this trick */ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[8]; + XXH_memcpy(acc, state->acc, sizeof(acc)); +#else + xxh_u64* XXH_RESTRICT const acc = state->acc; +#endif + state->totalLen += len; + XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE); -/* ************************************************************** -* Includes -****************************************************************/ -/**** skipping file: ../common/zstd_deps.h ****/ -/**** skipping file: ../common/compiler.h ****/ -/**** skipping file: ../common/bitstream.h ****/ -/**** skipping file: hist.h ****/ -#define FSE_STATIC_LINKING_ONLY /* FSE_optimalTableLog_internal */ -/**** skipping file: ../common/fse.h ****/ -#define HUF_STATIC_LINKING_ONLY -/**** skipping file: ../common/huf.h ****/ -/**** skipping file: ../common/error_private.h ****/ + /* small input : just fill in tmp buffer */ + if (len <= XXH3_INTERNALBUFFER_SIZE - state->bufferedSize) { + XXH_memcpy(state->buffer + state->bufferedSize, input, len); + state->bufferedSize += (XXH32_hash_t)len; + return XXH_OK; + } + /* total input is now > XXH3_INTERNALBUFFER_SIZE */ + #define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN) + XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0); /* clean multiple */ -/* ************************************************************** -* Error Management -****************************************************************/ -#define HUF_isError ERR_isError -#define HUF_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */ + /* + * Internal buffer is partially filled (always, except at beginning) + * Complete it, then consume it. + */ + if (state->bufferedSize) { + size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize; + XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize); + input += loadSize; + XXH3_consumeStripes(acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, XXH3_INTERNALBUFFER_STRIPES, + secret, state->secretLimit, + f_acc, f_scramble); + state->bufferedSize = 0; + } + XXH_ASSERT(input < bEnd); + if (bEnd - input > XXH3_INTERNALBUFFER_SIZE) { + size_t nbStripes = (size_t)(bEnd - 1 - input) / XXH_STRIPE_LEN; + input = XXH3_consumeStripes(acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + input, nbStripes, + secret, state->secretLimit, + f_acc, f_scramble); + XXH_memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN); + + } + /* Some remaining input (always) : buffer it */ + XXH_ASSERT(input < bEnd); + XXH_ASSERT(bEnd - input <= XXH3_INTERNALBUFFER_SIZE); + XXH_ASSERT(state->bufferedSize == 0); + XXH_memcpy(state->buffer, input, (size_t)(bEnd-input)); + state->bufferedSize = (XXH32_hash_t)(bEnd-input); +#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* save stack accumulators into state */ + XXH_memcpy(state->acc, acc, sizeof(acc)); +#endif + } + return XXH_OK; +} -/* ************************************************************** -* Utils -****************************************************************/ -unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len) { - return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1); + return XXH3_update(state, (const xxh_u8*)input, len, + XXH3_accumulate, XXH3_scrambleAcc); } -/* ******************************************************* -* HUF : Huffman block compression -*********************************************************/ -/* HUF_compressWeights() : - * Same as FSE_compress(), but dedicated to huff0's weights compression. - * The use case needs much less stack memory. - * Note : all elements within weightTable are supposed to be <= HUF_TABLELOG_MAX. - */ -#define MAX_FSE_TABLELOG_FOR_HUFF_HEADER 6 -static size_t HUF_compressWeights (void* dst, size_t dstSize, const void* weightTable, size_t wtSize) +XXH_FORCE_INLINE void +XXH3_digest_long (XXH64_hash_t* acc, + const XXH3_state_t* state, + const unsigned char* secret) { - BYTE* const ostart = (BYTE*) dst; - BYTE* op = ostart; - BYTE* const oend = ostart + dstSize; + xxh_u8 lastStripe[XXH_STRIPE_LEN]; + const xxh_u8* lastStripePtr; - unsigned maxSymbolValue = HUF_TABLELOG_MAX; - U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER; + /* + * Digest on a local copy. This way, the state remains unaltered, and it can + * continue ingesting more input afterwards. + */ + XXH_memcpy(acc, state->acc, sizeof(state->acc)); + if (state->bufferedSize >= XXH_STRIPE_LEN) { + /* Consume remaining stripes then point to remaining data in buffer */ + size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN; + size_t nbStripesSoFar = state->nbStripesSoFar; + XXH3_consumeStripes(acc, + &nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, nbStripes, + secret, state->secretLimit, + XXH3_accumulate, XXH3_scrambleAcc); + lastStripePtr = state->buffer + state->bufferedSize - XXH_STRIPE_LEN; + } else { /* bufferedSize < XXH_STRIPE_LEN */ + /* Copy to temp buffer */ + size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize; + XXH_ASSERT(state->bufferedSize > 0); /* there is always some input buffered */ + XXH_memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, catchupSize); + XXH_memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize); + lastStripePtr = lastStripe; + } + /* Last stripe */ + XXH3_accumulate_512(acc, + lastStripePtr, + secret + state->secretLimit - XXH_SECRET_LASTACC_START); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + return XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + } + /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */ + if (state->useSeed) + return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} +#endif /* !XXH_NO_STREAM */ + + +/* ========================================== + * XXH3 128 bits (a.k.a XXH128) + * ========================================== + * XXH3's 128-bit variant has better mixing and strength than the 64-bit variant, + * even without counting the significantly larger output size. + * + * For example, extra steps are taken to avoid the seed-dependent collisions + * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B). + * + * This strength naturally comes at the cost of some speed, especially on short + * lengths. Note that longer hashes are about as fast as the 64-bit version + * due to it using only a slight modification of the 64-bit loop. + * + * XXH128 is also more oriented towards 64-bit machines. It is still extremely + * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64). + */ + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_1to3_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + /* A doubled version of 1to3_64b with different constants. */ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combinedl = { input[0], 0x01, input[0], input[0] } + * len = 2: combinedl = { input[1], 0x02, input[0], input[1] } + * len = 3: combinedl = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combinedl = ((xxh_u32)c1 <<16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13); + xxh_u64 const bitflipl = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const bitfliph = (XXH_readLE32(secret+8) ^ XXH_readLE32(secret+12)) - seed; + xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl; + xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph; + XXH128_hash_t h128; + h128.low64 = XXH64_avalanche(keyed_lo); + h128.high64 = XXH64_avalanche(keyed_hi); + return h128; + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_4to8_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input_lo = XXH_readLE32(input); + xxh_u32 const input_hi = XXH_readLE32(input + len - 4); + xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32); + xxh_u64 const bitflip = (XXH_readLE64(secret+16) ^ XXH_readLE64(secret+24)) + seed; + xxh_u64 const keyed = input_64 ^ bitflip; + + /* Shift len to the left to ensure it is even, this avoids even multiplies. */ + XXH128_hash_t m128 = XXH_mult64to128(keyed, XXH_PRIME64_1 + (len << 2)); + + m128.high64 += (m128.low64 << 1); + m128.low64 ^= (m128.high64 >> 3); + + m128.low64 = XXH_xorshift64(m128.low64, 35); + m128.low64 *= PRIME_MX2; + m128.low64 = XXH_xorshift64(m128.low64, 28); + m128.high64 = XXH3_avalanche(m128.high64); + return m128; + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_9to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflipl = (XXH_readLE64(secret+32) ^ XXH_readLE64(secret+40)) - seed; + xxh_u64 const bitfliph = (XXH_readLE64(secret+48) ^ XXH_readLE64(secret+56)) + seed; + xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 input_hi = XXH_readLE64(input + len - 8); + XXH128_hash_t m128 = XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1); + /* + * Put len in the middle of m128 to ensure that the length gets mixed to + * both the low and high bits in the 128x64 multiply below. + */ + m128.low64 += (xxh_u64)(len - 1) << 54; + input_hi ^= bitfliph; + /* + * Add the high 32 bits of input_hi to the high 32 bits of m128, then + * add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to + * the high 64 bits of m128. + * + * The best approach to this operation is different on 32-bit and 64-bit. + */ + if (sizeof(void *) < sizeof(xxh_u64)) { /* 32-bit */ + /* + * 32-bit optimized version, which is more readable. + * + * On 32-bit, it removes an ADC and delays a dependency between the two + * halves of m128.high64, but it generates an extra mask on 64-bit. + */ + m128.high64 += (input_hi & 0xFFFFFFFF00000000ULL) + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2); + } else { + /* + * 64-bit optimized (albeit more confusing) version. + * + * Uses some properties of addition and multiplication to remove the mask: + * + * Let: + * a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF) + * b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000) + * c = XXH_PRIME32_2 + * + * a + (b * c) + * Inverse Property: x + y - x == y + * a + (b * (1 + c - 1)) + * Distributive Property: x * (y + z) == (x * y) + (x * z) + * a + (b * 1) + (b * (c - 1)) + * Identity Property: x * 1 == x + * a + b + (b * (c - 1)) + * + * Substitute a, b, and c: + * input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + * + * Since input_hi.hi + input_hi.lo == input_hi, we get this: + * input_hi + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + */ + m128.high64 += input_hi + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2 - 1); + } + /* m128 ^= XXH_swap64(m128 >> 64); */ + m128.low64 ^= XXH_swap64(m128.high64); - FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)]; - U32 scratchBuffer[FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(HUF_TABLELOG_MAX, MAX_FSE_TABLELOG_FOR_HUFF_HEADER)]; + { /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */ + XXH128_hash_t h128 = XXH_mult64to128(m128.low64, XXH_PRIME64_2); + h128.high64 += m128.high64 * XXH_PRIME64_2; - unsigned count[HUF_TABLELOG_MAX+1]; - S16 norm[HUF_TABLELOG_MAX+1]; + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = XXH3_avalanche(h128.high64); + return h128; + } } +} - /* init conditions */ - if (wtSize <= 1) return 0; /* Not compressible */ +/* + * Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN + */ +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_0to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed); + if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed); + if (len) return XXH3_len_1to3_128b(input, len, secret, seed); + { XXH128_hash_t h128; + xxh_u64 const bitflipl = XXH_readLE64(secret+64) ^ XXH_readLE64(secret+72); + xxh_u64 const bitfliph = XXH_readLE64(secret+80) ^ XXH_readLE64(secret+88); + h128.low64 = XXH64_avalanche(seed ^ bitflipl); + h128.high64 = XXH64_avalanche( seed ^ bitfliph); + return h128; + } } +} - /* Scan input and build symbol stats */ - { unsigned const maxCount = HIST_count_simple(count, &maxSymbolValue, weightTable, wtSize); /* never fails */ - if (maxCount == wtSize) return 1; /* only a single symbol in src : rle */ - if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */ +/* + * A bit slower than XXH3_mix16B, but handles multiply by zero better. + */ +XXH_FORCE_INLINE XXH128_hash_t +XXH128_mix32B(XXH128_hash_t acc, const xxh_u8* input_1, const xxh_u8* input_2, + const xxh_u8* secret, XXH64_hash_t seed) +{ + acc.low64 += XXH3_mix16B (input_1, secret+0, seed); + acc.low64 ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8); + acc.high64 += XXH3_mix16B (input_2, secret+16, seed); + acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8); + return acc; +} + + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_17to128_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { XXH128_hash_t acc; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + +#if XXH_SIZE_OPT >= 1 + { + /* Smaller, but slightly slower. */ + unsigned int i = (unsigned int)(len - 1) / 32; + do { + acc = XXH128_mix32B(acc, input+16*i, input+len-16*(i+1), secret+32*i, seed); + } while (i-- != 0); + } +#else + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc = XXH128_mix32B(acc, input+48, input+len-64, secret+96, seed); + } + acc = XXH128_mix32B(acc, input+32, input+len-48, secret+64, seed); + } + acc = XXH128_mix32B(acc, input+16, input+len-32, secret+32, seed); + } + acc = XXH128_mix32B(acc, input, input+len-16, secret, seed); +#endif + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } } +} - tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue); - CHECK_F( FSE_normalizeCount(norm, tableLog, count, wtSize, maxSymbolValue, /* useLowProbCount */ 0) ); +XXH_NO_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_129to240_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); - /* Write table description header */ - { CHECK_V_F(hSize, FSE_writeNCount(op, (size_t)(oend-op), norm, maxSymbolValue, tableLog) ); - op += hSize; + { XXH128_hash_t acc; + unsigned i; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + /* + * We set as `i` as offset + 32. We do this so that unchanged + * `len` can be used as upper bound. This reaches a sweet spot + * where both x86 and aarch64 get simple agen and good codegen + * for the loop. + */ + for (i = 32; i < 160; i += 32) { + acc = XXH128_mix32B(acc, + input + i - 32, + input + i - 16, + secret + i - 32, + seed); + } + acc.low64 = XXH3_avalanche(acc.low64); + acc.high64 = XXH3_avalanche(acc.high64); + /* + * NB: `i <= len` will duplicate the last 32-bytes if + * len % 32 was zero. This is an unfortunate necessity to keep + * the hash result stable. + */ + for (i=160; i <= len; i += 32) { + acc = XXH128_mix32B(acc, + input + i - 32, + input + i - 16, + secret + XXH3_MIDSIZE_STARTOFFSET + i - 160, + seed); + } + /* last bytes */ + acc = XXH128_mix32B(acc, + input + len - 16, + input + len - 32, + secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16, + (XXH64_hash_t)0 - seed); + + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } } +} - /* Compress */ - CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, sizeof(scratchBuffer)) ); - { CHECK_V_F(cSize, FSE_compress_usingCTable(op, (size_t)(oend - op), weightTable, wtSize, CTable) ); - if (cSize == 0) return 0; /* not enough space for compressed data */ - op += cSize; +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_internal(const void* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, secret, secretSize, f_acc, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)len * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs(acc, + secret + secretSize + - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)len * XXH_PRIME64_2)); + return h128; } +} - return (size_t)(op-ostart); +/* + * It's important for performance that XXH3_hashLong() is not inlined. + */ +XXH_NO_INLINE XXH_PUREF XXH128_hash_t +XXH3_hashLong_128b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_accumulate, XXH3_scrambleAcc); } +/* + * It's important for performance to pass @p secretLen (when it's static) + * to the compiler, so that it can properly optimize the vectorized loop. + * + * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE + * breaks -Og, this is XXH_NO_INLINE. + */ +XXH3_WITH_SECRET_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, secretLen, + XXH3_accumulate, XXH3_scrambleAcc); +} -/*! HUF_writeCTable() : - `CTable` : Huffman tree to save, using huf representation. - @return : size of saved CTable */ -size_t HUF_writeCTable (void* dst, size_t maxDstSize, - const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog) +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) { - BYTE bitsToWeight[HUF_TABLELOG_MAX + 1]; /* precomputed conversion table */ - BYTE huffWeight[HUF_SYMBOLVALUE_MAX]; - BYTE* op = (BYTE*)dst; - U32 n; + if (seed64 == 0) + return XXH3_hashLong_128b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc, f_scramble); + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed64); + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, sizeof(secret), + f_acc, f_scramble); + } +} - /* check conditions */ - if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge); +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_128b_withSeed_internal(input, len, seed64, + XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret); +} - /* convert to weight */ - bitsToWeight[0] = 0; - for (n=1; n1) & (hSize < maxSymbolValue/2)) { /* FSE compressed */ - op[0] = (BYTE)hSize; - return hSize+1; - } } +XXH_FORCE_INLINE XXH128_hash_t +XXH3_128bits_internal(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong128_f f_hl128) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secret` conditions are not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + */ + if (len <= 16) + return XXH3_len_0to16_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hl128(input, len, seed64, secret, secretLen); +} - /* write raw values as 4-bits (max : 15) */ - if (maxSymbolValue > (256-128)) return ERROR(GENERIC); /* should not happen : likely means source cannot be compressed */ - if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */ - op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue-1)); - huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause msan issue in final combination */ - for (n=0; n 0); +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_128b_withSecret(input, len, seed, secret, secretSize); +} - /* check result */ - if (tableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); - if (nbSymbols > *maxSymbolValuePtr+1) return ERROR(maxSymbolValue_tooSmall); +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_withSeed(input, len, seed); +} - /* Prepare base value per rank */ - { U32 n, nextRankStart = 0; - for (n=1; n<=tableLog; n++) { - U32 curr = nextRankStart; - nextRankStart += (rankVal[n] << (n-1)); - rankVal[n] = curr; - } } - /* fill nbBits */ - { U32 n; for (n=0; nn=tableLog+1 */ - U16 valPerRank[HUF_TABLELOG_MAX+2] = {0}; - { U32 n; for (n=0; n0; n--) { /* start at n=tablelog <-> w=1 */ - valPerRank[n] = min; /* get starting value within each rank */ - min += nbPerRank[n]; - min >>= 1; - } } - /* assign value within rank, symbol order */ - { U32 n; for (n=0; nextSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs(acc, + secret + state->secretLimit + XXH_STRIPE_LEN + - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)state->totalLen * XXH_PRIME64_2)); + return h128; + } } + /* len <= XXH3_MIDSIZE_MAX : short code */ + if (state->seed) + return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} +#endif /* !XXH_NO_STREAM */ +/* 128-bit utility functions */ - *maxSymbolValuePtr = nbSymbols - 1; - return readSize; +/* return : 1 is equal, 0 if different */ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2) +{ + /* note : XXH128_hash_t is compact, it has no padding byte */ + return !(memcmp(&h1, &h2, sizeof(h1))); } -U32 HUF_getNbBits(const void* symbolTable, U32 symbolValue) +/* This prototype is compatible with stdlib's qsort(). + * @return : >0 if *h128_1 > *h128_2 + * <0 if *h128_1 < *h128_2 + * =0 if *h128_1 == *h128_2 */ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2) { - const HUF_CElt* table = (const HUF_CElt*)symbolTable; - assert(symbolValue <= HUF_SYMBOLVALUE_MAX); - return table[symbolValue].nbBits; + XXH128_hash_t const h1 = *(const XXH128_hash_t*)h128_1; + XXH128_hash_t const h2 = *(const XXH128_hash_t*)h128_2; + int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64); + /* note : bets that, in most cases, hash values are different */ + if (hcmp) return hcmp; + return (h1.low64 > h2.low64) - (h2.low64 > h1.low64); } -typedef struct nodeElt_s { - U32 count; - U16 parent; - BYTE byte; - BYTE nbBits; -} nodeElt; +/*====== Canonical representation ======*/ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API void +XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) { + hash.high64 = XXH_swap64(hash.high64); + hash.low64 = XXH_swap64(hash.low64); + } + XXH_memcpy(dst, &hash.high64, sizeof(hash.high64)); + XXH_memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64)); +} -/** - * HUF_setMaxHeight(): - * Enforces maxNbBits on the Huffman tree described in huffNode. - * - * It sets all nodes with nbBits > maxNbBits to be maxNbBits. Then it adjusts - * the tree to so that it is a valid canonical Huffman tree. - * - * @pre The sum of the ranks of each symbol == 2^largestBits, - * where largestBits == huffNode[lastNonNull].nbBits. - * @post The sum of the ranks of each symbol == 2^largestBits, - * where largestBits is the return value <= maxNbBits. - * - * @param huffNode The Huffman tree modified in place to enforce maxNbBits. - * @param lastNonNull The symbol with the lowest count in the Huffman tree. - * @param maxNbBits The maximum allowed number of bits, which the Huffman tree - * may not respect. After this function the Huffman tree will - * respect maxNbBits. - * @return The maximum number of bits of the Huffman tree after adjustment, - * necessarily no more than maxNbBits. +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src) +{ + XXH128_hash_t h; + h.high64 = XXH_readBE64(src); + h.low64 = XXH_readBE64(src->digest + 8); + return h; +} + + + +/* ========================================== + * Secret generators + * ========================================== */ -static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) +#define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x)) + +XXH_FORCE_INLINE void XXH3_combine16(void* dst, XXH128_hash_t h128) { - const U32 largestBits = huffNode[lastNonNull].nbBits; - /* early exit : no elt > maxNbBits, so the tree is already valid. */ - if (largestBits <= maxNbBits) return largestBits; + XXH_writeLE64( dst, XXH_readLE64(dst) ^ h128.low64 ); + XXH_writeLE64( (char*)dst+8, XXH_readLE64((char*)dst+8) ^ h128.high64 ); +} - /* there are several too large elements (at least >= 2) */ - { int totalCost = 0; - const U32 baseCost = 1 << (largestBits - maxNbBits); - int n = (int)lastNonNull; +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize) +{ +#if (XXH_DEBUGLEVEL >= 1) + XXH_ASSERT(secretBuffer != NULL); + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); +#else + /* production mode, assert() are disabled */ + if (secretBuffer == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; +#endif - /* Adjust any ranks > maxNbBits to maxNbBits. - * Compute totalCost, which is how far the sum of the ranks is - * we are over 2^largestBits after adjust the offending ranks. - */ - while (huffNode[n].nbBits > maxNbBits) { - totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits)); - huffNode[n].nbBits = (BYTE)maxNbBits; - n--; + if (customSeedSize == 0) { + customSeed = XXH3_kSecret; + customSeedSize = XXH_SECRET_DEFAULT_SIZE; + } +#if (XXH_DEBUGLEVEL >= 1) + XXH_ASSERT(customSeed != NULL); +#else + if (customSeed == NULL) return XXH_ERROR; +#endif + + /* Fill secretBuffer with a copy of customSeed - repeat as needed */ + { size_t pos = 0; + while (pos < secretSize) { + size_t const toCopy = XXH_MIN((secretSize - pos), customSeedSize); + memcpy((char*)secretBuffer + pos, customSeed, toCopy); + pos += toCopy; + } } + + { size_t const nbSeg16 = secretSize / 16; + size_t n; + XXH128_canonical_t scrambler; + XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0)); + for (n=0; n>= (largestBits - maxNbBits); - assert(totalCost > 0); +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API void +XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer, XXH64_hash_t seed) +{ + XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + XXH3_initCustomSecret(secret, seed); + XXH_ASSERT(secretBuffer != NULL); + memcpy(secretBuffer, secret, XXH_SECRET_DEFAULT_SIZE); +} - /* repay normalized cost */ - { U32 const noSymbol = 0xF0F0F0F0; - U32 rankLast[HUF_TABLELOG_MAX+2]; - /* Get pos of last (smallest = lowest cum. count) symbol per rank */ - ZSTD_memset(rankLast, 0xF0, sizeof(rankLast)); - { U32 currentNbBits = maxNbBits; - int pos; - for (pos=n ; pos >= 0; pos--) { - if (huffNode[pos].nbBits >= currentNbBits) continue; - currentNbBits = huffNode[pos].nbBits; /* < maxNbBits */ - rankLast[maxNbBits-currentNbBits] = (U32)pos; - } } - while (totalCost > 0) { - /* Try to reduce the next power of 2 above totalCost because we - * gain back half the rank. - */ - U32 nBitsToDecrease = BIT_highbit32((U32)totalCost) + 1; - for ( ; nBitsToDecrease > 1; nBitsToDecrease--) { - U32 const highPos = rankLast[nBitsToDecrease]; - U32 const lowPos = rankLast[nBitsToDecrease-1]; - if (highPos == noSymbol) continue; - /* Decrease highPos if no symbols of lowPos or if it is - * not cheaper to remove 2 lowPos than highPos. - */ - if (lowPos == noSymbol) break; - { U32 const highTotal = huffNode[highPos].count; - U32 const lowTotal = 2 * huffNode[lowPos].count; - if (highTotal <= lowTotal) break; - } } - /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */ - assert(rankLast[nBitsToDecrease] != noSymbol || nBitsToDecrease == 1); - /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */ - while ((nBitsToDecrease<=HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol)) - nBitsToDecrease++; - assert(rankLast[nBitsToDecrease] != noSymbol); - /* Increase the number of bits to gain back half the rank cost. */ - totalCost -= 1 << (nBitsToDecrease-1); - huffNode[rankLast[nBitsToDecrease]].nbBits++; +/* Pop our optimization override from above */ +#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ + && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__OPTIMIZE__) && XXH_SIZE_OPT <= 0 /* respect -O0 and -Os */ +# pragma GCC pop_options +#endif - /* Fix up the new rank. - * If the new rank was empty, this symbol is now its smallest. - * Otherwise, this symbol will be the largest in the new rank so no adjustment. - */ - if (rankLast[nBitsToDecrease-1] == noSymbol) - rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease]; - /* Fix up the old rank. - * If the symbol was at position 0, meaning it was the highest weight symbol in the tree, - * it must be the only symbol in its rank, so the old rank now has no symbols. - * Otherwise, since the Huffman nodes are sorted by count, the previous position is now - * the smallest node in the rank. If the previous position belongs to a different rank, - * then the rank is now empty. - */ - if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */ - rankLast[nBitsToDecrease] = noSymbol; - else { - rankLast[nBitsToDecrease]--; - if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease) - rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */ - } - } /* while (totalCost > 0) */ - /* If we've removed too much weight, then we have to add it back. - * To avoid overshooting again, we only adjust the smallest rank. - * We take the largest nodes from the lowest rank 0 and move them - * to rank 1. There's guaranteed to be enough rank 0 symbols because - * TODO. - */ - while (totalCost < 0) { /* Sometimes, cost correction overshoot */ - /* special case : no rank 1 symbol (using maxNbBits-1); - * let's create one from largest rank 0 (using maxNbBits). - */ - if (rankLast[1] == noSymbol) { - while (huffNode[n].nbBits == maxNbBits) n--; - huffNode[n+1].nbBits--; - assert(n >= 0); - rankLast[1] = (U32)(n+1); - totalCost++; - continue; - } - huffNode[ rankLast[1] + 1 ].nbBits--; - rankLast[1]++; - totalCost ++; - } - } /* repay normalized cost */ - } /* there are several too large elements (at least >= 2) */ +#if defined (__cplusplus) +} /* extern "C" */ +#endif - return maxNbBits; -} +#endif /* XXH_NO_LONG_LONG */ +#endif /* XXH_NO_XXH3 */ -typedef struct { - U32 base; - U32 curr; -} rankPos; +/*! + * @} + */ +#endif /* XXH_IMPLEMENTATION */ +/**** ended inlining xxhash.h ****/ +#ifndef ZSTD_NO_TRACE +/**** start inlining zstd_trace.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_TRACE_H +#define ZSTD_TRACE_H + +#include -typedef nodeElt huffNodeTable[HUF_CTABLE_WORKSPACE_SIZE_U32]; +/* weak symbol support + * For now, enable conservatively: + * - Only GNUC + * - Only ELF + * - Only x86-64, i386, aarch64 and risc-v. + * Also, explicitly disable on platforms known not to work so they aren't + * forgotten in the future. + */ +#if !defined(ZSTD_HAVE_WEAK_SYMBOLS) && \ + defined(__GNUC__) && defined(__ELF__) && \ + (defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || \ + defined(_M_IX86) || defined(__aarch64__) || defined(__riscv)) && \ + !defined(__APPLE__) && !defined(_WIN32) && !defined(__MINGW32__) && \ + !defined(__CYGWIN__) && !defined(_AIX) +# define ZSTD_HAVE_WEAK_SYMBOLS 1 +#else +# define ZSTD_HAVE_WEAK_SYMBOLS 0 +#endif +#if ZSTD_HAVE_WEAK_SYMBOLS +# define ZSTD_WEAK_ATTR __attribute__((__weak__)) +#else +# define ZSTD_WEAK_ATTR +#endif + +/* Only enable tracing when weak symbols are available. */ +#ifndef ZSTD_TRACE +# define ZSTD_TRACE ZSTD_HAVE_WEAK_SYMBOLS +#endif + +#if ZSTD_TRACE -#define RANK_POSITION_TABLE_SIZE 32 +struct ZSTD_CCtx_s; +struct ZSTD_DCtx_s; +struct ZSTD_CCtx_params_s; typedef struct { - huffNodeTable huffNodeTbl; - rankPos rankPosition[RANK_POSITION_TABLE_SIZE]; -} HUF_buildCTable_wksp_tables; + /** + * ZSTD_VERSION_NUMBER + * + * This is guaranteed to be the first member of ZSTD_trace. + * Otherwise, this struct is not stable between versions. If + * the version number does not match your expectation, you + * should not interpret the rest of the struct. + */ + unsigned version; + /** + * Non-zero if streaming (de)compression is used. + */ + int streaming; + /** + * The dictionary ID. + */ + unsigned dictionaryID; + /** + * Is the dictionary cold? + * Only set on decompression. + */ + int dictionaryIsCold; + /** + * The dictionary size or zero if no dictionary. + */ + size_t dictionarySize; + /** + * The uncompressed size of the data. + */ + size_t uncompressedSize; + /** + * The compressed size of the data. + */ + size_t compressedSize; + /** + * The fully resolved CCtx parameters (NULL on decompression). + */ + struct ZSTD_CCtx_params_s const* params; + /** + * The ZSTD_CCtx pointer (NULL on decompression). + */ + struct ZSTD_CCtx_s const* cctx; + /** + * The ZSTD_DCtx pointer (NULL on compression). + */ + struct ZSTD_DCtx_s const* dctx; +} ZSTD_Trace; /** - * HUF_sort(): - * Sorts the symbols [0, maxSymbolValue] by count[symbol] in decreasing order. + * A tracing context. It must be 0 when tracing is disabled. + * Otherwise, any non-zero value returned by a tracing begin() + * function is presented to any subsequent calls to end(). * - * @param[out] huffNode Sorted symbols by decreasing count. Only members `.count` and `.byte` are filled. - * Must have (maxSymbolValue + 1) entries. - * @param[in] count Histogram of the symbols. - * @param[in] maxSymbolValue Maximum symbol value. - * @param rankPosition This is a scratch workspace. Must have RANK_POSITION_TABLE_SIZE entries. + * Any non-zero value is treated as tracing is enabled and not + * interpreted by the library. + * + * Two possible uses are: + * * A timestamp for when the begin() function was called. + * * A unique key identifying the (de)compression, like the + * address of the [dc]ctx pointer if you need to track + * more information than just a timestamp. */ -static void HUF_sort(nodeElt* huffNode, const unsigned* count, U32 maxSymbolValue, rankPos* rankPosition) -{ - int n; - int const maxSymbolValue1 = (int)maxSymbolValue + 1; +typedef unsigned long long ZSTD_TraceCtx; - /* Compute base and set curr to base. - * For symbol s let lowerRank = BIT_highbit32(count[n]+1) and rank = lowerRank + 1. - * Then 2^lowerRank <= count[n]+1 <= 2^rank. - * We attribute each symbol to lowerRank's base value, because we want to know where - * each rank begins in the output, so for rank R we want to count ranks R+1 and above. - */ - ZSTD_memset(rankPosition, 0, sizeof(*rankPosition) * RANK_POSITION_TABLE_SIZE); - for (n = 0; n < maxSymbolValue1; ++n) { - U32 lowerRank = BIT_highbit32(count[n] + 1); - rankPosition[lowerRank].base++; - } - assert(rankPosition[RANK_POSITION_TABLE_SIZE - 1].base == 0); - for (n = RANK_POSITION_TABLE_SIZE - 1; n > 0; --n) { - rankPosition[n-1].base += rankPosition[n].base; - rankPosition[n-1].curr = rankPosition[n-1].base; - } - /* Sort */ - for (n = 0; n < maxSymbolValue1; ++n) { - U32 const c = count[n]; - U32 const r = BIT_highbit32(c+1) + 1; - U32 pos = rankPosition[r].curr++; - /* Insert into the correct position in the rank. - * We have at most 256 symbols, so this insertion should be fine. - */ - while ((pos > rankPosition[r].base) && (c > huffNode[pos-1].count)) { - huffNode[pos] = huffNode[pos-1]; - pos--; - } - huffNode[pos].count = c; - huffNode[pos].byte = (BYTE)n; - } -} +/** + * Trace the beginning of a compression call. + * @param cctx The dctx pointer for the compression. + * It can be used as a key to map begin() to end(). + * @returns Non-zero if tracing is enabled. The return value is + * passed to ZSTD_trace_compress_end(). + */ +ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_compress_begin( + struct ZSTD_CCtx_s const* cctx); +/** + * Trace the end of a compression call. + * @param ctx The return value of ZSTD_trace_compress_begin(). + * @param trace The zstd tracing info. + */ +ZSTD_WEAK_ATTR void ZSTD_trace_compress_end( + ZSTD_TraceCtx ctx, + ZSTD_Trace const* trace); -/** HUF_buildCTable_wksp() : - * Same as HUF_buildCTable(), but using externally allocated scratch buffer. - * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as sizeof(HUF_buildCTable_wksp_tables). +/** + * Trace the beginning of a decompression call. + * @param dctx The dctx pointer for the decompression. + * It can be used as a key to map begin() to end(). + * @returns Non-zero if tracing is enabled. The return value is + * passed to ZSTD_trace_compress_end(). */ -#define STARTNODE (HUF_SYMBOLVALUE_MAX+1) +ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_decompress_begin( + struct ZSTD_DCtx_s const* dctx); -/* HUF_buildTree(): - * Takes the huffNode array sorted by HUF_sort() and builds an unlimited-depth Huffman tree. - * - * @param huffNode The array sorted by HUF_sort(). Builds the Huffman tree in this array. - * @param maxSymbolValue The maximum symbol value. - * @return The smallest node in the Huffman tree (by count). +/** + * Trace the end of a decompression call. + * @param ctx The return value of ZSTD_trace_decompress_begin(). + * @param trace The zstd tracing info. */ -static int HUF_buildTree(nodeElt* huffNode, U32 maxSymbolValue) -{ - nodeElt* const huffNode0 = huffNode - 1; - int nonNullRank; - int lowS, lowN; - int nodeNb = STARTNODE; - int n, nodeRoot; - /* init for parents */ - nonNullRank = (int)maxSymbolValue; - while(huffNode[nonNullRank].count == 0) nonNullRank--; - lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb; - huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count; - huffNode[lowS].parent = huffNode[lowS-1].parent = (U16)nodeNb; - nodeNb++; lowS-=2; - for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30); - huffNode0[0].count = (U32)(1U<<31); /* fake entry, strong barrier */ +ZSTD_WEAK_ATTR void ZSTD_trace_decompress_end( + ZSTD_TraceCtx ctx, + ZSTD_Trace const* trace); - /* create parents */ - while (nodeNb <= nodeRoot) { - int const n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; - int const n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; - huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count; - huffNode[n1].parent = huffNode[n2].parent = (U16)nodeNb; - nodeNb++; - } +#endif /* ZSTD_TRACE */ - /* distribute weights (unlimited tree height) */ - huffNode[nodeRoot].nbBits = 0; - for (n=nodeRoot-1; n>=STARTNODE; n--) - huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; - for (n=0; n<=nonNullRank; n++) - huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; +#endif /* ZSTD_TRACE_H */ +/**** ended inlining zstd_trace.h ****/ +#else +# define ZSTD_TRACE 0 +#endif - return nonNullRank; -} +/* ---- static assert (debug) --- */ +#define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) +#define ZSTD_isError ERR_isError /* for inlining */ +#define FSE_isError ERR_isError +#define HUF_isError ERR_isError -/** - * HUF_buildCTableFromTree(): - * Build the CTable given the Huffman tree in huffNode. - * - * @param[out] CTable The output Huffman CTable. - * @param huffNode The Huffman tree. - * @param nonNullRank The last and smallest node in the Huffman tree. - * @param maxSymbolValue The maximum symbol value. - * @param maxNbBits The exact maximum number of bits used in the Huffman tree. - */ -static void HUF_buildCTableFromTree(HUF_CElt* CTable, nodeElt const* huffNode, int nonNullRank, U32 maxSymbolValue, U32 maxNbBits) -{ - /* fill result into ctable (val, nbBits) */ - int n; - U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0}; - U16 valPerRank[HUF_TABLELOG_MAX+1] = {0}; - int const alphabetSize = (int)(maxSymbolValue + 1); - for (n=0; n<=nonNullRank; n++) - nbPerRank[huffNode[n].nbBits]++; - /* determine starting value per rank */ - { U16 min = 0; - for (n=(int)maxNbBits; n>0; n--) { - valPerRank[n] = min; /* get starting value within each rank */ - min += nbPerRank[n]; - min >>= 1; - } } - for (n=0; nhuffNodeTbl; - nodeElt* const huffNode = huffNode0+1; - int nonNullRank; +/*-************************************* +* shared macros +***************************************/ +#undef MIN +#undef MAX +#define MIN(a,b) ((a)<(b) ? (a) : (b)) +#define MAX(a,b) ((a)>(b) ? (a) : (b)) +#define BOUNDED(min,val,max) (MAX(min,MIN(val,max))) - /* safety checks */ - if (((size_t)workSpace & 3) != 0) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */ - if (wkspSize < sizeof(HUF_buildCTable_wksp_tables)) - return ERROR(workSpace_tooSmall); - if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT; - if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) - return ERROR(maxSymbolValue_tooLarge); - ZSTD_memset(huffNode0, 0, sizeof(huffNodeTable)); - /* sort, decreasing order */ - HUF_sort(huffNode, count, maxSymbolValue, wksp_tables->rankPosition); +/*-************************************* +* Common constants +***************************************/ +#define ZSTD_OPT_NUM (1<<12) - /* build tree */ - nonNullRank = HUF_buildTree(huffNode, maxSymbolValue); +#define ZSTD_REP_NUM 3 /* number of repcodes */ +static UNUSED_ATTR const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 }; - /* enforce maxTableLog */ - maxNbBits = HUF_setMaxHeight(huffNode, (U32)nonNullRank, maxNbBits); - if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */ +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) - HUF_buildCTableFromTree(tree, huffNode, nonNullRank, maxSymbolValue, maxNbBits); +#define BIT7 128 +#define BIT6 64 +#define BIT5 32 +#define BIT4 16 +#define BIT1 2 +#define BIT0 1 - return maxNbBits; -} +#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10 +static UNUSED_ATTR const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 }; +static UNUSED_ATTR const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 }; -size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) -{ - size_t nbBits = 0; - int s; - for (s = 0; s <= (int)maxSymbolValue; ++s) { - nbBits += CTable[s].nbBits * count[s]; - } - return nbBits >> 3; -} +#define ZSTD_FRAMEIDSIZE 4 /* magic number size */ -int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) { - int bad = 0; - int s; - for (s = 0; s <= (int)maxSymbolValue; ++s) { - bad |= (count[s] != 0) & (CTable[s].nbBits == 0); - } - return !bad; -} +#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ +static UNUSED_ATTR const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; +typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e; -size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); } +#define ZSTD_FRAMECHECKSUMSIZE 4 -FORCE_INLINE_TEMPLATE void -HUF_encodeSymbol(BIT_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable) -{ - BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits); -} +#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */ +#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */) /* for a non-null block */ +#define MIN_LITERALS_FOR_4_STREAMS 6 -#define HUF_FLUSHBITS(s) BIT_flushBits(s) +typedef enum { set_basic, set_rle, set_compressed, set_repeat } SymbolEncodingType_e; -#define HUF_FLUSHBITS_1(stream) \ - if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*2+7) HUF_FLUSHBITS(stream) +#define LONGNBSEQ 0x7F00 -#define HUF_FLUSHBITS_2(stream) \ - if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*4+7) HUF_FLUSHBITS(stream) +#define MINMATCH 3 -FORCE_INLINE_TEMPLATE size_t -HUF_compress1X_usingCTable_internal_body(void* dst, size_t dstSize, - const void* src, size_t srcSize, - const HUF_CElt* CTable) -{ - const BYTE* ip = (const BYTE*) src; - BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = ostart + dstSize; - BYTE* op = ostart; - size_t n; - BIT_CStream_t bitC; +#define Litbits 8 +#define LitHufLog 11 +#define MaxLit ((1<0; n-=4) { /* note : n&3==0 at this stage */ - HUF_encodeSymbol(&bitC, ip[n- 1], CTable); - HUF_FLUSHBITS_1(&bitC); - HUF_encodeSymbol(&bitC, ip[n- 2], CTable); - HUF_FLUSHBITS_2(&bitC); - HUF_encodeSymbol(&bitC, ip[n- 3], CTable); - HUF_FLUSHBITS_1(&bitC); - HUF_encodeSymbol(&bitC, ip[n- 4], CTable); - HUF_FLUSHBITS(&bitC); - } +static UNUSED_ATTR const U8 ML_bits[MaxML+1] = { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 2, 2, 3, 3, + 4, 4, 5, 7, 8, 9,10,11, + 12,13,14,15,16 +}; +static UNUSED_ATTR const S16 ML_defaultNorm[MaxML+1] = { + 1, 4, 3, 2, 2, 2, 2, 2, + 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1,-1,-1, + -1,-1,-1,-1,-1 +}; +#define ML_DEFAULTNORMLOG 6 /* for static allocation */ +static UNUSED_ATTR const U32 ML_defaultNormLog = ML_DEFAULTNORMLOG; - return BIT_closeCStream(&bitC); -} +static UNUSED_ATTR const S16 OF_defaultNorm[DefaultMaxOff+1] = { + 1, 1, 1, 1, 1, 1, 2, 2, + 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + -1,-1,-1,-1,-1 +}; +#define OF_DEFAULTNORMLOG 5 /* for static allocation */ +static UNUSED_ATTR const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG; -#if DYNAMIC_BMI2 -static TARGET_ATTRIBUTE("bmi2") size_t -HUF_compress1X_usingCTable_internal_bmi2(void* dst, size_t dstSize, - const void* src, size_t srcSize, - const HUF_CElt* CTable) -{ - return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable); +/*-******************************************* +* Shared functions to include for inlining +*********************************************/ +static void ZSTD_copy8(void* dst, const void* src) { +#if defined(ZSTD_ARCH_ARM_NEON) + vst1_u8((uint8_t*)dst, vld1_u8((const uint8_t*)src)); +#else + ZSTD_memcpy(dst, src, 8); +#endif } +#define COPY8(d,s) do { ZSTD_copy8(d,s); d+=8; s+=8; } while (0) -static size_t -HUF_compress1X_usingCTable_internal_default(void* dst, size_t dstSize, - const void* src, size_t srcSize, - const HUF_CElt* CTable) -{ - return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable); +/* Need to use memmove here since the literal buffer can now be located within + the dst buffer. In circumstances where the op "catches up" to where the + literal buffer is, there can be partial overlaps in this call on the final + copy if the literal is being shifted by less than 16 bytes. */ +static void ZSTD_copy16(void* dst, const void* src) { +#if defined(ZSTD_ARCH_ARM_NEON) + vst1q_u8((uint8_t*)dst, vld1q_u8((const uint8_t*)src)); +#elif defined(ZSTD_ARCH_X86_SSE2) + _mm_storeu_si128((__m128i*)dst, _mm_loadu_si128((const __m128i*)src)); +#elif defined(__clang__) + ZSTD_memmove(dst, src, 16); +#else + /* ZSTD_memmove is not inlined properly by gcc */ + BYTE copy16_buf[16]; + ZSTD_memcpy(copy16_buf, src, 16); + ZSTD_memcpy(dst, copy16_buf, 16); +#endif } +#define COPY16(d,s) do { ZSTD_copy16(d,s); d+=16; s+=16; } while (0) -static size_t -HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize, - const void* src, size_t srcSize, - const HUF_CElt* CTable, const int bmi2) -{ - if (bmi2) { - return HUF_compress1X_usingCTable_internal_bmi2(dst, dstSize, src, srcSize, CTable); - } - return HUF_compress1X_usingCTable_internal_default(dst, dstSize, src, srcSize, CTable); -} +#define WILDCOPY_OVERLENGTH 32 +#define WILDCOPY_VECLEN 16 -#else +typedef enum { + ZSTD_no_overlap, + ZSTD_overlap_src_before_dst + /* ZSTD_overlap_dst_before_src, */ +} ZSTD_overlap_e; -static size_t -HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize, - const void* src, size_t srcSize, - const HUF_CElt* CTable, const int bmi2) +/*! ZSTD_wildcopy() : + * Custom version of ZSTD_memcpy(), can over read/write up to WILDCOPY_OVERLENGTH bytes (if length==0) + * @param ovtype controls the overlap detection + * - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart. + * - ZSTD_overlap_src_before_dst: The src and dst may overlap, but they MUST be at least 8 bytes apart. + * The src buffer must be before the dst buffer. + */ +MEM_STATIC FORCE_INLINE_ATTR +void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e const ovtype) { - (void)bmi2; - return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable); -} + ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src; + const BYTE* ip = (const BYTE*)src; + BYTE* op = (BYTE*)dst; + BYTE* const oend = op + length; -#endif + if (ovtype == ZSTD_overlap_src_before_dst && diff < WILDCOPY_VECLEN) { + /* Handle short offset copies. */ + do { + COPY8(op, ip); + } while (op < oend); + } else { + assert(diff >= WILDCOPY_VECLEN || diff <= -WILDCOPY_VECLEN); + /* Separate out the first COPY16() call because the copy length is + * almost certain to be short, so the branches have different + * probabilities. Since it is almost certain to be short, only do + * one COPY16() in the first call. Then, do two calls per loop since + * at that point it is more likely to have a high trip count. + */ + ZSTD_copy16(op, ip); + if (16 >= length) return; + op += 16; + ip += 16; + do { + COPY16(op, ip); + COPY16(op, ip); + } + while (op < oend); + } +} -size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) +MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) { - return HUF_compress1X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, /* bmi2 */ 0); + size_t const length = MIN(dstCapacity, srcSize); + if (length > 0) { + ZSTD_memcpy(dst, src, length); + } + return length; } +/* define "workspace is too large" as this number of times larger than needed */ +#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 -static size_t -HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize, - const void* src, size_t srcSize, - const HUF_CElt* CTable, int bmi2) -{ - size_t const segmentSize = (srcSize+3)/4; /* first 3 segments */ - const BYTE* ip = (const BYTE*) src; - const BYTE* const iend = ip + srcSize; - BYTE* const ostart = (BYTE*) dst; - BYTE* const oend = ostart + dstSize; - BYTE* op = ostart; - - if (dstSize < 6 + 1 + 1 + 1 + 8) return 0; /* minimum space to compress successfully */ - if (srcSize < 12) return 0; /* no saving possible : too small input */ - op += 6; /* jumpTable */ - - assert(op <= oend); - { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, bmi2) ); - if (cSize==0) return 0; - assert(cSize <= 65535); - MEM_writeLE16(ostart, (U16)cSize); - op += cSize; - } - - ip += segmentSize; - assert(op <= oend); - { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, bmi2) ); - if (cSize==0) return 0; - assert(cSize <= 65535); - MEM_writeLE16(ostart+2, (U16)cSize); - op += cSize; - } - - ip += segmentSize; - assert(op <= oend); - { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, bmi2) ); - if (cSize==0) return 0; - assert(cSize <= 65535); - MEM_writeLE16(ostart+4, (U16)cSize); - op += cSize; - } - - ip += segmentSize; - assert(op <= oend); - assert(ip <= iend); - { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, (size_t)(iend-ip), CTable, bmi2) ); - if (cSize==0) return 0; - op += cSize; - } - - return (size_t)(op-ostart); -} +/* when workspace is continuously too large + * during at least this number of times, + * context's memory usage is considered wasteful, + * because it's sized to handle a worst case scenario which rarely happens. + * In which case, resize it down to free some memory */ +#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 -size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) -{ - return HUF_compress4X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, /* bmi2 */ 0); -} +/* Controls whether the input/output buffer is buffered or stable. */ +typedef enum { + ZSTD_bm_buffered = 0, /* Buffer the input/output */ + ZSTD_bm_stable = 1 /* ZSTD_inBuffer/ZSTD_outBuffer is stable */ +} ZSTD_bufferMode_e; -typedef enum { HUF_singleStream, HUF_fourStreams } HUF_nbStreams_e; -static size_t HUF_compressCTable_internal( - BYTE* const ostart, BYTE* op, BYTE* const oend, - const void* src, size_t srcSize, - HUF_nbStreams_e nbStreams, const HUF_CElt* CTable, const int bmi2) -{ - size_t const cSize = (nbStreams==HUF_singleStream) ? - HUF_compress1X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, bmi2) : - HUF_compress4X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, bmi2); - if (HUF_isError(cSize)) { return cSize; } - if (cSize==0) { return 0; } /* uncompressible */ - op += cSize; - /* check compressibility */ - assert(op >= ostart); - if ((size_t)(op-ostart) >= srcSize-1) { return 0; } - return (size_t)(op-ostart); -} +/*-******************************************* +* Private declarations +*********************************************/ +/** + * Contains the compressed frame size and an upper-bound for the decompressed frame size. + * Note: before using `compressedSize`, check for errors using ZSTD_isError(). + * similarly, before using `decompressedBound`, check for errors using: + * `decompressedBound != ZSTD_CONTENTSIZE_ERROR` + */ typedef struct { - unsigned count[HUF_SYMBOLVALUE_MAX + 1]; - HUF_CElt CTable[HUF_SYMBOLVALUE_MAX + 1]; - HUF_buildCTable_wksp_tables buildCTable_wksp; -} HUF_compress_tables_t; - -/* HUF_compress_internal() : - * `workSpace_align4` must be aligned on 4-bytes boundaries, - * and occupies the same space as a table of HUF_WORKSPACE_SIZE_U32 unsigned */ -static size_t -HUF_compress_internal (void* dst, size_t dstSize, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned huffLog, - HUF_nbStreams_e nbStreams, - void* workSpace_align4, size_t wkspSize, - HUF_CElt* oldHufTable, HUF_repeat* repeat, int preferRepeat, - const int bmi2) -{ - HUF_compress_tables_t* const table = (HUF_compress_tables_t*)workSpace_align4; - BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = ostart + dstSize; - BYTE* op = ostart; - - HUF_STATIC_ASSERT(sizeof(*table) <= HUF_WORKSPACE_SIZE); - assert(((size_t)workSpace_align4 & 3) == 0); /* must be aligned on 4-bytes boundaries */ - - /* checks & inits */ - if (wkspSize < HUF_WORKSPACE_SIZE) return ERROR(workSpace_tooSmall); - if (!srcSize) return 0; /* Uncompressed */ - if (!dstSize) return 0; /* cannot fit anything within dst budget */ - if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); /* current block size limit */ - if (huffLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); - if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge); - if (!maxSymbolValue) maxSymbolValue = HUF_SYMBOLVALUE_MAX; - if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT; + size_t nbBlocks; + size_t compressedSize; + unsigned long long decompressedBound; +} ZSTD_frameSizeInfo; /* decompress & legacy */ - /* Heuristic : If old table is valid, use it for small inputs */ - if (preferRepeat && repeat && *repeat == HUF_repeat_valid) { - return HUF_compressCTable_internal(ostart, op, oend, - src, srcSize, - nbStreams, oldHufTable, bmi2); - } +/* ZSTD_invalidateRepCodes() : + * ensures next compression will not use repcodes from previous block. + * Note : only works with regular variant; + * do not use with extDict variant ! */ +void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx); /* zstdmt, adaptive_compression (shouldn't get this definition from here) */ - /* Scan input and build symbol stats */ - { CHECK_V_F(largest, HIST_count_wksp (table->count, &maxSymbolValue, (const BYTE*)src, srcSize, workSpace_align4, wkspSize) ); - if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */ - if (largest <= (srcSize >> 7)+4) return 0; /* heuristic : probably not compressible enough */ - } - /* Check validity of previous table */ - if ( repeat - && *repeat == HUF_repeat_check - && !HUF_validateCTable(oldHufTable, table->count, maxSymbolValue)) { - *repeat = HUF_repeat_none; - } - /* Heuristic : use existing table for small inputs */ - if (preferRepeat && repeat && *repeat != HUF_repeat_none) { - return HUF_compressCTable_internal(ostart, op, oend, - src, srcSize, - nbStreams, oldHufTable, bmi2); - } +typedef struct { + blockType_e blockType; + U32 lastBlock; + U32 origSize; +} blockProperties_t; /* declared here for decompress and fullbench */ - /* Build Huffman Tree */ - huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue); - { size_t const maxBits = HUF_buildCTable_wksp(table->CTable, table->count, - maxSymbolValue, huffLog, - &table->buildCTable_wksp, sizeof(table->buildCTable_wksp)); - CHECK_F(maxBits); - huffLog = (U32)maxBits; - /* Zero unused symbols in CTable, so we can check it for validity */ - ZSTD_memset(table->CTable + (maxSymbolValue + 1), 0, - sizeof(table->CTable) - ((maxSymbolValue + 1) * sizeof(HUF_CElt))); - } +/*! ZSTD_getcBlockSize() : + * Provides the size of compressed block from block header `src` */ +/* Used by: decompress, fullbench */ +size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, + blockProperties_t* bpPtr); - /* Write table description header */ - { CHECK_V_F(hSize, HUF_writeCTable (op, dstSize, table->CTable, maxSymbolValue, huffLog) ); - /* Check if using previous huffman table is beneficial */ - if (repeat && *repeat != HUF_repeat_none) { - size_t const oldSize = HUF_estimateCompressedSize(oldHufTable, table->count, maxSymbolValue); - size_t const newSize = HUF_estimateCompressedSize(table->CTable, table->count, maxSymbolValue); - if (oldSize <= hSize + newSize || hSize + 12 >= srcSize) { - return HUF_compressCTable_internal(ostart, op, oend, - src, srcSize, - nbStreams, oldHufTable, bmi2); - } } +/*! ZSTD_decodeSeqHeaders() : + * decode sequence header from src */ +/* Used by: zstd_decompress_block, fullbench */ +size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, + const void* src, size_t srcSize); - /* Use the new huffman table */ - if (hSize + 12ul >= srcSize) { return 0; } - op += hSize; - if (repeat) { *repeat = HUF_repeat_none; } - if (oldHufTable) - ZSTD_memcpy(oldHufTable, table->CTable, sizeof(table->CTable)); /* Save new table */ - } - return HUF_compressCTable_internal(ostart, op, oend, - src, srcSize, - nbStreams, table->CTable, bmi2); +/** + * @returns true iff the CPU supports dynamic BMI2 dispatch. + */ +MEM_STATIC int ZSTD_cpuSupportsBmi2(void) +{ + ZSTD_cpuid_t cpuid = ZSTD_cpuid(); + return ZSTD_cpuid_bmi1(cpuid) && ZSTD_cpuid_bmi2(cpuid); } +#endif /* ZSTD_CCOMMON_H_MODULE */ +/**** ended inlining zstd_internal.h ****/ -size_t HUF_compress1X_wksp (void* dst, size_t dstSize, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned huffLog, - void* workSpace, size_t wkspSize) -{ - return HUF_compress_internal(dst, dstSize, src, srcSize, - maxSymbolValue, huffLog, HUF_singleStream, - workSpace, wkspSize, - NULL, NULL, 0, 0 /*bmi2*/); -} -size_t HUF_compress1X_repeat (void* dst, size_t dstSize, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned huffLog, - void* workSpace, size_t wkspSize, - HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2) -{ - return HUF_compress_internal(dst, dstSize, src, srcSize, - maxSymbolValue, huffLog, HUF_singleStream, - workSpace, wkspSize, hufTable, - repeat, preferRepeat, bmi2); -} +/*-**************************************** +* Version +******************************************/ +unsigned ZSTD_versionNumber(void) { return ZSTD_VERSION_NUMBER; } -/* HUF_compress4X_repeat(): - * compress input using 4 streams. - * provide workspace to generate compression tables */ -size_t HUF_compress4X_wksp (void* dst, size_t dstSize, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned huffLog, - void* workSpace, size_t wkspSize) -{ - return HUF_compress_internal(dst, dstSize, src, srcSize, - maxSymbolValue, huffLog, HUF_fourStreams, - workSpace, wkspSize, - NULL, NULL, 0, 0 /*bmi2*/); -} +const char* ZSTD_versionString(void) { return ZSTD_VERSION_STRING; } -/* HUF_compress4X_repeat(): - * compress input using 4 streams. - * re-use an existing huffman compression table */ -size_t HUF_compress4X_repeat (void* dst, size_t dstSize, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned huffLog, - void* workSpace, size_t wkspSize, - HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2) -{ - return HUF_compress_internal(dst, dstSize, src, srcSize, - maxSymbolValue, huffLog, HUF_fourStreams, - workSpace, wkspSize, - hufTable, repeat, preferRepeat, bmi2); -} -#ifndef ZSTD_NO_UNUSED_FUNCTIONS -/** HUF_buildCTable() : - * @return : maxNbBits - * Note : count is used before tree is written, so they can safely overlap - */ -size_t HUF_buildCTable (HUF_CElt* tree, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits) -{ - HUF_buildCTable_wksp_tables workspace; - return HUF_buildCTable_wksp(tree, count, maxSymbolValue, maxNbBits, &workspace, sizeof(workspace)); -} +/*-**************************************** +* ZSTD Error Management +******************************************/ +#undef ZSTD_isError /* defined within zstd_internal.h */ +/*! ZSTD_isError() : + * tells if a return value is an error code + * symbol is required for external callers */ +unsigned ZSTD_isError(size_t code) { return ERR_isError(code); } -size_t HUF_compress1X (void* dst, size_t dstSize, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned huffLog) -{ - unsigned workSpace[HUF_WORKSPACE_SIZE_U32]; - return HUF_compress1X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace)); -} +/*! ZSTD_getErrorName() : + * provides error code string from function result (useful for debugging) */ +const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); } -size_t HUF_compress2 (void* dst, size_t dstSize, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned huffLog) -{ - unsigned workSpace[HUF_WORKSPACE_SIZE_U32]; - return HUF_compress4X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace)); -} +/*! ZSTD_getError() : + * convert a `size_t` function result into a proper ZSTD_errorCode enum */ +ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); } -size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize) -{ - return HUF_compress2(dst, maxDstSize, src, srcSize, 255, HUF_TABLELOG_DEFAULT); -} -#endif -/**** ended inlining compress/huf_compress.c ****/ -/**** start inlining compress/zstd_compress_literals.c ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. +/*! ZSTD_getErrorString() : + * provides error code string from enum */ +const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); } +/**** ended inlining common/zstd_common.c ****/ + +/**** start inlining compress/fse_compress.c ****/ +/* ****************************************************************** + * FSE : Finite State Entropy encoder + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c * * This source code is licensed under both the BSD-style license (found in the * LICENSE file in the root directory of this source tree) and the GPLv2 (found * in the COPYING file in the root directory of this source tree). * You may select, at your option, one of the above-listed licenses. - */ +****************************************************************** */ - /*-************************************* - * Dependencies - ***************************************/ -/**** start inlining zstd_compress_literals.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -#ifndef ZSTD_COMPRESS_LITERALS_H -#define ZSTD_COMPRESS_LITERALS_H - -/**** start inlining zstd_compress_internal.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. +/* ************************************************************** +* Includes +****************************************************************/ +/**** skipping file: ../common/compiler.h ****/ +/**** skipping file: ../common/mem.h ****/ +/**** skipping file: ../common/debug.h ****/ +/**** start inlining hist.h ****/ +/* ****************************************************************** + * hist : Histogram functions + * part of Finite State Entropy project + * Copyright (c) Meta Platforms, Inc. and affiliates. * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -/* This header contains definitions - * that shall **only** be used by modules within lib/compress. - */ - -#ifndef ZSTD_COMPRESS_H -#define ZSTD_COMPRESS_H - -/*-************************************* -* Dependencies -***************************************/ -/**** skipping file: ../common/zstd_internal.h ****/ -/**** start inlining ../common/zstd_trace.h ****/ -/* - * Copyright (c) 2016-2021, Facebook, Inc. - * All rights reserved. + * You can contact the author at : + * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c * * This source code is licensed under both the BSD-style license (found in the * LICENSE file in the root directory of this source tree) and the GPLv2 (found * in the COPYING file in the root directory of this source tree). * You may select, at your option, one of the above-listed licenses. - */ - -#ifndef ZSTD_TRACE_H -#define ZSTD_TRACE_H +****************************************************************** */ -#if defined (__cplusplus) -extern "C" { -#endif +/* --- dependencies --- */ +/**** skipping file: ../common/zstd_deps.h ****/ -#include -/* weak symbol support */ -#if !defined(ZSTD_HAVE_WEAK_SYMBOLS) && defined(__GNUC__) && \ - !defined(__APPLE__) && !defined(_WIN32) && !defined(__MINGW32__) && \ - !defined(__CYGWIN__) -# define ZSTD_HAVE_WEAK_SYMBOLS 1 -#else -# define ZSTD_HAVE_WEAK_SYMBOLS 0 -#endif -#if ZSTD_HAVE_WEAK_SYMBOLS -# define ZSTD_WEAK_ATTR __attribute__((__weak__)) -#else -# define ZSTD_WEAK_ATTR -#endif +/* --- simple histogram functions --- */ -/* Only enable tracing when weak symbols are available. */ -#ifndef ZSTD_TRACE -# define ZSTD_TRACE ZSTD_HAVE_WEAK_SYMBOLS -#endif +/*! HIST_count(): + * Provides the precise count of each byte within a table 'count'. + * 'count' is a table of unsigned int, of minimum size (*maxSymbolValuePtr+1). + * Updates *maxSymbolValuePtr with actual largest symbol value detected. + * @return : count of the most frequent symbol (which isn't identified). + * or an error code, which can be tested using HIST_isError(). + * note : if return == srcSize, there is only one symbol. + */ +size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize); -#if ZSTD_TRACE +unsigned HIST_isError(size_t code); /**< tells if a return value is an error code */ -struct ZSTD_CCtx_s; -struct ZSTD_DCtx_s; -struct ZSTD_CCtx_params_s; -typedef struct { - /** - * ZSTD_VERSION_NUMBER - * - * This is guaranteed to be the first member of ZSTD_trace. - * Otherwise, this struct is not stable between versions. If - * the version number does not match your expectation, you - * should not interpret the rest of the struct. - */ - unsigned version; - /** - * Non-zero if streaming (de)compression is used. - */ - unsigned streaming; - /** - * The dictionary ID. - */ - unsigned dictionaryID; - /** - * Is the dictionary cold? - * Only set on decompression. - */ - unsigned dictionaryIsCold; - /** - * The dictionary size or zero if no dictionary. - */ - size_t dictionarySize; - /** - * The uncompressed size of the data. - */ - size_t uncompressedSize; - /** - * The compressed size of the data. - */ - size_t compressedSize; - /** - * The fully resolved CCtx parameters (NULL on decompression). - */ - struct ZSTD_CCtx_params_s const* params; - /** - * The ZSTD_CCtx pointer (NULL on decompression). - */ - struct ZSTD_CCtx_s const* cctx; - /** - * The ZSTD_DCtx pointer (NULL on compression). - */ - struct ZSTD_DCtx_s const* dctx; -} ZSTD_Trace; +/* --- advanced histogram functions --- */ -/** - * A tracing context. It must be 0 when tracing is disabled. - * Otherwise, any non-zero value returned by a tracing begin() - * function is presented to any subsequent calls to end(). - * - * Any non-zero value is treated as tracing is enabled and not - * interpreted by the library. - * - * Two possible uses are: - * * A timestamp for when the begin() function was called. - * * A unique key identifying the (de)compression, like the - * address of the [dc]ctx pointer if you need to track - * more information than just a timestamp. +#define HIST_WKSP_SIZE_U32 1024 +#define HIST_WKSP_SIZE (HIST_WKSP_SIZE_U32 * sizeof(unsigned)) +/** HIST_count_wksp() : + * Same as HIST_count(), but using an externally provided scratch buffer. + * Benefit is this function will use very little stack space. + * `workSpace` is a writable buffer which must be 4-bytes aligned, + * `workSpaceSize` must be >= HIST_WKSP_SIZE */ -typedef unsigned long long ZSTD_TraceCtx; +size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize, + void* workSpace, size_t workSpaceSize); -/** - * Trace the beginning of a compression call. - * @param cctx The dctx pointer for the compression. - * It can be used as a key to map begin() to end(). - * @returns Non-zero if tracing is enabled. The return value is - * passed to ZSTD_trace_compress_end(). +/** HIST_countFast() : + * same as HIST_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr. + * This function is unsafe, and will segfault if any value within `src` is `> *maxSymbolValuePtr` */ -ZSTD_TraceCtx ZSTD_trace_compress_begin(struct ZSTD_CCtx_s const* cctx); +size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize); -/** - * Trace the end of a compression call. - * @param ctx The return value of ZSTD_trace_compress_begin(). - * @param trace The zstd tracing info. +/** HIST_countFast_wksp() : + * Same as HIST_countFast(), but using an externally provided scratch buffer. + * `workSpace` is a writable buffer which must be 4-bytes aligned, + * `workSpaceSize` must be >= HIST_WKSP_SIZE */ -void ZSTD_trace_compress_end( - ZSTD_TraceCtx ctx, - ZSTD_Trace const* trace); +size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize, + void* workSpace, size_t workSpaceSize); -/** - * Trace the beginning of a decompression call. - * @param dctx The dctx pointer for the decompression. - * It can be used as a key to map begin() to end(). - * @returns Non-zero if tracing is enabled. The return value is - * passed to ZSTD_trace_compress_end(). +/*! HIST_count_simple() : + * Same as HIST_countFast(), this function is unsafe, + * and will segfault if any value within `src` is `> *maxSymbolValuePtr`. + * It is also a bit slower for large inputs. + * However, it does not need any additional memory (not even on stack). + * @return : count of the most frequent symbol. + * Note this function doesn't produce any error (i.e. it must succeed). */ -ZSTD_TraceCtx ZSTD_trace_decompress_begin(struct ZSTD_DCtx_s const* dctx); +unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize); -/** - * Trace the end of a decompression call. - * @param ctx The return value of ZSTD_trace_decompress_begin(). - * @param trace The zstd tracing info. +/*! HIST_add() : + * Lowest level: just add nb of occurrences of characters from @src into @count. + * @count is not reset. @count array is presumed large enough (i.e. 1 KB). + @ This function does not need any additional stack memory. */ -void ZSTD_trace_decompress_end( - ZSTD_TraceCtx ctx, - ZSTD_Trace const* trace); - -#endif /* ZSTD_TRACE */ +void HIST_add(unsigned* count, const void* src, size_t srcSize); +/**** ended inlining hist.h ****/ +/**** skipping file: ../common/bitstream.h ****/ +#define FSE_STATIC_LINKING_ONLY +/**** skipping file: ../common/fse.h ****/ +/**** skipping file: ../common/error_private.h ****/ +#define ZSTD_DEPS_NEED_MALLOC +#define ZSTD_DEPS_NEED_MATH64 +/**** skipping file: ../common/zstd_deps.h ****/ +/**** skipping file: ../common/bits.h ****/ -#if defined (__cplusplus) -} -#endif -#endif /* ZSTD_TRACE_H */ -/**** ended inlining ../common/zstd_trace.h ****/ -/**** start inlining zstd_cwksp.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ +/* ************************************************************** +* Error Management +****************************************************************/ +#define FSE_isError ERR_isError -#ifndef ZSTD_CWKSP_H -#define ZSTD_CWKSP_H -/*-************************************* -* Dependencies -***************************************/ -/**** skipping file: ../common/zstd_internal.h ****/ +/* ************************************************************** +* Templates +****************************************************************/ +/* + designed to be included + for type-specific functions (template emulation in C) + Objective is to write these functions only once, for improved maintenance +*/ -#if defined (__cplusplus) -extern "C" { +/* safety checks */ +#ifndef FSE_FUNCTION_EXTENSION +# error "FSE_FUNCTION_EXTENSION must be defined" #endif - -/*-************************************* -* Constants -***************************************/ - -/* Since the workspace is effectively its own little malloc implementation / - * arena, when we run under ASAN, we should similarly insert redzones between - * each internal element of the workspace, so ASAN will catch overruns that - * reach outside an object but that stay inside the workspace. - * - * This defines the size of that redzone. - */ -#ifndef ZSTD_CWKSP_ASAN_REDZONE_SIZE -#define ZSTD_CWKSP_ASAN_REDZONE_SIZE 128 +#ifndef FSE_FUNCTION_TYPE +# error "FSE_FUNCTION_TYPE must be defined" #endif -/*-************************************* -* Structures -***************************************/ -typedef enum { - ZSTD_cwksp_alloc_objects, - ZSTD_cwksp_alloc_buffers, - ZSTD_cwksp_alloc_aligned -} ZSTD_cwksp_alloc_phase_e; +/* Function names */ +#define FSE_CAT(X,Y) X##Y +#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) +#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) -/** - * Used to describe whether the workspace is statically allocated (and will not - * necessarily ever be freed), or if it's dynamically allocated and we can - * expect a well-formed caller to free this. - */ -typedef enum { - ZSTD_cwksp_dynamic_alloc, - ZSTD_cwksp_static_alloc -} ZSTD_cwksp_static_alloc_e; -/** - * Zstd fits all its internal datastructures into a single continuous buffer, - * so that it only needs to perform a single OS allocation (or so that a buffer - * can be provided to it and it can perform no allocations at all). This buffer - * is called the workspace. - * - * Several optimizations complicate that process of allocating memory ranges - * from this workspace for each internal datastructure: - * - * - These different internal datastructures have different setup requirements: - * - * - The static objects need to be cleared once and can then be trivially - * reused for each compression. - * - * - Various buffers don't need to be initialized at all--they are always - * written into before they're read. - * - * - The matchstate tables have a unique requirement that they don't need - * their memory to be totally cleared, but they do need the memory to have - * some bound, i.e., a guarantee that all values in the memory they've been - * allocated is less than some maximum value (which is the starting value - * for the indices that they will then use for compression). When this - * guarantee is provided to them, they can use the memory without any setup - * work. When it can't, they have to clear the area. - * - * - These buffers also have different alignment requirements. - * - * - We would like to reuse the objects in the workspace for multiple - * compressions without having to perform any expensive reallocation or - * reinitialization work. - * - * - We would like to be able to efficiently reuse the workspace across - * multiple compressions **even when the compression parameters change** and - * we need to resize some of the objects (where possible). - * - * To attempt to manage this buffer, given these constraints, the ZSTD_cwksp - * abstraction was created. It works as follows: - * - * Workspace Layout: - * - * [ ... workspace ... ] - * [objects][tables ... ->] free space [<- ... aligned][<- ... buffers] - * - * The various objects that live in the workspace are divided into the - * following categories, and are allocated separately: - * - * - Static objects: this is optionally the enclosing ZSTD_CCtx or ZSTD_CDict, - * so that literally everything fits in a single buffer. Note: if present, - * this must be the first object in the workspace, since ZSTD_customFree{CCtx, - * CDict}() rely on a pointer comparison to see whether one or two frees are - * required. - * - * - Fixed size objects: these are fixed-size, fixed-count objects that are - * nonetheless "dynamically" allocated in the workspace so that we can - * control how they're initialized separately from the broader ZSTD_CCtx. - * Examples: - * - Entropy Workspace - * - 2 x ZSTD_compressedBlockState_t - * - CDict dictionary contents - * - * - Tables: these are any of several different datastructures (hash tables, - * chain tables, binary trees) that all respect a common format: they are - * uint32_t arrays, all of whose values are between 0 and (nextSrc - base). - * Their sizes depend on the cparams. - * - * - Aligned: these buffers are used for various purposes that require 4 byte - * alignment, but don't require any initialization before they're used. - * - * - Buffers: these buffers are used for various purposes that don't require - * any alignment or initialization before they're used. This means they can - * be moved around at no cost for a new compression. - * - * Allocating Memory: - * - * The various types of objects must be allocated in order, so they can be - * correctly packed into the workspace buffer. That order is: - * - * 1. Objects - * 2. Buffers - * 3. Aligned - * 4. Tables - * - * Attempts to reserve objects of different types out of order will fail. - */ -typedef struct { - void* workspace; - void* workspaceEnd; +/* Function templates */ - void* objectEnd; - void* tableEnd; - void* tableValidEnd; - void* allocStart; +/* FSE_buildCTable_wksp() : + * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). + * wkspSize should be sized to handle worst case situation, which is `1<>1 : 1) ; + FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT); + U32 const step = FSE_TABLESTEP(tableSize); + U32 const maxSV1 = maxSymbolValue+1; - BYTE allocFailed; - int workspaceOversizedDuration; - ZSTD_cwksp_alloc_phase_e phase; - ZSTD_cwksp_static_alloc_e isStatic; -} ZSTD_cwksp; + U16* cumul = (U16*)workSpace; /* size = maxSV1 */ + FSE_FUNCTION_TYPE* const tableSymbol = (FSE_FUNCTION_TYPE*)(cumul + (maxSV1+1)); /* size = tableSize */ -/*-************************************* -* Functions -***************************************/ + U32 highThreshold = tableSize-1; -MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws); + assert(((size_t)workSpace & 1) == 0); /* Must be 2 bytes-aligned */ + if (FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) > wkspSize) return ERROR(tableLog_tooLarge); + /* CTable header */ + tableU16[-2] = (U16) tableLog; + tableU16[-1] = (U16) maxSymbolValue; + assert(tableLog < 16); /* required for threshold strategy to work */ -MEM_STATIC void ZSTD_cwksp_assert_internal_consistency(ZSTD_cwksp* ws) { - (void)ws; - assert(ws->workspace <= ws->objectEnd); - assert(ws->objectEnd <= ws->tableEnd); - assert(ws->objectEnd <= ws->tableValidEnd); - assert(ws->tableEnd <= ws->allocStart); - assert(ws->tableValidEnd <= ws->allocStart); - assert(ws->allocStart <= ws->workspaceEnd); -} + /* For explanations on how to distribute symbol values over the table : + * https://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */ -/** - * Align must be a power of 2. - */ -MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t const align) { - size_t const mask = align - 1; - assert((align & mask) == 0); - return (size + mask) & ~mask; -} + #ifdef __clang_analyzer__ + ZSTD_memset(tableSymbol, 0, sizeof(*tableSymbol) * tableSize); /* useless initialization, just to keep scan-build happy */ + #endif -/** - * Use this to determine how much space in the workspace we will consume to - * allocate this object. (Normally it should be exactly the size of the object, - * but under special conditions, like ASAN, where we pad each object, it might - * be larger.) - * - * Since tables aren't currently redzoned, you don't need to call through this - * to figure out how much space you need for the matchState tables. Everything - * else is though. - */ -MEM_STATIC size_t ZSTD_cwksp_alloc_size(size_t size) { - if (size == 0) - return 0; -#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) - return size + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; -#else - return size; -#endif -} + /* symbol start positions */ + { U32 u; + cumul[0] = 0; + for (u=1; u <= maxSV1; u++) { + if (normalizedCounter[u-1]==-1) { /* Low proba symbol */ + cumul[u] = cumul[u-1] + 1; + tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u-1); + } else { + assert(normalizedCounter[u-1] >= 0); + cumul[u] = cumul[u-1] + (U16)normalizedCounter[u-1]; + assert(cumul[u] >= cumul[u-1]); /* no overflow */ + } } + cumul[maxSV1] = (U16)(tableSize+1); + } -MEM_STATIC void ZSTD_cwksp_internal_advance_phase( - ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase) { - assert(phase >= ws->phase); - if (phase > ws->phase) { - if (ws->phase < ZSTD_cwksp_alloc_buffers && - phase >= ZSTD_cwksp_alloc_buffers) { - ws->tableValidEnd = ws->objectEnd; + /* Spread symbols */ + if (highThreshold == tableSize - 1) { + /* Case for no low prob count symbols. Lay down 8 bytes at a time + * to reduce branch misses since we are operating on a small block + */ + BYTE* const spread = tableSymbol + tableSize; /* size = tableSize + 8 (may write beyond tableSize) */ + { U64 const add = 0x0101010101010101ull; + size_t pos = 0; + U64 sv = 0; + U32 s; + for (s=0; s=0); + pos += (size_t)n; + } } - if (ws->phase < ZSTD_cwksp_alloc_aligned && - phase >= ZSTD_cwksp_alloc_aligned) { - /* If unaligned allocations down from a too-large top have left us - * unaligned, we need to realign our alloc ptr. Technically, this - * can consume space that is unaccounted for in the neededSpace - * calculation. However, I believe this can only happen when the - * workspace is too large, and specifically when it is too large - * by a larger margin than the space that will be consumed. */ - /* TODO: cleaner, compiler warning friendly way to do this??? */ - ws->allocStart = (BYTE*)ws->allocStart - ((size_t)ws->allocStart & (sizeof(U32)-1)); - if (ws->allocStart < ws->tableValidEnd) { - ws->tableValidEnd = ws->allocStart; + /* Spread symbols across the table. Lack of lowprob symbols means that + * we don't need variable sized inner loop, so we can unroll the loop and + * reduce branch misses. + */ + { size_t position = 0; + size_t s; + size_t const unroll = 2; /* Experimentally determined optimal unroll */ + assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */ + for (s = 0; s < (size_t)tableSize; s += unroll) { + size_t u; + for (u = 0; u < unroll; ++u) { + size_t const uPosition = (position + (u * step)) & tableMask; + tableSymbol[uPosition] = spread[s + u]; + } + position = (position + (unroll * step)) & tableMask; } + assert(position == 0); /* Must have initialized all positions */ } - ws->phase = phase; + } else { + U32 position = 0; + U32 symbol; + for (symbol=0; symbol highThreshold) + position = (position + step) & tableMask; /* Low proba area */ + } } + assert(position==0); /* Must have initialized all positions */ } -} - -/** - * Returns whether this object/buffer/etc was allocated in this workspace. - */ -MEM_STATIC int ZSTD_cwksp_owns_buffer(const ZSTD_cwksp* ws, const void* ptr) { - return (ptr != NULL) && (ws->workspace <= ptr) && (ptr <= ws->workspaceEnd); -} - -/** - * Internal function. Do not use directly. - */ -MEM_STATIC void* ZSTD_cwksp_reserve_internal( - ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) { - void* alloc; - void* bottom = ws->tableEnd; - ZSTD_cwksp_internal_advance_phase(ws, phase); - alloc = (BYTE *)ws->allocStart - bytes; - if (bytes == 0) - return NULL; - -#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) - /* over-reserve space */ - alloc = (BYTE *)alloc - 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; -#endif + /* Build table */ + { U32 u; for (u=0; u= bottom); - if (alloc < bottom) { - DEBUGLOG(4, "cwksp: alloc failed!"); - ws->allocFailed = 1; - return NULL; - } - if (alloc < ws->tableValidEnd) { - ws->tableValidEnd = alloc; - } - ws->allocStart = alloc; + /* Build Symbol Transformation Table */ + { unsigned total = 0; + unsigned s; + for (s=0; s<=maxSymbolValue; s++) { + switch (normalizedCounter[s]) + { + case 0: + /* filling nonetheless, for compatibility with FSE_getMaxNbBits() */ + symbolTT[s].deltaNbBits = ((tableLog+1) << 16) - (1<isStatic == ZSTD_cwksp_dynamic_alloc) { - __asan_unpoison_memory_region(alloc, bytes); - } + case -1: + case 1: + symbolTT[s].deltaNbBits = (tableLog << 16) - (1< 1); + { U32 const maxBitsOut = tableLog - ZSTD_highbit32 ((U32)normalizedCounter[s]-1); + U32 const minStatePlus = (U32)normalizedCounter[s] << maxBitsOut; + symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus; + symbolTT[s].deltaFindState = (int)(total - (unsigned)normalizedCounter[s]); + total += (unsigned)normalizedCounter[s]; + } } } } + +#if 0 /* debug : symbol costs */ + DEBUGLOG(5, "\n --- table statistics : "); + { U32 symbol; + for (symbol=0; symbol<=maxSymbolValue; symbol++) { + DEBUGLOG(5, "%3u: w=%3i, maxBits=%u, fracBits=%.2f", + symbol, normalizedCounter[symbol], + FSE_getMaxNbBits(symbolTT, symbol), + (double)FSE_bitCost(symbolTT, tableLog, symbol, 8) / 256); + } } #endif - return alloc; + return 0; } -/** - * Reserves and returns unaligned memory. - */ -MEM_STATIC BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes) { - return (BYTE*)ZSTD_cwksp_reserve_internal(ws, bytes, ZSTD_cwksp_alloc_buffers); -} -/** - * Reserves and returns memory sized on and aligned on sizeof(unsigned). - */ -MEM_STATIC void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes) { - assert((bytes & (sizeof(U32)-1)) == 0); - return ZSTD_cwksp_reserve_internal(ws, ZSTD_cwksp_align(bytes, sizeof(U32)), ZSTD_cwksp_alloc_aligned); -} -/** - * Aligned on sizeof(unsigned). These buffers have the special property that - * their values remain constrained, allowing us to re-use them without - * memset()-ing them. - */ -MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { - const ZSTD_cwksp_alloc_phase_e phase = ZSTD_cwksp_alloc_aligned; - void* alloc = ws->tableEnd; - void* end = (BYTE *)alloc + bytes; - void* top = ws->allocStart; +#ifndef FSE_COMMONDEFS_ONLY - DEBUGLOG(5, "cwksp: reserving %p table %zd bytes, %zd bytes remaining", - alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes); - assert((bytes & (sizeof(U32)-1)) == 0); - ZSTD_cwksp_internal_advance_phase(ws, phase); - ZSTD_cwksp_assert_internal_consistency(ws); - assert(end <= top); - if (end > top) { - DEBUGLOG(4, "cwksp: table alloc failed!"); - ws->allocFailed = 1; - return NULL; - } - ws->tableEnd = end; +/*-************************************************************** +* FSE NCount encoding +****************************************************************/ +size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog) +{ + size_t const maxHeaderSize = (((maxSymbolValue+1) * tableLog + + 4 /* bitCount initialized at 4 */ + + 2 /* first two symbols may use one additional bit each */) / 8) + + 1 /* round up to whole nb bytes */ + + 2 /* additional two bytes for bitstream flush */; + return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */ +} -#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) - if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { - __asan_unpoison_memory_region(alloc, bytes); - } -#endif +static size_t +FSE_writeNCount_generic (void* header, size_t headerBufferSize, + const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, + unsigned writeIsSafe) +{ + BYTE* const ostart = (BYTE*) header; + BYTE* out = ostart; + BYTE* const oend = ostart + headerBufferSize; + int nbBits; + const int tableSize = 1 << tableLog; + int remaining; + int threshold; + U32 bitStream = 0; + int bitCount = 0; + unsigned symbol = 0; + unsigned const alphabetSize = maxSymbolValue + 1; + int previousIs0 = 0; - return alloc; -} + /* Table Size */ + bitStream += (tableLog-FSE_MIN_TABLELOG) << bitCount; + bitCount += 4; -/** - * Aligned on sizeof(void*). - */ -MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { - size_t roundedBytes = ZSTD_cwksp_align(bytes, sizeof(void*)); - void* alloc = ws->objectEnd; - void* end = (BYTE*)alloc + roundedBytes; + /* Init */ + remaining = tableSize+1; /* +1 for extra accuracy */ + threshold = tableSize; + nbBits = (int)tableLog+1; -#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) - /* over-reserve space */ - end = (BYTE *)end + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; -#endif + while ((symbol < alphabetSize) && (remaining>1)) { /* stops at 1 */ + if (previousIs0) { + unsigned start = symbol; + while ((symbol < alphabetSize) && !normalizedCounter[symbol]) symbol++; + if (symbol == alphabetSize) break; /* incorrect distribution */ + while (symbol >= start+24) { + start+=24; + bitStream += 0xFFFFU << bitCount; + if ((!writeIsSafe) && (out > oend-2)) + return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE) bitStream; + out[1] = (BYTE)(bitStream>>8); + out+=2; + bitStream>>=16; + } + while (symbol >= start+3) { + start+=3; + bitStream += 3U << bitCount; + bitCount += 2; + } + bitStream += (symbol-start) << bitCount; + bitCount += 2; + if (bitCount>16) { + if ((!writeIsSafe) && (out > oend - 2)) + return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream>>8); + out += 2; + bitStream >>= 16; + bitCount -= 16; + } } + { int count = normalizedCounter[symbol++]; + int const max = (2*threshold-1) - remaining; + remaining -= count < 0 ? -count : count; + count++; /* +1 for extra accuracy */ + if (count>=threshold) + count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */ + bitStream += (U32)count << bitCount; + bitCount += nbBits; + bitCount -= (count>=1; } + } + if (bitCount>16) { + if ((!writeIsSafe) && (out > oend - 2)) + return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream>>8); + out += 2; + bitStream >>= 16; + bitCount -= 16; + } } - DEBUGLOG(5, - "cwksp: reserving %p object %zd bytes (rounded to %zd), %zd bytes remaining", - alloc, bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes); - assert(((size_t)alloc & (sizeof(void*)-1)) == 0); - assert((bytes & (sizeof(void*)-1)) == 0); - ZSTD_cwksp_assert_internal_consistency(ws); - /* we must be in the first phase, no advance is possible */ - if (ws->phase != ZSTD_cwksp_alloc_objects || end > ws->workspaceEnd) { - DEBUGLOG(4, "cwksp: object alloc failed!"); - ws->allocFailed = 1; - return NULL; - } - ws->objectEnd = end; - ws->tableEnd = end; - ws->tableValidEnd = end; + if (remaining != 1) + return ERROR(GENERIC); /* incorrect normalized distribution */ + assert(symbol <= alphabetSize); -#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) - /* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on - * either size. */ - alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE; - if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { - __asan_unpoison_memory_region(alloc, bytes); - } -#endif + /* flush remaining bitStream */ + if ((!writeIsSafe) && (out > oend - 2)) + return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream>>8); + out+= (bitCount+7) /8; - return alloc; + assert(out >= ostart); + return (size_t)(out-ostart); } -MEM_STATIC void ZSTD_cwksp_mark_tables_dirty(ZSTD_cwksp* ws) { - DEBUGLOG(4, "cwksp: ZSTD_cwksp_mark_tables_dirty"); -#if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) - /* To validate that the table re-use logic is sound, and that we don't - * access table space that we haven't cleaned, we re-"poison" the table - * space every time we mark it dirty. */ - { - size_t size = (BYTE*)ws->tableValidEnd - (BYTE*)ws->objectEnd; - assert(__msan_test_shadow(ws->objectEnd, size) == -1); - __msan_poison(ws->objectEnd, size); - } -#endif +size_t FSE_writeNCount (void* buffer, size_t bufferSize, + const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) +{ + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported */ + if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported */ - assert(ws->tableValidEnd >= ws->objectEnd); - assert(ws->tableValidEnd <= ws->allocStart); - ws->tableValidEnd = ws->objectEnd; - ZSTD_cwksp_assert_internal_consistency(ws); + if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog)) + return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0); + + return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1 /* write in buffer is safe */); } -MEM_STATIC void ZSTD_cwksp_mark_tables_clean(ZSTD_cwksp* ws) { - DEBUGLOG(4, "cwksp: ZSTD_cwksp_mark_tables_clean"); - assert(ws->tableValidEnd >= ws->objectEnd); - assert(ws->tableValidEnd <= ws->allocStart); - if (ws->tableValidEnd < ws->tableEnd) { - ws->tableValidEnd = ws->tableEnd; - } - ZSTD_cwksp_assert_internal_consistency(ws); + +/*-************************************************************** +* FSE Compression Code +****************************************************************/ + +/* provides the minimum logSize to safely represent a distribution */ +static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue) +{ + U32 minBitsSrc = ZSTD_highbit32((U32)(srcSize)) + 1; + U32 minBitsSymbols = ZSTD_highbit32(maxSymbolValue) + 2; + U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols; + assert(srcSize > 1); /* Not supported, RLE should be used instead */ + return minBits; } -/** - * Zero the part of the allocated tables not already marked clean. - */ -MEM_STATIC void ZSTD_cwksp_clean_tables(ZSTD_cwksp* ws) { - DEBUGLOG(4, "cwksp: ZSTD_cwksp_clean_tables"); - assert(ws->tableValidEnd >= ws->objectEnd); - assert(ws->tableValidEnd <= ws->allocStart); - if (ws->tableValidEnd < ws->tableEnd) { - ZSTD_memset(ws->tableValidEnd, 0, (BYTE*)ws->tableEnd - (BYTE*)ws->tableValidEnd); - } - ZSTD_cwksp_mark_tables_clean(ws); +unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus) +{ + U32 maxBitsSrc = ZSTD_highbit32((U32)(srcSize - 1)) - minus; + U32 tableLog = maxTableLog; + U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue); + assert(srcSize > 1); /* Not supported, RLE should be used instead */ + if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; + if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */ + if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */ + if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG; + if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG; + return tableLog; } -/** - * Invalidates table allocations. - * All other allocations remain valid. - */ -MEM_STATIC void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) { - DEBUGLOG(4, "cwksp: clearing tables!"); +unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) +{ + return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2); +} -#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) - /* We don't do this when the workspace is statically allocated, because - * when that is the case, we have no capability to hook into the end of the - * workspace's lifecycle to unpoison the memory. - */ - if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { - size_t size = (BYTE*)ws->tableValidEnd - (BYTE*)ws->objectEnd; - __asan_poison_memory_region(ws->objectEnd, size); - } -#endif +/* Secondary normalization method. + To be used when primary method fails. */ - ws->tableEnd = ws->objectEnd; - ZSTD_cwksp_assert_internal_consistency(ws); -} +static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue, short lowProbCount) +{ + short const NOT_YET_ASSIGNED = -2; + U32 s; + U32 distributed = 0; + U32 ToDistribute; -/** - * Invalidates all buffer, aligned, and table allocations. - * Object allocations remain valid. - */ -MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) { - DEBUGLOG(4, "cwksp: clearing!"); + /* Init */ + U32 const lowThreshold = (U32)(total >> tableLog); + U32 lowOne = (U32)((total * 3) >> (tableLog + 1)); -#if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) - /* To validate that the context re-use logic is sound, and that we don't - * access stuff that this compression hasn't initialized, we re-"poison" - * the workspace (or at least the non-static, non-table parts of it) - * every time we start a new compression. */ - { - size_t size = (BYTE*)ws->workspaceEnd - (BYTE*)ws->tableValidEnd; - __msan_poison(ws->tableValidEnd, size); + for (s=0; s<=maxSymbolValue; s++) { + if (count[s] == 0) { + norm[s]=0; + continue; + } + if (count[s] <= lowThreshold) { + norm[s] = lowProbCount; + distributed++; + total -= count[s]; + continue; + } + if (count[s] <= lowOne) { + norm[s] = 1; + distributed++; + total -= count[s]; + continue; + } + + norm[s]=NOT_YET_ASSIGNED; } -#endif + ToDistribute = (1 << tableLog) - distributed; -#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) - /* We don't do this when the workspace is statically allocated, because - * when that is the case, we have no capability to hook into the end of the - * workspace's lifecycle to unpoison the memory. - */ - if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { - size_t size = (BYTE*)ws->workspaceEnd - (BYTE*)ws->objectEnd; - __asan_poison_memory_region(ws->objectEnd, size); + if (ToDistribute == 0) + return 0; + + if ((total / ToDistribute) > lowOne) { + /* risk of rounding to zero */ + lowOne = (U32)((total * 3) / (ToDistribute * 2)); + for (s=0; s<=maxSymbolValue; s++) { + if ((norm[s] == NOT_YET_ASSIGNED) && (count[s] <= lowOne)) { + norm[s] = 1; + distributed++; + total -= count[s]; + continue; + } } + ToDistribute = (1 << tableLog) - distributed; } -#endif - ws->tableEnd = ws->objectEnd; - ws->allocStart = ws->workspaceEnd; - ws->allocFailed = 0; - if (ws->phase > ZSTD_cwksp_alloc_buffers) { - ws->phase = ZSTD_cwksp_alloc_buffers; + if (distributed == maxSymbolValue+1) { + /* all values are pretty poor; + probably incompressible data (should have already been detected); + find max, then give all remaining points to max */ + U32 maxV = 0, maxC = 0; + for (s=0; s<=maxSymbolValue; s++) + if (count[s] > maxC) { maxV=s; maxC=count[s]; } + norm[maxV] += (short)ToDistribute; + return 0; } - ZSTD_cwksp_assert_internal_consistency(ws); -} -/** - * The provided workspace takes ownership of the buffer [start, start+size). - * Any existing values in the workspace are ignored (the previously managed - * buffer, if present, must be separately freed). - */ -MEM_STATIC void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size, ZSTD_cwksp_static_alloc_e isStatic) { - DEBUGLOG(4, "cwksp: init'ing workspace with %zd bytes", size); - assert(((size_t)start & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ - ws->workspace = start; - ws->workspaceEnd = (BYTE*)start + size; - ws->objectEnd = ws->workspace; - ws->tableValidEnd = ws->objectEnd; - ws->phase = ZSTD_cwksp_alloc_objects; - ws->isStatic = isStatic; - ZSTD_cwksp_clear(ws); - ws->workspaceOversizedDuration = 0; - ZSTD_cwksp_assert_internal_consistency(ws); -} + if (total == 0) { + /* all of the symbols were low enough for the lowOne or lowThreshold */ + for (s=0; ToDistribute > 0; s = (s+1)%(maxSymbolValue+1)) + if (norm[s] > 0) { ToDistribute--; norm[s]++; } + return 0; + } + + { U64 const vStepLog = 62 - tableLog; + U64 const mid = (1ULL << (vStepLog-1)) - 1; + U64 const rStep = ZSTD_div64((((U64)1<> vStepLog); + U32 const sEnd = (U32)(end >> vStepLog); + U32 const weight = sEnd - sStart; + if (weight < 1) + return ERROR(GENERIC); + norm[s] = (short)weight; + tmpTotal = end; + } } } -MEM_STATIC size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem) { - void* workspace = ZSTD_customMalloc(size, customMem); - DEBUGLOG(4, "cwksp: creating new workspace with %zd bytes", size); - RETURN_ERROR_IF(workspace == NULL, memory_allocation, "NULL pointer!"); - ZSTD_cwksp_init(ws, workspace, size, ZSTD_cwksp_dynamic_alloc); return 0; } -MEM_STATIC void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) { - void *ptr = ws->workspace; - DEBUGLOG(4, "cwksp: freeing workspace"); - ZSTD_memset(ws, 0, sizeof(ZSTD_cwksp)); - ZSTD_customFree(ptr, customMem); -} +size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog, + const unsigned* count, size_t total, + unsigned maxSymbolValue, unsigned useLowProbCount) +{ + /* Sanity checks */ + if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; + if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported size */ + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported size */ + if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */ -/** - * Moves the management of a workspace from one cwksp to another. The src cwksp - * is left in an invalid state (src must be re-init()'ed before it's used again). - */ -MEM_STATIC void ZSTD_cwksp_move(ZSTD_cwksp* dst, ZSTD_cwksp* src) { - *dst = *src; - ZSTD_memset(src, 0, sizeof(ZSTD_cwksp)); -} + { static U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 }; + short const lowProbCount = useLowProbCount ? -1 : 1; + U64 const scale = 62 - tableLog; + U64 const step = ZSTD_div64((U64)1<<62, (U32)total); /* <== here, one division ! */ + U64 const vStep = 1ULL<<(scale-20); + int stillToDistribute = 1<> tableLog); -MEM_STATIC size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) { - return (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace); -} + for (s=0; s<=maxSymbolValue; s++) { + if (count[s] == total) return 0; /* rle special case */ + if (count[s] == 0) { normalizedCounter[s]=0; continue; } + if (count[s] <= lowThreshold) { + normalizedCounter[s] = lowProbCount; + stillToDistribute--; + } else { + short proba = (short)((count[s]*step) >> scale); + if (proba<8) { + U64 restToBeat = vStep * rtbTable[proba]; + proba += (count[s]*step) - ((U64)proba< restToBeat; + } + if (proba > largestP) { largestP=proba; largest=s; } + normalizedCounter[s] = proba; + stillToDistribute -= proba; + } } + if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) { + /* corner case, need another normalization method */ + size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue, lowProbCount); + if (FSE_isError(errorCode)) return errorCode; + } + else normalizedCounter[largest] += (short)stillToDistribute; + } -MEM_STATIC size_t ZSTD_cwksp_used(const ZSTD_cwksp* ws) { - return (size_t)((BYTE*)ws->tableEnd - (BYTE*)ws->workspace) - + (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->allocStart); -} +#if 0 + { /* Print Table (debug) */ + U32 s; + U32 nTotal = 0; + for (s=0; s<=maxSymbolValue; s++) + RAWLOG(2, "%3i: %4i \n", s, normalizedCounter[s]); + for (s=0; s<=maxSymbolValue; s++) + nTotal += abs(normalizedCounter[s]); + if (nTotal != (1U<allocFailed; + return tableLog; } -/*-************************************* -* Functions Checking Free Space -***************************************/ +/* fake FSE_CTable, for rle input (always same symbol) */ +size_t FSE_buildCTable_rle (FSE_CTable* ct, BYTE symbolValue) +{ + void* ptr = ct; + U16* tableU16 = ( (U16*) ptr) + 2; + void* FSCTptr = (U32*)ptr + 2; + FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) FSCTptr; -MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws) { - return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd); -} + /* header */ + tableU16[-2] = (U16) 0; + tableU16[-1] = (U16) symbolValue; -MEM_STATIC int ZSTD_cwksp_check_available(ZSTD_cwksp* ws, size_t additionalNeededSpace) { - return ZSTD_cwksp_available_space(ws) >= additionalNeededSpace; -} + /* Build table */ + tableU16[0] = 0; + tableU16[1] = 0; /* just in case */ -MEM_STATIC int ZSTD_cwksp_check_too_large(ZSTD_cwksp* ws, size_t additionalNeededSpace) { - return ZSTD_cwksp_check_available( - ws, additionalNeededSpace * ZSTD_WORKSPACETOOLARGE_FACTOR); -} + /* Build Symbol Transformation Table */ + symbolTT[symbolValue].deltaNbBits = 0; + symbolTT[symbolValue].deltaFindState = 0; -MEM_STATIC int ZSTD_cwksp_check_wasteful(ZSTD_cwksp* ws, size_t additionalNeededSpace) { - return ZSTD_cwksp_check_too_large(ws, additionalNeededSpace) - && ws->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION; + return 0; } -MEM_STATIC void ZSTD_cwksp_bump_oversized_duration( - ZSTD_cwksp* ws, size_t additionalNeededSpace) { - if (ZSTD_cwksp_check_too_large(ws, additionalNeededSpace)) { - ws->workspaceOversizedDuration++; - } else { - ws->workspaceOversizedDuration = 0; - } -} -#if defined (__cplusplus) -} -#endif - -#endif /* ZSTD_CWKSP_H */ -/**** ended inlining zstd_cwksp.h ****/ -#ifdef ZSTD_MULTITHREAD -/**** start inlining zstdmt_compress.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ +static size_t FSE_compress_usingCTable_generic (void* dst, size_t dstSize, + const void* src, size_t srcSize, + const FSE_CTable* ct, const unsigned fast) +{ + const BYTE* const istart = (const BYTE*) src; + const BYTE* const iend = istart + srcSize; + const BYTE* ip=iend; - #ifndef ZSTDMT_COMPRESS_H - #define ZSTDMT_COMPRESS_H + BIT_CStream_t bitC; + FSE_CState_t CState1, CState2; - #if defined (__cplusplus) - extern "C" { - #endif + /* init */ + if (srcSize <= 2) return 0; + { size_t const initError = BIT_initCStream(&bitC, dst, dstSize); + if (FSE_isError(initError)) return 0; /* not enough space available to write a bitstream */ } +#define FSE_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s)) -/* Note : This is an internal API. - * These APIs used to be exposed with ZSTDLIB_API, - * because it used to be the only way to invoke MT compression. - * Now, you must use ZSTD_compress2 and ZSTD_compressStream2() instead. - * - * This API requires ZSTD_MULTITHREAD to be defined during compilation, - * otherwise ZSTDMT_createCCtx*() will fail. - */ + if (srcSize & 1) { + FSE_initCState2(&CState1, ct, *--ip); + FSE_initCState2(&CState2, ct, *--ip); + FSE_encodeSymbol(&bitC, &CState1, *--ip); + FSE_FLUSHBITS(&bitC); + } else { + FSE_initCState2(&CState2, ct, *--ip); + FSE_initCState2(&CState1, ct, *--ip); + } -/* === Dependencies === */ -/**** skipping file: ../common/zstd_deps.h ****/ -#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */ -/**** skipping file: ../zstd.h ****/ + /* join to mod 4 */ + srcSize -= 2; + if ((sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) && (srcSize & 2)) { /* test bit 2 */ + FSE_encodeSymbol(&bitC, &CState2, *--ip); + FSE_encodeSymbol(&bitC, &CState1, *--ip); + FSE_FLUSHBITS(&bitC); + } + /* 2 or 4 encoding per loop */ + while ( ip>istart ) { -/* === Constants === */ -#ifndef ZSTDMT_NBWORKERS_MAX -# define ZSTDMT_NBWORKERS_MAX 200 -#endif -#ifndef ZSTDMT_JOBSIZE_MIN -# define ZSTDMT_JOBSIZE_MIN (1 MB) -#endif -#define ZSTDMT_JOBLOG_MAX (MEM_32bits() ? 29 : 30) -#define ZSTDMT_JOBSIZE_MAX (MEM_32bits() ? (512 MB) : (1024 MB)) + FSE_encodeSymbol(&bitC, &CState2, *--ip); + if (sizeof(bitC.bitContainer)*8 < FSE_MAX_TABLELOG*2+7 ) /* this test must be static */ + FSE_FLUSHBITS(&bitC); -/* ======================================================== - * === Private interface, for use by ZSTD_compress.c === - * === Not exposed in libzstd. Never invoke directly === - * ======================================================== */ + FSE_encodeSymbol(&bitC, &CState1, *--ip); -/* === Memory management === */ -typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx; -/* Requires ZSTD_MULTITHREAD to be defined during compilation, otherwise it will return NULL. */ -ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, - ZSTD_customMem cMem, - ZSTD_threadPool *pool); -size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx); + if (sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) { /* this test must be static */ + FSE_encodeSymbol(&bitC, &CState2, *--ip); + FSE_encodeSymbol(&bitC, &CState1, *--ip); + } -size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx); + FSE_FLUSHBITS(&bitC); + } -/* === Streaming functions === */ + FSE_flushCState(&bitC, &CState2); + FSE_flushCState(&bitC, &CState1); + return BIT_closeCStream(&bitC); +} -size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx); +size_t FSE_compress_usingCTable (void* dst, size_t dstSize, + const void* src, size_t srcSize, + const FSE_CTable* ct) +{ + unsigned const fast = (dstSize >= FSE_BLOCKBOUND(srcSize)); -/*! ZSTDMT_initCStream_internal() : - * Private use only. Init streaming operation. - * expects params to be valid. - * must receive dict, or cdict, or none, but not both. - * @return : 0, or an error code */ -size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs, - const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType, - const ZSTD_CDict* cdict, - ZSTD_CCtx_params params, unsigned long long pledgedSrcSize); + if (fast) + return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1); + else + return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0); +} -/*! ZSTDMT_compressStream_generic() : - * Combines ZSTDMT_compressStream() with optional ZSTDMT_flushStream() or ZSTDMT_endStream() - * depending on flush directive. - * @return : minimum amount of data still to be flushed - * 0 if fully flushed - * or an error code - * note : needs to be init using any ZSTD_initCStream*() variant */ -size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, - ZSTD_outBuffer* output, - ZSTD_inBuffer* input, - ZSTD_EndDirective endOp); - /*! ZSTDMT_toFlushNow() - * Tell how many bytes are ready to be flushed immediately. - * Probe the oldest active job (not yet entirely flushed) and check its output buffer. - * If return 0, it means there is no active job, - * or, it means oldest job is still active, but everything produced has been flushed so far, - * therefore flushing is limited by speed of oldest job. */ -size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx); +size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); } -/*! ZSTDMT_updateCParams_whileCompressing() : - * Updates only a selected set of compression parameters, to remain compatible with current frame. - * New parameters will be applied to next compression job. */ -void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_params* cctxParams); +#endif /* FSE_COMMONDEFS_ONLY */ +/**** ended inlining compress/fse_compress.c ****/ +/**** start inlining compress/hist.c ****/ +/* ****************************************************************** + * hist : Histogram functions + * part of Finite State Entropy project + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ -/*! ZSTDMT_getFrameProgression(): - * tells how much data has been consumed (input) and produced (output) for current frame. - * able to count progression inside worker threads. - */ -ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx); +/* --- dependencies --- */ +/**** skipping file: ../common/mem.h ****/ +/**** skipping file: ../common/debug.h ****/ +/**** skipping file: ../common/error_private.h ****/ +/**** skipping file: hist.h ****/ -#if defined (__cplusplus) -} -#endif +/* --- Error management --- */ +unsigned HIST_isError(size_t code) { return ERR_isError(code); } -#endif /* ZSTDMT_COMPRESS_H */ -/**** ended inlining zstdmt_compress.h ****/ -#endif +/*-************************************************************** + * Histogram functions + ****************************************************************/ +void HIST_add(unsigned* count, const void* src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + const BYTE* const end = ip + srcSize; -#if defined (__cplusplus) -extern "C" { -#endif + while (ip largestCount) largestCount = count[s]; + } -typedef struct { - HUF_CElt CTable[HUF_CTABLE_SIZE_U32(255)]; - HUF_repeat repeatMode; -} ZSTD_hufCTables_t; + return largestCount; +} -typedef struct { - FSE_CTable offcodeCTable[FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)]; - FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)]; - FSE_CTable litlengthCTable[FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)]; - FSE_repeat offcode_repeatMode; - FSE_repeat matchlength_repeatMode; - FSE_repeat litlength_repeatMode; -} ZSTD_fseCTables_t; +typedef enum { trustInput, checkMaxSymbolValue } HIST_checkInput_e; -typedef struct { - ZSTD_hufCTables_t huf; - ZSTD_fseCTables_t fse; -} ZSTD_entropyCTables_t; +/* HIST_count_parallel_wksp() : + * store histogram into 4 intermediate tables, recombined at the end. + * this design makes better use of OoO cpus, + * and is noticeably faster when some values are heavily repeated. + * But it needs some additional workspace for intermediate tables. + * `workSpace` must be a U32 table of size >= HIST_WKSP_SIZE_U32. + * @return : largest histogram frequency, + * or an error code (notably when histogram's alphabet is larger than *maxSymbolValuePtr) */ +static size_t HIST_count_parallel_wksp( + unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize, + HIST_checkInput_e check, + U32* const workSpace) +{ + const BYTE* ip = (const BYTE*)source; + const BYTE* const iend = ip+sourceSize; + size_t const countSize = (*maxSymbolValuePtr + 1) * sizeof(*count); + unsigned max=0; + U32* const Counting1 = workSpace; + U32* const Counting2 = Counting1 + 256; + U32* const Counting3 = Counting2 + 256; + U32* const Counting4 = Counting3 + 256; -typedef struct { - U32 off; /* Offset code (offset + ZSTD_REP_MOVE) for the match */ - U32 len; /* Raw length of match */ -} ZSTD_match_t; + /* safety checks */ + assert(*maxSymbolValuePtr <= 255); + if (!sourceSize) { + ZSTD_memset(count, 0, countSize); + *maxSymbolValuePtr = 0; + return 0; + } + ZSTD_memset(workSpace, 0, 4*256*sizeof(unsigned)); -typedef struct { - U32 offset; /* Offset of sequence */ - U32 litLength; /* Length of literals prior to match */ - U32 matchLength; /* Raw length of match */ -} rawSeq; + /* by stripes of 16 bytes */ + { U32 cached = MEM_read32(ip); ip += 4; + while (ip < iend-15) { + U32 c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + } + ip-=4; + } -typedef struct { - rawSeq* seq; /* The start of the sequences */ - size_t pos; /* The index in seq where reading stopped. pos <= size. */ - size_t posInSequence; /* The position within the sequence at seq[pos] where reading - stopped. posInSequence <= seq[pos].litLength + seq[pos].matchLength */ - size_t size; /* The number of sequences. <= capacity. */ - size_t capacity; /* The capacity starting from `seq` pointer */ -} rawSeqStore_t; + /* finish last symbols */ + while (ip max) max = Counting1[s]; + } } -typedef struct { - int price; - U32 off; - U32 mlen; - U32 litlen; - U32 rep[ZSTD_REP_NUM]; -} ZSTD_optimal_t; + { unsigned maxSymbolValue = 255; + while (!Counting1[maxSymbolValue]) maxSymbolValue--; + if (check && maxSymbolValue > *maxSymbolValuePtr) return ERROR(maxSymbolValue_tooSmall); + *maxSymbolValuePtr = maxSymbolValue; + ZSTD_memmove(count, Counting1, countSize); /* in case count & Counting1 are overlapping */ + } + return (size_t)max; +} -typedef enum { zop_dynamic=0, zop_predef } ZSTD_OptPrice_e; +/* HIST_countFast_wksp() : + * Same as HIST_countFast(), but using an externally provided scratch buffer. + * `workSpace` is a writable buffer which must be 4-bytes aligned, + * `workSpaceSize` must be >= HIST_WKSP_SIZE + */ +size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize, + void* workSpace, size_t workSpaceSize) +{ + if (sourceSize < 1500) /* heuristic threshold */ + return HIST_count_simple(count, maxSymbolValuePtr, source, sourceSize); + if ((size_t)workSpace & 3) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */ + if (workSpaceSize < HIST_WKSP_SIZE) return ERROR(workSpace_tooSmall); + return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, trustInput, (U32*)workSpace); +} -typedef struct { - /* All tables are allocated inside cctx->workspace by ZSTD_resetCCtx_internal() */ - unsigned* litFreq; /* table of literals statistics, of size 256 */ - unsigned* litLengthFreq; /* table of litLength statistics, of size (MaxLL+1) */ - unsigned* matchLengthFreq; /* table of matchLength statistics, of size (MaxML+1) */ - unsigned* offCodeFreq; /* table of offCode statistics, of size (MaxOff+1) */ - ZSTD_match_t* matchTable; /* list of found matches, of size ZSTD_OPT_NUM+1 */ - ZSTD_optimal_t* priceTable; /* All positions tracked by optimal parser, of size ZSTD_OPT_NUM+1 */ +/* HIST_count_wksp() : + * Same as HIST_count(), but using an externally provided scratch buffer. + * `workSpace` size must be table of >= HIST_WKSP_SIZE_U32 unsigned */ +size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize, + void* workSpace, size_t workSpaceSize) +{ + if ((size_t)workSpace & 3) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */ + if (workSpaceSize < HIST_WKSP_SIZE) return ERROR(workSpace_tooSmall); + if (*maxSymbolValuePtr < 255) + return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, checkMaxSymbolValue, (U32*)workSpace); + *maxSymbolValuePtr = 255; + return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace, workSpaceSize); +} - U32 litSum; /* nb of literals */ - U32 litLengthSum; /* nb of litLength codes */ - U32 matchLengthSum; /* nb of matchLength codes */ - U32 offCodeSum; /* nb of offset codes */ - U32 litSumBasePrice; /* to compare to log2(litfreq) */ - U32 litLengthSumBasePrice; /* to compare to log2(llfreq) */ - U32 matchLengthSumBasePrice;/* to compare to log2(mlfreq) */ - U32 offCodeSumBasePrice; /* to compare to log2(offreq) */ - ZSTD_OptPrice_e priceType; /* prices can be determined dynamically, or follow a pre-defined cost structure */ - const ZSTD_entropyCTables_t* symbolCosts; /* pre-calculated dictionary statistics */ - ZSTD_literalCompressionMode_e literalCompressionMode; -} optState_t; +#ifndef ZSTD_NO_UNUSED_FUNCTIONS +/* fast variant (unsafe : won't check if src contains values beyond count[] limit) */ +size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize) +{ + unsigned tmpCounters[HIST_WKSP_SIZE_U32]; + return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, tmpCounters, sizeof(tmpCounters)); +} -typedef struct { - ZSTD_entropyCTables_t entropy; - U32 rep[ZSTD_REP_NUM]; -} ZSTD_compressedBlockState_t; +size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize) +{ + unsigned tmpCounters[HIST_WKSP_SIZE_U32]; + return HIST_count_wksp(count, maxSymbolValuePtr, src, srcSize, tmpCounters, sizeof(tmpCounters)); +} +#endif +/**** ended inlining compress/hist.c ****/ +/**** start inlining compress/huf_compress.c ****/ +/* ****************************************************************** + * Huffman encoder, part of New Generation Entropy library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ -typedef struct { - BYTE const* nextSrc; /* next block here to continue on current prefix */ - BYTE const* base; /* All regular indexes relative to this position */ - BYTE const* dictBase; /* extDict indexes relative to this position */ - U32 dictLimit; /* below that point, need extDict */ - U32 lowLimit; /* below that point, no more valid data */ -} ZSTD_window_t; +/* ************************************************************** +* Compiler specifics +****************************************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif -typedef struct ZSTD_matchState_t ZSTD_matchState_t; -struct ZSTD_matchState_t { - ZSTD_window_t window; /* State for window round buffer management */ - U32 loadedDictEnd; /* index of end of dictionary, within context's referential. - * When loadedDictEnd != 0, a dictionary is in use, and still valid. - * This relies on a mechanism to set loadedDictEnd=0 when dictionary is no longer within distance. - * Such mechanism is provided within ZSTD_window_enforceMaxDist() and ZSTD_checkDictValidity(). - * When dict referential is copied into active context (i.e. not attached), - * loadedDictEnd == dictSize, since referential starts from zero. - */ - U32 nextToUpdate; /* index from which to continue table update */ - U32 hashLog3; /* dispatch table for matches of len==3 : larger == faster, more memory */ - U32* hashTable; - U32* hashTable3; - U32* chainTable; - int dedicatedDictSearch; /* Indicates whether this matchState is using the - * dedicated dictionary search structure. - */ - optState_t opt; /* optimal parser state */ - const ZSTD_matchState_t* dictMatchState; - ZSTD_compressionParameters cParams; - const rawSeqStore_t* ldmSeqStore; -}; -typedef struct { - ZSTD_compressedBlockState_t* prevCBlock; - ZSTD_compressedBlockState_t* nextCBlock; - ZSTD_matchState_t matchState; -} ZSTD_blockState_t; +/* ************************************************************** +* Includes +****************************************************************/ +/**** skipping file: ../common/zstd_deps.h ****/ +/**** skipping file: ../common/compiler.h ****/ +/**** skipping file: ../common/bitstream.h ****/ +/**** skipping file: hist.h ****/ +#define FSE_STATIC_LINKING_ONLY /* FSE_optimalTableLog_internal */ +/**** skipping file: ../common/fse.h ****/ +/**** skipping file: ../common/huf.h ****/ +/**** skipping file: ../common/error_private.h ****/ +/**** skipping file: ../common/bits.h ****/ -typedef struct { - U32 offset; - U32 checksum; -} ldmEntry_t; -typedef struct { - BYTE const* split; - U32 hash; - U32 checksum; - ldmEntry_t* bucket; -} ldmMatchCandidate_t; +/* ************************************************************** +* Error Management +****************************************************************/ +#define HUF_isError ERR_isError +#define HUF_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */ -#define LDM_BATCH_SIZE 64 -typedef struct { - ZSTD_window_t window; /* State for the window round buffer management */ - ldmEntry_t* hashTable; - U32 loadedDictEnd; - BYTE* bucketOffsets; /* Next position in bucket to insert entry */ - size_t splitIndices[LDM_BATCH_SIZE]; - ldmMatchCandidate_t matchCandidates[LDM_BATCH_SIZE]; -} ldmState_t; +/* ************************************************************** +* Required declarations +****************************************************************/ +typedef struct nodeElt_s { + U32 count; + U16 parent; + BYTE byte; + BYTE nbBits; +} nodeElt; -typedef struct { - U32 enableLdm; /* 1 if enable long distance matching */ - U32 hashLog; /* Log size of hashTable */ - U32 bucketSizeLog; /* Log bucket size for collision resolution, at most 8 */ - U32 minMatchLength; /* Minimum match length */ - U32 hashRateLog; /* Log number of entries to skip */ - U32 windowLog; /* Window log for the LDM */ -} ldmParams_t; -typedef struct { - int collectSequences; - ZSTD_Sequence* seqStart; - size_t seqIndex; - size_t maxSequences; -} SeqCollector; +/* ************************************************************** +* Debug Traces +****************************************************************/ -struct ZSTD_CCtx_params_s { - ZSTD_format_e format; - ZSTD_compressionParameters cParams; - ZSTD_frameParameters fParams; +#if DEBUGLEVEL >= 2 - int compressionLevel; - int forceWindow; /* force back-references to respect limit of - * 1<= add) { + assert(add < align); + assert(((size_t)aligned & mask) == 0); + *workspaceSizePtr -= add; + return aligned; + } else { + *workspaceSizePtr = 0; + return NULL; + } +} -struct ZSTD_CCtx_s { - ZSTD_compressionStage_e stage; - int cParamsChanged; /* == 1 if cParams(except wlog) or compression level are changed in requestedParams. Triggers transmission of new params to ZSTDMT (if available) then reset to 0. */ - int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */ - ZSTD_CCtx_params requestedParams; - ZSTD_CCtx_params appliedParams; - U32 dictID; - size_t dictContentSize; - ZSTD_cwksp workspace; /* manages buffer for dynamic allocations */ - size_t blockSize; - unsigned long long pledgedSrcSizePlusOne; /* this way, 0 (default) == unknown */ - unsigned long long consumedSrcSize; - unsigned long long producedCSize; - XXH64_state_t xxhState; - ZSTD_customMem customMem; - ZSTD_threadPool* pool; - size_t staticSize; - SeqCollector seqCollector; - int isFirstBlock; - int initialized; +/* HUF_compressWeights() : + * Same as FSE_compress(), but dedicated to huff0's weights compression. + * The use case needs much less stack memory. + * Note : all elements within weightTable are supposed to be <= HUF_TABLELOG_MAX. + */ +#define MAX_FSE_TABLELOG_FOR_HUFF_HEADER 6 - seqStore_t seqStore; /* sequences storage ptrs */ - ldmState_t ldmState; /* long distance matching state */ - rawSeq* ldmSequences; /* Storage for the ldm output sequences */ - size_t maxNbLdmSequences; - rawSeqStore_t externSeqStore; /* Mutable reference to external sequences */ - ZSTD_blockState_t blockState; - U32* entropyWorkspace; /* entropy workspace of ENTROPY_WORKSPACE_SIZE bytes */ +typedef struct { + FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)]; + U32 scratchBuffer[FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(HUF_TABLELOG_MAX, MAX_FSE_TABLELOG_FOR_HUFF_HEADER)]; + unsigned count[HUF_TABLELOG_MAX+1]; + S16 norm[HUF_TABLELOG_MAX+1]; +} HUF_CompressWeightsWksp; - /* Wether we are streaming or not */ - ZSTD_buffered_policy_e bufferedPolicy; +static size_t +HUF_compressWeights(void* dst, size_t dstSize, + const void* weightTable, size_t wtSize, + void* workspace, size_t workspaceSize) +{ + BYTE* const ostart = (BYTE*) dst; + BYTE* op = ostart; + BYTE* const oend = ostart + dstSize; - /* streaming */ - char* inBuff; - size_t inBuffSize; - size_t inToCompress; - size_t inBuffPos; - size_t inBuffTarget; - char* outBuff; - size_t outBuffSize; - size_t outBuffContentSize; - size_t outBuffFlushedSize; - ZSTD_cStreamStage streamStage; - U32 frameEnded; + unsigned maxSymbolValue = HUF_TABLELOG_MAX; + U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER; + HUF_CompressWeightsWksp* wksp = (HUF_CompressWeightsWksp*)HUF_alignUpWorkspace(workspace, &workspaceSize, ZSTD_ALIGNOF(U32)); - /* Stable in/out buffer verification */ - ZSTD_inBuffer expectedInBuffer; - size_t expectedOutBufferSize; + if (workspaceSize < sizeof(HUF_CompressWeightsWksp)) return ERROR(GENERIC); - /* Dictionary */ - ZSTD_localDict localDict; - const ZSTD_CDict* cdict; - ZSTD_prefixDict prefixDict; /* single-usage dictionary */ + /* init conditions */ + if (wtSize <= 1) return 0; /* Not compressible */ - /* Multi-threading */ -#ifdef ZSTD_MULTITHREAD - ZSTDMT_CCtx* mtctx; -#endif + /* Scan input and build symbol stats */ + { unsigned const maxCount = HIST_count_simple(wksp->count, &maxSymbolValue, weightTable, wtSize); /* never fails */ + if (maxCount == wtSize) return 1; /* only a single symbol in src : rle */ + if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */ + } - /* Tracing */ -#if ZSTD_TRACE - ZSTD_TraceCtx traceCtx; -#endif -}; + tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue); + CHECK_F( FSE_normalizeCount(wksp->norm, tableLog, wksp->count, wtSize, maxSymbolValue, /* useLowProbCount */ 0) ); -typedef enum { ZSTD_dtlm_fast, ZSTD_dtlm_full } ZSTD_dictTableLoadMethod_e; + /* Write table description header */ + { CHECK_V_F(hSize, FSE_writeNCount(op, (size_t)(oend-op), wksp->norm, maxSymbolValue, tableLog) ); + op += hSize; + } -typedef enum { - ZSTD_noDict = 0, - ZSTD_extDict = 1, - ZSTD_dictMatchState = 2, - ZSTD_dedicatedDictSearch = 3 -} ZSTD_dictMode_e; + /* Compress */ + CHECK_F( FSE_buildCTable_wksp(wksp->CTable, wksp->norm, maxSymbolValue, tableLog, wksp->scratchBuffer, sizeof(wksp->scratchBuffer)) ); + { CHECK_V_F(cSize, FSE_compress_usingCTable(op, (size_t)(oend - op), weightTable, wtSize, wksp->CTable) ); + if (cSize == 0) return 0; /* not enough space for compressed data */ + op += cSize; + } -typedef enum { - ZSTD_cpm_noAttachDict = 0, /* Compression with ZSTD_noDict or ZSTD_extDict. - * In this mode we use both the srcSize and the dictSize - * when selecting and adjusting parameters. - */ - ZSTD_cpm_attachDict = 1, /* Compression with ZSTD_dictMatchState or ZSTD_dedicatedDictSearch. - * In this mode we only take the srcSize into account when selecting - * and adjusting parameters. - */ - ZSTD_cpm_createCDict = 2, /* Creating a CDict. - * In this mode we take both the source size and the dictionary size - * into account when selecting and adjusting the parameters. - */ - ZSTD_cpm_unknown = 3, /* ZSTD_getCParams, ZSTD_getParams, ZSTD_adjustParams. - * We don't know what these parameters are for. We default to the legacy - * behavior of taking both the source size and the dict size into account - * when selecting and adjusting parameters. - */ -} ZSTD_cParamMode_e; + return (size_t)(op-ostart); +} -typedef size_t (*ZSTD_blockCompressor) ( - ZSTD_matchState_t* bs, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); -ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_dictMode_e dictMode); +static size_t HUF_getNbBits(HUF_CElt elt) +{ + return elt & 0xFF; +} +static size_t HUF_getNbBitsFast(HUF_CElt elt) +{ + return elt; +} -MEM_STATIC U32 ZSTD_LLcode(U32 litLength) +static size_t HUF_getValue(HUF_CElt elt) { - static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, - 16, 16, 17, 17, 18, 18, 19, 19, - 20, 20, 20, 20, 21, 21, 21, 21, - 22, 22, 22, 22, 22, 22, 22, 22, - 23, 23, 23, 23, 23, 23, 23, 23, - 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24 }; - static const U32 LL_deltaCode = 19; - return (litLength > 63) ? ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength]; + return elt & ~(size_t)0xFF; } -/* ZSTD_MLcode() : - * note : mlBase = matchLength - MINMATCH; - * because it's the format it's stored in seqStore->sequences */ -MEM_STATIC U32 ZSTD_MLcode(U32 mlBase) +static size_t HUF_getValueFast(HUF_CElt elt) { - static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, - 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, - 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, - 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, - 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, - 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 }; - static const U32 ML_deltaCode = 36; - return (mlBase > 127) ? ZSTD_highbit32(mlBase) + ML_deltaCode : ML_Code[mlBase]; + return elt; } -typedef struct repcodes_s { - U32 rep[3]; -} repcodes_t; +static void HUF_setNbBits(HUF_CElt* elt, size_t nbBits) +{ + assert(nbBits <= HUF_TABLELOG_ABSOLUTEMAX); + *elt = nbBits; +} -MEM_STATIC repcodes_t ZSTD_updateRep(U32 const rep[3], U32 const offset, U32 const ll0) +static void HUF_setValue(HUF_CElt* elt, size_t value) { - repcodes_t newReps; - if (offset >= ZSTD_REP_NUM) { /* full offset */ - newReps.rep[2] = rep[1]; - newReps.rep[1] = rep[0]; - newReps.rep[0] = offset - ZSTD_REP_MOVE; - } else { /* repcode */ - U32 const repCode = offset + ll0; - if (repCode > 0) { /* note : if repCode==0, no change */ - U32 const currentOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode]; - newReps.rep[2] = (repCode >= 2) ? rep[1] : rep[2]; - newReps.rep[1] = rep[0]; - newReps.rep[0] = currentOffset; - } else { /* repCode == 0 */ - ZSTD_memcpy(&newReps, rep, sizeof(newReps)); - } + size_t const nbBits = HUF_getNbBits(*elt); + if (nbBits > 0) { + assert((value >> nbBits) == 0); + *elt |= value << (sizeof(HUF_CElt) * 8 - nbBits); } - return newReps; } -/* ZSTD_cParam_withinBounds: - * @return 1 if value is within cParam bounds, - * 0 otherwise */ -MEM_STATIC int ZSTD_cParam_withinBounds(ZSTD_cParameter cParam, int value) +HUF_CTableHeader HUF_readCTableHeader(HUF_CElt const* ctable) { - ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); - if (ZSTD_isError(bounds.error)) return 0; - if (value < bounds.lowerBound) return 0; - if (value > bounds.upperBound) return 0; - return 1; + HUF_CTableHeader header; + ZSTD_memcpy(&header, ctable, sizeof(header)); + return header; } -/* ZSTD_noCompressBlock() : - * Writes uncompressed block to dst buffer from given src. - * Returns the size of the block */ -MEM_STATIC size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 lastBlock) +static void HUF_writeCTableHeader(HUF_CElt* ctable, U32 tableLog, U32 maxSymbolValue) { - U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw)<<1) + (U32)(srcSize << 3); - RETURN_ERROR_IF(srcSize + ZSTD_blockHeaderSize > dstCapacity, - dstSize_tooSmall, "dst buf too small for uncompressed block"); - MEM_writeLE24(dst, cBlockHeader24); - ZSTD_memcpy((BYTE*)dst + ZSTD_blockHeaderSize, src, srcSize); - return ZSTD_blockHeaderSize + srcSize; + HUF_CTableHeader header; + HUF_STATIC_ASSERT(sizeof(ctable[0]) == sizeof(header)); + ZSTD_memset(&header, 0, sizeof(header)); + assert(tableLog < 256); + header.tableLog = (BYTE)tableLog; + assert(maxSymbolValue < 256); + header.maxSymbolValue = (BYTE)maxSymbolValue; + ZSTD_memcpy(ctable, &header, sizeof(header)); } -MEM_STATIC size_t ZSTD_rleCompressBlock (void* dst, size_t dstCapacity, BYTE src, size_t srcSize, U32 lastBlock) +typedef struct { + HUF_CompressWeightsWksp wksp; + BYTE bitsToWeight[HUF_TABLELOG_MAX + 1]; /* precomputed conversion table */ + BYTE huffWeight[HUF_SYMBOLVALUE_MAX]; +} HUF_WriteCTableWksp; + +size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, + const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, + void* workspace, size_t workspaceSize) { - BYTE* const op = (BYTE*)dst; - U32 const cBlockHeader = lastBlock + (((U32)bt_rle)<<1) + (U32)(srcSize << 3); - RETURN_ERROR_IF(dstCapacity < 4, dstSize_tooSmall, ""); - MEM_writeLE24(op, cBlockHeader); - op[3] = src; - return 4; -} + HUF_CElt const* const ct = CTable + 1; + BYTE* op = (BYTE*)dst; + U32 n; + HUF_WriteCTableWksp* wksp = (HUF_WriteCTableWksp*)HUF_alignUpWorkspace(workspace, &workspaceSize, ZSTD_ALIGNOF(U32)); + HUF_STATIC_ASSERT(HUF_CTABLE_WORKSPACE_SIZE >= sizeof(HUF_WriteCTableWksp)); -/* ZSTD_minGain() : - * minimum compression required - * to generate a compress block or a compressed literals section. - * note : use same formula for both situations */ -MEM_STATIC size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat) -{ - U32 const minlog = (strat>=ZSTD_btultra) ? (U32)(strat) - 1 : 6; - ZSTD_STATIC_ASSERT(ZSTD_btultra == 8); - assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, strat)); - return (srcSize >> minlog) + 2; -} - -MEM_STATIC int ZSTD_disableLiteralsCompression(const ZSTD_CCtx_params* cctxParams) -{ - switch (cctxParams->literalCompressionMode) { - case ZSTD_lcm_huffman: - return 0; - case ZSTD_lcm_uncompressed: - return 1; - default: - assert(0 /* impossible: pre-validated */); - /* fall-through */ - case ZSTD_lcm_auto: - return (cctxParams->cParams.strategy == ZSTD_fast) && (cctxParams->cParams.targetLength > 0); - } -} - -/*! ZSTD_safecopyLiterals() : - * memcpy() function that won't read beyond more than WILDCOPY_OVERLENGTH bytes past ilimit_w. - * Only called when the sequence ends past ilimit_w, so it only needs to be optimized for single - * large copies. - */ -static void ZSTD_safecopyLiterals(BYTE* op, BYTE const* ip, BYTE const* const iend, BYTE const* ilimit_w) { - assert(iend > ilimit_w); - if (ip <= ilimit_w) { - ZSTD_wildcopy(op, ip, ilimit_w - ip, ZSTD_no_overlap); - op += ilimit_w - ip; - ip = ilimit_w; - } - while (ip < iend) *op++ = *ip++; -} - -/*! ZSTD_storeSeq() : - * Store a sequence (litlen, litPtr, offCode and mlBase) into seqStore_t. - * `offCode` : distance to match + ZSTD_REP_MOVE (values <= ZSTD_REP_MOVE are repCodes). - * `mlBase` : matchLength - MINMATCH - * Allowed to overread literals up to litLimit. -*/ -HINT_INLINE UNUSED_ATTR -void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, const BYTE* litLimit, U32 offCode, size_t mlBase) -{ - BYTE const* const litLimit_w = litLimit - WILDCOPY_OVERLENGTH; - BYTE const* const litEnd = literals + litLength; -#if defined(DEBUGLEVEL) && (DEBUGLEVEL >= 6) - static const BYTE* g_start = NULL; - if (g_start==NULL) g_start = (const BYTE*)literals; /* note : index only works for compression within a single segment */ - { U32 const pos = (U32)((const BYTE*)literals - g_start); - DEBUGLOG(6, "Cpos%7u :%3u literals, match%4u bytes at offCode%7u", - pos, (U32)litLength, (U32)mlBase+MINMATCH, (U32)offCode); - } -#endif - assert((size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart) < seqStorePtr->maxNbSeq); - /* copy Literals */ - assert(seqStorePtr->maxNbLit <= 128 KB); - assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + seqStorePtr->maxNbLit); - assert(literals + litLength <= litLimit); - if (litEnd <= litLimit_w) { - /* Common case we can use wildcopy. - * First copy 16 bytes, because literals are likely short. - */ - assert(WILDCOPY_OVERLENGTH >= 16); - ZSTD_copy16(seqStorePtr->lit, literals); - if (litLength > 16) { - ZSTD_wildcopy(seqStorePtr->lit+16, literals+16, (ptrdiff_t)litLength-16, ZSTD_no_overlap); - } - } else { - ZSTD_safecopyLiterals(seqStorePtr->lit, literals, litEnd, litLimit_w); - } - seqStorePtr->lit += litLength; - - /* literal Length */ - if (litLength>0xFFFF) { - assert(seqStorePtr->longLengthID == 0); /* there can only be a single long length */ - seqStorePtr->longLengthID = 1; - seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); - } - seqStorePtr->sequences[0].litLength = (U16)litLength; - - /* match offset */ - seqStorePtr->sequences[0].offset = offCode + 1; - - /* match Length */ - if (mlBase>0xFFFF) { - assert(seqStorePtr->longLengthID == 0); /* there can only be a single long length */ - seqStorePtr->longLengthID = 2; - seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); - } - seqStorePtr->sequences[0].matchLength = (U16)mlBase; + assert(HUF_readCTableHeader(CTable).maxSymbolValue == maxSymbolValue); + assert(HUF_readCTableHeader(CTable).tableLog == huffLog); - seqStorePtr->sequences++; -} + /* check conditions */ + if (workspaceSize < sizeof(HUF_WriteCTableWksp)) return ERROR(GENERIC); + if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge); + /* convert to weight */ + wksp->bitsToWeight[0] = 0; + for (n=1; nbitsToWeight[n] = (BYTE)(huffLog + 1 - n); + for (n=0; nhuffWeight[n] = wksp->bitsToWeight[HUF_getNbBits(ct[n])]; -/*-************************************* -* Match length counter -***************************************/ -static unsigned ZSTD_NbCommonBytes (size_t val) -{ - if (MEM_isLittleEndian()) { - if (MEM_64bits()) { -# if defined(_MSC_VER) && defined(_WIN64) -# if STATIC_BMI2 - return _tzcnt_u64(val) >> 3; -# else - unsigned long r = 0; - return _BitScanForward64( &r, (U64)val ) ? (unsigned)(r >> 3) : 0; -# endif -# elif defined(__GNUC__) && (__GNUC__ >= 4) - return (__builtin_ctzll((U64)val) >> 3); -# else - static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, - 0, 3, 1, 3, 1, 4, 2, 7, - 0, 2, 3, 6, 1, 5, 3, 5, - 1, 3, 4, 4, 2, 5, 6, 7, - 7, 0, 1, 2, 3, 3, 4, 6, - 2, 6, 5, 5, 3, 4, 5, 6, - 7, 1, 2, 4, 6, 4, 4, 5, - 7, 2, 6, 5, 7, 6, 7, 7 }; - return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; -# endif - } else { /* 32 bits */ -# if defined(_MSC_VER) - unsigned long r=0; - return _BitScanForward( &r, (U32)val ) ? (unsigned)(r >> 3) : 0; -# elif defined(__GNUC__) && (__GNUC__ >= 3) - return (__builtin_ctz((U32)val) >> 3); -# else - static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, - 3, 2, 2, 1, 3, 2, 0, 1, - 3, 3, 1, 2, 2, 2, 2, 0, - 3, 1, 2, 0, 1, 0, 1, 1 }; - return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; -# endif - } - } else { /* Big Endian CPU */ - if (MEM_64bits()) { -# if defined(_MSC_VER) && defined(_WIN64) -# if STATIC_BMI2 - return _lzcnt_u64(val) >> 3; -# else - unsigned long r = 0; - return _BitScanReverse64(&r, (U64)val) ? (unsigned)(r >> 3) : 0; -# endif -# elif defined(__GNUC__) && (__GNUC__ >= 4) - return (__builtin_clzll(val) >> 3); -# else - unsigned r; - const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */ - if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; } - if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } - r += (!val); - return r; -# endif - } else { /* 32 bits */ -# if defined(_MSC_VER) - unsigned long r = 0; - return _BitScanReverse( &r, (unsigned long)val ) ? (unsigned)(r >> 3) : 0; -# elif defined(__GNUC__) && (__GNUC__ >= 3) - return (__builtin_clz((U32)val) >> 3); -# else - unsigned r; - if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } - r += (!val); - return r; -# endif + /* attempt weights compression by FSE */ + if (maxDstSize < 1) return ERROR(dstSize_tooSmall); + { CHECK_V_F(hSize, HUF_compressWeights(op+1, maxDstSize-1, wksp->huffWeight, maxSymbolValue, &wksp->wksp, sizeof(wksp->wksp)) ); + if ((hSize>1) & (hSize < maxSymbolValue/2)) { /* FSE compressed */ + op[0] = (BYTE)hSize; + return hSize+1; } } -} - - -MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* const pInLimit) -{ - const BYTE* const pStart = pIn; - const BYTE* const pInLoopLimit = pInLimit - (sizeof(size_t)-1); - if (pIn < pInLoopLimit) { - { size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn); - if (diff) return ZSTD_NbCommonBytes(diff); } - pIn+=sizeof(size_t); pMatch+=sizeof(size_t); - while (pIn < pInLoopLimit) { - size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn); - if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; } - pIn += ZSTD_NbCommonBytes(diff); - return (size_t)(pIn - pStart); - } } - if (MEM_64bits() && (pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; } - if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; } - if ((pIn (256-128)) return ERROR(GENERIC); /* should not happen : likely means source cannot be compressed */ + if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */ + op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue-1)); + wksp->huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause msan issue in final combination */ + for (n=0; nhuffWeight[n] << 4) + wksp->huffWeight[n+1]); + return ((maxSymbolValue+1)/2) + 1; } -/** ZSTD_count_2segments() : - * can count match length with `ip` & `match` in 2 different segments. - * convention : on reaching mEnd, match count continue starting from iStart - */ -MEM_STATIC size_t -ZSTD_count_2segments(const BYTE* ip, const BYTE* match, - const BYTE* iEnd, const BYTE* mEnd, const BYTE* iStart) -{ - const BYTE* const vEnd = MIN( ip + (mEnd - match), iEnd); - size_t const matchLength = ZSTD_count(ip, match, vEnd); - if (match + matchLength != mEnd) return matchLength; - DEBUGLOG(7, "ZSTD_count_2segments: found a 2-parts match (current length==%zu)", matchLength); - DEBUGLOG(7, "distance from match beginning to end dictionary = %zi", mEnd - match); - DEBUGLOG(7, "distance from current pos to end buffer = %zi", iEnd - ip); - DEBUGLOG(7, "next byte : ip==%02X, istart==%02X", ip[matchLength], *iStart); - DEBUGLOG(7, "final match length = %zu", matchLength + ZSTD_count(ip+matchLength, iStart, iEnd)); - return matchLength + ZSTD_count(ip+matchLength, iStart, iEnd); -} +size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned* hasZeroWeights) +{ + BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; /* init not required, even though some static analyzer may complain */ + U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */ + U32 tableLog = 0; + U32 nbSymbols = 0; + HUF_CElt* const ct = CTable + 1; -/*-************************************* - * Hashes - ***************************************/ -static const U32 prime3bytes = 506832829U; -static U32 ZSTD_hash3(U32 u, U32 h) { return ((u << (32-24)) * prime3bytes) >> (32-h) ; } -MEM_STATIC size_t ZSTD_hash3Ptr(const void* ptr, U32 h) { return ZSTD_hash3(MEM_readLE32(ptr), h); } /* only in zstd_opt.h */ + /* get symbol weights */ + CHECK_V_F(readSize, HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX+1, rankVal, &nbSymbols, &tableLog, src, srcSize)); + *hasZeroWeights = (rankVal[0] > 0); -static const U32 prime4bytes = 2654435761U; -static U32 ZSTD_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32-h) ; } -static size_t ZSTD_hash4Ptr(const void* ptr, U32 h) { return ZSTD_hash4(MEM_read32(ptr), h); } + /* check result */ + if (tableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); + if (nbSymbols > *maxSymbolValuePtr+1) return ERROR(maxSymbolValue_tooSmall); -static const U64 prime5bytes = 889523592379ULL; -static size_t ZSTD_hash5(U64 u, U32 h) { return (size_t)(((u << (64-40)) * prime5bytes) >> (64-h)) ; } -static size_t ZSTD_hash5Ptr(const void* p, U32 h) { return ZSTD_hash5(MEM_readLE64(p), h); } + *maxSymbolValuePtr = nbSymbols - 1; -static const U64 prime6bytes = 227718039650203ULL; -static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64-48)) * prime6bytes) >> (64-h)) ; } -static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h); } + HUF_writeCTableHeader(CTable, tableLog, *maxSymbolValuePtr); -static const U64 prime7bytes = 58295818150454627ULL; -static size_t ZSTD_hash7(U64 u, U32 h) { return (size_t)(((u << (64-56)) * prime7bytes) >> (64-h)) ; } -static size_t ZSTD_hash7Ptr(const void* p, U32 h) { return ZSTD_hash7(MEM_readLE64(p), h); } + /* Prepare base value per rank */ + { U32 n, nextRankStart = 0; + for (n=1; n<=tableLog; n++) { + U32 curr = nextRankStart; + nextRankStart += (rankVal[n] << (n-1)); + rankVal[n] = curr; + } } -static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL; -static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; } -static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); } + /* fill nbBits */ + { U32 n; for (n=0; nn=tableLog+1 */ + U16 valPerRank[HUF_TABLELOG_MAX+2] = {0}; + { U32 n; for (n=0; n0; n--) { /* start at n=tablelog <-> w=1 */ + valPerRank[n] = min; /* get starting value within each rank */ + min += nbPerRank[n]; + min >>= 1; + } } + /* assign value within rank, symbol order */ + { U32 n; for (n=0; n>= 1; - base *= base; - } - return power; + const HUF_CElt* const ct = CTable + 1; + assert(symbolValue <= HUF_SYMBOLVALUE_MAX); + if (symbolValue > HUF_readCTableHeader(CTable).maxSymbolValue) + return 0; + return (U32)HUF_getNbBits(ct[symbolValue]); } -#define ZSTD_ROLL_HASH_CHAR_OFFSET 10 -/** ZSTD_rollingHash_append() : - * Add the buffer to the hash value. +/** + * HUF_setMaxHeight(): + * Try to enforce @targetNbBits on the Huffman tree described in @huffNode. + * + * It attempts to convert all nodes with nbBits > @targetNbBits + * to employ @targetNbBits instead. Then it adjusts the tree + * so that it remains a valid canonical Huffman tree. + * + * @pre The sum of the ranks of each symbol == 2^largestBits, + * where largestBits == huffNode[lastNonNull].nbBits. + * @post The sum of the ranks of each symbol == 2^largestBits, + * where largestBits is the return value (expected <= targetNbBits). + * + * @param huffNode The Huffman tree modified in place to enforce targetNbBits. + * It's presumed sorted, from most frequent to rarest symbol. + * @param lastNonNull The symbol with the lowest count in the Huffman tree. + * @param targetNbBits The allowed number of bits, which the Huffman tree + * may not respect. After this function the Huffman tree will + * respect targetNbBits. + * @return The maximum number of bits of the Huffman tree after adjustment. */ -static U64 ZSTD_rollingHash_append(U64 hash, void const* buf, size_t size) +static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 targetNbBits) { - BYTE const* istart = (BYTE const*)buf; - size_t pos; - for (pos = 0; pos < size; ++pos) { - hash *= prime8bytes; - hash += istart[pos] + ZSTD_ROLL_HASH_CHAR_OFFSET; - } - return hash; -} + const U32 largestBits = huffNode[lastNonNull].nbBits; + /* early exit : no elt > targetNbBits, so the tree is already valid. */ + if (largestBits <= targetNbBits) return largestBits; -/** ZSTD_rollingHash_compute() : - * Compute the rolling hash value of the buffer. - */ -MEM_STATIC U64 ZSTD_rollingHash_compute(void const* buf, size_t size) -{ - return ZSTD_rollingHash_append(0, buf, size); -} + DEBUGLOG(5, "HUF_setMaxHeight (targetNbBits = %u)", targetNbBits); -/** ZSTD_rollingHash_primePower() : - * Compute the primePower to be passed to ZSTD_rollingHash_rotate() for a hash - * over a window of length bytes. - */ -MEM_STATIC U64 ZSTD_rollingHash_primePower(U32 length) -{ - return ZSTD_ipow(prime8bytes, length - 1); -} + /* there are several too large elements (at least >= 2) */ + { int totalCost = 0; + const U32 baseCost = 1 << (largestBits - targetNbBits); + int n = (int)lastNonNull; -/** ZSTD_rollingHash_rotate() : - * Rotate the rolling hash by one byte. - */ -MEM_STATIC U64 ZSTD_rollingHash_rotate(U64 hash, BYTE toRemove, BYTE toAdd, U64 primePower) -{ - hash -= (toRemove + ZSTD_ROLL_HASH_CHAR_OFFSET) * primePower; - hash *= prime8bytes; - hash += toAdd + ZSTD_ROLL_HASH_CHAR_OFFSET; - return hash; + /* Adjust any ranks > targetNbBits to targetNbBits. + * Compute totalCost, which is how far the sum of the ranks is + * we are over 2^largestBits after adjust the offending ranks. + */ + while (huffNode[n].nbBits > targetNbBits) { + totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits)); + huffNode[n].nbBits = (BYTE)targetNbBits; + n--; + } + /* n stops at huffNode[n].nbBits <= targetNbBits */ + assert(huffNode[n].nbBits <= targetNbBits); + /* n end at index of smallest symbol using < targetNbBits */ + while (huffNode[n].nbBits == targetNbBits) --n; + + /* renorm totalCost from 2^largestBits to 2^targetNbBits + * note : totalCost is necessarily a multiple of baseCost */ + assert(((U32)totalCost & (baseCost - 1)) == 0); + totalCost >>= (largestBits - targetNbBits); + assert(totalCost > 0); + + /* repay normalized cost */ + { U32 const noSymbol = 0xF0F0F0F0; + U32 rankLast[HUF_TABLELOG_MAX+2]; + + /* Get pos of last (smallest = lowest cum. count) symbol per rank */ + ZSTD_memset(rankLast, 0xF0, sizeof(rankLast)); + { U32 currentNbBits = targetNbBits; + int pos; + for (pos=n ; pos >= 0; pos--) { + if (huffNode[pos].nbBits >= currentNbBits) continue; + currentNbBits = huffNode[pos].nbBits; /* < targetNbBits */ + rankLast[targetNbBits-currentNbBits] = (U32)pos; + } } + + while (totalCost > 0) { + /* Try to reduce the next power of 2 above totalCost because we + * gain back half the rank. + */ + U32 nBitsToDecrease = ZSTD_highbit32((U32)totalCost) + 1; + for ( ; nBitsToDecrease > 1; nBitsToDecrease--) { + U32 const highPos = rankLast[nBitsToDecrease]; + U32 const lowPos = rankLast[nBitsToDecrease-1]; + if (highPos == noSymbol) continue; + /* Decrease highPos if no symbols of lowPos or if it is + * not cheaper to remove 2 lowPos than highPos. + */ + if (lowPos == noSymbol) break; + { U32 const highTotal = huffNode[highPos].count; + U32 const lowTotal = 2 * huffNode[lowPos].count; + if (highTotal <= lowTotal) break; + } } + /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */ + assert(rankLast[nBitsToDecrease] != noSymbol || nBitsToDecrease == 1); + /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */ + while ((nBitsToDecrease<=HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol)) + nBitsToDecrease++; + assert(rankLast[nBitsToDecrease] != noSymbol); + /* Increase the number of bits to gain back half the rank cost. */ + totalCost -= 1 << (nBitsToDecrease-1); + huffNode[rankLast[nBitsToDecrease]].nbBits++; + + /* Fix up the new rank. + * If the new rank was empty, this symbol is now its smallest. + * Otherwise, this symbol will be the largest in the new rank so no adjustment. + */ + if (rankLast[nBitsToDecrease-1] == noSymbol) + rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease]; + /* Fix up the old rank. + * If the symbol was at position 0, meaning it was the highest weight symbol in the tree, + * it must be the only symbol in its rank, so the old rank now has no symbols. + * Otherwise, since the Huffman nodes are sorted by count, the previous position is now + * the smallest node in the rank. If the previous position belongs to a different rank, + * then the rank is now empty. + */ + if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */ + rankLast[nBitsToDecrease] = noSymbol; + else { + rankLast[nBitsToDecrease]--; + if (huffNode[rankLast[nBitsToDecrease]].nbBits != targetNbBits-nBitsToDecrease) + rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */ + } + } /* while (totalCost > 0) */ + + /* If we've removed too much weight, then we have to add it back. + * To avoid overshooting again, we only adjust the smallest rank. + * We take the largest nodes from the lowest rank 0 and move them + * to rank 1. There's guaranteed to be enough rank 0 symbols because + * TODO. + */ + while (totalCost < 0) { /* Sometimes, cost correction overshoot */ + /* special case : no rank 1 symbol (using targetNbBits-1); + * let's create one from largest rank 0 (using targetNbBits). + */ + if (rankLast[1] == noSymbol) { + while (huffNode[n].nbBits == targetNbBits) n--; + huffNode[n+1].nbBits--; + assert(n >= 0); + rankLast[1] = (U32)(n+1); + totalCost++; + continue; + } + huffNode[ rankLast[1] + 1 ].nbBits--; + rankLast[1]++; + totalCost ++; + } + } /* repay normalized cost */ + } /* there are several too large elements (at least >= 2) */ + + return targetNbBits; } -/*-************************************* -* Round buffer management -***************************************/ -#if (ZSTD_WINDOWLOG_MAX_64 > 31) -# error "ZSTD_WINDOWLOG_MAX is too large : would overflow ZSTD_CURRENT_MAX" -#endif -/* Max current allowed */ -#define ZSTD_CURRENT_MAX ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX)) -/* Maximum chunk size before overflow correction needs to be called again */ -#define ZSTD_CHUNKSIZE_MAX \ - ( ((U32)-1) /* Maximum ending current index */ \ - - ZSTD_CURRENT_MAX) /* Maximum beginning lowLimit */ +typedef struct { + U16 base; + U16 curr; +} rankPos; -/** - * ZSTD_window_clear(): - * Clears the window containing the history by simply setting it to empty. +typedef nodeElt huffNodeTable[2 * (HUF_SYMBOLVALUE_MAX + 1)]; + +/* Number of buckets available for HUF_sort() */ +#define RANK_POSITION_TABLE_SIZE 192 + +typedef struct { + huffNodeTable huffNodeTbl; + rankPos rankPosition[RANK_POSITION_TABLE_SIZE]; +} HUF_buildCTable_wksp_tables; + +/* RANK_POSITION_DISTINCT_COUNT_CUTOFF == Cutoff point in HUF_sort() buckets for which we use log2 bucketing. + * Strategy is to use as many buckets as possible for representing distinct + * counts while using the remainder to represent all "large" counts. + * + * To satisfy this requirement for 192 buckets, we can do the following: + * Let buckets 0-166 represent distinct counts of [0, 166] + * Let buckets 166 to 192 represent all remaining counts up to RANK_POSITION_MAX_COUNT_LOG using log2 bucketing. */ -MEM_STATIC void ZSTD_window_clear(ZSTD_window_t* window) -{ - size_t const endT = (size_t)(window->nextSrc - window->base); - U32 const end = (U32)endT; +#define RANK_POSITION_MAX_COUNT_LOG 32 +#define RANK_POSITION_LOG_BUCKETS_BEGIN ((RANK_POSITION_TABLE_SIZE - 1) - RANK_POSITION_MAX_COUNT_LOG - 1 /* == 158 */) +#define RANK_POSITION_DISTINCT_COUNT_CUTOFF (RANK_POSITION_LOG_BUCKETS_BEGIN + ZSTD_highbit32(RANK_POSITION_LOG_BUCKETS_BEGIN) /* == 166 */) - window->lowLimit = end; - window->dictLimit = end; +/* Return the appropriate bucket index for a given count. See definition of + * RANK_POSITION_DISTINCT_COUNT_CUTOFF for explanation of bucketing strategy. + */ +static U32 HUF_getIndex(U32 const count) { + return (count < RANK_POSITION_DISTINCT_COUNT_CUTOFF) + ? count + : ZSTD_highbit32(count) + RANK_POSITION_LOG_BUCKETS_BEGIN; } -/** - * ZSTD_window_hasExtDict(): - * Returns non-zero if the window has a non-empty extDict. - */ -MEM_STATIC U32 ZSTD_window_hasExtDict(ZSTD_window_t const window) -{ - return window.lowLimit < window.dictLimit; +/* Helper swap function for HUF_quickSortPartition() */ +static void HUF_swapNodes(nodeElt* a, nodeElt* b) { + nodeElt tmp = *a; + *a = *b; + *b = tmp; } -/** - * ZSTD_matchState_dictMode(): - * Inspects the provided matchState and figures out what dictMode should be - * passed to the compressor. - */ -MEM_STATIC ZSTD_dictMode_e ZSTD_matchState_dictMode(const ZSTD_matchState_t *ms) -{ - return ZSTD_window_hasExtDict(ms->window) ? - ZSTD_extDict : - ms->dictMatchState != NULL ? - (ms->dictMatchState->dedicatedDictSearch ? ZSTD_dedicatedDictSearch : ZSTD_dictMatchState) : - ZSTD_noDict; +/* Returns 0 if the huffNode array is not sorted by descending count */ +MEM_STATIC int HUF_isSorted(nodeElt huffNode[], U32 const maxSymbolValue1) { + U32 i; + for (i = 1; i < maxSymbolValue1; ++i) { + if (huffNode[i].count > huffNode[i-1].count) { + return 0; + } + } + return 1; } -/** - * ZSTD_window_needOverflowCorrection(): - * Returns non-zero if the indices are getting too large and need overflow - * protection. +/* Insertion sort by descending order */ +HINT_INLINE void HUF_insertionSort(nodeElt huffNode[], int const low, int const high) { + int i; + int const size = high-low+1; + huffNode += low; + for (i = 1; i < size; ++i) { + nodeElt const key = huffNode[i]; + int j = i - 1; + while (j >= 0 && huffNode[j].count < key.count) { + huffNode[j + 1] = huffNode[j]; + j--; + } + huffNode[j + 1] = key; + } +} + +/* Pivot helper function for quicksort. */ +static int HUF_quickSortPartition(nodeElt arr[], int const low, int const high) { + /* Simply select rightmost element as pivot. "Better" selectors like + * median-of-three don't experimentally appear to have any benefit. + */ + U32 const pivot = arr[high].count; + int i = low - 1; + int j = low; + for ( ; j < high; j++) { + if (arr[j].count > pivot) { + i++; + HUF_swapNodes(&arr[i], &arr[j]); + } + } + HUF_swapNodes(&arr[i + 1], &arr[high]); + return i + 1; +} + +/* Classic quicksort by descending with partially iterative calls + * to reduce worst case callstack size. */ -MEM_STATIC U32 ZSTD_window_needOverflowCorrection(ZSTD_window_t const window, - void const* srcEnd) -{ - U32 const curr = (U32)((BYTE const*)srcEnd - window.base); - return curr > ZSTD_CURRENT_MAX; +static void HUF_simpleQuickSort(nodeElt arr[], int low, int high) { + int const kInsertionSortThreshold = 8; + if (high - low < kInsertionSortThreshold) { + HUF_insertionSort(arr, low, high); + return; + } + while (low < high) { + int const idx = HUF_quickSortPartition(arr, low, high); + if (idx - low < high - idx) { + HUF_simpleQuickSort(arr, low, idx - 1); + low = idx + 1; + } else { + HUF_simpleQuickSort(arr, idx + 1, high); + high = idx - 1; + } + } } /** - * ZSTD_window_correctOverflow(): - * Reduces the indices to protect from index overflow. - * Returns the correction made to the indices, which must be applied to every - * stored index. + * HUF_sort(): + * Sorts the symbols [0, maxSymbolValue] by count[symbol] in decreasing order. + * This is a typical bucket sorting strategy that uses either quicksort or insertion sort to sort each bucket. * - * The least significant cycleLog bits of the indices must remain the same, - * which may be 0. Every index up to maxDist in the past must be valid. - * NOTE: (maxDist & cycleMask) must be zero. + * @param[out] huffNode Sorted symbols by decreasing count. Only members `.count` and `.byte` are filled. + * Must have (maxSymbolValue + 1) entries. + * @param[in] count Histogram of the symbols. + * @param[in] maxSymbolValue Maximum symbol value. + * @param rankPosition This is a scratch workspace. Must have RANK_POSITION_TABLE_SIZE entries. */ -MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog, - U32 maxDist, void const* src) -{ - /* preemptive overflow correction: - * 1. correction is large enough: - * lowLimit > (3<<29) ==> current > 3<<29 + 1< (3<<29 + 1< (3<<29) - (1< (3<<29) - (1<<30) (NOTE: chainLog <= 30) - * > 1<<29 - * - * 2. (ip+ZSTD_CHUNKSIZE_MAX - cctx->base) doesn't overflow: - * After correction, current is less than (1<base < 1<<32. - * 3. (cctx->lowLimit + 1< 3<<29 + 1<base); - U32 const currentCycle0 = curr & cycleMask; - /* Exclude zero so that newCurrent - maxDist >= 1. */ - U32 const currentCycle1 = currentCycle0 == 0 ? (1U << cycleLog) : currentCycle0; - U32 const newCurrent = currentCycle1 + maxDist; - U32 const correction = curr - newCurrent; - assert((maxDist & cycleMask) == 0); - assert(curr > newCurrent); - /* Loose bound, should be around 1<<29 (see above) */ - assert(correction > 1<<28); + ZSTD_memset(rankPosition, 0, sizeof(*rankPosition) * RANK_POSITION_TABLE_SIZE); + for (n = 0; n < maxSymbolValue1; ++n) { + U32 lowerRank = HUF_getIndex(count[n]); + assert(lowerRank < RANK_POSITION_TABLE_SIZE - 1); + rankPosition[lowerRank].base++; + } - window->base += correction; - window->dictBase += correction; - if (window->lowLimit <= correction) window->lowLimit = 1; - else window->lowLimit -= correction; - if (window->dictLimit <= correction) window->dictLimit = 1; - else window->dictLimit -= correction; + assert(rankPosition[RANK_POSITION_TABLE_SIZE - 1].base == 0); + /* Set up the rankPosition table */ + for (n = RANK_POSITION_TABLE_SIZE - 1; n > 0; --n) { + rankPosition[n-1].base += rankPosition[n].base; + rankPosition[n-1].curr = rankPosition[n-1].base; + } - /* Ensure we can still reference the full window. */ - assert(newCurrent >= maxDist); - assert(newCurrent - maxDist >= 1); - /* Ensure that lowLimit and dictLimit didn't underflow. */ - assert(window->lowLimit <= newCurrent); - assert(window->dictLimit <= newCurrent); + /* Insert each symbol into their appropriate bucket, setting up rankPosition table. */ + for (n = 0; n < maxSymbolValue1; ++n) { + U32 const c = count[n]; + U32 const r = HUF_getIndex(c) + 1; + U32 const pos = rankPosition[r].curr++; + assert(pos < maxSymbolValue1); + huffNode[pos].count = c; + huffNode[pos].byte = (BYTE)n; + } - DEBUGLOG(4, "Correction of 0x%x bytes to lowLimit=0x%x", correction, - window->lowLimit); - return correction; + /* Sort each bucket. */ + for (n = RANK_POSITION_DISTINCT_COUNT_CUTOFF; n < RANK_POSITION_TABLE_SIZE - 1; ++n) { + int const bucketSize = rankPosition[n].curr - rankPosition[n].base; + U32 const bucketStartIdx = rankPosition[n].base; + if (bucketSize > 1) { + assert(bucketStartIdx < maxSymbolValue1); + HUF_simpleQuickSort(huffNode + bucketStartIdx, 0, bucketSize-1); + } + } + + assert(HUF_isSorted(huffNode, maxSymbolValue1)); } -/** - * ZSTD_window_enforceMaxDist(): - * Updates lowLimit so that: - * (srcEnd - base) - lowLimit == maxDist + loadedDictEnd + +/** HUF_buildCTable_wksp() : + * Same as HUF_buildCTable(), but using externally allocated scratch buffer. + * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as sizeof(HUF_buildCTable_wksp_tables). + */ +#define STARTNODE (HUF_SYMBOLVALUE_MAX+1) + +/* HUF_buildTree(): + * Takes the huffNode array sorted by HUF_sort() and builds an unlimited-depth Huffman tree. * - * It ensures index is valid as long as index >= lowLimit. - * This must be called before a block compression call. - * - * loadedDictEnd is only defined if a dictionary is in use for current compression. - * As the name implies, loadedDictEnd represents the index at end of dictionary. - * The value lies within context's referential, it can be directly compared to blockEndIdx. - * - * If loadedDictEndPtr is NULL, no dictionary is in use, and we use loadedDictEnd == 0. - * If loadedDictEndPtr is not NULL, we set it to zero after updating lowLimit. - * This is because dictionaries are allowed to be referenced fully - * as long as the last byte of the dictionary is in the window. - * Once input has progressed beyond window size, dictionary cannot be referenced anymore. - * - * In normal dict mode, the dictionary lies between lowLimit and dictLimit. - * In dictMatchState mode, lowLimit and dictLimit are the same, - * and the dictionary is below them. - * forceWindow and dictMatchState are therefore incompatible. + * @param huffNode The array sorted by HUF_sort(). Builds the Huffman tree in this array. + * @param maxSymbolValue The maximum symbol value. + * @return The smallest node in the Huffman tree (by count). */ -MEM_STATIC void -ZSTD_window_enforceMaxDist(ZSTD_window_t* window, - const void* blockEnd, - U32 maxDist, - U32* loadedDictEndPtr, - const ZSTD_matchState_t** dictMatchStatePtr) +static int HUF_buildTree(nodeElt* huffNode, U32 maxSymbolValue) { - U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base); - U32 const loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0; - DEBUGLOG(5, "ZSTD_window_enforceMaxDist: blockEndIdx=%u, maxDist=%u, loadedDictEnd=%u", - (unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd); + nodeElt* const huffNode0 = huffNode - 1; + int nonNullRank; + int lowS, lowN; + int nodeNb = STARTNODE; + int n, nodeRoot; + DEBUGLOG(5, "HUF_buildTree (alphabet size = %u)", maxSymbolValue + 1); + /* init for parents */ + nonNullRank = (int)maxSymbolValue; + while(huffNode[nonNullRank].count == 0) nonNullRank--; + lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb; + huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count; + huffNode[lowS].parent = huffNode[lowS-1].parent = (U16)nodeNb; + nodeNb++; lowS-=2; + for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30); + huffNode0[0].count = (U32)(1U<<31); /* fake entry, strong barrier */ - /* - When there is no dictionary : loadedDictEnd == 0. - In which case, the test (blockEndIdx > maxDist) is merely to avoid - overflowing next operation `newLowLimit = blockEndIdx - maxDist`. - - When there is a standard dictionary : - Index referential is copied from the dictionary, - which means it starts from 0. - In which case, loadedDictEnd == dictSize, - and it makes sense to compare `blockEndIdx > maxDist + dictSize` - since `blockEndIdx` also starts from zero. - - When there is an attached dictionary : - loadedDictEnd is expressed within the referential of the context, - so it can be directly compared against blockEndIdx. - */ - if (blockEndIdx > maxDist + loadedDictEnd) { - U32 const newLowLimit = blockEndIdx - maxDist; - if (window->lowLimit < newLowLimit) window->lowLimit = newLowLimit; - if (window->dictLimit < window->lowLimit) { - DEBUGLOG(5, "Update dictLimit to match lowLimit, from %u to %u", - (unsigned)window->dictLimit, (unsigned)window->lowLimit); - window->dictLimit = window->lowLimit; - } - /* On reaching window size, dictionaries are invalidated */ - if (loadedDictEndPtr) *loadedDictEndPtr = 0; - if (dictMatchStatePtr) *dictMatchStatePtr = NULL; + /* create parents */ + while (nodeNb <= nodeRoot) { + int const n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; + int const n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; + huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count; + huffNode[n1].parent = huffNode[n2].parent = (U16)nodeNb; + nodeNb++; } -} -/* Similar to ZSTD_window_enforceMaxDist(), - * but only invalidates dictionary - * when input progresses beyond window size. - * assumption : loadedDictEndPtr and dictMatchStatePtr are valid (non NULL) - * loadedDictEnd uses same referential as window->base - * maxDist is the window size */ -MEM_STATIC void -ZSTD_checkDictValidity(const ZSTD_window_t* window, - const void* blockEnd, - U32 maxDist, - U32* loadedDictEndPtr, - const ZSTD_matchState_t** dictMatchStatePtr) -{ - assert(loadedDictEndPtr != NULL); - assert(dictMatchStatePtr != NULL); - { U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base); - U32 const loadedDictEnd = *loadedDictEndPtr; - DEBUGLOG(5, "ZSTD_checkDictValidity: blockEndIdx=%u, maxDist=%u, loadedDictEnd=%u", - (unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd); - assert(blockEndIdx >= loadedDictEnd); + /* distribute weights (unlimited tree height) */ + huffNode[nodeRoot].nbBits = 0; + for (n=nodeRoot-1; n>=STARTNODE; n--) + huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; + for (n=0; n<=nonNullRank; n++) + huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; - if (blockEndIdx > loadedDictEnd + maxDist) { - /* On reaching window size, dictionaries are invalidated. - * For simplification, if window size is reached anywhere within next block, - * the dictionary is invalidated for the full block. - */ - DEBUGLOG(6, "invalidating dictionary for current block (distance > windowSize)"); - *loadedDictEndPtr = 0; - *dictMatchStatePtr = NULL; - } else { - if (*loadedDictEndPtr != 0) { - DEBUGLOG(6, "dictionary considered valid for current block"); - } } } -} + DEBUGLOG(6, "Initial distribution of bits completed (%zu sorted symbols)", showHNodeBits(huffNode, maxSymbolValue+1)); -MEM_STATIC void ZSTD_window_init(ZSTD_window_t* window) { - ZSTD_memset(window, 0, sizeof(*window)); - window->base = (BYTE const*)""; - window->dictBase = (BYTE const*)""; - window->dictLimit = 1; /* start from 1, so that 1st position is valid */ - window->lowLimit = 1; /* it ensures first and later CCtx usages compress the same */ - window->nextSrc = window->base + 1; /* see issue #1241 */ + return nonNullRank; } /** - * ZSTD_window_update(): - * Updates the window by appending [src, src + srcSize) to the window. - * If it is not contiguous, the current prefix becomes the extDict, and we - * forget about the extDict. Handles overlap of the prefix and extDict. - * Returns non-zero if the segment is contiguous. + * HUF_buildCTableFromTree(): + * Build the CTable given the Huffman tree in huffNode. + * + * @param[out] CTable The output Huffman CTable. + * @param huffNode The Huffman tree. + * @param nonNullRank The last and smallest node in the Huffman tree. + * @param maxSymbolValue The maximum symbol value. + * @param maxNbBits The exact maximum number of bits used in the Huffman tree. */ -MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window, - void const* src, size_t srcSize) +static void HUF_buildCTableFromTree(HUF_CElt* CTable, nodeElt const* huffNode, int nonNullRank, U32 maxSymbolValue, U32 maxNbBits) { - BYTE const* const ip = (BYTE const*)src; - U32 contiguous = 1; - DEBUGLOG(5, "ZSTD_window_update"); - if (srcSize == 0) - return contiguous; - assert(window->base != NULL); - assert(window->dictBase != NULL); - /* Check if blocks follow each other */ - if (src != window->nextSrc) { - /* not contiguous */ - size_t const distanceFromBase = (size_t)(window->nextSrc - window->base); - DEBUGLOG(5, "Non contiguous blocks, new segment starts at %u", window->dictLimit); - window->lowLimit = window->dictLimit; - assert(distanceFromBase == (size_t)(U32)distanceFromBase); /* should never overflow */ - window->dictLimit = (U32)distanceFromBase; - window->dictBase = window->base; - window->base = ip - distanceFromBase; - /* ms->nextToUpdate = window->dictLimit; */ - if (window->dictLimit - window->lowLimit < HASH_READ_SIZE) window->lowLimit = window->dictLimit; /* too small extDict */ - contiguous = 0; - } - window->nextSrc = ip + srcSize; - /* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */ - if ( (ip+srcSize > window->dictBase + window->lowLimit) - & (ip < window->dictBase + window->dictLimit)) { - ptrdiff_t const highInputIdx = (ip + srcSize) - window->dictBase; - U32 const lowLimitMax = (highInputIdx > (ptrdiff_t)window->dictLimit) ? window->dictLimit : (U32)highInputIdx; - window->lowLimit = lowLimitMax; - DEBUGLOG(5, "Overlapping extDict and input : new lowLimit = %u", window->lowLimit); - } - return contiguous; -} + HUF_CElt* const ct = CTable + 1; + /* fill result into ctable (val, nbBits) */ + int n; + U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0}; + U16 valPerRank[HUF_TABLELOG_MAX+1] = {0}; + int const alphabetSize = (int)(maxSymbolValue + 1); + for (n=0; n<=nonNullRank; n++) + nbPerRank[huffNode[n].nbBits]++; + /* determine starting value per rank */ + { U16 min = 0; + for (n=(int)maxNbBits; n>0; n--) { + valPerRank[n] = min; /* get starting value within each rank */ + min += nbPerRank[n]; + min >>= 1; + } } + for (n=0; nwindow.lowLimit; - U32 const withinWindow = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid; - U32 const isDictionary = (ms->loadedDictEnd != 0); - /* When using a dictionary the entire dictionary is valid if a single byte of the dictionary - * is within the window. We invalidate the dictionary (and set loadedDictEnd to 0) when it isn't - * valid for the entire block. So this check is sufficient to find the lowest valid match index. - */ - U32 const matchLowest = isDictionary ? lowestValid : withinWindow; - return matchLowest; + HUF_writeCTableHeader(CTable, maxNbBits, maxSymbolValue); } -/** - * Returns the lowest allowed match index in the prefix. - */ -MEM_STATIC U32 ZSTD_getLowestPrefixIndex(const ZSTD_matchState_t* ms, U32 curr, unsigned windowLog) +size_t +HUF_buildCTable_wksp(HUF_CElt* CTable, const unsigned* count, U32 maxSymbolValue, U32 maxNbBits, + void* workSpace, size_t wkspSize) { - U32 const maxDistance = 1U << windowLog; - U32 const lowestValid = ms->window.dictLimit; - U32 const withinWindow = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid; - U32 const isDictionary = (ms->loadedDictEnd != 0); - /* When computing the lowest prefix index we need to take the dictionary into account to handle - * the edge case where the dictionary and the source are contiguous in memory. - */ - U32 const matchLowest = isDictionary ? lowestValid : withinWindow; - return matchLowest; -} + HUF_buildCTable_wksp_tables* const wksp_tables = + (HUF_buildCTable_wksp_tables*)HUF_alignUpWorkspace(workSpace, &wkspSize, ZSTD_ALIGNOF(U32)); + nodeElt* const huffNode0 = wksp_tables->huffNodeTbl; + nodeElt* const huffNode = huffNode0+1; + int nonNullRank; + HUF_STATIC_ASSERT(HUF_CTABLE_WORKSPACE_SIZE == sizeof(HUF_buildCTable_wksp_tables)); + DEBUGLOG(5, "HUF_buildCTable_wksp (alphabet size = %u)", maxSymbolValue+1); -/* debug functions */ -#if (DEBUGLEVEL>=2) + /* safety checks */ + if (wkspSize < sizeof(HUF_buildCTable_wksp_tables)) + return ERROR(workSpace_tooSmall); + if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT; + if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) + return ERROR(maxSymbolValue_tooLarge); + ZSTD_memset(huffNode0, 0, sizeof(huffNodeTable)); -MEM_STATIC double ZSTD_fWeight(U32 rawStat) -{ - U32 const fp_accuracy = 8; - U32 const fp_multiplier = (1 << fp_accuracy); - U32 const newStat = rawStat + 1; - U32 const hb = ZSTD_highbit32(newStat); - U32 const BWeight = hb * fp_multiplier; - U32 const FWeight = (newStat << fp_accuracy) >> hb; - U32 const weight = BWeight + FWeight; - assert(hb + fp_accuracy < 31); - return (double)weight / fp_multiplier; + /* sort, decreasing order */ + HUF_sort(huffNode, count, maxSymbolValue, wksp_tables->rankPosition); + DEBUGLOG(6, "sorted symbols completed (%zu symbols)", showHNodeSymbols(huffNode, maxSymbolValue+1)); + + /* build tree */ + nonNullRank = HUF_buildTree(huffNode, maxSymbolValue); + + /* determine and enforce maxTableLog */ + maxNbBits = HUF_setMaxHeight(huffNode, (U32)nonNullRank, maxNbBits); + if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */ + + HUF_buildCTableFromTree(CTable, huffNode, nonNullRank, maxSymbolValue, maxNbBits); + + return maxNbBits; } -/* display a table content, - * listing each element, its frequency, and its predicted bit cost */ -MEM_STATIC void ZSTD_debugTable(const U32* table, U32 max) +size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) { - unsigned u, sum; - for (u=0, sum=0; u<=max; u++) sum += table[u]; - DEBUGLOG(2, "total nb elts: %u", sum); - for (u=0; u<=max; u++) { - DEBUGLOG(2, "%2u: %5u (%.2f)", - u, table[u], ZSTD_fWeight(sum) - ZSTD_fWeight(table[u]) ); + HUF_CElt const* ct = CTable + 1; + size_t nbBits = 0; + int s; + for (s = 0; s <= (int)maxSymbolValue; ++s) { + nbBits += HUF_getNbBits(ct[s]) * count[s]; } + return nbBits >> 3; } -#endif +int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) { + HUF_CTableHeader header = HUF_readCTableHeader(CTable); + HUF_CElt const* ct = CTable + 1; + int bad = 0; + int s; + assert(header.tableLog <= HUF_TABLELOG_ABSOLUTEMAX); -#if defined (__cplusplus) -} -#endif + if (header.maxSymbolValue < maxSymbolValue) + return 0; -/* =============================================================== - * Shared internal declarations - * These prototypes may be called from sources not in lib/compress - * =============================================================== */ - -/* ZSTD_loadCEntropy() : - * dict : must point at beginning of a valid zstd dictionary. - * return : size of dictionary header (size of magic number + dict ID + entropy tables) - * assumptions : magic number supposed already checked - * and dictSize >= 8 */ -size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, - const void* const dict, size_t dictSize); - -void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs); - -/* ============================================================== - * Private declarations - * These prototypes shall only be called from within lib/compress - * ============================================================== */ - -/* ZSTD_getCParamsFromCCtxParams() : - * cParams are built depending on compressionLevel, src size hints, - * LDM and manually set compression parameters. - * Note: srcSizeHint == 0 means 0! - */ -ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( - const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode); - -/*! ZSTD_initCStream_internal() : - * Private use only. Init streaming operation. - * expects params to be valid. - * must receive dict, or cdict, or none, but not both. - * @return : 0, or an error code */ -size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, - const void* dict, size_t dictSize, - const ZSTD_CDict* cdict, - const ZSTD_CCtx_params* params, unsigned long long pledgedSrcSize); + for (s = 0; s <= (int)maxSymbolValue; ++s) { + bad |= (count[s] != 0) & (HUF_getNbBits(ct[s]) == 0); + } + return !bad; +} -void ZSTD_resetSeqStore(seqStore_t* ssPtr); +size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); } -/*! ZSTD_getCParamsFromCDict() : - * as the name implies */ -ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict); +/** HUF_CStream_t: + * Huffman uses its own BIT_CStream_t implementation. + * There are three major differences from BIT_CStream_t: + * 1. HUF_addBits() takes a HUF_CElt (size_t) which is + * the pair (nbBits, value) in the format: + * format: + * - Bits [0, 4) = nbBits + * - Bits [4, 64 - nbBits) = 0 + * - Bits [64 - nbBits, 64) = value + * 2. The bitContainer is built from the upper bits and + * right shifted. E.g. to add a new value of N bits + * you right shift the bitContainer by N, then or in + * the new value into the N upper bits. + * 3. The bitstream has two bit containers. You can add + * bits to the second container and merge them into + * the first container. + */ + +#define HUF_BITS_IN_CONTAINER (sizeof(size_t) * 8) -/* ZSTD_compressBegin_advanced_internal() : - * Private use only. To be called from zstdmt_compress.c. */ -size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx, - const void* dict, size_t dictSize, - ZSTD_dictContentType_e dictContentType, - ZSTD_dictTableLoadMethod_e dtlm, - const ZSTD_CDict* cdict, - const ZSTD_CCtx_params* params, - unsigned long long pledgedSrcSize); +typedef struct { + size_t bitContainer[2]; + size_t bitPos[2]; -/* ZSTD_compress_advanced_internal() : - * Private use only. To be called from zstdmt_compress.c. */ -size_t ZSTD_compress_advanced_internal(ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize, - const ZSTD_CCtx_params* params); + BYTE* startPtr; + BYTE* ptr; + BYTE* endPtr; +} HUF_CStream_t; +/**! HUF_initCStream(): + * Initializes the bitstream. + * @returns 0 or an error code. + */ +static size_t HUF_initCStream(HUF_CStream_t* bitC, + void* startPtr, size_t dstCapacity) +{ + ZSTD_memset(bitC, 0, sizeof(*bitC)); + bitC->startPtr = (BYTE*)startPtr; + bitC->ptr = bitC->startPtr; + bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->bitContainer[0]); + if (dstCapacity <= sizeof(bitC->bitContainer[0])) return ERROR(dstSize_tooSmall); + return 0; +} -/* ZSTD_writeLastEmptyBlock() : - * output an empty Block with end-of-frame mark to complete a frame - * @return : size of data written into `dst` (== ZSTD_blockHeaderSize (defined in zstd_internal.h)) - * or an error code if `dstCapacity` is too small (bitContainer[idx] >>= HUF_getNbBits(elt); + bitC->bitContainer[idx] |= kFast ? HUF_getValueFast(elt) : HUF_getValue(elt); + /* We only read the low 8 bits of bitC->bitPos[idx] so it + * doesn't matter that the high bits have noise from the value. + */ + bitC->bitPos[idx] += HUF_getNbBitsFast(elt); + assert((bitC->bitPos[idx] & 0xFF) <= HUF_BITS_IN_CONTAINER); + /* The last 4-bits of elt are dirty if fast is set, + * so we must not be overwriting bits that have already been + * inserted into the bit container. + */ +#if DEBUGLEVEL >= 1 + { + size_t const nbBits = HUF_getNbBits(elt); + size_t const dirtyBits = nbBits == 0 ? 0 : ZSTD_highbit32((U32)nbBits) + 1; + (void)dirtyBits; + /* Middle bits are 0. */ + assert(((elt >> dirtyBits) << (dirtyBits + nbBits)) == 0); + /* We didn't overwrite any bits in the bit container. */ + assert(!kFast || (bitC->bitPos[idx] & 0xFF) <= HUF_BITS_IN_CONTAINER); + (void)dirtyBits; + } +#endif +} +FORCE_INLINE_TEMPLATE void HUF_zeroIndex1(HUF_CStream_t* bitC) +{ + bitC->bitContainer[1] = 0; + bitC->bitPos[1] = 0; +} -/* ZSTD_referenceExternalSequences() : - * Must be called before starting a compression operation. - * seqs must parse a prefix of the source. - * This cannot be used when long range matching is enabled. - * Zstd will use these sequences, and pass the literals to a secondary block - * compressor. - * @return : An error code on failure. - * NOTE: seqs are not verified! Invalid sequences can cause out-of-bounds memory - * access and data corruption. +/*! HUF_mergeIndex1() : + * Merges the bit container @ index 1 into the bit container @ index 0 + * and zeros the bit container @ index 1. */ -size_t ZSTD_referenceExternalSequences(ZSTD_CCtx* cctx, rawSeq* seq, size_t nbSeq); +FORCE_INLINE_TEMPLATE void HUF_mergeIndex1(HUF_CStream_t* bitC) +{ + assert((bitC->bitPos[1] & 0xFF) < HUF_BITS_IN_CONTAINER); + bitC->bitContainer[0] >>= (bitC->bitPos[1] & 0xFF); + bitC->bitContainer[0] |= bitC->bitContainer[1]; + bitC->bitPos[0] += bitC->bitPos[1]; + assert((bitC->bitPos[0] & 0xFF) <= HUF_BITS_IN_CONTAINER); +} -/** ZSTD_cycleLog() : - * condition for correct operation : hashLog > 1 */ -U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat); +/*! HUF_flushBits() : +* Flushes the bits in the bit container @ index 0. +* +* @post bitPos will be < 8. +* @param kFast If kFast is set then we must know a-priori that +* the bit container will not overflow. +*/ +FORCE_INLINE_TEMPLATE void HUF_flushBits(HUF_CStream_t* bitC, int kFast) +{ + /* The upper bits of bitPos are noisy, so we must mask by 0xFF. */ + size_t const nbBits = bitC->bitPos[0] & 0xFF; + size_t const nbBytes = nbBits >> 3; + /* The top nbBits bits of bitContainer are the ones we need. */ + size_t const bitContainer = bitC->bitContainer[0] >> (HUF_BITS_IN_CONTAINER - nbBits); + /* Mask bitPos to account for the bytes we consumed. */ + bitC->bitPos[0] &= 7; + assert(nbBits > 0); + assert(nbBits <= sizeof(bitC->bitContainer[0]) * 8); + assert(bitC->ptr <= bitC->endPtr); + MEM_writeLEST(bitC->ptr, bitContainer); + bitC->ptr += nbBytes; + assert(!kFast || bitC->ptr <= bitC->endPtr); + if (!kFast && bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr; + /* bitContainer doesn't need to be modified because the leftover + * bits are already the top bitPos bits. And we don't care about + * noise in the lower values. + */ +} -/** ZSTD_CCtx_trace() : - * Trace the end of a compression call. +/*! HUF_endMark() + * @returns The Huffman stream end mark: A 1-bit value = 1. */ -void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize); +static HUF_CElt HUF_endMark(void) +{ + HUF_CElt endMark; + HUF_setNbBits(&endMark, 1); + HUF_setValue(&endMark, 1); + return endMark; +} -#endif /* ZSTD_COMPRESS_H */ -/**** ended inlining zstd_compress_internal.h ****/ +/*! HUF_closeCStream() : + * @return Size of CStream, in bytes, + * or 0 if it could not fit into dstBuffer */ +static size_t HUF_closeCStream(HUF_CStream_t* bitC) +{ + HUF_addBits(bitC, HUF_endMark(), /* idx */ 0, /* kFast */ 0); + HUF_flushBits(bitC, /* kFast */ 0); + { + size_t const nbBits = bitC->bitPos[0] & 0xFF; + if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */ + return (size_t)(bitC->ptr - bitC->startPtr) + (nbBits > 0); + } +} +FORCE_INLINE_TEMPLATE void +HUF_encodeSymbol(HUF_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable, int idx, int fast) +{ + HUF_addBits(bitCPtr, CTable[symbol], idx, fast); +} -size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize); +FORCE_INLINE_TEMPLATE void +HUF_compress1X_usingCTable_internal_body_loop(HUF_CStream_t* bitC, + const BYTE* ip, size_t srcSize, + const HUF_CElt* ct, + int kUnroll, int kFastFlush, int kLastFast) +{ + /* Join to kUnroll */ + int n = (int)srcSize; + int rem = n % kUnroll; + if (rem > 0) { + for (; rem > 0; --rem) { + HUF_encodeSymbol(bitC, ip[--n], ct, 0, /* fast */ 0); + } + HUF_flushBits(bitC, kFastFlush); + } + assert(n % kUnroll == 0); + + /* Join to 2 * kUnroll */ + if (n % (2 * kUnroll)) { + int u; + for (u = 1; u < kUnroll; ++u) { + HUF_encodeSymbol(bitC, ip[n - u], ct, 0, 1); + } + HUF_encodeSymbol(bitC, ip[n - kUnroll], ct, 0, kLastFast); + HUF_flushBits(bitC, kFastFlush); + n -= kUnroll; + } + assert(n % (2 * kUnroll) == 0); + + for (; n>0; n-= 2 * kUnroll) { + /* Encode kUnroll symbols into the bitstream @ index 0. */ + int u; + for (u = 1; u < kUnroll; ++u) { + HUF_encodeSymbol(bitC, ip[n - u], ct, /* idx */ 0, /* fast */ 1); + } + HUF_encodeSymbol(bitC, ip[n - kUnroll], ct, /* idx */ 0, /* fast */ kLastFast); + HUF_flushBits(bitC, kFastFlush); + /* Encode kUnroll symbols into the bitstream @ index 1. + * This allows us to start filling the bit container + * without any data dependencies. + */ + HUF_zeroIndex1(bitC); + for (u = 1; u < kUnroll; ++u) { + HUF_encodeSymbol(bitC, ip[n - kUnroll - u], ct, /* idx */ 1, /* fast */ 1); + } + HUF_encodeSymbol(bitC, ip[n - kUnroll - kUnroll], ct, /* idx */ 1, /* fast */ kLastFast); + /* Merge bitstream @ index 1 into the bitstream @ index 0 */ + HUF_mergeIndex1(bitC); + HUF_flushBits(bitC, kFastFlush); + } + assert(n == 0); -size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize); +} -size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, - ZSTD_hufCTables_t* nextHuf, - ZSTD_strategy strategy, int disableLiteralCompression, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - void* entropyWorkspace, size_t entropyWorkspaceSize, - const int bmi2); +/** + * Returns a tight upper bound on the output space needed by Huffman + * with 8 bytes buffer to handle over-writes. If the output is at least + * this large we don't need to do bounds checks during Huffman encoding. + */ +static size_t HUF_tightCompressBound(size_t srcSize, size_t tableLog) +{ + return ((srcSize * tableLog) >> 3) + 8; +} -#endif /* ZSTD_COMPRESS_LITERALS_H */ -/**** ended inlining zstd_compress_literals.h ****/ -size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize) +FORCE_INLINE_TEMPLATE size_t +HUF_compress1X_usingCTable_internal_body(void* dst, size_t dstSize, + const void* src, size_t srcSize, + const HUF_CElt* CTable) { + U32 const tableLog = HUF_readCTableHeader(CTable).tableLog; + HUF_CElt const* ct = CTable + 1; + const BYTE* ip = (const BYTE*) src; BYTE* const ostart = (BYTE*)dst; - U32 const flSize = 1 + (srcSize>31) + (srcSize>4095); + BYTE* const oend = ostart + dstSize; + HUF_CStream_t bitC; - RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall, ""); + /* init */ + if (dstSize < 8) return 0; /* not enough space to compress */ + { BYTE* op = ostart; + size_t const initErr = HUF_initCStream(&bitC, op, (size_t)(oend-op)); + if (HUF_isError(initErr)) return 0; } - switch(flSize) - { - case 1: /* 2 - 1 - 5 */ - ostart[0] = (BYTE)((U32)set_basic + (srcSize<<3)); - break; - case 2: /* 2 - 2 - 12 */ - MEM_writeLE16(ostart, (U16)((U32)set_basic + (1<<2) + (srcSize<<4))); - break; - case 3: /* 2 - 2 - 20 */ - MEM_writeLE32(ostart, (U32)((U32)set_basic + (3<<2) + (srcSize<<4))); - break; - default: /* not necessary : flSize is {1,2,3} */ - assert(0); + if (dstSize < HUF_tightCompressBound(srcSize, (size_t)tableLog) || tableLog > 11) + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ MEM_32bits() ? 2 : 4, /* kFast */ 0, /* kLastFast */ 0); + else { + if (MEM_32bits()) { + switch (tableLog) { + case 11: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 2, /* kFastFlush */ 1, /* kLastFast */ 0); + break; + case 10: ZSTD_FALLTHROUGH; + case 9: ZSTD_FALLTHROUGH; + case 8: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 2, /* kFastFlush */ 1, /* kLastFast */ 1); + break; + case 7: ZSTD_FALLTHROUGH; + default: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 3, /* kFastFlush */ 1, /* kLastFast */ 1); + break; + } + } else { + switch (tableLog) { + case 11: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 5, /* kFastFlush */ 1, /* kLastFast */ 0); + break; + case 10: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 5, /* kFastFlush */ 1, /* kLastFast */ 1); + break; + case 9: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 6, /* kFastFlush */ 1, /* kLastFast */ 0); + break; + case 8: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 7, /* kFastFlush */ 1, /* kLastFast */ 0); + break; + case 7: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 8, /* kFastFlush */ 1, /* kLastFast */ 0); + break; + case 6: ZSTD_FALLTHROUGH; + default: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 9, /* kFastFlush */ 1, /* kLastFast */ 1); + break; + } + } } + assert(bitC.ptr <= bitC.endPtr); - ZSTD_memcpy(ostart + flSize, src, srcSize); - DEBUGLOG(5, "Raw literals: %u -> %u", (U32)srcSize, (U32)(srcSize + flSize)); - return srcSize + flSize; + return HUF_closeCStream(&bitC); } -size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize) +#if DYNAMIC_BMI2 + +static BMI2_TARGET_ATTRIBUTE size_t +HUF_compress1X_usingCTable_internal_bmi2(void* dst, size_t dstSize, + const void* src, size_t srcSize, + const HUF_CElt* CTable) { - BYTE* const ostart = (BYTE*)dst; - U32 const flSize = 1 + (srcSize>31) + (srcSize>4095); + return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable); +} - (void)dstCapacity; /* dstCapacity already guaranteed to be >=4, hence large enough */ +static size_t +HUF_compress1X_usingCTable_internal_default(void* dst, size_t dstSize, + const void* src, size_t srcSize, + const HUF_CElt* CTable) +{ + return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable); +} - switch(flSize) - { - case 1: /* 2 - 1 - 5 */ - ostart[0] = (BYTE)((U32)set_rle + (srcSize<<3)); - break; - case 2: /* 2 - 2 - 12 */ - MEM_writeLE16(ostart, (U16)((U32)set_rle + (1<<2) + (srcSize<<4))); - break; - case 3: /* 2 - 2 - 20 */ - MEM_writeLE32(ostart, (U32)((U32)set_rle + (3<<2) + (srcSize<<4))); - break; - default: /* not necessary : flSize is {1,2,3} */ - assert(0); +static size_t +HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize, + const void* src, size_t srcSize, + const HUF_CElt* CTable, const int flags) +{ + if (flags & HUF_flags_bmi2) { + return HUF_compress1X_usingCTable_internal_bmi2(dst, dstSize, src, srcSize, CTable); } - - ostart[flSize] = *(const BYTE*)src; - DEBUGLOG(5, "RLE literals: %u -> %u", (U32)srcSize, (U32)flSize + 1); - return flSize+1; + return HUF_compress1X_usingCTable_internal_default(dst, dstSize, src, srcSize, CTable); } -size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, - ZSTD_hufCTables_t* nextHuf, - ZSTD_strategy strategy, int disableLiteralCompression, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - void* entropyWorkspace, size_t entropyWorkspaceSize, - const int bmi2) -{ - size_t const minGain = ZSTD_minGain(srcSize, strategy); - size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB); - BYTE* const ostart = (BYTE*)dst; - U32 singleStream = srcSize < 256; - symbolEncodingType_e hType = set_compressed; - size_t cLitSize; +#else - DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i srcSize=%u)", - disableLiteralCompression, (U32)srcSize); +static size_t +HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize, + const void* src, size_t srcSize, + const HUF_CElt* CTable, const int flags) +{ + (void)flags; + return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable); +} - /* Prepare nextEntropy assuming reusing the existing table */ - ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); +#endif - if (disableLiteralCompression) - return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags) +{ + return HUF_compress1X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, flags); +} - /* small ? don't even attempt compression (speed opt) */ -# define COMPRESS_LITERALS_SIZE_MIN 63 - { size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN; - if (srcSize <= minLitSize) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); - } +static size_t +HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize, + const void* src, size_t srcSize, + const HUF_CElt* CTable, int flags) +{ + size_t const segmentSize = (srcSize+3)/4; /* first 3 segments */ + const BYTE* ip = (const BYTE*) src; + const BYTE* const iend = ip + srcSize; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; - RETURN_ERROR_IF(dstCapacity < lhSize+1, dstSize_tooSmall, "not enough space for compression"); - { HUF_repeat repeat = prevHuf->repeatMode; - int const preferRepeat = strategy < ZSTD_lazy ? srcSize <= 1024 : 0; - if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1; - cLitSize = singleStream ? - HUF_compress1X_repeat( - ostart+lhSize, dstCapacity-lhSize, src, srcSize, - HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_DEFAULT, entropyWorkspace, entropyWorkspaceSize, - (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2) : - HUF_compress4X_repeat( - ostart+lhSize, dstCapacity-lhSize, src, srcSize, - HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_DEFAULT, entropyWorkspace, entropyWorkspaceSize, - (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2); - if (repeat != HUF_repeat_none) { - /* reused the existing table */ - DEBUGLOG(5, "Reusing previous huffman table"); - hType = set_repeat; - } + if (dstSize < 6 + 1 + 1 + 1 + 8) return 0; /* minimum space to compress successfully */ + if (srcSize < 12) return 0; /* no saving possible : too small input */ + op += 6; /* jumpTable */ + + assert(op <= oend); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) ); + if (cSize == 0 || cSize > 65535) return 0; + MEM_writeLE16(ostart, (U16)cSize); + op += cSize; } - if ((cLitSize==0) | (cLitSize >= srcSize - minGain) | ERR_isError(cLitSize)) { - ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); - return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + ip += segmentSize; + assert(op <= oend); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) ); + if (cSize == 0 || cSize > 65535) return 0; + MEM_writeLE16(ostart+2, (U16)cSize); + op += cSize; } - if (cLitSize==1) { - ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); - return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize); + + ip += segmentSize; + assert(op <= oend); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) ); + if (cSize == 0 || cSize > 65535) return 0; + MEM_writeLE16(ostart+4, (U16)cSize); + op += cSize; } - if (hType == set_compressed) { - /* using a newly constructed table */ - nextHuf->repeatMode = HUF_repeat_check; + ip += segmentSize; + assert(op <= oend); + assert(ip <= iend); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, (size_t)(iend-ip), CTable, flags) ); + if (cSize == 0 || cSize > 65535) return 0; + op += cSize; } - /* Build header */ - switch(lhSize) - { - case 3: /* 2 - 2 - 10 - 10 */ - { U32 const lhc = hType + ((!singleStream) << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<14); - MEM_writeLE24(ostart, lhc); - break; + return (size_t)(op-ostart); +} + +size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags) +{ + return HUF_compress4X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, flags); +} + +typedef enum { HUF_singleStream, HUF_fourStreams } HUF_nbStreams_e; + +static size_t HUF_compressCTable_internal( + BYTE* const ostart, BYTE* op, BYTE* const oend, + const void* src, size_t srcSize, + HUF_nbStreams_e nbStreams, const HUF_CElt* CTable, const int flags) +{ + size_t const cSize = (nbStreams==HUF_singleStream) ? + HUF_compress1X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, flags) : + HUF_compress4X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, flags); + if (HUF_isError(cSize)) { return cSize; } + if (cSize==0) { return 0; } /* uncompressible */ + op += cSize; + /* check compressibility */ + assert(op >= ostart); + if ((size_t)(op-ostart) >= srcSize-1) { return 0; } + return (size_t)(op-ostart); +} + +typedef struct { + unsigned count[HUF_SYMBOLVALUE_MAX + 1]; + HUF_CElt CTable[HUF_CTABLE_SIZE_ST(HUF_SYMBOLVALUE_MAX)]; + union { + HUF_buildCTable_wksp_tables buildCTable_wksp; + HUF_WriteCTableWksp writeCTable_wksp; + U32 hist_wksp[HIST_WKSP_SIZE_U32]; + } wksps; +} HUF_compress_tables_t; + +#define SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE 4096 +#define SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO 10 /* Must be >= 2 */ + +unsigned HUF_cardinality(const unsigned* count, unsigned maxSymbolValue) +{ + unsigned cardinality = 0; + unsigned i; + + for (i = 0; i < maxSymbolValue + 1; i++) { + if (count[i] != 0) cardinality += 1; + } + + return cardinality; +} + +unsigned HUF_minTableLog(unsigned symbolCardinality) +{ + U32 minBitsSymbols = ZSTD_highbit32(symbolCardinality) + 1; + return minBitsSymbols; +} + +unsigned HUF_optimalTableLog( + unsigned maxTableLog, + size_t srcSize, + unsigned maxSymbolValue, + void* workSpace, size_t wkspSize, + HUF_CElt* table, + const unsigned* count, + int flags) +{ + assert(srcSize > 1); /* Not supported, RLE should be used instead */ + assert(wkspSize >= sizeof(HUF_buildCTable_wksp_tables)); + + if (!(flags & HUF_flags_optimalDepth)) { + /* cheap evaluation, based on FSE */ + return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1); + } + + { BYTE* dst = (BYTE*)workSpace + sizeof(HUF_WriteCTableWksp); + size_t dstSize = wkspSize - sizeof(HUF_WriteCTableWksp); + size_t hSize, newSize; + const unsigned symbolCardinality = HUF_cardinality(count, maxSymbolValue); + const unsigned minTableLog = HUF_minTableLog(symbolCardinality); + size_t optSize = ((size_t) ~0) - 1; + unsigned optLog = maxTableLog, optLogGuess; + + DEBUGLOG(6, "HUF_optimalTableLog: probing huf depth (srcSize=%zu)", srcSize); + + /* Search until size increases */ + for (optLogGuess = minTableLog; optLogGuess <= maxTableLog; optLogGuess++) { + DEBUGLOG(7, "checking for huffLog=%u", optLogGuess); + + { size_t maxBits = HUF_buildCTable_wksp(table, count, maxSymbolValue, optLogGuess, workSpace, wkspSize); + if (ERR_isError(maxBits)) continue; + + if (maxBits < optLogGuess && optLogGuess > minTableLog) break; + + hSize = HUF_writeCTable_wksp(dst, dstSize, table, maxSymbolValue, (U32)maxBits, workSpace, wkspSize); + } + + if (ERR_isError(hSize)) continue; + + newSize = HUF_estimateCompressedSize(table, count, maxSymbolValue) + hSize; + + if (newSize > optSize + 1) { + break; + } + + if (newSize < optSize) { + optSize = newSize; + optLog = optLogGuess; + } } - case 4: /* 2 - 2 - 14 - 14 */ - { U32 const lhc = hType + (2 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<18); - MEM_writeLE32(ostart, lhc); - break; + assert(optLog <= HUF_TABLELOG_MAX); + return optLog; + } +} + +/* HUF_compress_internal() : + * `workSpace_align4` must be aligned on 4-bytes boundaries, + * and occupies the same space as a table of HUF_WORKSPACE_SIZE_U64 unsigned */ +static size_t +HUF_compress_internal (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog, + HUF_nbStreams_e nbStreams, + void* workSpace, size_t wkspSize, + HUF_CElt* oldHufTable, HUF_repeat* repeat, int flags) +{ + HUF_compress_tables_t* const table = (HUF_compress_tables_t*)HUF_alignUpWorkspace(workSpace, &wkspSize, ZSTD_ALIGNOF(size_t)); + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; + + DEBUGLOG(5, "HUF_compress_internal (srcSize=%zu)", srcSize); + HUF_STATIC_ASSERT(sizeof(*table) + HUF_WORKSPACE_MAX_ALIGNMENT <= HUF_WORKSPACE_SIZE); + + /* checks & inits */ + if (wkspSize < sizeof(*table)) return ERROR(workSpace_tooSmall); + if (!srcSize) return 0; /* Uncompressed */ + if (!dstSize) return 0; /* cannot fit anything within dst budget */ + if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); /* current block size limit */ + if (huffLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); + if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge); + if (!maxSymbolValue) maxSymbolValue = HUF_SYMBOLVALUE_MAX; + if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT; + + /* Heuristic : If old table is valid, use it for small inputs */ + if ((flags & HUF_flags_preferRepeat) && repeat && *repeat == HUF_repeat_valid) { + return HUF_compressCTable_internal(ostart, op, oend, + src, srcSize, + nbStreams, oldHufTable, flags); + } + + /* If uncompressible data is suspected, do a smaller sampling first */ + DEBUG_STATIC_ASSERT(SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO >= 2); + if ((flags & HUF_flags_suspectUncompressible) && srcSize >= (SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE * SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO)) { + size_t largestTotal = 0; + DEBUGLOG(5, "input suspected incompressible : sampling to check"); + { unsigned maxSymbolValueBegin = maxSymbolValue; + CHECK_V_F(largestBegin, HIST_count_simple (table->count, &maxSymbolValueBegin, (const BYTE*)src, SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) ); + largestTotal += largestBegin; } - case 5: /* 2 - 2 - 18 - 18 */ - { U32 const lhc = hType + (3 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<22); - MEM_writeLE32(ostart, lhc); - ostart[4] = (BYTE)(cLitSize >> 10); - break; + { unsigned maxSymbolValueEnd = maxSymbolValue; + CHECK_V_F(largestEnd, HIST_count_simple (table->count, &maxSymbolValueEnd, (const BYTE*)src + srcSize - SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE, SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) ); + largestTotal += largestEnd; } - default: /* not possible : lhSize is {3,4,5} */ - assert(0); + if (largestTotal <= ((2 * SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) >> 7)+4) return 0; /* heuristic : probably not compressible enough */ } - DEBUGLOG(5, "Compressed literals: %u -> %u", (U32)srcSize, (U32)(lhSize+cLitSize)); - return lhSize+cLitSize; + + /* Scan input and build symbol stats */ + { CHECK_V_F(largest, HIST_count_wksp (table->count, &maxSymbolValue, (const BYTE*)src, srcSize, table->wksps.hist_wksp, sizeof(table->wksps.hist_wksp)) ); + if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */ + if (largest <= (srcSize >> 7)+4) return 0; /* heuristic : probably not compressible enough */ + } + DEBUGLOG(6, "histogram detail completed (%zu symbols)", showU32(table->count, maxSymbolValue+1)); + + /* Check validity of previous table */ + if ( repeat + && *repeat == HUF_repeat_check + && !HUF_validateCTable(oldHufTable, table->count, maxSymbolValue)) { + *repeat = HUF_repeat_none; + } + /* Heuristic : use existing table for small inputs */ + if ((flags & HUF_flags_preferRepeat) && repeat && *repeat != HUF_repeat_none) { + return HUF_compressCTable_internal(ostart, op, oend, + src, srcSize, + nbStreams, oldHufTable, flags); + } + + /* Build Huffman Tree */ + huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, &table->wksps, sizeof(table->wksps), table->CTable, table->count, flags); + { size_t const maxBits = HUF_buildCTable_wksp(table->CTable, table->count, + maxSymbolValue, huffLog, + &table->wksps.buildCTable_wksp, sizeof(table->wksps.buildCTable_wksp)); + CHECK_F(maxBits); + huffLog = (U32)maxBits; + DEBUGLOG(6, "bit distribution completed (%zu symbols)", showCTableBits(table->CTable + 1, maxSymbolValue+1)); + } + + /* Write table description header */ + { CHECK_V_F(hSize, HUF_writeCTable_wksp(op, dstSize, table->CTable, maxSymbolValue, huffLog, + &table->wksps.writeCTable_wksp, sizeof(table->wksps.writeCTable_wksp)) ); + /* Check if using previous huffman table is beneficial */ + if (repeat && *repeat != HUF_repeat_none) { + size_t const oldSize = HUF_estimateCompressedSize(oldHufTable, table->count, maxSymbolValue); + size_t const newSize = HUF_estimateCompressedSize(table->CTable, table->count, maxSymbolValue); + if (oldSize <= hSize + newSize || hSize + 12 >= srcSize) { + return HUF_compressCTable_internal(ostart, op, oend, + src, srcSize, + nbStreams, oldHufTable, flags); + } } + + /* Use the new huffman table */ + if (hSize + 12ul >= srcSize) { return 0; } + op += hSize; + if (repeat) { *repeat = HUF_repeat_none; } + if (oldHufTable) + ZSTD_memcpy(oldHufTable, table->CTable, sizeof(table->CTable)); /* Save new table */ + } + return HUF_compressCTable_internal(ostart, op, oend, + src, srcSize, + nbStreams, table->CTable, flags); } -/**** ended inlining compress/zstd_compress_literals.c ****/ -/**** start inlining compress/zstd_compress_sequences.c ****/ + +size_t HUF_compress1X_repeat (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog, + void* workSpace, size_t wkspSize, + HUF_CElt* hufTable, HUF_repeat* repeat, int flags) +{ + DEBUGLOG(5, "HUF_compress1X_repeat (srcSize = %zu)", srcSize); + return HUF_compress_internal(dst, dstSize, src, srcSize, + maxSymbolValue, huffLog, HUF_singleStream, + workSpace, wkspSize, hufTable, + repeat, flags); +} + +/* HUF_compress4X_repeat(): + * compress input using 4 streams. + * consider skipping quickly + * reuse an existing huffman compression table */ +size_t HUF_compress4X_repeat (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog, + void* workSpace, size_t wkspSize, + HUF_CElt* hufTable, HUF_repeat* repeat, int flags) +{ + DEBUGLOG(5, "HUF_compress4X_repeat (srcSize = %zu)", srcSize); + return HUF_compress_internal(dst, dstSize, src, srcSize, + maxSymbolValue, huffLog, HUF_fourStreams, + workSpace, wkspSize, + hufTable, repeat, flags); +} +/**** ended inlining compress/huf_compress.c ****/ +/**** start inlining compress/zstd_compress_literals.c ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -12534,9 +15036,9 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, /*-************************************* * Dependencies ***************************************/ -/**** start inlining zstd_compress_sequences.h ****/ +/**** start inlining zstd_compress_literals.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -12545,4612 +15047,9102 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, * You may select, at your option, one of the above-listed licenses. */ -#ifndef ZSTD_COMPRESS_SEQUENCES_H -#define ZSTD_COMPRESS_SEQUENCES_H +#ifndef ZSTD_COMPRESS_LITERALS_H +#define ZSTD_COMPRESS_LITERALS_H -/**** skipping file: ../common/fse.h ****/ -/**** skipping file: ../common/zstd_internal.h ****/ +/**** start inlining zstd_compress_internal.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ -typedef enum { - ZSTD_defaultDisallowed = 0, - ZSTD_defaultAllowed = 1 -} ZSTD_defaultPolicy_e; +/* This header contains definitions + * that shall **only** be used by modules within lib/compress. + */ -symbolEncodingType_e -ZSTD_selectEncodingType( - FSE_repeat* repeatMode, unsigned const* count, unsigned const max, - size_t const mostFrequent, size_t nbSeq, unsigned const FSELog, - FSE_CTable const* prevCTable, - short const* defaultNorm, U32 defaultNormLog, - ZSTD_defaultPolicy_e const isDefaultAllowed, - ZSTD_strategy const strategy); +#ifndef ZSTD_COMPRESS_H +#define ZSTD_COMPRESS_H -size_t -ZSTD_buildCTable(void* dst, size_t dstCapacity, - FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type, - unsigned* count, U32 max, - const BYTE* codeTable, size_t nbSeq, - const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax, - const FSE_CTable* prevCTable, size_t prevCTableSize, - void* entropyWorkspace, size_t entropyWorkspaceSize); +/*-************************************* +* Dependencies +***************************************/ +/**** skipping file: ../common/zstd_internal.h ****/ +/**** start inlining zstd_cwksp.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ -size_t ZSTD_encodeSequences( - void* dst, size_t dstCapacity, - FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, - FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, - FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, - seqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2); +#ifndef ZSTD_CWKSP_H +#define ZSTD_CWKSP_H -size_t ZSTD_fseBitCost( - FSE_CTable const* ctable, - unsigned const* count, - unsigned const max); +/*-************************************* +* Dependencies +***************************************/ +/**** skipping file: ../common/allocations.h ****/ +/**** skipping file: ../common/zstd_internal.h ****/ +/**** skipping file: ../common/portability_macros.h ****/ +/**** skipping file: ../common/compiler.h ****/ -size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog, - unsigned const* count, unsigned const max); -#endif /* ZSTD_COMPRESS_SEQUENCES_H */ -/**** ended inlining zstd_compress_sequences.h ****/ +/*-************************************* +* Constants +***************************************/ + +/* Since the workspace is effectively its own little malloc implementation / + * arena, when we run under ASAN, we should similarly insert redzones between + * each internal element of the workspace, so ASAN will catch overruns that + * reach outside an object but that stay inside the workspace. + * + * This defines the size of that redzone. + */ +#ifndef ZSTD_CWKSP_ASAN_REDZONE_SIZE +#define ZSTD_CWKSP_ASAN_REDZONE_SIZE 128 +#endif + + +/* Set our tables and aligneds to align by 64 bytes */ +#define ZSTD_CWKSP_ALIGNMENT_BYTES 64 + +/*-************************************* +* Structures +***************************************/ +typedef enum { + ZSTD_cwksp_alloc_objects, + ZSTD_cwksp_alloc_aligned_init_once, + ZSTD_cwksp_alloc_aligned, + ZSTD_cwksp_alloc_buffers +} ZSTD_cwksp_alloc_phase_e; /** - * -log2(x / 256) lookup table for x in [0, 256). - * If x == 0: Return 0 - * Else: Return floor(-log2(x / 256) * 256) + * Used to describe whether the workspace is statically allocated (and will not + * necessarily ever be freed), or if it's dynamically allocated and we can + * expect a well-formed caller to free this. */ -static unsigned const kInverseProbabilityLog256[256] = { - 0, 2048, 1792, 1642, 1536, 1453, 1386, 1329, 1280, 1236, 1197, 1162, - 1130, 1100, 1073, 1047, 1024, 1001, 980, 960, 941, 923, 906, 889, - 874, 859, 844, 830, 817, 804, 791, 779, 768, 756, 745, 734, - 724, 714, 704, 694, 685, 676, 667, 658, 650, 642, 633, 626, - 618, 610, 603, 595, 588, 581, 574, 567, 561, 554, 548, 542, - 535, 529, 523, 517, 512, 506, 500, 495, 489, 484, 478, 473, - 468, 463, 458, 453, 448, 443, 438, 434, 429, 424, 420, 415, - 411, 407, 402, 398, 394, 390, 386, 382, 377, 373, 370, 366, - 362, 358, 354, 350, 347, 343, 339, 336, 332, 329, 325, 322, - 318, 315, 311, 308, 305, 302, 298, 295, 292, 289, 286, 282, - 279, 276, 273, 270, 267, 264, 261, 258, 256, 253, 250, 247, - 244, 241, 239, 236, 233, 230, 228, 225, 222, 220, 217, 215, - 212, 209, 207, 204, 202, 199, 197, 194, 192, 190, 187, 185, - 182, 180, 178, 175, 173, 171, 168, 166, 164, 162, 159, 157, - 155, 153, 151, 149, 146, 144, 142, 140, 138, 136, 134, 132, - 130, 128, 126, 123, 121, 119, 117, 115, 114, 112, 110, 108, - 106, 104, 102, 100, 98, 96, 94, 93, 91, 89, 87, 85, - 83, 82, 80, 78, 76, 74, 73, 71, 69, 67, 66, 64, - 62, 61, 59, 57, 55, 54, 52, 50, 49, 47, 46, 44, - 42, 41, 39, 37, 36, 34, 33, 31, 30, 28, 26, 25, - 23, 22, 20, 19, 17, 16, 14, 13, 11, 10, 8, 7, - 5, 4, 2, 1, -}; +typedef enum { + ZSTD_cwksp_dynamic_alloc, + ZSTD_cwksp_static_alloc +} ZSTD_cwksp_static_alloc_e; -static unsigned ZSTD_getFSEMaxSymbolValue(FSE_CTable const* ctable) { - void const* ptr = ctable; - U16 const* u16ptr = (U16 const*)ptr; - U32 const maxSymbolValue = MEM_read16(u16ptr + 1); - return maxSymbolValue; +/** + * Zstd fits all its internal datastructures into a single continuous buffer, + * so that it only needs to perform a single OS allocation (or so that a buffer + * can be provided to it and it can perform no allocations at all). This buffer + * is called the workspace. + * + * Several optimizations complicate that process of allocating memory ranges + * from this workspace for each internal datastructure: + * + * - These different internal datastructures have different setup requirements: + * + * - The static objects need to be cleared once and can then be trivially + * reused for each compression. + * + * - Various buffers don't need to be initialized at all--they are always + * written into before they're read. + * + * - The matchstate tables have a unique requirement that they don't need + * their memory to be totally cleared, but they do need the memory to have + * some bound, i.e., a guarantee that all values in the memory they've been + * allocated is less than some maximum value (which is the starting value + * for the indices that they will then use for compression). When this + * guarantee is provided to them, they can use the memory without any setup + * work. When it can't, they have to clear the area. + * + * - These buffers also have different alignment requirements. + * + * - We would like to reuse the objects in the workspace for multiple + * compressions without having to perform any expensive reallocation or + * reinitialization work. + * + * - We would like to be able to efficiently reuse the workspace across + * multiple compressions **even when the compression parameters change** and + * we need to resize some of the objects (where possible). + * + * To attempt to manage this buffer, given these constraints, the ZSTD_cwksp + * abstraction was created. It works as follows: + * + * Workspace Layout: + * + * [ ... workspace ... ] + * [objects][tables ->] free space [<- buffers][<- aligned][<- init once] + * + * The various objects that live in the workspace are divided into the + * following categories, and are allocated separately: + * + * - Static objects: this is optionally the enclosing ZSTD_CCtx or ZSTD_CDict, + * so that literally everything fits in a single buffer. Note: if present, + * this must be the first object in the workspace, since ZSTD_customFree{CCtx, + * CDict}() rely on a pointer comparison to see whether one or two frees are + * required. + * + * - Fixed size objects: these are fixed-size, fixed-count objects that are + * nonetheless "dynamically" allocated in the workspace so that we can + * control how they're initialized separately from the broader ZSTD_CCtx. + * Examples: + * - Entropy Workspace + * - 2 x ZSTD_compressedBlockState_t + * - CDict dictionary contents + * + * - Tables: these are any of several different datastructures (hash tables, + * chain tables, binary trees) that all respect a common format: they are + * uint32_t arrays, all of whose values are between 0 and (nextSrc - base). + * Their sizes depend on the cparams. These tables are 64-byte aligned. + * + * - Init once: these buffers require to be initialized at least once before + * use. They should be used when we want to skip memory initialization + * while not triggering memory checkers (like Valgrind) when reading from + * from this memory without writing to it first. + * These buffers should be used carefully as they might contain data + * from previous compressions. + * Buffers are aligned to 64 bytes. + * + * - Aligned: these buffers don't require any initialization before they're + * used. The user of the buffer should make sure they write into a buffer + * location before reading from it. + * Buffers are aligned to 64 bytes. + * + * - Buffers: these buffers are used for various purposes that don't require + * any alignment or initialization before they're used. This means they can + * be moved around at no cost for a new compression. + * + * Allocating Memory: + * + * The various types of objects must be allocated in order, so they can be + * correctly packed into the workspace buffer. That order is: + * + * 1. Objects + * 2. Init once / Tables + * 3. Aligned / Tables + * 4. Buffers / Tables + * + * Attempts to reserve objects of different types out of order will fail. + */ +typedef struct { + void* workspace; + void* workspaceEnd; + + void* objectEnd; + void* tableEnd; + void* tableValidEnd; + void* allocStart; + void* initOnceStart; + + BYTE allocFailed; + int workspaceOversizedDuration; + ZSTD_cwksp_alloc_phase_e phase; + ZSTD_cwksp_static_alloc_e isStatic; +} ZSTD_cwksp; + +/*-************************************* +* Functions +***************************************/ + +MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws); +MEM_STATIC void* ZSTD_cwksp_initialAllocStart(ZSTD_cwksp* ws); + +MEM_STATIC void ZSTD_cwksp_assert_internal_consistency(ZSTD_cwksp* ws) { + (void)ws; + assert(ws->workspace <= ws->objectEnd); + assert(ws->objectEnd <= ws->tableEnd); + assert(ws->objectEnd <= ws->tableValidEnd); + assert(ws->tableEnd <= ws->allocStart); + assert(ws->tableValidEnd <= ws->allocStart); + assert(ws->allocStart <= ws->workspaceEnd); + assert(ws->initOnceStart <= ZSTD_cwksp_initialAllocStart(ws)); + assert(ws->workspace <= ws->initOnceStart); +#if ZSTD_MEMORY_SANITIZER + { + intptr_t const offset = __msan_test_shadow(ws->initOnceStart, + (U8*)ZSTD_cwksp_initialAllocStart(ws) - (U8*)ws->initOnceStart); + (void)offset; +#if defined(ZSTD_MSAN_PRINT) + if(offset!=-1) { + __msan_print_shadow((U8*)ws->initOnceStart + offset - 8, 32); + } +#endif + assert(offset==-1); + }; +#endif } /** - * Returns true if we should use ncount=-1 else we should - * use ncount=1 for low probability symbols instead. + * Align must be a power of 2. */ -static unsigned ZSTD_useLowProbCount(size_t const nbSeq) -{ - /* Heuristic: This should cover most blocks <= 16K and - * start to fade out after 16K to about 32K depending on - * comprssibility. - */ - return nbSeq >= 2048; +MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t align) { + size_t const mask = align - 1; + assert(ZSTD_isPower2(align)); + return (size + mask) & ~mask; } /** - * Returns the cost in bytes of encoding the normalized count header. - * Returns an error if any of the helper functions return an error. + * Use this to determine how much space in the workspace we will consume to + * allocate this object. (Normally it should be exactly the size of the object, + * but under special conditions, like ASAN, where we pad each object, it might + * be larger.) + * + * Since tables aren't currently redzoned, you don't need to call through this + * to figure out how much space you need for the matchState tables. Everything + * else is though. + * + * Do not use for sizing aligned buffers. Instead, use ZSTD_cwksp_aligned64_alloc_size(). */ -static size_t ZSTD_NCountCost(unsigned const* count, unsigned const max, - size_t const nbSeq, unsigned const FSELog) -{ - BYTE wksp[FSE_NCOUNTBOUND]; - S16 norm[MaxSeq + 1]; - const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max); - FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq, max, ZSTD_useLowProbCount(nbSeq)), ""); - return FSE_writeNCount(wksp, sizeof(wksp), norm, max, tableLog); +MEM_STATIC size_t ZSTD_cwksp_alloc_size(size_t size) { + if (size == 0) + return 0; +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + return size + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; +#else + return size; +#endif +} + +MEM_STATIC size_t ZSTD_cwksp_aligned_alloc_size(size_t size, size_t alignment) { + return ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(size, alignment)); } /** - * Returns the cost in bits of encoding the distribution described by count - * using the entropy bound. + * Returns an adjusted alloc size that is the nearest larger multiple of 64 bytes. + * Used to determine the number of bytes required for a given "aligned". */ -static size_t ZSTD_entropyCost(unsigned const* count, unsigned const max, size_t const total) -{ - unsigned cost = 0; - unsigned s; - for (s = 0; s <= max; ++s) { - unsigned norm = (unsigned)((256 * count[s]) / total); - if (count[s] != 0 && norm == 0) - norm = 1; - assert(count[s] < total); - cost += count[s] * kInverseProbabilityLog256[norm]; - } - return cost >> 8; +MEM_STATIC size_t ZSTD_cwksp_aligned64_alloc_size(size_t size) { + return ZSTD_cwksp_aligned_alloc_size(size, ZSTD_CWKSP_ALIGNMENT_BYTES); } /** - * Returns the cost in bits of encoding the distribution in count using ctable. - * Returns an error if ctable cannot represent all the symbols in count. + * Returns the amount of additional space the cwksp must allocate + * for internal purposes (currently only alignment). */ -size_t ZSTD_fseBitCost( - FSE_CTable const* ctable, - unsigned const* count, - unsigned const max) -{ - unsigned const kAccuracyLog = 8; - size_t cost = 0; - unsigned s; - FSE_CState_t cstate; - FSE_initCState(&cstate, ctable); - if (ZSTD_getFSEMaxSymbolValue(ctable) < max) { - DEBUGLOG(5, "Repeat FSE_CTable has maxSymbolValue %u < %u", - ZSTD_getFSEMaxSymbolValue(ctable), max); - return ERROR(GENERIC); +MEM_STATIC size_t ZSTD_cwksp_slack_space_required(void) { + /* For alignment, the wksp will always allocate an additional 2*ZSTD_CWKSP_ALIGNMENT_BYTES + * bytes to align the beginning of tables section and end of buffers; + */ + size_t const slackSpace = ZSTD_CWKSP_ALIGNMENT_BYTES * 2; + return slackSpace; +} + + +/** + * Return the number of additional bytes required to align a pointer to the given number of bytes. + * alignBytes must be a power of two. + */ +MEM_STATIC size_t ZSTD_cwksp_bytes_to_align_ptr(void* ptr, const size_t alignBytes) { + size_t const alignBytesMask = alignBytes - 1; + size_t const bytes = (alignBytes - ((size_t)ptr & (alignBytesMask))) & alignBytesMask; + assert(ZSTD_isPower2(alignBytes)); + assert(bytes < alignBytes); + return bytes; +} + +/** + * Returns the initial value for allocStart which is used to determine the position from + * which we can allocate from the end of the workspace. + */ +MEM_STATIC void* ZSTD_cwksp_initialAllocStart(ZSTD_cwksp* ws) +{ + char* endPtr = (char*)ws->workspaceEnd; + assert(ZSTD_isPower2(ZSTD_CWKSP_ALIGNMENT_BYTES)); + endPtr = endPtr - ((size_t)endPtr % ZSTD_CWKSP_ALIGNMENT_BYTES); + return (void*)endPtr; +} + +/** + * Internal function. Do not use directly. + * Reserves the given number of bytes within the aligned/buffer segment of the wksp, + * which counts from the end of the wksp (as opposed to the object/table segment). + * + * Returns a pointer to the beginning of that space. + */ +MEM_STATIC void* +ZSTD_cwksp_reserve_internal_buffer_space(ZSTD_cwksp* ws, size_t const bytes) +{ + void* const alloc = (BYTE*)ws->allocStart - bytes; + void* const bottom = ws->tableEnd; + DEBUGLOG(5, "cwksp: reserving [0x%p]:%zd bytes; %zd bytes remaining", + alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes); + ZSTD_cwksp_assert_internal_consistency(ws); + assert(alloc >= bottom); + if (alloc < bottom) { + DEBUGLOG(4, "cwksp: alloc failed!"); + ws->allocFailed = 1; + return NULL; } - for (s = 0; s <= max; ++s) { - unsigned const tableLog = cstate.stateLog; - unsigned const badCost = (tableLog + 1) << kAccuracyLog; - unsigned const bitCost = FSE_bitCost(cstate.symbolTT, tableLog, s, kAccuracyLog); - if (count[s] == 0) - continue; - if (bitCost >= badCost) { - DEBUGLOG(5, "Repeat FSE_CTable has Prob[%u] == 0", s); - return ERROR(GENERIC); - } - cost += (size_t)count[s] * bitCost; + /* the area is reserved from the end of wksp. + * If it overlaps with tableValidEnd, it voids guarantees on values' range */ + if (alloc < ws->tableValidEnd) { + ws->tableValidEnd = alloc; } - return cost >> kAccuracyLog; + ws->allocStart = alloc; + return alloc; } /** - * Returns the cost in bits of encoding the distribution in count using the - * table described by norm. The max symbol support by norm is assumed >= max. - * norm must be valid for every symbol with non-zero probability in count. + * Moves the cwksp to the next phase, and does any necessary allocations. + * cwksp initialization must necessarily go through each phase in order. + * Returns a 0 on success, or zstd error */ -size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog, - unsigned const* count, unsigned const max) +MEM_STATIC size_t +ZSTD_cwksp_internal_advance_phase(ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase) { - unsigned const shift = 8 - accuracyLog; - size_t cost = 0; - unsigned s; - assert(accuracyLog <= 8); - for (s = 0; s <= max; ++s) { - unsigned const normAcc = (norm[s] != -1) ? (unsigned)norm[s] : 1; - unsigned const norm256 = normAcc << shift; - assert(norm256 > 0); - assert(norm256 < 256); - cost += count[s] * kInverseProbabilityLog256[norm256]; + assert(phase >= ws->phase); + if (phase > ws->phase) { + /* Going from allocating objects to allocating initOnce / tables */ + if (ws->phase < ZSTD_cwksp_alloc_aligned_init_once && + phase >= ZSTD_cwksp_alloc_aligned_init_once) { + ws->tableValidEnd = ws->objectEnd; + ws->initOnceStart = ZSTD_cwksp_initialAllocStart(ws); + + { /* Align the start of the tables to 64 bytes. Use [0, 63] bytes */ + void *const alloc = ws->objectEnd; + size_t const bytesToAlign = ZSTD_cwksp_bytes_to_align_ptr(alloc, ZSTD_CWKSP_ALIGNMENT_BYTES); + void *const objectEnd = (BYTE *) alloc + bytesToAlign; + DEBUGLOG(5, "reserving table alignment addtl space: %zu", bytesToAlign); + RETURN_ERROR_IF(objectEnd > ws->workspaceEnd, memory_allocation, + "table phase - alignment initial allocation failed!"); + ws->objectEnd = objectEnd; + ws->tableEnd = objectEnd; /* table area starts being empty */ + if (ws->tableValidEnd < ws->tableEnd) { + ws->tableValidEnd = ws->tableEnd; + } + } + } + ws->phase = phase; + ZSTD_cwksp_assert_internal_consistency(ws); } - return cost >> 8; + return 0; } -symbolEncodingType_e -ZSTD_selectEncodingType( - FSE_repeat* repeatMode, unsigned const* count, unsigned const max, - size_t const mostFrequent, size_t nbSeq, unsigned const FSELog, - FSE_CTable const* prevCTable, - short const* defaultNorm, U32 defaultNormLog, - ZSTD_defaultPolicy_e const isDefaultAllowed, - ZSTD_strategy const strategy) +/** + * Returns whether this object/buffer/etc was allocated in this workspace. + */ +MEM_STATIC int ZSTD_cwksp_owns_buffer(const ZSTD_cwksp* ws, const void* ptr) { - ZSTD_STATIC_ASSERT(ZSTD_defaultDisallowed == 0 && ZSTD_defaultAllowed != 0); - if (mostFrequent == nbSeq) { - *repeatMode = FSE_repeat_none; - if (isDefaultAllowed && nbSeq <= 2) { - /* Prefer set_basic over set_rle when there are 2 or less symbols, - * since RLE uses 1 byte, but set_basic uses 5-6 bits per symbol. - * If basic encoding isn't possible, always choose RLE. - */ - DEBUGLOG(5, "Selected set_basic"); - return set_basic; - } - DEBUGLOG(5, "Selected set_rle"); - return set_rle; + return (ptr != NULL) && (ws->workspace <= ptr) && (ptr < ws->workspaceEnd); +} + +/** + * Internal function. Do not use directly. + */ +MEM_STATIC void* +ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) +{ + void* alloc; + if (ZSTD_isError(ZSTD_cwksp_internal_advance_phase(ws, phase)) || bytes == 0) { + return NULL; } - if (strategy < ZSTD_lazy) { - if (isDefaultAllowed) { - size_t const staticFse_nbSeq_max = 1000; - size_t const mult = 10 - strategy; - size_t const baseLog = 3; - size_t const dynamicFse_nbSeq_min = (((size_t)1 << defaultNormLog) * mult) >> baseLog; /* 28-36 for offset, 56-72 for lengths */ - assert(defaultNormLog >= 5 && defaultNormLog <= 6); /* xx_DEFAULTNORMLOG */ - assert(mult <= 9 && mult >= 7); - if ( (*repeatMode == FSE_repeat_valid) - && (nbSeq < staticFse_nbSeq_max) ) { - DEBUGLOG(5, "Selected set_repeat"); - return set_repeat; - } - if ( (nbSeq < dynamicFse_nbSeq_min) - || (mostFrequent < (nbSeq >> (defaultNormLog-1))) ) { - DEBUGLOG(5, "Selected set_basic"); - /* The format allows default tables to be repeated, but it isn't useful. - * When using simple heuristics to select encoding type, we don't want - * to confuse these tables with dictionaries. When running more careful - * analysis, we don't need to waste time checking both repeating tables - * and default tables. - */ - *repeatMode = FSE_repeat_none; - return set_basic; - } - } - } else { - size_t const basicCost = isDefaultAllowed ? ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, count, max) : ERROR(GENERIC); - size_t const repeatCost = *repeatMode != FSE_repeat_none ? ZSTD_fseBitCost(prevCTable, count, max) : ERROR(GENERIC); - size_t const NCountCost = ZSTD_NCountCost(count, max, nbSeq, FSELog); - size_t const compressedCost = (NCountCost << 3) + ZSTD_entropyCost(count, max, nbSeq); - if (isDefaultAllowed) { - assert(!ZSTD_isError(basicCost)); - assert(!(*repeatMode == FSE_repeat_valid && ZSTD_isError(repeatCost))); - } - assert(!ZSTD_isError(NCountCost)); - assert(compressedCost < ERROR(maxCode)); - DEBUGLOG(5, "Estimated bit costs: basic=%u\trepeat=%u\tcompressed=%u", - (unsigned)basicCost, (unsigned)repeatCost, (unsigned)compressedCost); - if (basicCost <= repeatCost && basicCost <= compressedCost) { - DEBUGLOG(5, "Selected set_basic"); - assert(isDefaultAllowed); - *repeatMode = FSE_repeat_none; - return set_basic; - } - if (repeatCost <= compressedCost) { - DEBUGLOG(5, "Selected set_repeat"); - assert(!ZSTD_isError(repeatCost)); - return set_repeat; +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* over-reserve space */ + bytes += 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; +#endif + + alloc = ZSTD_cwksp_reserve_internal_buffer_space(ws, bytes); + +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on + * either size. */ + if (alloc) { + alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE; + if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { + /* We need to keep the redzone poisoned while unpoisoning the bytes that + * are actually allocated. */ + __asan_unpoison_memory_region(alloc, bytes - 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE); } - assert(compressedCost < basicCost && compressedCost < repeatCost); } - DEBUGLOG(5, "Selected set_compressed"); - *repeatMode = FSE_repeat_check; - return set_compressed; +#endif + + return alloc; } -size_t -ZSTD_buildCTable(void* dst, size_t dstCapacity, - FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type, - unsigned* count, U32 max, - const BYTE* codeTable, size_t nbSeq, - const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax, - const FSE_CTable* prevCTable, size_t prevCTableSize, - void* entropyWorkspace, size_t entropyWorkspaceSize) +/** + * Reserves and returns unaligned memory. + */ +MEM_STATIC BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes) { - BYTE* op = (BYTE*)dst; - const BYTE* const oend = op + dstCapacity; - DEBUGLOG(6, "ZSTD_buildCTable (dstCapacity=%u)", (unsigned)dstCapacity); + return (BYTE*)ZSTD_cwksp_reserve_internal(ws, bytes, ZSTD_cwksp_alloc_buffers); +} - switch (type) { - case set_rle: - FORWARD_IF_ERROR(FSE_buildCTable_rle(nextCTable, (BYTE)max), ""); - RETURN_ERROR_IF(dstCapacity==0, dstSize_tooSmall, "not enough space"); - *op = codeTable[0]; - return 1; - case set_repeat: - ZSTD_memcpy(nextCTable, prevCTable, prevCTableSize); - return 0; - case set_basic: - FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, entropyWorkspace, entropyWorkspaceSize), ""); /* note : could be pre-calculated */ - return 0; - case set_compressed: { - S16 norm[MaxSeq + 1]; - size_t nbSeq_1 = nbSeq; - const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max); - if (count[codeTable[nbSeq-1]] > 1) { - count[codeTable[nbSeq-1]]--; - nbSeq_1--; - } - assert(nbSeq_1 > 1); - assert(entropyWorkspaceSize >= FSE_BUILD_CTABLE_WORKSPACE_SIZE(MaxSeq, MaxFSELog)); - FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max, ZSTD_useLowProbCount(nbSeq_1)), ""); - { size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */ - FORWARD_IF_ERROR(NCountSize, "FSE_writeNCount failed"); - FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog, entropyWorkspace, entropyWorkspaceSize), ""); - return NCountSize; - } - } - default: assert(0); RETURN_ERROR(GENERIC, "impossible to reach"); +/** + * Reserves and returns memory sized on and aligned on ZSTD_CWKSP_ALIGNMENT_BYTES (64 bytes). + * This memory has been initialized at least once in the past. + * This doesn't mean it has been initialized this time, and it might contain data from previous + * operations. + * The main usage is for algorithms that might need read access into uninitialized memory. + * The algorithm must maintain safety under these conditions and must make sure it doesn't + * leak any of the past data (directly or in side channels). + */ +MEM_STATIC void* ZSTD_cwksp_reserve_aligned_init_once(ZSTD_cwksp* ws, size_t bytes) +{ + size_t const alignedBytes = ZSTD_cwksp_align(bytes, ZSTD_CWKSP_ALIGNMENT_BYTES); + void* ptr = ZSTD_cwksp_reserve_internal(ws, alignedBytes, ZSTD_cwksp_alloc_aligned_init_once); + assert(((size_t)ptr & (ZSTD_CWKSP_ALIGNMENT_BYTES-1)) == 0); + if(ptr && ptr < ws->initOnceStart) { + /* We assume the memory following the current allocation is either: + * 1. Not usable as initOnce memory (end of workspace) + * 2. Another initOnce buffer that has been allocated before (and so was previously memset) + * 3. An ASAN redzone, in which case we don't want to write on it + * For these reasons it should be fine to not explicitly zero every byte up to ws->initOnceStart. + * Note that we assume here that MSAN and ASAN cannot run in the same time. */ + ZSTD_memset(ptr, 0, MIN((size_t)((U8*)ws->initOnceStart - (U8*)ptr), alignedBytes)); + ws->initOnceStart = ptr; } +#if ZSTD_MEMORY_SANITIZER + assert(__msan_test_shadow(ptr, bytes) == -1); +#endif + return ptr; } -FORCE_INLINE_TEMPLATE size_t -ZSTD_encodeSequences_body( - void* dst, size_t dstCapacity, - FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, - FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, - FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, - seqDef const* sequences, size_t nbSeq, int longOffsets) +/** + * Reserves and returns memory sized on and aligned on ZSTD_CWKSP_ALIGNMENT_BYTES (64 bytes). + */ +MEM_STATIC void* ZSTD_cwksp_reserve_aligned64(ZSTD_cwksp* ws, size_t bytes) { - BIT_CStream_t blockStream; - FSE_CState_t stateMatchLength; - FSE_CState_t stateOffsetBits; - FSE_CState_t stateLitLength; + void* const ptr = ZSTD_cwksp_reserve_internal(ws, + ZSTD_cwksp_align(bytes, ZSTD_CWKSP_ALIGNMENT_BYTES), + ZSTD_cwksp_alloc_aligned); + assert(((size_t)ptr & (ZSTD_CWKSP_ALIGNMENT_BYTES-1)) == 0); + return ptr; +} - RETURN_ERROR_IF( - ERR_isError(BIT_initCStream(&blockStream, dst, dstCapacity)), - dstSize_tooSmall, "not enough space remaining"); - DEBUGLOG(6, "available space for bitstream : %i (dstCapacity=%u)", - (int)(blockStream.endPtr - blockStream.startPtr), - (unsigned)dstCapacity); +/** + * Aligned on 64 bytes. These buffers have the special property that + * their values remain constrained, allowing us to reuse them without + * memset()-ing them. + */ +MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) +{ + const ZSTD_cwksp_alloc_phase_e phase = ZSTD_cwksp_alloc_aligned_init_once; + void* alloc; + void* end; + void* top; - /* first symbols */ - FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq-1]); - FSE_initCState2(&stateOffsetBits, CTable_OffsetBits, ofCodeTable[nbSeq-1]); - FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq-1]); - BIT_addBits(&blockStream, sequences[nbSeq-1].litLength, LL_bits[llCodeTable[nbSeq-1]]); - if (MEM_32bits()) BIT_flushBits(&blockStream); - BIT_addBits(&blockStream, sequences[nbSeq-1].matchLength, ML_bits[mlCodeTable[nbSeq-1]]); - if (MEM_32bits()) BIT_flushBits(&blockStream); - if (longOffsets) { - U32 const ofBits = ofCodeTable[nbSeq-1]; - unsigned const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); - if (extraBits) { - BIT_addBits(&blockStream, sequences[nbSeq-1].offset, extraBits); - BIT_flushBits(&blockStream); + /* We can only start allocating tables after we are done reserving space for objects at the + * start of the workspace */ + if(ws->phase < phase) { + if (ZSTD_isError(ZSTD_cwksp_internal_advance_phase(ws, phase))) { + return NULL; } - BIT_addBits(&blockStream, sequences[nbSeq-1].offset >> extraBits, - ofBits - extraBits); - } else { - BIT_addBits(&blockStream, sequences[nbSeq-1].offset, ofCodeTable[nbSeq-1]); } - BIT_flushBits(&blockStream); + alloc = ws->tableEnd; + end = (BYTE *)alloc + bytes; + top = ws->allocStart; - { size_t n; - for (n=nbSeq-2 ; n= 64-7-(LLFSELog+MLFSELog+OffFSELog))) - BIT_flushBits(&blockStream); /* (7)*/ - BIT_addBits(&blockStream, sequences[n].litLength, llBits); - if (MEM_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream); - BIT_addBits(&blockStream, sequences[n].matchLength, mlBits); - if (MEM_32bits() || (ofBits+mlBits+llBits > 56)) BIT_flushBits(&blockStream); - if (longOffsets) { - unsigned const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); - if (extraBits) { - BIT_addBits(&blockStream, sequences[n].offset, extraBits); - BIT_flushBits(&blockStream); /* (7)*/ - } - BIT_addBits(&blockStream, sequences[n].offset >> extraBits, - ofBits - extraBits); /* 31 */ - } else { - BIT_addBits(&blockStream, sequences[n].offset, ofBits); /* 31 */ - } - BIT_flushBits(&blockStream); /* (7)*/ - DEBUGLOG(7, "remaining space : %i", (int)(blockStream.endPtr - blockStream.ptr)); - } } - - DEBUGLOG(6, "ZSTD_encodeSequences: flushing ML state with %u bits", stateMatchLength.stateLog); - FSE_flushCState(&blockStream, &stateMatchLength); - DEBUGLOG(6, "ZSTD_encodeSequences: flushing Off state with %u bits", stateOffsetBits.stateLog); - FSE_flushCState(&blockStream, &stateOffsetBits); - DEBUGLOG(6, "ZSTD_encodeSequences: flushing LL state with %u bits", stateLitLength.stateLog); - FSE_flushCState(&blockStream, &stateLitLength); + DEBUGLOG(5, "cwksp: reserving %p table %zd bytes, %zd bytes remaining", + alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes); + assert((bytes & (sizeof(U32)-1)) == 0); + ZSTD_cwksp_assert_internal_consistency(ws); + assert(end <= top); + if (end > top) { + DEBUGLOG(4, "cwksp: table alloc failed!"); + ws->allocFailed = 1; + return NULL; + } + ws->tableEnd = end; - { size_t const streamSize = BIT_closeCStream(&blockStream); - RETURN_ERROR_IF(streamSize==0, dstSize_tooSmall, "not enough space"); - return streamSize; +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { + __asan_unpoison_memory_region(alloc, bytes); } +#endif + + assert((bytes & (ZSTD_CWKSP_ALIGNMENT_BYTES-1)) == 0); + assert(((size_t)alloc & (ZSTD_CWKSP_ALIGNMENT_BYTES-1)) == 0); + return alloc; } -static size_t -ZSTD_encodeSequences_default( - void* dst, size_t dstCapacity, - FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, - FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, - FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, - seqDef const* sequences, size_t nbSeq, int longOffsets) +/** + * Aligned on sizeof(void*). + * Note : should happen only once, at workspace first initialization + */ +MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { - return ZSTD_encodeSequences_body(dst, dstCapacity, - CTable_MatchLength, mlCodeTable, - CTable_OffsetBits, ofCodeTable, - CTable_LitLength, llCodeTable, - sequences, nbSeq, longOffsets); -} + size_t const roundedBytes = ZSTD_cwksp_align(bytes, sizeof(void*)); + void* alloc = ws->objectEnd; + void* end = (BYTE*)alloc + roundedBytes; +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* over-reserve space */ + end = (BYTE *)end + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; +#endif -#if DYNAMIC_BMI2 + DEBUGLOG(4, + "cwksp: reserving %p object %zd bytes (rounded to %zd), %zd bytes remaining", + alloc, bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes); + assert((size_t)alloc % ZSTD_ALIGNOF(void*) == 0); + assert(bytes % ZSTD_ALIGNOF(void*) == 0); + ZSTD_cwksp_assert_internal_consistency(ws); + /* we must be in the first phase, no advance is possible */ + if (ws->phase != ZSTD_cwksp_alloc_objects || end > ws->workspaceEnd) { + DEBUGLOG(3, "cwksp: object alloc failed!"); + ws->allocFailed = 1; + return NULL; + } + ws->objectEnd = end; + ws->tableEnd = end; + ws->tableValidEnd = end; -static TARGET_ATTRIBUTE("bmi2") size_t -ZSTD_encodeSequences_bmi2( - void* dst, size_t dstCapacity, - FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, - FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, - FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, - seqDef const* sequences, size_t nbSeq, int longOffsets) +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on + * either size. */ + alloc = (BYTE*)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE; + if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { + __asan_unpoison_memory_region(alloc, bytes); + } +#endif + + return alloc; +} +/** + * with alignment control + * Note : should happen only once, at workspace first initialization + */ +MEM_STATIC void* ZSTD_cwksp_reserve_object_aligned(ZSTD_cwksp* ws, size_t byteSize, size_t alignment) { - return ZSTD_encodeSequences_body(dst, dstCapacity, - CTable_MatchLength, mlCodeTable, - CTable_OffsetBits, ofCodeTable, - CTable_LitLength, llCodeTable, - sequences, nbSeq, longOffsets); + size_t const mask = alignment - 1; + size_t const surplus = (alignment > sizeof(void*)) ? alignment - sizeof(void*) : 0; + void* const start = ZSTD_cwksp_reserve_object(ws, byteSize + surplus); + if (start == NULL) return NULL; + if (surplus == 0) return start; + assert(ZSTD_isPower2(alignment)); + return (void*)(((size_t)start + surplus) & ~mask); } -#endif - -size_t ZSTD_encodeSequences( - void* dst, size_t dstCapacity, - FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, - FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, - FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, - seqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2) +MEM_STATIC void ZSTD_cwksp_mark_tables_dirty(ZSTD_cwksp* ws) { - DEBUGLOG(5, "ZSTD_encodeSequences: dstCapacity = %u", (unsigned)dstCapacity); -#if DYNAMIC_BMI2 - if (bmi2) { - return ZSTD_encodeSequences_bmi2(dst, dstCapacity, - CTable_MatchLength, mlCodeTable, - CTable_OffsetBits, ofCodeTable, - CTable_LitLength, llCodeTable, - sequences, nbSeq, longOffsets); + DEBUGLOG(4, "cwksp: ZSTD_cwksp_mark_tables_dirty"); + +#if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) + /* To validate that the table reuse logic is sound, and that we don't + * access table space that we haven't cleaned, we re-"poison" the table + * space every time we mark it dirty. + * Since tableValidEnd space and initOnce space may overlap we don't poison + * the initOnce portion as it break its promise. This means that this poisoning + * check isn't always applied fully. */ + { + size_t size = (BYTE*)ws->tableValidEnd - (BYTE*)ws->objectEnd; + assert(__msan_test_shadow(ws->objectEnd, size) == -1); + if((BYTE*)ws->tableValidEnd < (BYTE*)ws->initOnceStart) { + __msan_poison(ws->objectEnd, size); + } else { + assert(ws->initOnceStart >= ws->objectEnd); + __msan_poison(ws->objectEnd, (BYTE*)ws->initOnceStart - (BYTE*)ws->objectEnd); + } } #endif - (void)bmi2; - return ZSTD_encodeSequences_default(dst, dstCapacity, - CTable_MatchLength, mlCodeTable, - CTable_OffsetBits, ofCodeTable, - CTable_LitLength, llCodeTable, - sequences, nbSeq, longOffsets); + + assert(ws->tableValidEnd >= ws->objectEnd); + assert(ws->tableValidEnd <= ws->allocStart); + ws->tableValidEnd = ws->objectEnd; + ZSTD_cwksp_assert_internal_consistency(ws); } -/**** ended inlining compress/zstd_compress_sequences.c ****/ -/**** start inlining compress/zstd_compress_superblock.c ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. + +MEM_STATIC void ZSTD_cwksp_mark_tables_clean(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: ZSTD_cwksp_mark_tables_clean"); + assert(ws->tableValidEnd >= ws->objectEnd); + assert(ws->tableValidEnd <= ws->allocStart); + if (ws->tableValidEnd < ws->tableEnd) { + ws->tableValidEnd = ws->tableEnd; + } + ZSTD_cwksp_assert_internal_consistency(ws); +} + +/** + * Zero the part of the allocated tables not already marked clean. */ +MEM_STATIC void ZSTD_cwksp_clean_tables(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: ZSTD_cwksp_clean_tables"); + assert(ws->tableValidEnd >= ws->objectEnd); + assert(ws->tableValidEnd <= ws->allocStart); + if (ws->tableValidEnd < ws->tableEnd) { + ZSTD_memset(ws->tableValidEnd, 0, (size_t)((BYTE*)ws->tableEnd - (BYTE*)ws->tableValidEnd)); + } + ZSTD_cwksp_mark_tables_clean(ws); +} - /*-************************************* - * Dependencies - ***************************************/ -/**** start inlining zstd_compress_superblock.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. +/** + * Invalidates table allocations. + * All other allocations remain valid. */ +MEM_STATIC void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) +{ + DEBUGLOG(4, "cwksp: clearing tables!"); -#ifndef ZSTD_COMPRESS_ADVANCED_H -#define ZSTD_COMPRESS_ADVANCED_H +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* We don't do this when the workspace is statically allocated, because + * when that is the case, we have no capability to hook into the end of the + * workspace's lifecycle to unpoison the memory. + */ + if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { + size_t size = (BYTE*)ws->tableValidEnd - (BYTE*)ws->objectEnd; + __asan_poison_memory_region(ws->objectEnd, size); + } +#endif -/*-************************************* -* Dependencies -***************************************/ + ws->tableEnd = ws->objectEnd; + ZSTD_cwksp_assert_internal_consistency(ws); +} -/**** skipping file: ../zstd.h ****/ +/** + * Invalidates all buffer, aligned, and table allocations. + * Object allocations remain valid. + */ +MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: clearing!"); -/*-************************************* -* Target Compressed Block Size -***************************************/ +#if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) + /* To validate that the context reuse logic is sound, and that we don't + * access stuff that this compression hasn't initialized, we re-"poison" + * the workspace except for the areas in which we expect memory reuse + * without initialization (objects, valid tables area and init once + * memory). */ + { + if((BYTE*)ws->tableValidEnd < (BYTE*)ws->initOnceStart) { + size_t size = (BYTE*)ws->initOnceStart - (BYTE*)ws->tableValidEnd; + __msan_poison(ws->tableValidEnd, size); + } + } +#endif -/* ZSTD_compressSuperBlock() : - * Used to compress a super block when targetCBlockSize is being used. - * The given block will be compressed into multiple sub blocks that are around targetCBlockSize. */ -size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc, - void* dst, size_t dstCapacity, - void const* src, size_t srcSize, - unsigned lastBlock); +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* We don't do this when the workspace is statically allocated, because + * when that is the case, we have no capability to hook into the end of the + * workspace's lifecycle to unpoison the memory. + */ + if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { + size_t size = (BYTE*)ws->workspaceEnd - (BYTE*)ws->objectEnd; + __asan_poison_memory_region(ws->objectEnd, size); + } +#endif -#endif /* ZSTD_COMPRESS_ADVANCED_H */ -/**** ended inlining zstd_compress_superblock.h ****/ + ws->tableEnd = ws->objectEnd; + ws->allocStart = ZSTD_cwksp_initialAllocStart(ws); + ws->allocFailed = 0; + if (ws->phase > ZSTD_cwksp_alloc_aligned_init_once) { + ws->phase = ZSTD_cwksp_alloc_aligned_init_once; + } + ZSTD_cwksp_assert_internal_consistency(ws); +} -/**** skipping file: ../common/zstd_internal.h ****/ -/**** skipping file: hist.h ****/ -/**** skipping file: zstd_compress_internal.h ****/ -/**** skipping file: zstd_compress_sequences.h ****/ -/**** skipping file: zstd_compress_literals.h ****/ +MEM_STATIC size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) { + return (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace); +} -/*-************************************* -* Superblock entropy buffer structs -***************************************/ -/** ZSTD_hufCTablesMetadata_t : - * Stores Literals Block Type for a super-block in hType, and - * huffman tree description in hufDesBuffer. - * hufDesSize refers to the size of huffman tree description in bytes. - * This metadata is populated in ZSTD_buildSuperBlockEntropy_literal() */ -typedef struct { - symbolEncodingType_e hType; - BYTE hufDesBuffer[ZSTD_MAX_HUF_HEADER_SIZE]; - size_t hufDesSize; -} ZSTD_hufCTablesMetadata_t; - -/** ZSTD_fseCTablesMetadata_t : - * Stores symbol compression modes for a super-block in {ll, ol, ml}Type, and - * fse tables in fseTablesBuffer. - * fseTablesSize refers to the size of fse tables in bytes. - * This metadata is populated in ZSTD_buildSuperBlockEntropy_sequences() */ -typedef struct { - symbolEncodingType_e llType; - symbolEncodingType_e ofType; - symbolEncodingType_e mlType; - BYTE fseTablesBuffer[ZSTD_MAX_FSE_HEADERS_SIZE]; - size_t fseTablesSize; - size_t lastCountSize; /* This is to account for bug in 1.3.4. More detail in ZSTD_compressSubBlock_sequences() */ -} ZSTD_fseCTablesMetadata_t; - -typedef struct { - ZSTD_hufCTablesMetadata_t hufMetadata; - ZSTD_fseCTablesMetadata_t fseMetadata; -} ZSTD_entropyCTablesMetadata_t; - - -/** ZSTD_buildSuperBlockEntropy_literal() : - * Builds entropy for the super-block literals. - * Stores literals block type (raw, rle, compressed, repeat) and - * huffman description table to hufMetadata. - * @return : size of huffman description table or error code */ -static size_t ZSTD_buildSuperBlockEntropy_literal(void* const src, size_t srcSize, - const ZSTD_hufCTables_t* prevHuf, - ZSTD_hufCTables_t* nextHuf, - ZSTD_hufCTablesMetadata_t* hufMetadata, - const int disableLiteralsCompression, - void* workspace, size_t wkspSize) -{ - BYTE* const wkspStart = (BYTE*)workspace; - BYTE* const wkspEnd = wkspStart + wkspSize; - BYTE* const countWkspStart = wkspStart; - unsigned* const countWksp = (unsigned*)workspace; - const size_t countWkspSize = (HUF_SYMBOLVALUE_MAX + 1) * sizeof(unsigned); - BYTE* const nodeWksp = countWkspStart + countWkspSize; - const size_t nodeWkspSize = wkspEnd-nodeWksp; - unsigned maxSymbolValue = 255; - unsigned huffLog = HUF_TABLELOG_DEFAULT; - HUF_repeat repeat = prevHuf->repeatMode; +MEM_STATIC size_t ZSTD_cwksp_used(const ZSTD_cwksp* ws) { + return (size_t)((BYTE*)ws->tableEnd - (BYTE*)ws->workspace) + + (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->allocStart); +} - DEBUGLOG(5, "ZSTD_buildSuperBlockEntropy_literal (srcSize=%zu)", srcSize); +/** + * The provided workspace takes ownership of the buffer [start, start+size). + * Any existing values in the workspace are ignored (the previously managed + * buffer, if present, must be separately freed). + */ +MEM_STATIC void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size, ZSTD_cwksp_static_alloc_e isStatic) { + DEBUGLOG(4, "cwksp: init'ing workspace with %zd bytes", size); + assert(((size_t)start & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ + ws->workspace = start; + ws->workspaceEnd = (BYTE*)start + size; + ws->objectEnd = ws->workspace; + ws->tableValidEnd = ws->objectEnd; + ws->initOnceStart = ZSTD_cwksp_initialAllocStart(ws); + ws->phase = ZSTD_cwksp_alloc_objects; + ws->isStatic = isStatic; + ZSTD_cwksp_clear(ws); + ws->workspaceOversizedDuration = 0; + ZSTD_cwksp_assert_internal_consistency(ws); +} - /* Prepare nextEntropy assuming reusing the existing table */ - ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); +MEM_STATIC size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem) { + void* workspace = ZSTD_customMalloc(size, customMem); + DEBUGLOG(4, "cwksp: creating new workspace with %zd bytes", size); + RETURN_ERROR_IF(workspace == NULL, memory_allocation, "NULL pointer!"); + ZSTD_cwksp_init(ws, workspace, size, ZSTD_cwksp_dynamic_alloc); + return 0; +} - if (disableLiteralsCompression) { - DEBUGLOG(5, "set_basic - disabled"); - hufMetadata->hType = set_basic; - return 0; +MEM_STATIC void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) { + void *ptr = ws->workspace; + DEBUGLOG(4, "cwksp: freeing workspace"); +#if ZSTD_MEMORY_SANITIZER && !defined(ZSTD_MSAN_DONT_POISON_WORKSPACE) + if (ptr != NULL && customMem.customFree != NULL) { + __msan_unpoison(ptr, ZSTD_cwksp_sizeof(ws)); } +#endif + ZSTD_memset(ws, 0, sizeof(ZSTD_cwksp)); + ZSTD_customFree(ptr, customMem); +} - /* small ? don't even attempt compression (speed opt) */ -# define COMPRESS_LITERALS_SIZE_MIN 63 - { size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN; - if (srcSize <= minLitSize) { - DEBUGLOG(5, "set_basic - too small"); - hufMetadata->hType = set_basic; - return 0; - } - } +/** + * Moves the management of a workspace from one cwksp to another. The src cwksp + * is left in an invalid state (src must be re-init()'ed before it's used again). + */ +MEM_STATIC void ZSTD_cwksp_move(ZSTD_cwksp* dst, ZSTD_cwksp* src) { + *dst = *src; + ZSTD_memset(src, 0, sizeof(ZSTD_cwksp)); +} - /* Scan input and build symbol stats */ - { size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)src, srcSize, workspace, wkspSize); - FORWARD_IF_ERROR(largest, "HIST_count_wksp failed"); - if (largest == srcSize) { - DEBUGLOG(5, "set_rle"); - hufMetadata->hType = set_rle; - return 0; - } - if (largest <= (srcSize >> 7)+4) { - DEBUGLOG(5, "set_basic - no gain"); - hufMetadata->hType = set_basic; - return 0; - } - } +MEM_STATIC int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) { + return ws->allocFailed; +} - /* Validate the previous Huffman table */ - if (repeat == HUF_repeat_check && !HUF_validateCTable((HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue)) { - repeat = HUF_repeat_none; - } +/*-************************************* +* Functions Checking Free Space +***************************************/ - /* Build Huffman Tree */ - ZSTD_memset(nextHuf->CTable, 0, sizeof(nextHuf->CTable)); - huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue); - { size_t const maxBits = HUF_buildCTable_wksp((HUF_CElt*)nextHuf->CTable, countWksp, - maxSymbolValue, huffLog, - nodeWksp, nodeWkspSize); - FORWARD_IF_ERROR(maxBits, "HUF_buildCTable_wksp"); - huffLog = (U32)maxBits; - { /* Build and write the CTable */ - size_t const newCSize = HUF_estimateCompressedSize( - (HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue); - size_t const hSize = HUF_writeCTable( - hufMetadata->hufDesBuffer, sizeof(hufMetadata->hufDesBuffer), - (HUF_CElt*)nextHuf->CTable, maxSymbolValue, huffLog); - /* Check against repeating the previous CTable */ - if (repeat != HUF_repeat_none) { - size_t const oldCSize = HUF_estimateCompressedSize( - (HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue); - if (oldCSize < srcSize && (oldCSize <= hSize + newCSize || hSize + 12 >= srcSize)) { - DEBUGLOG(5, "set_repeat - smaller"); - ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); - hufMetadata->hType = set_repeat; - return 0; - } - } - if (newCSize + hSize >= srcSize) { - DEBUGLOG(5, "set_basic - no gains"); - ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); - hufMetadata->hType = set_basic; - return 0; - } - DEBUGLOG(5, "set_compressed (hSize=%u)", (U32)hSize); - hufMetadata->hType = set_compressed; - nextHuf->repeatMode = HUF_repeat_check; - return hSize; - } - } +/* ZSTD_alignmentSpaceWithinBounds() : + * Returns if the estimated space needed for a wksp is within an acceptable limit of the + * actual amount of space used. + */ +MEM_STATIC int ZSTD_cwksp_estimated_space_within_bounds(const ZSTD_cwksp *const ws, size_t const estimatedSpace) { + /* We have an alignment space between objects and tables between tables and buffers, so we can have up to twice + * the alignment bytes difference between estimation and actual usage */ + return (estimatedSpace - ZSTD_cwksp_slack_space_required()) <= ZSTD_cwksp_used(ws) && + ZSTD_cwksp_used(ws) <= estimatedSpace; } -/** ZSTD_buildSuperBlockEntropy_sequences() : - * Builds entropy for the super-block sequences. - * Stores symbol compression modes and fse table to fseMetadata. - * @return : size of fse tables or error code */ -static size_t ZSTD_buildSuperBlockEntropy_sequences(seqStore_t* seqStorePtr, - const ZSTD_fseCTables_t* prevEntropy, - ZSTD_fseCTables_t* nextEntropy, - const ZSTD_CCtx_params* cctxParams, - ZSTD_fseCTablesMetadata_t* fseMetadata, - void* workspace, size_t wkspSize) -{ - BYTE* const wkspStart = (BYTE*)workspace; - BYTE* const wkspEnd = wkspStart + wkspSize; - BYTE* const countWkspStart = wkspStart; - unsigned* const countWksp = (unsigned*)workspace; - const size_t countWkspSize = (MaxSeq + 1) * sizeof(unsigned); - BYTE* const cTableWksp = countWkspStart + countWkspSize; - const size_t cTableWkspSize = wkspEnd-cTableWksp; - ZSTD_strategy const strategy = cctxParams->cParams.strategy; - FSE_CTable* CTable_LitLength = nextEntropy->litlengthCTable; - FSE_CTable* CTable_OffsetBits = nextEntropy->offcodeCTable; - FSE_CTable* CTable_MatchLength = nextEntropy->matchlengthCTable; - const BYTE* const ofCodeTable = seqStorePtr->ofCode; - const BYTE* const llCodeTable = seqStorePtr->llCode; - const BYTE* const mlCodeTable = seqStorePtr->mlCode; - size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart; - BYTE* const ostart = fseMetadata->fseTablesBuffer; - BYTE* const oend = ostart + sizeof(fseMetadata->fseTablesBuffer); - BYTE* op = ostart; - assert(cTableWkspSize >= (1 << MaxFSELog) * sizeof(FSE_FUNCTION_TYPE)); - DEBUGLOG(5, "ZSTD_buildSuperBlockEntropy_sequences (nbSeq=%zu)", nbSeq); - ZSTD_memset(workspace, 0, wkspSize); +MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws) { + return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd); +} - fseMetadata->lastCountSize = 0; - /* convert length/distances into codes */ - ZSTD_seqToCodes(seqStorePtr); - /* build CTable for Literal Lengths */ - { U32 LLtype; - unsigned max = MaxLL; - size_t const mostFrequent = HIST_countFast_wksp(countWksp, &max, llCodeTable, nbSeq, workspace, wkspSize); /* can't fail */ - DEBUGLOG(5, "Building LL table"); - nextEntropy->litlength_repeatMode = prevEntropy->litlength_repeatMode; - LLtype = ZSTD_selectEncodingType(&nextEntropy->litlength_repeatMode, - countWksp, max, mostFrequent, nbSeq, - LLFSELog, prevEntropy->litlengthCTable, - LL_defaultNorm, LL_defaultNormLog, - ZSTD_defaultAllowed, strategy); - assert(set_basic < set_compressed && set_rle < set_compressed); - assert(!(LLtype < set_compressed && nextEntropy->litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ - { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype, - countWksp, max, llCodeTable, nbSeq, LL_defaultNorm, LL_defaultNormLog, MaxLL, - prevEntropy->litlengthCTable, sizeof(prevEntropy->litlengthCTable), - cTableWksp, cTableWkspSize); - FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for LitLens failed"); - if (LLtype == set_compressed) - fseMetadata->lastCountSize = countSize; - op += countSize; - fseMetadata->llType = (symbolEncodingType_e) LLtype; - } } - /* build CTable for Offsets */ - { U32 Offtype; - unsigned max = MaxOff; - size_t const mostFrequent = HIST_countFast_wksp(countWksp, &max, ofCodeTable, nbSeq, workspace, wkspSize); /* can't fail */ - /* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */ - ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed; - DEBUGLOG(5, "Building OF table"); - nextEntropy->offcode_repeatMode = prevEntropy->offcode_repeatMode; - Offtype = ZSTD_selectEncodingType(&nextEntropy->offcode_repeatMode, - countWksp, max, mostFrequent, nbSeq, - OffFSELog, prevEntropy->offcodeCTable, - OF_defaultNorm, OF_defaultNormLog, - defaultPolicy, strategy); - assert(!(Offtype < set_compressed && nextEntropy->offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */ - { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype, - countWksp, max, ofCodeTable, nbSeq, OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, - prevEntropy->offcodeCTable, sizeof(prevEntropy->offcodeCTable), - cTableWksp, cTableWkspSize); - FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for Offsets failed"); - if (Offtype == set_compressed) - fseMetadata->lastCountSize = countSize; - op += countSize; - fseMetadata->ofType = (symbolEncodingType_e) Offtype; - } } - /* build CTable for MatchLengths */ - { U32 MLtype; - unsigned max = MaxML; - size_t const mostFrequent = HIST_countFast_wksp(countWksp, &max, mlCodeTable, nbSeq, workspace, wkspSize); /* can't fail */ - DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op)); - nextEntropy->matchlength_repeatMode = prevEntropy->matchlength_repeatMode; - MLtype = ZSTD_selectEncodingType(&nextEntropy->matchlength_repeatMode, - countWksp, max, mostFrequent, nbSeq, - MLFSELog, prevEntropy->matchlengthCTable, - ML_defaultNorm, ML_defaultNormLog, - ZSTD_defaultAllowed, strategy); - assert(!(MLtype < set_compressed && nextEntropy->matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ - { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype, - countWksp, max, mlCodeTable, nbSeq, ML_defaultNorm, ML_defaultNormLog, MaxML, - prevEntropy->matchlengthCTable, sizeof(prevEntropy->matchlengthCTable), - cTableWksp, cTableWkspSize); - FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for MatchLengths failed"); - if (MLtype == set_compressed) - fseMetadata->lastCountSize = countSize; - op += countSize; - fseMetadata->mlType = (symbolEncodingType_e) MLtype; - } } - assert((size_t) (op-ostart) <= sizeof(fseMetadata->fseTablesBuffer)); - return op-ostart; +MEM_STATIC int ZSTD_cwksp_check_available(ZSTD_cwksp* ws, size_t additionalNeededSpace) { + return ZSTD_cwksp_available_space(ws) >= additionalNeededSpace; } +MEM_STATIC int ZSTD_cwksp_check_too_large(ZSTD_cwksp* ws, size_t additionalNeededSpace) { + return ZSTD_cwksp_check_available( + ws, additionalNeededSpace * ZSTD_WORKSPACETOOLARGE_FACTOR); +} -/** ZSTD_buildSuperBlockEntropy() : - * Builds entropy for the super-block. - * @return : 0 on success or error code */ -static size_t -ZSTD_buildSuperBlockEntropy(seqStore_t* seqStorePtr, - const ZSTD_entropyCTables_t* prevEntropy, - ZSTD_entropyCTables_t* nextEntropy, - const ZSTD_CCtx_params* cctxParams, - ZSTD_entropyCTablesMetadata_t* entropyMetadata, - void* workspace, size_t wkspSize) -{ - size_t const litSize = seqStorePtr->lit - seqStorePtr->litStart; - DEBUGLOG(5, "ZSTD_buildSuperBlockEntropy"); - entropyMetadata->hufMetadata.hufDesSize = - ZSTD_buildSuperBlockEntropy_literal(seqStorePtr->litStart, litSize, - &prevEntropy->huf, &nextEntropy->huf, - &entropyMetadata->hufMetadata, - ZSTD_disableLiteralsCompression(cctxParams), - workspace, wkspSize); - FORWARD_IF_ERROR(entropyMetadata->hufMetadata.hufDesSize, "ZSTD_buildSuperBlockEntropy_literal failed"); - entropyMetadata->fseMetadata.fseTablesSize = - ZSTD_buildSuperBlockEntropy_sequences(seqStorePtr, - &prevEntropy->fse, &nextEntropy->fse, - cctxParams, - &entropyMetadata->fseMetadata, - workspace, wkspSize); - FORWARD_IF_ERROR(entropyMetadata->fseMetadata.fseTablesSize, "ZSTD_buildSuperBlockEntropy_sequences failed"); - return 0; +MEM_STATIC int ZSTD_cwksp_check_wasteful(ZSTD_cwksp* ws, size_t additionalNeededSpace) { + return ZSTD_cwksp_check_too_large(ws, additionalNeededSpace) + && ws->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION; } -/** ZSTD_compressSubBlock_literal() : - * Compresses literals section for a sub-block. - * When we have to write the Huffman table we will sometimes choose a header - * size larger than necessary. This is because we have to pick the header size - * before we know the table size + compressed size, so we have a bound on the - * table size. If we guessed incorrectly, we fall back to uncompressed literals. - * - * We write the header when writeEntropy=1 and set entropyWritten=1 when we succeeded - * in writing the header, otherwise it is set to 0. +MEM_STATIC void ZSTD_cwksp_bump_oversized_duration( + ZSTD_cwksp* ws, size_t additionalNeededSpace) { + if (ZSTD_cwksp_check_too_large(ws, additionalNeededSpace)) { + ws->workspaceOversizedDuration++; + } else { + ws->workspaceOversizedDuration = 0; + } +} + +#endif /* ZSTD_CWKSP_H */ +/**** ended inlining zstd_cwksp.h ****/ +#ifdef ZSTD_MULTITHREAD +/**** start inlining zstdmt_compress.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. * - * hufMetadata->hType has literals block type info. - * If it is set_basic, all sub-blocks literals section will be Raw_Literals_Block. - * If it is set_rle, all sub-blocks literals section will be RLE_Literals_Block. - * If it is set_compressed, first sub-block's literals section will be Compressed_Literals_Block - * If it is set_compressed, first sub-block's literals section will be Treeless_Literals_Block - * and the following sub-blocks' literals sections will be Treeless_Literals_Block. - * @return : compressed size of literals section of a sub-block - * Or 0 if it unable to compress. - * Or error code */ -static size_t ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable, - const ZSTD_hufCTablesMetadata_t* hufMetadata, - const BYTE* literals, size_t litSize, - void* dst, size_t dstSize, - const int bmi2, int writeEntropy, int* entropyWritten) -{ - size_t const header = writeEntropy ? 200 : 0; - size_t const lhSize = 3 + (litSize >= (1 KB - header)) + (litSize >= (16 KB - header)); - BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = ostart + dstSize; - BYTE* op = ostart + lhSize; - U32 const singleStream = lhSize == 3; - symbolEncodingType_e hType = writeEntropy ? hufMetadata->hType : set_repeat; - size_t cLitSize = 0; + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ - (void)bmi2; /* TODO bmi2... */ + #ifndef ZSTDMT_COMPRESS_H + #define ZSTDMT_COMPRESS_H - DEBUGLOG(5, "ZSTD_compressSubBlock_literal (litSize=%zu, lhSize=%zu, writeEntropy=%d)", litSize, lhSize, writeEntropy); +/* === Dependencies === */ +/**** skipping file: ../common/zstd_deps.h ****/ +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */ +/**** skipping file: ../zstd.h ****/ - *entropyWritten = 0; - if (litSize == 0 || hufMetadata->hType == set_basic) { - DEBUGLOG(5, "ZSTD_compressSubBlock_literal using raw literal"); - return ZSTD_noCompressLiterals(dst, dstSize, literals, litSize); - } else if (hufMetadata->hType == set_rle) { - DEBUGLOG(5, "ZSTD_compressSubBlock_literal using rle literal"); - return ZSTD_compressRleLiteralsBlock(dst, dstSize, literals, litSize); - } +/* Note : This is an internal API. + * These APIs used to be exposed with ZSTDLIB_API, + * because it used to be the only way to invoke MT compression. + * Now, you must use ZSTD_compress2 and ZSTD_compressStream2() instead. + * + * This API requires ZSTD_MULTITHREAD to be defined during compilation, + * otherwise ZSTDMT_createCCtx*() will fail. + */ - assert(litSize > 0); - assert(hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat); +/* === Constants === */ +#ifndef ZSTDMT_NBWORKERS_MAX /* a different value can be selected at compile time */ +# define ZSTDMT_NBWORKERS_MAX ((sizeof(void*)==4) /*32-bit*/ ? 64 : 256) +#endif +#ifndef ZSTDMT_JOBSIZE_MIN /* a different value can be selected at compile time */ +# define ZSTDMT_JOBSIZE_MIN (512 KB) +#endif +#define ZSTDMT_JOBLOG_MAX (MEM_32bits() ? 29 : 30) +#define ZSTDMT_JOBSIZE_MAX (MEM_32bits() ? (512 MB) : (1024 MB)) - if (writeEntropy && hufMetadata->hType == set_compressed) { - ZSTD_memcpy(op, hufMetadata->hufDesBuffer, hufMetadata->hufDesSize); - op += hufMetadata->hufDesSize; - cLitSize += hufMetadata->hufDesSize; - DEBUGLOG(5, "ZSTD_compressSubBlock_literal (hSize=%zu)", hufMetadata->hufDesSize); - } - /* TODO bmi2 */ - { const size_t cSize = singleStream ? HUF_compress1X_usingCTable(op, oend-op, literals, litSize, hufTable) - : HUF_compress4X_usingCTable(op, oend-op, literals, litSize, hufTable); - op += cSize; - cLitSize += cSize; - if (cSize == 0 || ERR_isError(cSize)) { - DEBUGLOG(5, "Failed to write entropy tables %s", ZSTD_getErrorName(cSize)); - return 0; - } - /* If we expand and we aren't writing a header then emit uncompressed */ - if (!writeEntropy && cLitSize >= litSize) { - DEBUGLOG(5, "ZSTD_compressSubBlock_literal using raw literal because uncompressible"); - return ZSTD_noCompressLiterals(dst, dstSize, literals, litSize); - } - /* If we are writing headers then allow expansion that doesn't change our header size. */ - if (lhSize < (size_t)(3 + (cLitSize >= 1 KB) + (cLitSize >= 16 KB))) { - assert(cLitSize > litSize); - DEBUGLOG(5, "Literals expanded beyond allowed header size"); - return ZSTD_noCompressLiterals(dst, dstSize, literals, litSize); - } - DEBUGLOG(5, "ZSTD_compressSubBlock_literal (cSize=%zu)", cSize); - } +/* ======================================================== + * === Private interface, for use by ZSTD_compress.c === + * === Not exposed in libzstd. Never invoke directly === + * ======================================================== */ - /* Build header */ - switch(lhSize) - { - case 3: /* 2 - 2 - 10 - 10 */ - { U32 const lhc = hType + ((!singleStream) << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<14); - MEM_writeLE24(ostart, lhc); - break; - } - case 4: /* 2 - 2 - 14 - 14 */ - { U32 const lhc = hType + (2 << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<18); - MEM_writeLE32(ostart, lhc); - break; - } - case 5: /* 2 - 2 - 18 - 18 */ - { U32 const lhc = hType + (3 << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<22); - MEM_writeLE32(ostart, lhc); - ostart[4] = (BYTE)(cLitSize >> 10); - break; - } - default: /* not possible : lhSize is {3,4,5} */ - assert(0); - } - *entropyWritten = 1; - DEBUGLOG(5, "Compressed literals: %u -> %u", (U32)litSize, (U32)(op-ostart)); - return op-ostart; -} +/* === Memory management === */ +typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx; +/* Requires ZSTD_MULTITHREAD to be defined during compilation, otherwise it will return NULL. */ +ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, + ZSTD_customMem cMem, + ZSTD_threadPool *pool); +size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx); -static size_t ZSTD_seqDecompressedSize(seqStore_t const* seqStore, const seqDef* sequences, size_t nbSeq, size_t litSize, int lastSequence) { - const seqDef* const sstart = sequences; - const seqDef* const send = sequences + nbSeq; - const seqDef* sp = sstart; - size_t matchLengthSum = 0; - size_t litLengthSum = 0; - (void)litLengthSum; - while (send-sp > 0) { - ZSTD_sequenceLength const seqLen = ZSTD_getSequenceLength(seqStore, sp); - litLengthSum += seqLen.litLength; - matchLengthSum += seqLen.matchLength; - sp++; - } - assert(litLengthSum <= litSize); - if (!lastSequence) { - assert(litLengthSum == litSize); - } - return matchLengthSum + litSize; -} +size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx); -/** ZSTD_compressSubBlock_sequences() : - * Compresses sequences section for a sub-block. - * fseMetadata->llType, fseMetadata->ofType, and fseMetadata->mlType have - * symbol compression modes for the super-block. - * The first successfully compressed block will have these in its header. - * We set entropyWritten=1 when we succeed in compressing the sequences. - * The following sub-blocks will always have repeat mode. - * @return : compressed size of sequences section of a sub-block - * Or 0 if it is unable to compress - * Or error code. */ -static size_t ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables, - const ZSTD_fseCTablesMetadata_t* fseMetadata, - const seqDef* sequences, size_t nbSeq, - const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode, - const ZSTD_CCtx_params* cctxParams, - void* dst, size_t dstCapacity, - const int bmi2, int writeEntropy, int* entropyWritten) -{ - const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN; - BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = ostart + dstCapacity; - BYTE* op = ostart; - BYTE* seqHead; +/* === Streaming functions === */ - DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (nbSeq=%zu, writeEntropy=%d, longOffsets=%d)", nbSeq, writeEntropy, longOffsets); +size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx); - *entropyWritten = 0; - /* Sequences Header */ - RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/, - dstSize_tooSmall, ""); - if (nbSeq < 0x7F) - *op++ = (BYTE)nbSeq; - else if (nbSeq < LONGNBSEQ) - op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2; - else - op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3; - if (nbSeq==0) { - return op - ostart; - } +/*! ZSTDMT_initCStream_internal() : + * Private use only. Init streaming operation. + * expects params to be valid. + * must receive dict, or cdict, or none, but not both. + * mtctx can be freshly constructed or reused from a prior compression. + * If mtctx is reused, memory allocations from the prior compression may not be freed, + * even if they are not needed for the current compression. + * @return : 0, or an error code */ +size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* mtctx, + const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType, + const ZSTD_CDict* cdict, + ZSTD_CCtx_params params, unsigned long long pledgedSrcSize); - /* seqHead : flags for FSE encoding type */ - seqHead = op++; +/*! ZSTDMT_compressStream_generic() : + * Combines ZSTDMT_compressStream() with optional ZSTDMT_flushStream() or ZSTDMT_endStream() + * depending on flush directive. + * @return : minimum amount of data still to be flushed + * 0 if fully flushed + * or an error code + * note : needs to be init using any ZSTD_initCStream*() variant */ +size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective endOp); - DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (seqHeadSize=%u)", (unsigned)(op-ostart)); + /*! ZSTDMT_toFlushNow() + * Tell how many bytes are ready to be flushed immediately. + * Probe the oldest active job (not yet entirely flushed) and check its output buffer. + * If return 0, it means there is no active job, + * or, it means oldest job is still active, but everything produced has been flushed so far, + * therefore flushing is limited by speed of oldest job. */ +size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx); - if (writeEntropy) { - const U32 LLtype = fseMetadata->llType; - const U32 Offtype = fseMetadata->ofType; - const U32 MLtype = fseMetadata->mlType; - DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (fseTablesSize=%zu)", fseMetadata->fseTablesSize); - *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2)); - ZSTD_memcpy(op, fseMetadata->fseTablesBuffer, fseMetadata->fseTablesSize); - op += fseMetadata->fseTablesSize; - } else { - const U32 repeat = set_repeat; - *seqHead = (BYTE)((repeat<<6) + (repeat<<4) + (repeat<<2)); - } +/*! ZSTDMT_updateCParams_whileCompressing() : + * Updates only a selected set of compression parameters, to remain compatible with current frame. + * New parameters will be applied to next compression job. */ +void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_params* cctxParams); - { size_t const bitstreamSize = ZSTD_encodeSequences( - op, oend - op, - fseTables->matchlengthCTable, mlCode, - fseTables->offcodeCTable, ofCode, - fseTables->litlengthCTable, llCode, - sequences, nbSeq, - longOffsets, bmi2); - FORWARD_IF_ERROR(bitstreamSize, "ZSTD_encodeSequences failed"); - op += bitstreamSize; - /* zstd versions <= 1.3.4 mistakenly report corruption when - * FSE_readNCount() receives a buffer < 4 bytes. - * Fixed by https://github.com/facebook/zstd/pull/1146. - * This can happen when the last set_compressed table present is 2 - * bytes and the bitstream is only one byte. - * In this exceedingly rare case, we will simply emit an uncompressed - * block, since it isn't worth optimizing. - */ -#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION - if (writeEntropy && fseMetadata->lastCountSize && fseMetadata->lastCountSize + bitstreamSize < 4) { - /* NCountSize >= 2 && bitstreamSize > 0 ==> lastCountSize == 3 */ - assert(fseMetadata->lastCountSize + bitstreamSize == 3); - DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.3.4 by " - "emitting an uncompressed block."); - return 0; - } -#endif - DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (bitstreamSize=%zu)", bitstreamSize); - } +/*! ZSTDMT_getFrameProgression(): + * tells how much data has been consumed (input) and produced (output) for current frame. + * able to count progression inside worker threads. + */ +ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx); - /* zstd versions <= 1.4.0 mistakenly report error when - * sequences section body size is less than 3 bytes. - * Fixed by https://github.com/facebook/zstd/pull/1664. - * This can happen when the previous sequences section block is compressed - * with rle mode and the current block's sequences section is compressed - * with repeat mode where sequences section body size can be 1 byte. - */ -#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION - if (op-seqHead < 4) { - DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.4.0 by emitting " - "an uncompressed block when sequences are < 4 bytes"); - return 0; - } +#endif /* ZSTDMT_COMPRESS_H */ +/**** ended inlining zstdmt_compress.h ****/ #endif +/**** skipping file: ../common/bits.h ****/ +/**** start inlining zstd_preSplit.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ - *entropyWritten = 1; - return op - ostart; -} +#ifndef ZSTD_PRESPLIT_H +#define ZSTD_PRESPLIT_H -/** ZSTD_compressSubBlock() : - * Compresses a single sub-block. - * @return : compressed size of the sub-block - * Or 0 if it failed to compress. */ -static size_t ZSTD_compressSubBlock(const ZSTD_entropyCTables_t* entropy, - const ZSTD_entropyCTablesMetadata_t* entropyMetadata, - const seqDef* sequences, size_t nbSeq, - const BYTE* literals, size_t litSize, - const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode, - const ZSTD_CCtx_params* cctxParams, - void* dst, size_t dstCapacity, - const int bmi2, - int writeLitEntropy, int writeSeqEntropy, - int* litEntropyWritten, int* seqEntropyWritten, - U32 lastBlock) -{ - BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = ostart + dstCapacity; - BYTE* op = ostart + ZSTD_blockHeaderSize; - DEBUGLOG(5, "ZSTD_compressSubBlock (litSize=%zu, nbSeq=%zu, writeLitEntropy=%d, writeSeqEntropy=%d, lastBlock=%d)", - litSize, nbSeq, writeLitEntropy, writeSeqEntropy, lastBlock); - { size_t cLitSize = ZSTD_compressSubBlock_literal((const HUF_CElt*)entropy->huf.CTable, - &entropyMetadata->hufMetadata, literals, litSize, - op, oend-op, bmi2, writeLitEntropy, litEntropyWritten); - FORWARD_IF_ERROR(cLitSize, "ZSTD_compressSubBlock_literal failed"); - if (cLitSize == 0) return 0; - op += cLitSize; - } - { size_t cSeqSize = ZSTD_compressSubBlock_sequences(&entropy->fse, - &entropyMetadata->fseMetadata, - sequences, nbSeq, - llCode, mlCode, ofCode, - cctxParams, - op, oend-op, - bmi2, writeSeqEntropy, seqEntropyWritten); - FORWARD_IF_ERROR(cSeqSize, "ZSTD_compressSubBlock_sequences failed"); - if (cSeqSize == 0) return 0; - op += cSeqSize; - } - /* Write block header */ - { size_t cSize = (op-ostart)-ZSTD_blockHeaderSize; - U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); - MEM_writeLE24(ostart, cBlockHeader24); - } - return op-ostart; -} +#include /* size_t */ -static size_t ZSTD_estimateSubBlockSize_literal(const BYTE* literals, size_t litSize, - const ZSTD_hufCTables_t* huf, - const ZSTD_hufCTablesMetadata_t* hufMetadata, - void* workspace, size_t wkspSize, - int writeEntropy) -{ - unsigned* const countWksp = (unsigned*)workspace; - unsigned maxSymbolValue = 255; - size_t literalSectionHeaderSize = 3; /* Use hard coded size of 3 bytes */ +#define ZSTD_SLIPBLOCK_WORKSPACESIZE 8208 - if (hufMetadata->hType == set_basic) return litSize; - else if (hufMetadata->hType == set_rle) return 1; - else if (hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat) { - size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)literals, litSize, workspace, wkspSize); - if (ZSTD_isError(largest)) return litSize; - { size_t cLitSizeEstimate = HUF_estimateCompressedSize((const HUF_CElt*)huf->CTable, countWksp, maxSymbolValue); - if (writeEntropy) cLitSizeEstimate += hufMetadata->hufDesSize; - return cLitSizeEstimate + literalSectionHeaderSize; - } } - assert(0); /* impossible */ - return 0; -} +/* ZSTD_splitBlock(): + * @level must be a value between 0 and 4. + * higher levels spend more energy to detect block boundaries. + * @workspace must be aligned for size_t. + * @wkspSize must be at least >= ZSTD_SLIPBLOCK_WORKSPACESIZE + * note: + * For the time being, this function only accepts full 128 KB blocks. + * Therefore, @blockSize must be == 128 KB. + * While this could be extended to smaller sizes in the future, + * it is not yet clear if this would be useful. TBD. + */ +size_t ZSTD_splitBlock(const void* blockStart, size_t blockSize, + int level, + void* workspace, size_t wkspSize); -static size_t ZSTD_estimateSubBlockSize_symbolType(symbolEncodingType_e type, - const BYTE* codeTable, unsigned maxCode, - size_t nbSeq, const FSE_CTable* fseCTable, - const U32* additionalBits, - short const* defaultNorm, U32 defaultNormLog, U32 defaultMax, - void* workspace, size_t wkspSize) -{ - unsigned* const countWksp = (unsigned*)workspace; - const BYTE* ctp = codeTable; - const BYTE* const ctStart = ctp; - const BYTE* const ctEnd = ctStart + nbSeq; - size_t cSymbolTypeSizeEstimateInBits = 0; - unsigned max = maxCode; +#endif /* ZSTD_PRESPLIT_H */ +/**** ended inlining zstd_preSplit.h ****/ - HIST_countFast_wksp(countWksp, &max, codeTable, nbSeq, workspace, wkspSize); /* can't fail */ - if (type == set_basic) { - /* We selected this encoding type, so it must be valid. */ - assert(max <= defaultMax); - cSymbolTypeSizeEstimateInBits = max <= defaultMax - ? ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, countWksp, max) - : ERROR(GENERIC); - } else if (type == set_rle) { - cSymbolTypeSizeEstimateInBits = 0; - } else if (type == set_compressed || type == set_repeat) { - cSymbolTypeSizeEstimateInBits = ZSTD_fseBitCost(fseCTable, countWksp, max); - } - if (ZSTD_isError(cSymbolTypeSizeEstimateInBits)) return nbSeq * 10; - while (ctp < ctEnd) { - if (additionalBits) cSymbolTypeSizeEstimateInBits += additionalBits[*ctp]; - else cSymbolTypeSizeEstimateInBits += *ctp; /* for offset, offset code is also the number of additional bits */ - ctp++; - } - return cSymbolTypeSizeEstimateInBits / 8; -} +/*-************************************* +* Constants +***************************************/ +#define kSearchStrength 8 +#define HASH_READ_SIZE 8 +#define ZSTD_DUBT_UNSORTED_MARK 1 /* For btlazy2 strategy, index ZSTD_DUBT_UNSORTED_MARK==1 means "unsorted". + It could be confused for a real successor at index "1", if sorted as larger than its predecessor. + It's not a big deal though : candidate will just be sorted again. + Additionally, candidate position 1 will be lost. + But candidate 1 cannot hide a large tree of candidates, so it's a minimal loss. + The benefit is that ZSTD_DUBT_UNSORTED_MARK cannot be mishandled after table reuse with a different strategy. + This constant is required by ZSTD_compressBlock_btlazy2() and ZSTD_reduceTable_internal() */ -static size_t ZSTD_estimateSubBlockSize_sequences(const BYTE* ofCodeTable, - const BYTE* llCodeTable, - const BYTE* mlCodeTable, - size_t nbSeq, - const ZSTD_fseCTables_t* fseTables, - const ZSTD_fseCTablesMetadata_t* fseMetadata, - void* workspace, size_t wkspSize, - int writeEntropy) -{ - size_t sequencesSectionHeaderSize = 3; /* Use hard coded size of 3 bytes */ - size_t cSeqSizeEstimate = 0; - cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, MaxOff, - nbSeq, fseTables->offcodeCTable, NULL, - OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, - workspace, wkspSize); - cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->llType, llCodeTable, MaxLL, - nbSeq, fseTables->litlengthCTable, LL_bits, - LL_defaultNorm, LL_defaultNormLog, MaxLL, - workspace, wkspSize); - cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->mlType, mlCodeTable, MaxML, - nbSeq, fseTables->matchlengthCTable, ML_bits, - ML_defaultNorm, ML_defaultNormLog, MaxML, - workspace, wkspSize); - if (writeEntropy) cSeqSizeEstimate += fseMetadata->fseTablesSize; - return cSeqSizeEstimate + sequencesSectionHeaderSize; -} -static size_t ZSTD_estimateSubBlockSize(const BYTE* literals, size_t litSize, - const BYTE* ofCodeTable, - const BYTE* llCodeTable, - const BYTE* mlCodeTable, - size_t nbSeq, - const ZSTD_entropyCTables_t* entropy, - const ZSTD_entropyCTablesMetadata_t* entropyMetadata, - void* workspace, size_t wkspSize, - int writeLitEntropy, int writeSeqEntropy) { - size_t cSizeEstimate = 0; - cSizeEstimate += ZSTD_estimateSubBlockSize_literal(literals, litSize, - &entropy->huf, &entropyMetadata->hufMetadata, - workspace, wkspSize, writeLitEntropy); - cSizeEstimate += ZSTD_estimateSubBlockSize_sequences(ofCodeTable, llCodeTable, mlCodeTable, - nbSeq, &entropy->fse, &entropyMetadata->fseMetadata, - workspace, wkspSize, writeSeqEntropy); - return cSizeEstimate + ZSTD_blockHeaderSize; -} +/*-************************************* +* Context memory management +***************************************/ +typedef enum { ZSTDcs_created=0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e; +typedef enum { zcss_init=0, zcss_load, zcss_flush } ZSTD_cStreamStage; -static int ZSTD_needSequenceEntropyTables(ZSTD_fseCTablesMetadata_t const* fseMetadata) -{ - if (fseMetadata->llType == set_compressed || fseMetadata->llType == set_rle) - return 1; - if (fseMetadata->mlType == set_compressed || fseMetadata->mlType == set_rle) - return 1; - if (fseMetadata->ofType == set_compressed || fseMetadata->ofType == set_rle) - return 1; - return 0; -} +typedef struct ZSTD_prefixDict_s { + const void* dict; + size_t dictSize; + ZSTD_dictContentType_e dictContentType; +} ZSTD_prefixDict; -/** ZSTD_compressSubBlock_multi() : - * Breaks super-block into multiple sub-blocks and compresses them. - * Entropy will be written to the first block. - * The following blocks will use repeat mode to compress. - * All sub-blocks are compressed blocks (no raw or rle blocks). - * @return : compressed size of the super block (which is multiple ZSTD blocks) - * Or 0 if it failed to compress. */ -static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr, - const ZSTD_compressedBlockState_t* prevCBlock, - ZSTD_compressedBlockState_t* nextCBlock, - const ZSTD_entropyCTablesMetadata_t* entropyMetadata, - const ZSTD_CCtx_params* cctxParams, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const int bmi2, U32 lastBlock, - void* workspace, size_t wkspSize) -{ - const seqDef* const sstart = seqStorePtr->sequencesStart; - const seqDef* const send = seqStorePtr->sequences; - const seqDef* sp = sstart; - const BYTE* const lstart = seqStorePtr->litStart; - const BYTE* const lend = seqStorePtr->lit; - const BYTE* lp = lstart; - BYTE const* ip = (BYTE const*)src; - BYTE const* const iend = ip + srcSize; - BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = ostart + dstCapacity; - BYTE* op = ostart; - const BYTE* llCodePtr = seqStorePtr->llCode; - const BYTE* mlCodePtr = seqStorePtr->mlCode; - const BYTE* ofCodePtr = seqStorePtr->ofCode; - size_t targetCBlockSize = cctxParams->targetCBlockSize; - size_t litSize, seqCount; - int writeLitEntropy = entropyMetadata->hufMetadata.hType == set_compressed; - int writeSeqEntropy = 1; - int lastSequence = 0; +typedef struct { + void* dictBuffer; + void const* dict; + size_t dictSize; + ZSTD_dictContentType_e dictContentType; + ZSTD_CDict* cdict; +} ZSTD_localDict; - DEBUGLOG(5, "ZSTD_compressSubBlock_multi (litSize=%u, nbSeq=%u)", - (unsigned)(lend-lp), (unsigned)(send-sstart)); +typedef struct { + HUF_CElt CTable[HUF_CTABLE_SIZE_ST(255)]; + HUF_repeat repeatMode; +} ZSTD_hufCTables_t; - litSize = 0; - seqCount = 0; - do { - size_t cBlockSizeEstimate = 0; - if (sstart == send) { - lastSequence = 1; - } else { - const seqDef* const sequence = sp + seqCount; - lastSequence = sequence == send - 1; - litSize += ZSTD_getSequenceLength(seqStorePtr, sequence).litLength; - seqCount++; - } - if (lastSequence) { - assert(lp <= lend); - assert(litSize <= (size_t)(lend - lp)); - litSize = (size_t)(lend - lp); - } - /* I think there is an optimization opportunity here. - * Calling ZSTD_estimateSubBlockSize for every sequence can be wasteful - * since it recalculates estimate from scratch. - * For example, it would recount literal distribution and symbol codes everytime. - */ - cBlockSizeEstimate = ZSTD_estimateSubBlockSize(lp, litSize, ofCodePtr, llCodePtr, mlCodePtr, seqCount, - &nextCBlock->entropy, entropyMetadata, - workspace, wkspSize, writeLitEntropy, writeSeqEntropy); - if (cBlockSizeEstimate > targetCBlockSize || lastSequence) { - int litEntropyWritten = 0; - int seqEntropyWritten = 0; - const size_t decompressedSize = ZSTD_seqDecompressedSize(seqStorePtr, sp, seqCount, litSize, lastSequence); - const size_t cSize = ZSTD_compressSubBlock(&nextCBlock->entropy, entropyMetadata, - sp, seqCount, - lp, litSize, - llCodePtr, mlCodePtr, ofCodePtr, - cctxParams, - op, oend-op, - bmi2, writeLitEntropy, writeSeqEntropy, - &litEntropyWritten, &seqEntropyWritten, - lastBlock && lastSequence); - FORWARD_IF_ERROR(cSize, "ZSTD_compressSubBlock failed"); - if (cSize > 0 && cSize < decompressedSize) { - DEBUGLOG(5, "Committed the sub-block"); - assert(ip + decompressedSize <= iend); - ip += decompressedSize; - sp += seqCount; - lp += litSize; - op += cSize; - llCodePtr += seqCount; - mlCodePtr += seqCount; - ofCodePtr += seqCount; - litSize = 0; - seqCount = 0; - /* Entropy only needs to be written once */ - if (litEntropyWritten) { - writeLitEntropy = 0; - } - if (seqEntropyWritten) { - writeSeqEntropy = 0; - } - } - } - } while (!lastSequence); - if (writeLitEntropy) { - DEBUGLOG(5, "ZSTD_compressSubBlock_multi has literal entropy tables unwritten"); - ZSTD_memcpy(&nextCBlock->entropy.huf, &prevCBlock->entropy.huf, sizeof(prevCBlock->entropy.huf)); - } - if (writeSeqEntropy && ZSTD_needSequenceEntropyTables(&entropyMetadata->fseMetadata)) { - /* If we haven't written our entropy tables, then we've violated our contract and - * must emit an uncompressed block. - */ - DEBUGLOG(5, "ZSTD_compressSubBlock_multi has sequence entropy tables unwritten"); - return 0; - } - if (ip < iend) { - size_t const cSize = ZSTD_noCompressBlock(op, oend - op, ip, iend - ip, lastBlock); - DEBUGLOG(5, "ZSTD_compressSubBlock_multi last sub-block uncompressed, %zu bytes", (size_t)(iend - ip)); - FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed"); - assert(cSize != 0); - op += cSize; - /* We have to regenerate the repcodes because we've skipped some sequences */ - if (sp < send) { - seqDef const* seq; - repcodes_t rep; - ZSTD_memcpy(&rep, prevCBlock->rep, sizeof(rep)); - for (seq = sstart; seq < sp; ++seq) { - rep = ZSTD_updateRep(rep.rep, seq->offset - 1, ZSTD_getSequenceLength(seqStorePtr, seq).litLength == 0); - } - ZSTD_memcpy(nextCBlock->rep, &rep, sizeof(rep)); - } - } - DEBUGLOG(5, "ZSTD_compressSubBlock_multi compressed"); - return op-ostart; -} +typedef struct { + FSE_CTable offcodeCTable[FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)]; + FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)]; + FSE_CTable litlengthCTable[FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)]; + FSE_repeat offcode_repeatMode; + FSE_repeat matchlength_repeatMode; + FSE_repeat litlength_repeatMode; +} ZSTD_fseCTables_t; -size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc, - void* dst, size_t dstCapacity, - void const* src, size_t srcSize, - unsigned lastBlock) { - ZSTD_entropyCTablesMetadata_t entropyMetadata; +typedef struct { + ZSTD_hufCTables_t huf; + ZSTD_fseCTables_t fse; +} ZSTD_entropyCTables_t; - FORWARD_IF_ERROR(ZSTD_buildSuperBlockEntropy(&zc->seqStore, - &zc->blockState.prevCBlock->entropy, - &zc->blockState.nextCBlock->entropy, - &zc->appliedParams, - &entropyMetadata, - zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */), ""); +/*********************************************** +* Sequences * +***********************************************/ +typedef struct SeqDef_s { + U32 offBase; /* offBase == Offset + ZSTD_REP_NUM, or repcode 1,2,3 */ + U16 litLength; + U16 mlBase; /* mlBase == matchLength - MINMATCH */ +} SeqDef; - return ZSTD_compressSubBlock_multi(&zc->seqStore, - zc->blockState.prevCBlock, - zc->blockState.nextCBlock, - &entropyMetadata, - &zc->appliedParams, - dst, dstCapacity, - src, srcSize, - zc->bmi2, lastBlock, - zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */); -} -/**** ended inlining compress/zstd_compress_superblock.c ****/ -/**** start inlining compress/zstd_compress.c ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ +/* Controls whether seqStore has a single "long" litLength or matchLength. See SeqStore_t. */ +typedef enum { + ZSTD_llt_none = 0, /* no longLengthType */ + ZSTD_llt_literalLength = 1, /* represents a long literal */ + ZSTD_llt_matchLength = 2 /* represents a long match */ +} ZSTD_longLengthType_e; -/*-************************************* -* Dependencies -***************************************/ -/**** skipping file: ../common/zstd_deps.h ****/ -/**** start inlining ../common/cpu.h ****/ -/* - * Copyright (c) 2018-2021, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ +typedef struct { + SeqDef* sequencesStart; + SeqDef* sequences; /* ptr to end of sequences */ + BYTE* litStart; + BYTE* lit; /* ptr to end of literals */ + BYTE* llCode; + BYTE* mlCode; + BYTE* ofCode; + size_t maxNbSeq; + size_t maxNbLit; -#ifndef ZSTD_COMMON_CPU_H -#define ZSTD_COMMON_CPU_H + /* longLengthPos and longLengthType to allow us to represent either a single litLength or matchLength + * in the seqStore that has a value larger than U16 (if it exists). To do so, we increment + * the existing value of the litLength or matchLength by 0x10000. + */ + ZSTD_longLengthType_e longLengthType; + U32 longLengthPos; /* Index of the sequence to apply long length modification to */ +} SeqStore_t; + +typedef struct { + U32 litLength; + U32 matchLength; +} ZSTD_SequenceLength; /** - * Implementation taken from folly/CpuId.h - * https://github.com/facebook/folly/blob/master/folly/CpuId.h + * Returns the ZSTD_SequenceLength for the given sequences. It handles the decoding of long sequences + * indicated by longLengthPos and longLengthType, and adds MINMATCH back to matchLength. */ +MEM_STATIC ZSTD_SequenceLength ZSTD_getSequenceLength(SeqStore_t const* seqStore, SeqDef const* seq) +{ + ZSTD_SequenceLength seqLen; + seqLen.litLength = seq->litLength; + seqLen.matchLength = seq->mlBase + MINMATCH; + if (seqStore->longLengthPos == (U32)(seq - seqStore->sequencesStart)) { + if (seqStore->longLengthType == ZSTD_llt_literalLength) { + seqLen.litLength += 0x10000; + } + if (seqStore->longLengthType == ZSTD_llt_matchLength) { + seqLen.matchLength += 0x10000; + } + } + return seqLen; +} -/**** skipping file: mem.h ****/ +const SeqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBuilder */ +int ZSTD_seqToCodes(const SeqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */ -#ifdef _MSC_VER -#include -#endif +/*********************************************** +* Entropy buffer statistics structs and funcs * +***********************************************/ +/** ZSTD_hufCTablesMetadata_t : + * Stores Literals Block Type for a super-block in hType, and + * huffman tree description in hufDesBuffer. + * hufDesSize refers to the size of huffman tree description in bytes. + * This metadata is populated in ZSTD_buildBlockEntropyStats_literals() */ typedef struct { - U32 f1c; - U32 f1d; - U32 f7b; - U32 f7c; -} ZSTD_cpuid_t; + SymbolEncodingType_e hType; + BYTE hufDesBuffer[ZSTD_MAX_HUF_HEADER_SIZE]; + size_t hufDesSize; +} ZSTD_hufCTablesMetadata_t; -MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) { - U32 f1c = 0; - U32 f1d = 0; - U32 f7b = 0; - U32 f7c = 0; -#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) - int reg[4]; - __cpuid((int*)reg, 0); - { - int const n = reg[0]; - if (n >= 1) { - __cpuid((int*)reg, 1); - f1c = (U32)reg[2]; - f1d = (U32)reg[3]; - } - if (n >= 7) { - __cpuidex((int*)reg, 7, 0); - f7b = (U32)reg[1]; - f7c = (U32)reg[2]; - } - } -#elif defined(__i386__) && defined(__PIC__) && !defined(__clang__) && defined(__GNUC__) - /* The following block like the normal cpuid branch below, but gcc - * reserves ebx for use of its pic register so we must specially - * handle the save and restore to avoid clobbering the register - */ - U32 n; - __asm__( - "pushl %%ebx\n\t" - "cpuid\n\t" - "popl %%ebx\n\t" - : "=a"(n) - : "a"(0) - : "ecx", "edx"); - if (n >= 1) { - U32 f1a; - __asm__( - "pushl %%ebx\n\t" - "cpuid\n\t" - "popl %%ebx\n\t" - : "=a"(f1a), "=c"(f1c), "=d"(f1d) - : "a"(1)); - } - if (n >= 7) { - __asm__( - "pushl %%ebx\n\t" - "cpuid\n\t" - "movl %%ebx, %%eax\n\t" - "popl %%ebx" - : "=a"(f7b), "=c"(f7c) - : "a"(7), "c"(0) - : "edx"); - } -#elif defined(__x86_64__) || defined(_M_X64) || defined(__i386__) - U32 n; - __asm__("cpuid" : "=a"(n) : "a"(0) : "ebx", "ecx", "edx"); - if (n >= 1) { - U32 f1a; - __asm__("cpuid" : "=a"(f1a), "=c"(f1c), "=d"(f1d) : "a"(1) : "ebx"); - } - if (n >= 7) { - U32 f7a; - __asm__("cpuid" - : "=a"(f7a), "=b"(f7b), "=c"(f7c) - : "a"(7), "c"(0) - : "edx"); - } -#endif - { - ZSTD_cpuid_t cpuid; - cpuid.f1c = f1c; - cpuid.f1d = f1d; - cpuid.f7b = f7b; - cpuid.f7c = f7c; - return cpuid; - } -} +/** ZSTD_fseCTablesMetadata_t : + * Stores symbol compression modes for a super-block in {ll, ol, ml}Type, and + * fse tables in fseTablesBuffer. + * fseTablesSize refers to the size of fse tables in bytes. + * This metadata is populated in ZSTD_buildBlockEntropyStats_sequences() */ +typedef struct { + SymbolEncodingType_e llType; + SymbolEncodingType_e ofType; + SymbolEncodingType_e mlType; + BYTE fseTablesBuffer[ZSTD_MAX_FSE_HEADERS_SIZE]; + size_t fseTablesSize; + size_t lastCountSize; /* This is to account for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */ +} ZSTD_fseCTablesMetadata_t; -#define X(name, r, bit) \ - MEM_STATIC int ZSTD_cpuid_##name(ZSTD_cpuid_t const cpuid) { \ - return ((cpuid.r) & (1U << bit)) != 0; \ - } +typedef struct { + ZSTD_hufCTablesMetadata_t hufMetadata; + ZSTD_fseCTablesMetadata_t fseMetadata; +} ZSTD_entropyCTablesMetadata_t; -/* cpuid(1): Processor Info and Feature Bits. */ -#define C(name, bit) X(name, f1c, bit) - C(sse3, 0) - C(pclmuldq, 1) - C(dtes64, 2) - C(monitor, 3) - C(dscpl, 4) - C(vmx, 5) - C(smx, 6) - C(eist, 7) - C(tm2, 8) - C(ssse3, 9) - C(cnxtid, 10) - C(fma, 12) - C(cx16, 13) - C(xtpr, 14) - C(pdcm, 15) - C(pcid, 17) - C(dca, 18) - C(sse41, 19) - C(sse42, 20) - C(x2apic, 21) - C(movbe, 22) - C(popcnt, 23) - C(tscdeadline, 24) - C(aes, 25) - C(xsave, 26) - C(osxsave, 27) - C(avx, 28) - C(f16c, 29) - C(rdrand, 30) -#undef C -#define D(name, bit) X(name, f1d, bit) - D(fpu, 0) - D(vme, 1) - D(de, 2) - D(pse, 3) - D(tsc, 4) - D(msr, 5) - D(pae, 6) - D(mce, 7) - D(cx8, 8) - D(apic, 9) - D(sep, 11) - D(mtrr, 12) - D(pge, 13) - D(mca, 14) - D(cmov, 15) - D(pat, 16) - D(pse36, 17) - D(psn, 18) - D(clfsh, 19) - D(ds, 21) - D(acpi, 22) - D(mmx, 23) - D(fxsr, 24) - D(sse, 25) - D(sse2, 26) - D(ss, 27) - D(htt, 28) - D(tm, 29) - D(pbe, 31) -#undef D +/** ZSTD_buildBlockEntropyStats() : + * Builds entropy for the block. + * @return : 0 on success or error code */ +size_t ZSTD_buildBlockEntropyStats( + const SeqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + ZSTD_entropyCTablesMetadata_t* entropyMetadata, + void* workspace, size_t wkspSize); + +/********************************* +* Compression internals structs * +*********************************/ -/* cpuid(7): Extended Features. */ -#define B(name, bit) X(name, f7b, bit) - B(bmi1, 3) - B(hle, 4) - B(avx2, 5) - B(smep, 7) - B(bmi2, 8) - B(erms, 9) - B(invpcid, 10) - B(rtm, 11) - B(mpx, 14) - B(avx512f, 16) - B(avx512dq, 17) - B(rdseed, 18) - B(adx, 19) - B(smap, 20) - B(avx512ifma, 21) - B(pcommit, 22) - B(clflushopt, 23) - B(clwb, 24) - B(avx512pf, 26) - B(avx512er, 27) - B(avx512cd, 28) - B(sha, 29) - B(avx512bw, 30) - B(avx512vl, 31) -#undef B -#define C(name, bit) X(name, f7c, bit) - C(prefetchwt1, 0) - C(avx512vbmi, 1) -#undef C +typedef struct { + U32 off; /* Offset sumtype code for the match, using ZSTD_storeSeq() format */ + U32 len; /* Raw length of match */ +} ZSTD_match_t; -#undef X +typedef struct { + U32 offset; /* Offset of sequence */ + U32 litLength; /* Length of literals prior to match */ + U32 matchLength; /* Raw length of match */ +} rawSeq; + +typedef struct { + rawSeq* seq; /* The start of the sequences */ + size_t pos; /* The index in seq where reading stopped. pos <= size. */ + size_t posInSequence; /* The position within the sequence at seq[pos] where reading + stopped. posInSequence <= seq[pos].litLength + seq[pos].matchLength */ + size_t size; /* The number of sequences. <= capacity. */ + size_t capacity; /* The capacity starting from `seq` pointer */ +} RawSeqStore_t; + +UNUSED_ATTR static const RawSeqStore_t kNullRawSeqStore = {NULL, 0, 0, 0, 0}; + +typedef struct { + int price; /* price from beginning of segment to this position */ + U32 off; /* offset of previous match */ + U32 mlen; /* length of previous match */ + U32 litlen; /* nb of literals since previous match */ + U32 rep[ZSTD_REP_NUM]; /* offset history after previous match */ +} ZSTD_optimal_t; + +typedef enum { zop_dynamic=0, zop_predef } ZSTD_OptPrice_e; + +#define ZSTD_OPT_SIZE (ZSTD_OPT_NUM+3) +typedef struct { + /* All tables are allocated inside cctx->workspace by ZSTD_resetCCtx_internal() */ + unsigned* litFreq; /* table of literals statistics, of size 256 */ + unsigned* litLengthFreq; /* table of litLength statistics, of size (MaxLL+1) */ + unsigned* matchLengthFreq; /* table of matchLength statistics, of size (MaxML+1) */ + unsigned* offCodeFreq; /* table of offCode statistics, of size (MaxOff+1) */ + ZSTD_match_t* matchTable; /* list of found matches, of size ZSTD_OPT_SIZE */ + ZSTD_optimal_t* priceTable; /* All positions tracked by optimal parser, of size ZSTD_OPT_SIZE */ + + U32 litSum; /* nb of literals */ + U32 litLengthSum; /* nb of litLength codes */ + U32 matchLengthSum; /* nb of matchLength codes */ + U32 offCodeSum; /* nb of offset codes */ + U32 litSumBasePrice; /* to compare to log2(litfreq) */ + U32 litLengthSumBasePrice; /* to compare to log2(llfreq) */ + U32 matchLengthSumBasePrice;/* to compare to log2(mlfreq) */ + U32 offCodeSumBasePrice; /* to compare to log2(offreq) */ + ZSTD_OptPrice_e priceType; /* prices can be determined dynamically, or follow a pre-defined cost structure */ + const ZSTD_entropyCTables_t* symbolCosts; /* pre-calculated dictionary statistics */ + ZSTD_ParamSwitch_e literalCompressionMode; +} optState_t; + +typedef struct { + ZSTD_entropyCTables_t entropy; + U32 rep[ZSTD_REP_NUM]; +} ZSTD_compressedBlockState_t; + +typedef struct { + BYTE const* nextSrc; /* next block here to continue on current prefix */ + BYTE const* base; /* All regular indexes relative to this position */ + BYTE const* dictBase; /* extDict indexes relative to this position */ + U32 dictLimit; /* below that point, need extDict */ + U32 lowLimit; /* below that point, no more valid data */ + U32 nbOverflowCorrections; /* Number of times overflow correction has run since + * ZSTD_window_init(). Useful for debugging coredumps + * and for ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY. + */ +} ZSTD_window_t; + +#define ZSTD_WINDOW_START_INDEX 2 + +typedef struct ZSTD_MatchState_t ZSTD_MatchState_t; + +#define ZSTD_ROW_HASH_CACHE_SIZE 8 /* Size of prefetching hash cache for row-based matchfinder */ + +struct ZSTD_MatchState_t { + ZSTD_window_t window; /* State for window round buffer management */ + U32 loadedDictEnd; /* index of end of dictionary, within context's referential. + * When loadedDictEnd != 0, a dictionary is in use, and still valid. + * This relies on a mechanism to set loadedDictEnd=0 when dictionary is no longer within distance. + * Such mechanism is provided within ZSTD_window_enforceMaxDist() and ZSTD_checkDictValidity(). + * When dict referential is copied into active context (i.e. not attached), + * loadedDictEnd == dictSize, since referential starts from zero. + */ + U32 nextToUpdate; /* index from which to continue table update */ + U32 hashLog3; /* dispatch table for matches of len==3 : larger == faster, more memory */ + + U32 rowHashLog; /* For row-based matchfinder: Hashlog based on nb of rows in the hashTable.*/ + BYTE* tagTable; /* For row-based matchFinder: A row-based table containing the hashes and head index. */ + U32 hashCache[ZSTD_ROW_HASH_CACHE_SIZE]; /* For row-based matchFinder: a cache of hashes to improve speed */ + U64 hashSalt; /* For row-based matchFinder: salts the hash for reuse of tag table */ + U32 hashSaltEntropy; /* For row-based matchFinder: collects entropy for salt generation */ + + U32* hashTable; + U32* hashTable3; + U32* chainTable; + + int forceNonContiguous; /* Non-zero if we should force non-contiguous load for the next window update. */ + + int dedicatedDictSearch; /* Indicates whether this matchState is using the + * dedicated dictionary search structure. + */ + optState_t opt; /* optimal parser state */ + const ZSTD_MatchState_t* dictMatchState; + ZSTD_compressionParameters cParams; + const RawSeqStore_t* ldmSeqStore; + + /* Controls prefetching in some dictMatchState matchfinders. + * This behavior is controlled from the cctx ms. + * This parameter has no effect in the cdict ms. */ + int prefetchCDictTables; + + /* When == 0, lazy match finders insert every position. + * When != 0, lazy match finders only insert positions they search. + * This allows them to skip much faster over incompressible data, + * at a small cost to compression ratio. + */ + int lazySkipping; +}; + +typedef struct { + ZSTD_compressedBlockState_t* prevCBlock; + ZSTD_compressedBlockState_t* nextCBlock; + ZSTD_MatchState_t matchState; +} ZSTD_blockState_t; + +typedef struct { + U32 offset; + U32 checksum; +} ldmEntry_t; + +typedef struct { + BYTE const* split; + U32 hash; + U32 checksum; + ldmEntry_t* bucket; +} ldmMatchCandidate_t; + +#define LDM_BATCH_SIZE 64 + +typedef struct { + ZSTD_window_t window; /* State for the window round buffer management */ + ldmEntry_t* hashTable; + U32 loadedDictEnd; + BYTE* bucketOffsets; /* Next position in bucket to insert entry */ + size_t splitIndices[LDM_BATCH_SIZE]; + ldmMatchCandidate_t matchCandidates[LDM_BATCH_SIZE]; +} ldmState_t; + +typedef struct { + ZSTD_ParamSwitch_e enableLdm; /* ZSTD_ps_enable to enable LDM. ZSTD_ps_auto by default */ + U32 hashLog; /* Log size of hashTable */ + U32 bucketSizeLog; /* Log bucket size for collision resolution, at most 8 */ + U32 minMatchLength; /* Minimum match length */ + U32 hashRateLog; /* Log number of entries to skip */ + U32 windowLog; /* Window log for the LDM */ +} ldmParams_t; + +typedef struct { + int collectSequences; + ZSTD_Sequence* seqStart; + size_t seqIndex; + size_t maxSequences; +} SeqCollector; + +struct ZSTD_CCtx_params_s { + ZSTD_format_e format; + ZSTD_compressionParameters cParams; + ZSTD_frameParameters fParams; + + int compressionLevel; + int forceWindow; /* force back-references to respect limit of + * 1< 63) ? ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength]; +} + +/* ZSTD_MLcode() : + * note : mlBase = matchLength - MINMATCH; + * because it's the format it's stored in seqStore->sequences */ +MEM_STATIC U32 ZSTD_MLcode(U32 mlBase) +{ + static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, + 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, + 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, + 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 }; + static const U32 ML_deltaCode = 36; + return (mlBase > 127) ? ZSTD_highbit32(mlBase) + ML_deltaCode : ML_Code[mlBase]; +} + +/* ZSTD_cParam_withinBounds: + * @return 1 if value is within cParam bounds, + * 0 otherwise */ +MEM_STATIC int ZSTD_cParam_withinBounds(ZSTD_cParameter cParam, int value) +{ + ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); + if (ZSTD_isError(bounds.error)) return 0; + if (value < bounds.lowerBound) return 0; + if (value > bounds.upperBound) return 0; + return 1; +} + +/* ZSTD_selectAddr: + * @return index >= lowLimit ? candidate : backup, + * tries to force branchless codegen. */ +MEM_STATIC const BYTE* +ZSTD_selectAddr(U32 index, U32 lowLimit, const BYTE* candidate, const BYTE* backup) +{ +#if defined(__GNUC__) && defined(__x86_64__) + __asm__ ( + "cmp %1, %2\n" + "cmova %3, %0\n" + : "+r"(candidate) + : "r"(index), "r"(lowLimit), "r"(backup) + ); + return candidate; +#else + return index >= lowLimit ? candidate : backup; +#endif +} + +/* ZSTD_noCompressBlock() : + * Writes uncompressed block to dst buffer from given src. + * Returns the size of the block */ +MEM_STATIC size_t +ZSTD_noCompressBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 lastBlock) +{ + U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw)<<1) + (U32)(srcSize << 3); + DEBUGLOG(5, "ZSTD_noCompressBlock (srcSize=%zu, dstCapacity=%zu)", srcSize, dstCapacity); + RETURN_ERROR_IF(srcSize + ZSTD_blockHeaderSize > dstCapacity, + dstSize_tooSmall, "dst buf too small for uncompressed block"); + MEM_writeLE24(dst, cBlockHeader24); + ZSTD_memcpy((BYTE*)dst + ZSTD_blockHeaderSize, src, srcSize); + return ZSTD_blockHeaderSize + srcSize; +} + +MEM_STATIC size_t +ZSTD_rleCompressBlock(void* dst, size_t dstCapacity, BYTE src, size_t srcSize, U32 lastBlock) +{ + BYTE* const op = (BYTE*)dst; + U32 const cBlockHeader = lastBlock + (((U32)bt_rle)<<1) + (U32)(srcSize << 3); + RETURN_ERROR_IF(dstCapacity < 4, dstSize_tooSmall, ""); + MEM_writeLE24(op, cBlockHeader); + op[3] = src; + return 4; +} + + +/* ZSTD_minGain() : + * minimum compression required + * to generate a compress block or a compressed literals section. + * note : use same formula for both situations */ +MEM_STATIC size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat) +{ + U32 const minlog = (strat>=ZSTD_btultra) ? (U32)(strat) - 1 : 6; + ZSTD_STATIC_ASSERT(ZSTD_btultra == 8); + assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, (int)strat)); + return (srcSize >> minlog) + 2; +} + +MEM_STATIC int ZSTD_literalsCompressionIsDisabled(const ZSTD_CCtx_params* cctxParams) +{ + switch (cctxParams->literalCompressionMode) { + case ZSTD_ps_enable: + return 0; + case ZSTD_ps_disable: + return 1; + default: + assert(0 /* impossible: pre-validated */); + ZSTD_FALLTHROUGH; + case ZSTD_ps_auto: + return (cctxParams->cParams.strategy == ZSTD_fast) && (cctxParams->cParams.targetLength > 0); + } +} + +/*! ZSTD_safecopyLiterals() : + * memcpy() function that won't read beyond more than WILDCOPY_OVERLENGTH bytes past ilimit_w. + * Only called when the sequence ends past ilimit_w, so it only needs to be optimized for single + * large copies. + */ +static void +ZSTD_safecopyLiterals(BYTE* op, BYTE const* ip, BYTE const* const iend, BYTE const* ilimit_w) +{ + assert(iend > ilimit_w); + if (ip <= ilimit_w) { + ZSTD_wildcopy(op, ip, ilimit_w - ip, ZSTD_no_overlap); + op += ilimit_w - ip; + ip = ilimit_w; + } + while (ip < iend) *op++ = *ip++; +} + + +#define REPCODE1_TO_OFFBASE REPCODE_TO_OFFBASE(1) +#define REPCODE2_TO_OFFBASE REPCODE_TO_OFFBASE(2) +#define REPCODE3_TO_OFFBASE REPCODE_TO_OFFBASE(3) +#define REPCODE_TO_OFFBASE(r) (assert((r)>=1), assert((r)<=ZSTD_REP_NUM), (r)) /* accepts IDs 1,2,3 */ +#define OFFSET_TO_OFFBASE(o) (assert((o)>0), o + ZSTD_REP_NUM) +#define OFFBASE_IS_OFFSET(o) ((o) > ZSTD_REP_NUM) +#define OFFBASE_IS_REPCODE(o) ( 1 <= (o) && (o) <= ZSTD_REP_NUM) +#define OFFBASE_TO_OFFSET(o) (assert(OFFBASE_IS_OFFSET(o)), (o) - ZSTD_REP_NUM) +#define OFFBASE_TO_REPCODE(o) (assert(OFFBASE_IS_REPCODE(o)), (o)) /* returns ID 1,2,3 */ + +/*! ZSTD_storeSeqOnly() : + * Store a sequence (litlen, litPtr, offBase and matchLength) into SeqStore_t. + * Literals themselves are not copied, but @litPtr is updated. + * @offBase : Users should employ macros REPCODE_TO_OFFBASE() and OFFSET_TO_OFFBASE(). + * @matchLength : must be >= MINMATCH +*/ +HINT_INLINE UNUSED_ATTR void +ZSTD_storeSeqOnly(SeqStore_t* seqStorePtr, + size_t litLength, + U32 offBase, + size_t matchLength) +{ + assert((size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart) < seqStorePtr->maxNbSeq); + + /* literal Length */ + assert(litLength <= ZSTD_BLOCKSIZE_MAX); + if (UNLIKELY(litLength>0xFFFF)) { + assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */ + seqStorePtr->longLengthType = ZSTD_llt_literalLength; + seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + } + seqStorePtr->sequences[0].litLength = (U16)litLength; + + /* match offset */ + seqStorePtr->sequences[0].offBase = offBase; + + /* match Length */ + assert(matchLength <= ZSTD_BLOCKSIZE_MAX); + assert(matchLength >= MINMATCH); + { size_t const mlBase = matchLength - MINMATCH; + if (UNLIKELY(mlBase>0xFFFF)) { + assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */ + seqStorePtr->longLengthType = ZSTD_llt_matchLength; + seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + } + seqStorePtr->sequences[0].mlBase = (U16)mlBase; + } + + seqStorePtr->sequences++; +} + +/*! ZSTD_storeSeq() : + * Store a sequence (litlen, litPtr, offBase and matchLength) into SeqStore_t. + * @offBase : Users should employ macros REPCODE_TO_OFFBASE() and OFFSET_TO_OFFBASE(). + * @matchLength : must be >= MINMATCH + * Allowed to over-read literals up to litLimit. +*/ +HINT_INLINE UNUSED_ATTR void +ZSTD_storeSeq(SeqStore_t* seqStorePtr, + size_t litLength, const BYTE* literals, const BYTE* litLimit, + U32 offBase, + size_t matchLength) +{ + BYTE const* const litLimit_w = litLimit - WILDCOPY_OVERLENGTH; + BYTE const* const litEnd = literals + litLength; +#if defined(DEBUGLEVEL) && (DEBUGLEVEL >= 6) + static const BYTE* g_start = NULL; + if (g_start==NULL) g_start = (const BYTE*)literals; /* note : index only works for compression within a single segment */ + { U32 const pos = (U32)((const BYTE*)literals - g_start); + DEBUGLOG(6, "Cpos%7u :%3u literals, match%4u bytes at offBase%7u", + pos, (U32)litLength, (U32)matchLength, (U32)offBase); + } +#endif + assert((size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart) < seqStorePtr->maxNbSeq); + /* copy Literals */ + assert(seqStorePtr->maxNbLit <= 128 KB); + assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + seqStorePtr->maxNbLit); + assert(literals + litLength <= litLimit); + if (litEnd <= litLimit_w) { + /* Common case we can use wildcopy. + * First copy 16 bytes, because literals are likely short. + */ + ZSTD_STATIC_ASSERT(WILDCOPY_OVERLENGTH >= 16); + ZSTD_copy16(seqStorePtr->lit, literals); + if (litLength > 16) { + ZSTD_wildcopy(seqStorePtr->lit+16, literals+16, (ptrdiff_t)litLength-16, ZSTD_no_overlap); + } + } else { + ZSTD_safecopyLiterals(seqStorePtr->lit, literals, litEnd, litLimit_w); + } + seqStorePtr->lit += litLength; + + ZSTD_storeSeqOnly(seqStorePtr, litLength, offBase, matchLength); +} + +/* ZSTD_updateRep() : + * updates in-place @rep (array of repeat offsets) + * @offBase : sum-type, using numeric representation of ZSTD_storeSeq() + */ +MEM_STATIC void +ZSTD_updateRep(U32 rep[ZSTD_REP_NUM], U32 const offBase, U32 const ll0) +{ + if (OFFBASE_IS_OFFSET(offBase)) { /* full offset */ + rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = OFFBASE_TO_OFFSET(offBase); + } else { /* repcode */ + U32 const repCode = OFFBASE_TO_REPCODE(offBase) - 1 + ll0; + if (repCode > 0) { /* note : if repCode==0, no change */ + U32 const currentOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode]; + rep[2] = (repCode >= 2) ? rep[1] : rep[2]; + rep[1] = rep[0]; + rep[0] = currentOffset; + } else { /* repCode == 0 */ + /* nothing to do */ + } + } +} + +typedef struct repcodes_s { + U32 rep[3]; +} Repcodes_t; + +MEM_STATIC Repcodes_t +ZSTD_newRep(U32 const rep[ZSTD_REP_NUM], U32 const offBase, U32 const ll0) +{ + Repcodes_t newReps; + ZSTD_memcpy(&newReps, rep, sizeof(newReps)); + ZSTD_updateRep(newReps.rep, offBase, ll0); + return newReps; +} + + +/*-************************************* +* Match length counter +***************************************/ +MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* const pInLimit) +{ + const BYTE* const pStart = pIn; + const BYTE* const pInLoopLimit = pInLimit - (sizeof(size_t)-1); + + if (pIn < pInLoopLimit) { + { size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn); + if (diff) return ZSTD_NbCommonBytes(diff); } + pIn+=sizeof(size_t); pMatch+=sizeof(size_t); + while (pIn < pInLoopLimit) { + size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn); + if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; } + pIn += ZSTD_NbCommonBytes(diff); + return (size_t)(pIn - pStart); + } } + if (MEM_64bits() && (pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; } + if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; } + if ((pIn> (32-h) ; } +MEM_STATIC size_t ZSTD_hash3Ptr(const void* ptr, U32 h) { return ZSTD_hash3(MEM_readLE32(ptr), h, 0); } /* only in zstd_opt.h */ +MEM_STATIC size_t ZSTD_hash3PtrS(const void* ptr, U32 h, U32 s) { return ZSTD_hash3(MEM_readLE32(ptr), h, s); } + +static const U32 prime4bytes = 2654435761U; +static U32 ZSTD_hash4(U32 u, U32 h, U32 s) { assert(h <= 32); return ((u * prime4bytes) ^ s) >> (32-h) ; } +static size_t ZSTD_hash4Ptr(const void* ptr, U32 h) { return ZSTD_hash4(MEM_readLE32(ptr), h, 0); } +static size_t ZSTD_hash4PtrS(const void* ptr, U32 h, U32 s) { return ZSTD_hash4(MEM_readLE32(ptr), h, s); } + +static const U64 prime5bytes = 889523592379ULL; +static size_t ZSTD_hash5(U64 u, U32 h, U64 s) { assert(h <= 64); return (size_t)((((u << (64-40)) * prime5bytes) ^ s) >> (64-h)) ; } +static size_t ZSTD_hash5Ptr(const void* p, U32 h) { return ZSTD_hash5(MEM_readLE64(p), h, 0); } +static size_t ZSTD_hash5PtrS(const void* p, U32 h, U64 s) { return ZSTD_hash5(MEM_readLE64(p), h, s); } + +static const U64 prime6bytes = 227718039650203ULL; +static size_t ZSTD_hash6(U64 u, U32 h, U64 s) { assert(h <= 64); return (size_t)((((u << (64-48)) * prime6bytes) ^ s) >> (64-h)) ; } +static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h, 0); } +static size_t ZSTD_hash6PtrS(const void* p, U32 h, U64 s) { return ZSTD_hash6(MEM_readLE64(p), h, s); } + +static const U64 prime7bytes = 58295818150454627ULL; +static size_t ZSTD_hash7(U64 u, U32 h, U64 s) { assert(h <= 64); return (size_t)((((u << (64-56)) * prime7bytes) ^ s) >> (64-h)) ; } +static size_t ZSTD_hash7Ptr(const void* p, U32 h) { return ZSTD_hash7(MEM_readLE64(p), h, 0); } +static size_t ZSTD_hash7PtrS(const void* p, U32 h, U64 s) { return ZSTD_hash7(MEM_readLE64(p), h, s); } + +static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL; +static size_t ZSTD_hash8(U64 u, U32 h, U64 s) { assert(h <= 64); return (size_t)((((u) * prime8bytes) ^ s) >> (64-h)) ; } +static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h, 0); } +static size_t ZSTD_hash8PtrS(const void* p, U32 h, U64 s) { return ZSTD_hash8(MEM_readLE64(p), h, s); } + + +MEM_STATIC FORCE_INLINE_ATTR +size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls) +{ + /* Although some of these hashes do support hBits up to 64, some do not. + * To be on the safe side, always avoid hBits > 32. */ + assert(hBits <= 32); + + switch(mls) + { + default: + case 4: return ZSTD_hash4Ptr(p, hBits); + case 5: return ZSTD_hash5Ptr(p, hBits); + case 6: return ZSTD_hash6Ptr(p, hBits); + case 7: return ZSTD_hash7Ptr(p, hBits); + case 8: return ZSTD_hash8Ptr(p, hBits); + } +} + +MEM_STATIC FORCE_INLINE_ATTR +size_t ZSTD_hashPtrSalted(const void* p, U32 hBits, U32 mls, const U64 hashSalt) { + /* Although some of these hashes do support hBits up to 64, some do not. + * To be on the safe side, always avoid hBits > 32. */ + assert(hBits <= 32); + + switch(mls) + { + default: + case 4: return ZSTD_hash4PtrS(p, hBits, (U32)hashSalt); + case 5: return ZSTD_hash5PtrS(p, hBits, hashSalt); + case 6: return ZSTD_hash6PtrS(p, hBits, hashSalt); + case 7: return ZSTD_hash7PtrS(p, hBits, hashSalt); + case 8: return ZSTD_hash8PtrS(p, hBits, hashSalt); + } +} + + +/** ZSTD_ipow() : + * Return base^exponent. + */ +static U64 ZSTD_ipow(U64 base, U64 exponent) +{ + U64 power = 1; + while (exponent) { + if (exponent & 1) power *= base; + exponent >>= 1; + base *= base; + } + return power; +} + +#define ZSTD_ROLL_HASH_CHAR_OFFSET 10 + +/** ZSTD_rollingHash_append() : + * Add the buffer to the hash value. + */ +static U64 ZSTD_rollingHash_append(U64 hash, void const* buf, size_t size) +{ + BYTE const* istart = (BYTE const*)buf; + size_t pos; + for (pos = 0; pos < size; ++pos) { + hash *= prime8bytes; + hash += istart[pos] + ZSTD_ROLL_HASH_CHAR_OFFSET; + } + return hash; +} + +/** ZSTD_rollingHash_compute() : + * Compute the rolling hash value of the buffer. + */ +MEM_STATIC U64 ZSTD_rollingHash_compute(void const* buf, size_t size) +{ + return ZSTD_rollingHash_append(0, buf, size); +} + +/** ZSTD_rollingHash_primePower() : + * Compute the primePower to be passed to ZSTD_rollingHash_rotate() for a hash + * over a window of length bytes. + */ +MEM_STATIC U64 ZSTD_rollingHash_primePower(U32 length) +{ + return ZSTD_ipow(prime8bytes, length - 1); +} + +/** ZSTD_rollingHash_rotate() : + * Rotate the rolling hash by one byte. + */ +MEM_STATIC U64 ZSTD_rollingHash_rotate(U64 hash, BYTE toRemove, BYTE toAdd, U64 primePower) +{ + hash -= (toRemove + ZSTD_ROLL_HASH_CHAR_OFFSET) * primePower; + hash *= prime8bytes; + hash += toAdd + ZSTD_ROLL_HASH_CHAR_OFFSET; + return hash; +} + +/*-************************************* +* Round buffer management +***************************************/ +/* Max @current value allowed: + * In 32-bit mode: we want to avoid crossing the 2 GB limit, + * reducing risks of side effects in case of signed operations on indexes. + * In 64-bit mode: we want to ensure that adding the maximum job size (512 MB) + * doesn't overflow U32 index capacity (4 GB) */ +#define ZSTD_CURRENT_MAX (MEM_64bits() ? 3500U MB : 2000U MB) +/* Maximum chunk size before overflow correction needs to be called again */ +#define ZSTD_CHUNKSIZE_MAX \ + ( ((U32)-1) /* Maximum ending current index */ \ + - ZSTD_CURRENT_MAX) /* Maximum beginning lowLimit */ + +/** + * ZSTD_window_clear(): + * Clears the window containing the history by simply setting it to empty. + */ +MEM_STATIC void ZSTD_window_clear(ZSTD_window_t* window) +{ + size_t const endT = (size_t)(window->nextSrc - window->base); + U32 const end = (U32)endT; + + window->lowLimit = end; + window->dictLimit = end; +} + +MEM_STATIC U32 ZSTD_window_isEmpty(ZSTD_window_t const window) +{ + return window.dictLimit == ZSTD_WINDOW_START_INDEX && + window.lowLimit == ZSTD_WINDOW_START_INDEX && + (window.nextSrc - window.base) == ZSTD_WINDOW_START_INDEX; +} + +/** + * ZSTD_window_hasExtDict(): + * Returns non-zero if the window has a non-empty extDict. + */ +MEM_STATIC U32 ZSTD_window_hasExtDict(ZSTD_window_t const window) +{ + return window.lowLimit < window.dictLimit; +} + +/** + * ZSTD_matchState_dictMode(): + * Inspects the provided matchState and figures out what dictMode should be + * passed to the compressor. + */ +MEM_STATIC ZSTD_dictMode_e ZSTD_matchState_dictMode(const ZSTD_MatchState_t *ms) +{ + return ZSTD_window_hasExtDict(ms->window) ? + ZSTD_extDict : + ms->dictMatchState != NULL ? + (ms->dictMatchState->dedicatedDictSearch ? ZSTD_dedicatedDictSearch : ZSTD_dictMatchState) : + ZSTD_noDict; +} + +/* Defining this macro to non-zero tells zstd to run the overflow correction + * code much more frequently. This is very inefficient, and should only be + * used for tests and fuzzers. + */ +#ifndef ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY +# ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION +# define ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY 1 +# else +# define ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY 0 +# endif +#endif + +/** + * ZSTD_window_canOverflowCorrect(): + * Returns non-zero if the indices are large enough for overflow correction + * to work correctly without impacting compression ratio. + */ +MEM_STATIC U32 ZSTD_window_canOverflowCorrect(ZSTD_window_t const window, + U32 cycleLog, + U32 maxDist, + U32 loadedDictEnd, + void const* src) +{ + U32 const cycleSize = 1u << cycleLog; + U32 const curr = (U32)((BYTE const*)src - window.base); + U32 const minIndexToOverflowCorrect = cycleSize + + MAX(maxDist, cycleSize) + + ZSTD_WINDOW_START_INDEX; + + /* Adjust the min index to backoff the overflow correction frequency, + * so we don't waste too much CPU in overflow correction. If this + * computation overflows we don't really care, we just need to make + * sure it is at least minIndexToOverflowCorrect. + */ + U32 const adjustment = window.nbOverflowCorrections + 1; + U32 const adjustedIndex = MAX(minIndexToOverflowCorrect * adjustment, + minIndexToOverflowCorrect); + U32 const indexLargeEnough = curr > adjustedIndex; + + /* Only overflow correct early if the dictionary is invalidated already, + * so we don't hurt compression ratio. + */ + U32 const dictionaryInvalidated = curr > maxDist + loadedDictEnd; + + return indexLargeEnough && dictionaryInvalidated; +} + +/** + * ZSTD_window_needOverflowCorrection(): + * Returns non-zero if the indices are getting too large and need overflow + * protection. + */ +MEM_STATIC U32 ZSTD_window_needOverflowCorrection(ZSTD_window_t const window, + U32 cycleLog, + U32 maxDist, + U32 loadedDictEnd, + void const* src, + void const* srcEnd) +{ + U32 const curr = (U32)((BYTE const*)srcEnd - window.base); + if (ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY) { + if (ZSTD_window_canOverflowCorrect(window, cycleLog, maxDist, loadedDictEnd, src)) { + return 1; + } + } + return curr > ZSTD_CURRENT_MAX; +} + +/** + * ZSTD_window_correctOverflow(): + * Reduces the indices to protect from index overflow. + * Returns the correction made to the indices, which must be applied to every + * stored index. + * + * The least significant cycleLog bits of the indices must remain the same, + * which may be 0. Every index up to maxDist in the past must be valid. + */ +MEM_STATIC +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog, + U32 maxDist, void const* src) +{ + /* preemptive overflow correction: + * 1. correction is large enough: + * lowLimit > (3<<29) ==> current > 3<<29 + 1< (3<<29 + 1< (3<<29) - (1< (3<<29) - (1<<30) (NOTE: chainLog <= 30) + * > 1<<29 + * + * 2. (ip+ZSTD_CHUNKSIZE_MAX - cctx->base) doesn't overflow: + * After correction, current is less than (1<base < 1<<32. + * 3. (cctx->lowLimit + 1< 3<<29 + 1<base); + U32 const currentCycle = curr & cycleMask; + /* Ensure newCurrent - maxDist >= ZSTD_WINDOW_START_INDEX. */ + U32 const currentCycleCorrection = currentCycle < ZSTD_WINDOW_START_INDEX + ? MAX(cycleSize, ZSTD_WINDOW_START_INDEX) + : 0; + U32 const newCurrent = currentCycle + + currentCycleCorrection + + MAX(maxDist, cycleSize); + U32 const correction = curr - newCurrent; + /* maxDist must be a power of two so that: + * (newCurrent & cycleMask) == (curr & cycleMask) + * This is required to not corrupt the chains / binary tree. + */ + assert((maxDist & (maxDist - 1)) == 0); + assert((curr & cycleMask) == (newCurrent & cycleMask)); + assert(curr > newCurrent); + if (!ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY) { + /* Loose bound, should be around 1<<29 (see above) */ + assert(correction > 1<<28); + } + + window->base += correction; + window->dictBase += correction; + if (window->lowLimit < correction + ZSTD_WINDOW_START_INDEX) { + window->lowLimit = ZSTD_WINDOW_START_INDEX; + } else { + window->lowLimit -= correction; + } + if (window->dictLimit < correction + ZSTD_WINDOW_START_INDEX) { + window->dictLimit = ZSTD_WINDOW_START_INDEX; + } else { + window->dictLimit -= correction; + } + + /* Ensure we can still reference the full window. */ + assert(newCurrent >= maxDist); + assert(newCurrent - maxDist >= ZSTD_WINDOW_START_INDEX); + /* Ensure that lowLimit and dictLimit didn't underflow. */ + assert(window->lowLimit <= newCurrent); + assert(window->dictLimit <= newCurrent); + + ++window->nbOverflowCorrections; + + DEBUGLOG(4, "Correction of 0x%x bytes to lowLimit=0x%x", correction, + window->lowLimit); + return correction; +} + +/** + * ZSTD_window_enforceMaxDist(): + * Updates lowLimit so that: + * (srcEnd - base) - lowLimit == maxDist + loadedDictEnd + * + * It ensures index is valid as long as index >= lowLimit. + * This must be called before a block compression call. + * + * loadedDictEnd is only defined if a dictionary is in use for current compression. + * As the name implies, loadedDictEnd represents the index at end of dictionary. + * The value lies within context's referential, it can be directly compared to blockEndIdx. + * + * If loadedDictEndPtr is NULL, no dictionary is in use, and we use loadedDictEnd == 0. + * If loadedDictEndPtr is not NULL, we set it to zero after updating lowLimit. + * This is because dictionaries are allowed to be referenced fully + * as long as the last byte of the dictionary is in the window. + * Once input has progressed beyond window size, dictionary cannot be referenced anymore. + * + * In normal dict mode, the dictionary lies between lowLimit and dictLimit. + * In dictMatchState mode, lowLimit and dictLimit are the same, + * and the dictionary is below them. + * forceWindow and dictMatchState are therefore incompatible. + */ +MEM_STATIC void +ZSTD_window_enforceMaxDist(ZSTD_window_t* window, + const void* blockEnd, + U32 maxDist, + U32* loadedDictEndPtr, + const ZSTD_MatchState_t** dictMatchStatePtr) +{ + U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base); + U32 const loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0; + DEBUGLOG(5, "ZSTD_window_enforceMaxDist: blockEndIdx=%u, maxDist=%u, loadedDictEnd=%u", + (unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd); + + /* - When there is no dictionary : loadedDictEnd == 0. + In which case, the test (blockEndIdx > maxDist) is merely to avoid + overflowing next operation `newLowLimit = blockEndIdx - maxDist`. + - When there is a standard dictionary : + Index referential is copied from the dictionary, + which means it starts from 0. + In which case, loadedDictEnd == dictSize, + and it makes sense to compare `blockEndIdx > maxDist + dictSize` + since `blockEndIdx` also starts from zero. + - When there is an attached dictionary : + loadedDictEnd is expressed within the referential of the context, + so it can be directly compared against blockEndIdx. + */ + if (blockEndIdx > maxDist + loadedDictEnd) { + U32 const newLowLimit = blockEndIdx - maxDist; + if (window->lowLimit < newLowLimit) window->lowLimit = newLowLimit; + if (window->dictLimit < window->lowLimit) { + DEBUGLOG(5, "Update dictLimit to match lowLimit, from %u to %u", + (unsigned)window->dictLimit, (unsigned)window->lowLimit); + window->dictLimit = window->lowLimit; + } + /* On reaching window size, dictionaries are invalidated */ + if (loadedDictEndPtr) *loadedDictEndPtr = 0; + if (dictMatchStatePtr) *dictMatchStatePtr = NULL; + } +} + +/* Similar to ZSTD_window_enforceMaxDist(), + * but only invalidates dictionary + * when input progresses beyond window size. + * assumption : loadedDictEndPtr and dictMatchStatePtr are valid (non NULL) + * loadedDictEnd uses same referential as window->base + * maxDist is the window size */ +MEM_STATIC void +ZSTD_checkDictValidity(const ZSTD_window_t* window, + const void* blockEnd, + U32 maxDist, + U32* loadedDictEndPtr, + const ZSTD_MatchState_t** dictMatchStatePtr) +{ + assert(loadedDictEndPtr != NULL); + assert(dictMatchStatePtr != NULL); + { U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base); + U32 const loadedDictEnd = *loadedDictEndPtr; + DEBUGLOG(5, "ZSTD_checkDictValidity: blockEndIdx=%u, maxDist=%u, loadedDictEnd=%u", + (unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd); + assert(blockEndIdx >= loadedDictEnd); + + if (blockEndIdx > loadedDictEnd + maxDist || loadedDictEnd != window->dictLimit) { + /* On reaching window size, dictionaries are invalidated. + * For simplification, if window size is reached anywhere within next block, + * the dictionary is invalidated for the full block. + * + * We also have to invalidate the dictionary if ZSTD_window_update() has detected + * non-contiguous segments, which means that loadedDictEnd != window->dictLimit. + * loadedDictEnd may be 0, if forceWindow is true, but in that case we never use + * dictMatchState, so setting it to NULL is not a problem. + */ + DEBUGLOG(6, "invalidating dictionary for current block (distance > windowSize)"); + *loadedDictEndPtr = 0; + *dictMatchStatePtr = NULL; + } else { + if (*loadedDictEndPtr != 0) { + DEBUGLOG(6, "dictionary considered valid for current block"); + } } } +} + +MEM_STATIC void ZSTD_window_init(ZSTD_window_t* window) { + ZSTD_memset(window, 0, sizeof(*window)); + window->base = (BYTE const*)" "; + window->dictBase = (BYTE const*)" "; + ZSTD_STATIC_ASSERT(ZSTD_DUBT_UNSORTED_MARK < ZSTD_WINDOW_START_INDEX); /* Start above ZSTD_DUBT_UNSORTED_MARK */ + window->dictLimit = ZSTD_WINDOW_START_INDEX; /* start from >0, so that 1st position is valid */ + window->lowLimit = ZSTD_WINDOW_START_INDEX; /* it ensures first and later CCtx usages compress the same */ + window->nextSrc = window->base + ZSTD_WINDOW_START_INDEX; /* see issue #1241 */ + window->nbOverflowCorrections = 0; +} + +/** + * ZSTD_window_update(): + * Updates the window by appending [src, src + srcSize) to the window. + * If it is not contiguous, the current prefix becomes the extDict, and we + * forget about the extDict. Handles overlap of the prefix and extDict. + * Returns non-zero if the segment is contiguous. + */ +MEM_STATIC +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +U32 ZSTD_window_update(ZSTD_window_t* window, + const void* src, size_t srcSize, + int forceNonContiguous) +{ + BYTE const* const ip = (BYTE const*)src; + U32 contiguous = 1; + DEBUGLOG(5, "ZSTD_window_update"); + if (srcSize == 0) + return contiguous; + assert(window->base != NULL); + assert(window->dictBase != NULL); + /* Check if blocks follow each other */ + if (src != window->nextSrc || forceNonContiguous) { + /* not contiguous */ + size_t const distanceFromBase = (size_t)(window->nextSrc - window->base); + DEBUGLOG(5, "Non contiguous blocks, new segment starts at %u", window->dictLimit); + window->lowLimit = window->dictLimit; + assert(distanceFromBase == (size_t)(U32)distanceFromBase); /* should never overflow */ + window->dictLimit = (U32)distanceFromBase; + window->dictBase = window->base; + window->base = ip - distanceFromBase; + /* ms->nextToUpdate = window->dictLimit; */ + if (window->dictLimit - window->lowLimit < HASH_READ_SIZE) window->lowLimit = window->dictLimit; /* too small extDict */ + contiguous = 0; + } + window->nextSrc = ip + srcSize; + /* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */ + if ( (ip+srcSize > window->dictBase + window->lowLimit) + & (ip < window->dictBase + window->dictLimit)) { + size_t const highInputIdx = (size_t)((ip + srcSize) - window->dictBase); + U32 const lowLimitMax = (highInputIdx > (size_t)window->dictLimit) ? window->dictLimit : (U32)highInputIdx; + assert(highInputIdx < UINT_MAX); + window->lowLimit = lowLimitMax; + DEBUGLOG(5, "Overlapping extDict and input : new lowLimit = %u", window->lowLimit); + } + return contiguous; +} + +/** + * Returns the lowest allowed match index. It may either be in the ext-dict or the prefix. + */ +MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_MatchState_t* ms, U32 curr, unsigned windowLog) +{ + U32 const maxDistance = 1U << windowLog; + U32 const lowestValid = ms->window.lowLimit; + U32 const withinWindow = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid; + U32 const isDictionary = (ms->loadedDictEnd != 0); + /* When using a dictionary the entire dictionary is valid if a single byte of the dictionary + * is within the window. We invalidate the dictionary (and set loadedDictEnd to 0) when it isn't + * valid for the entire block. So this check is sufficient to find the lowest valid match index. + */ + U32 const matchLowest = isDictionary ? lowestValid : withinWindow; + return matchLowest; +} + +/** + * Returns the lowest allowed match index in the prefix. + */ +MEM_STATIC U32 ZSTD_getLowestPrefixIndex(const ZSTD_MatchState_t* ms, U32 curr, unsigned windowLog) +{ + U32 const maxDistance = 1U << windowLog; + U32 const lowestValid = ms->window.dictLimit; + U32 const withinWindow = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid; + U32 const isDictionary = (ms->loadedDictEnd != 0); + /* When computing the lowest prefix index we need to take the dictionary into account to handle + * the edge case where the dictionary and the source are contiguous in memory. + */ + U32 const matchLowest = isDictionary ? lowestValid : withinWindow; + return matchLowest; +} + +/* index_safety_check: + * intentional underflow : ensure repIndex isn't overlapping dict + prefix + * @return 1 if values are not overlapping, + * 0 otherwise */ +MEM_STATIC int ZSTD_index_overlap_check(const U32 prefixLowestIndex, const U32 repIndex) { + return ((U32)((prefixLowestIndex-1) - repIndex) >= 3); +} + + +/* debug functions */ +#if (DEBUGLEVEL>=2) + +MEM_STATIC double ZSTD_fWeight(U32 rawStat) +{ + U32 const fp_accuracy = 8; + U32 const fp_multiplier = (1 << fp_accuracy); + U32 const newStat = rawStat + 1; + U32 const hb = ZSTD_highbit32(newStat); + U32 const BWeight = hb * fp_multiplier; + U32 const FWeight = (newStat << fp_accuracy) >> hb; + U32 const weight = BWeight + FWeight; + assert(hb + fp_accuracy < 31); + return (double)weight / fp_multiplier; +} + +/* display a table content, + * listing each element, its frequency, and its predicted bit cost */ +MEM_STATIC void ZSTD_debugTable(const U32* table, U32 max) +{ + unsigned u, sum; + for (u=0, sum=0; u<=max; u++) sum += table[u]; + DEBUGLOG(2, "total nb elts: %u", sum); + for (u=0; u<=max; u++) { + DEBUGLOG(2, "%2u: %5u (%.2f)", + u, table[u], ZSTD_fWeight(sum) - ZSTD_fWeight(table[u]) ); + } +} + +#endif + +/* Short Cache */ + +/* Normally, zstd matchfinders follow this flow: + * 1. Compute hash at ip + * 2. Load index from hashTable[hash] + * 3. Check if *ip == *(base + index) + * In dictionary compression, loading *(base + index) is often an L2 or even L3 miss. + * + * Short cache is an optimization which allows us to avoid step 3 most of the time + * when the data doesn't actually match. With short cache, the flow becomes: + * 1. Compute (hash, currentTag) at ip. currentTag is an 8-bit independent hash at ip. + * 2. Load (index, matchTag) from hashTable[hash]. See ZSTD_writeTaggedIndex to understand how this works. + * 3. Only if currentTag == matchTag, check *ip == *(base + index). Otherwise, continue. + * + * Currently, short cache is only implemented in CDict hashtables. Thus, its use is limited to + * dictMatchState matchfinders. + */ +#define ZSTD_SHORT_CACHE_TAG_BITS 8 +#define ZSTD_SHORT_CACHE_TAG_MASK ((1u << ZSTD_SHORT_CACHE_TAG_BITS) - 1) + +/* Helper function for ZSTD_fillHashTable and ZSTD_fillDoubleHashTable. + * Unpacks hashAndTag into (hash, tag), then packs (index, tag) into hashTable[hash]. */ +MEM_STATIC void ZSTD_writeTaggedIndex(U32* const hashTable, size_t hashAndTag, U32 index) { + size_t const hash = hashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS; + U32 const tag = (U32)(hashAndTag & ZSTD_SHORT_CACHE_TAG_MASK); + assert(index >> (32 - ZSTD_SHORT_CACHE_TAG_BITS) == 0); + hashTable[hash] = (index << ZSTD_SHORT_CACHE_TAG_BITS) | tag; +} + +/* Helper function for short cache matchfinders. + * Unpacks tag1 and tag2 from lower bits of packedTag1 and packedTag2, then checks if the tags match. */ +MEM_STATIC int ZSTD_comparePackedTags(size_t packedTag1, size_t packedTag2) { + U32 const tag1 = packedTag1 & ZSTD_SHORT_CACHE_TAG_MASK; + U32 const tag2 = packedTag2 & ZSTD_SHORT_CACHE_TAG_MASK; + return tag1 == tag2; +} + +/* =============================================================== + * Shared internal declarations + * These prototypes may be called from sources not in lib/compress + * =============================================================== */ + +/* ZSTD_loadCEntropy() : + * dict : must point at beginning of a valid zstd dictionary. + * return : size of dictionary header (size of magic number + dict ID + entropy tables) + * assumptions : magic number supposed already checked + * and dictSize >= 8 */ +size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, + const void* const dict, size_t dictSize); + +void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs); + +typedef struct { + U32 idx; /* Index in array of ZSTD_Sequence */ + U32 posInSequence; /* Position within sequence at idx */ + size_t posInSrc; /* Number of bytes given by sequences provided so far */ +} ZSTD_SequencePosition; + +/* for benchmark */ +size_t ZSTD_convertBlockSequences(ZSTD_CCtx* cctx, + const ZSTD_Sequence* const inSeqs, size_t nbSequences, + int const repcodeResolution); + +typedef struct { + size_t nbSequences; + size_t blockSize; + size_t litSize; +} BlockSummary; + +BlockSummary ZSTD_get1BlockSummary(const ZSTD_Sequence* seqs, size_t nbSeqs); + +/* ============================================================== + * Private declarations + * These prototypes shall only be called from within lib/compress + * ============================================================== */ + +/* ZSTD_getCParamsFromCCtxParams() : + * cParams are built depending on compressionLevel, src size hints, + * LDM and manually set compression parameters. + * Note: srcSizeHint == 0 means 0! + */ +ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( + const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_CParamMode_e mode); + +/*! ZSTD_initCStream_internal() : + * Private use only. Init streaming operation. + * expects params to be valid. + * must receive dict, or cdict, or none, but not both. + * @return : 0, or an error code */ +size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, unsigned long long pledgedSrcSize); + +void ZSTD_resetSeqStore(SeqStore_t* ssPtr); + +/*! ZSTD_getCParamsFromCDict() : + * as the name implies */ +ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict); + +/* ZSTD_compressBegin_advanced_internal() : + * Private use only. To be called from zstdmt_compress.c. */ +size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_dictContentType_e dictContentType, + ZSTD_dictTableLoadMethod_e dtlm, + const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, + unsigned long long pledgedSrcSize); + +/* ZSTD_compress_advanced_internal() : + * Private use only. To be called from zstdmt_compress.c. */ +size_t ZSTD_compress_advanced_internal(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + const ZSTD_CCtx_params* params); + + +/* ZSTD_writeLastEmptyBlock() : + * output an empty Block with end-of-frame mark to complete a frame + * @return : size of data written into `dst` (== ZSTD_blockHeaderSize (defined in zstd_internal.h)) + * or an error code if `dstCapacity` is too small ( 1 */ +U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat); + +/** ZSTD_CCtx_trace() : + * Trace the end of a compression call. + */ +void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize); + +/* Returns 1 if an external sequence producer is registered, otherwise returns 0. */ +MEM_STATIC int ZSTD_hasExtSeqProd(const ZSTD_CCtx_params* params) { + return params->extSeqProdFunc != NULL; +} + +/* =============================================================== + * Deprecated definitions that are still used internally to avoid + * deprecation warnings. These functions are exactly equivalent to + * their public variants, but avoid the deprecation warnings. + * =============================================================== */ + +size_t ZSTD_compressBegin_usingCDict_deprecated(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); + +size_t ZSTD_compressContinue_public(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + +size_t ZSTD_compressEnd_public(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + +size_t ZSTD_compressBlock_deprecated(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + + +#endif /* ZSTD_COMPRESS_H */ +/**** ended inlining zstd_compress_internal.h ****/ + + +size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +/* ZSTD_compressRleLiteralsBlock() : + * Conditions : + * - All bytes in @src are identical + * - dstCapacity >= 4 */ +size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +/* ZSTD_compressLiterals(): + * @entropyWorkspace: must be aligned on 4-bytes boundaries + * @entropyWorkspaceSize : must be >= HUF_WORKSPACE_SIZE + * @suspectUncompressible: sampling checks, to potentially skip huffman coding + */ +size_t ZSTD_compressLiterals (void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + void* entropyWorkspace, size_t entropyWorkspaceSize, + const ZSTD_hufCTables_t* prevHuf, + ZSTD_hufCTables_t* nextHuf, + ZSTD_strategy strategy, int disableLiteralCompression, + int suspectUncompressible, + int bmi2); + +#endif /* ZSTD_COMPRESS_LITERALS_H */ +/**** ended inlining zstd_compress_literals.h ****/ + + +/* ************************************************************** +* Debug Traces +****************************************************************/ +#if DEBUGLEVEL >= 2 + +static size_t showHexa(const void* src, size_t srcSize) +{ + const BYTE* const ip = (const BYTE*)src; + size_t u; + for (u=0; u31) + (srcSize>4095); + + DEBUGLOG(5, "ZSTD_noCompressLiterals: srcSize=%zu, dstCapacity=%zu", srcSize, dstCapacity); + + RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall, ""); + + switch(flSize) + { + case 1: /* 2 - 1 - 5 */ + ostart[0] = (BYTE)((U32)set_basic + (srcSize<<3)); + break; + case 2: /* 2 - 2 - 12 */ + MEM_writeLE16(ostart, (U16)((U32)set_basic + (1<<2) + (srcSize<<4))); + break; + case 3: /* 2 - 2 - 20 */ + MEM_writeLE32(ostart, (U32)((U32)set_basic + (3<<2) + (srcSize<<4))); + break; + default: /* not necessary : flSize is {1,2,3} */ + assert(0); + } + + ZSTD_memcpy(ostart + flSize, src, srcSize); + DEBUGLOG(5, "Raw (uncompressed) literals: %u -> %u", (U32)srcSize, (U32)(srcSize + flSize)); + return srcSize + flSize; +} + +static int allBytesIdentical(const void* src, size_t srcSize) +{ + assert(srcSize >= 1); + assert(src != NULL); + { const BYTE b = ((const BYTE*)src)[0]; + size_t p; + for (p=1; p31) + (srcSize>4095); + + assert(dstCapacity >= 4); (void)dstCapacity; + assert(allBytesIdentical(src, srcSize)); + + switch(flSize) + { + case 1: /* 2 - 1 - 5 */ + ostart[0] = (BYTE)((U32)set_rle + (srcSize<<3)); + break; + case 2: /* 2 - 2 - 12 */ + MEM_writeLE16(ostart, (U16)((U32)set_rle + (1<<2) + (srcSize<<4))); + break; + case 3: /* 2 - 2 - 20 */ + MEM_writeLE32(ostart, (U32)((U32)set_rle + (3<<2) + (srcSize<<4))); + break; + default: /* not necessary : flSize is {1,2,3} */ + assert(0); + } + + ostart[flSize] = *(const BYTE*)src; + DEBUGLOG(5, "RLE : Repeated Literal (%02X: %u times) -> %u bytes encoded", ((const BYTE*)src)[0], (U32)srcSize, (U32)flSize + 1); + return flSize+1; +} + +/* ZSTD_minLiteralsToCompress() : + * returns minimal amount of literals + * for literal compression to even be attempted. + * Minimum is made tighter as compression strategy increases. + */ +static size_t +ZSTD_minLiteralsToCompress(ZSTD_strategy strategy, HUF_repeat huf_repeat) +{ + assert((int)strategy >= 0); + assert((int)strategy <= 9); + /* btultra2 : min 8 bytes; + * then 2x larger for each successive compression strategy + * max threshold 64 bytes */ + { int const shift = MIN(9-(int)strategy, 3); + size_t const mintc = (huf_repeat == HUF_repeat_valid) ? 6 : (size_t)8 << shift; + DEBUGLOG(7, "minLiteralsToCompress = %zu", mintc); + return mintc; + } +} + +size_t ZSTD_compressLiterals ( + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + void* entropyWorkspace, size_t entropyWorkspaceSize, + const ZSTD_hufCTables_t* prevHuf, + ZSTD_hufCTables_t* nextHuf, + ZSTD_strategy strategy, + int disableLiteralCompression, + int suspectUncompressible, + int bmi2) +{ + size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB); + BYTE* const ostart = (BYTE*)dst; + U32 singleStream = srcSize < 256; + SymbolEncodingType_e hType = set_compressed; + size_t cLitSize; + + DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i, srcSize=%u, dstCapacity=%zu)", + disableLiteralCompression, (U32)srcSize, dstCapacity); + + DEBUGLOG(6, "Completed literals listing (%zu bytes)", showHexa(src, srcSize)); + + /* Prepare nextEntropy assuming reusing the existing table */ + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + + if (disableLiteralCompression) + return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + + /* if too small, don't even attempt compression (speed opt) */ + if (srcSize < ZSTD_minLiteralsToCompress(strategy, prevHuf->repeatMode)) + return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + + RETURN_ERROR_IF(dstCapacity < lhSize+1, dstSize_tooSmall, "not enough space for compression"); + { HUF_repeat repeat = prevHuf->repeatMode; + int const flags = 0 + | (bmi2 ? HUF_flags_bmi2 : 0) + | (strategy < ZSTD_lazy && srcSize <= 1024 ? HUF_flags_preferRepeat : 0) + | (strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD ? HUF_flags_optimalDepth : 0) + | (suspectUncompressible ? HUF_flags_suspectUncompressible : 0); + + typedef size_t (*huf_compress_f)(void*, size_t, const void*, size_t, unsigned, unsigned, void*, size_t, HUF_CElt*, HUF_repeat*, int); + huf_compress_f huf_compress; + if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1; + huf_compress = singleStream ? HUF_compress1X_repeat : HUF_compress4X_repeat; + cLitSize = huf_compress(ostart+lhSize, dstCapacity-lhSize, + src, srcSize, + HUF_SYMBOLVALUE_MAX, LitHufLog, + entropyWorkspace, entropyWorkspaceSize, + (HUF_CElt*)nextHuf->CTable, + &repeat, flags); + DEBUGLOG(5, "%zu literals compressed into %zu bytes (before header)", srcSize, cLitSize); + if (repeat != HUF_repeat_none) { + /* reused the existing table */ + DEBUGLOG(5, "reusing statistics from previous huffman block"); + hType = set_repeat; + } + } + + { size_t const minGain = ZSTD_minGain(srcSize, strategy); + if ((cLitSize==0) || (cLitSize >= srcSize - minGain) || ERR_isError(cLitSize)) { + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + } } + if (cLitSize==1) { + /* A return value of 1 signals that the alphabet consists of a single symbol. + * However, in some rare circumstances, it could be the compressed size (a single byte). + * For that outcome to have a chance to happen, it's necessary that `srcSize < 8`. + * (it's also necessary to not generate statistics). + * Therefore, in such a case, actively check that all bytes are identical. */ + if ((srcSize >= 8) || allBytesIdentical(src, srcSize)) { + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize); + } } + + if (hType == set_compressed) { + /* using a newly constructed table */ + nextHuf->repeatMode = HUF_repeat_check; + } + + /* Build header */ + switch(lhSize) + { + case 3: /* 2 - 2 - 10 - 10 */ + if (!singleStream) assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS); + { U32 const lhc = hType + ((U32)(!singleStream) << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<14); + MEM_writeLE24(ostart, lhc); + break; + } + case 4: /* 2 - 2 - 14 - 14 */ + assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS); + { U32 const lhc = hType + (2 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<18); + MEM_writeLE32(ostart, lhc); + break; + } + case 5: /* 2 - 2 - 18 - 18 */ + assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS); + { U32 const lhc = hType + (3 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<22); + MEM_writeLE32(ostart, lhc); + ostart[4] = (BYTE)(cLitSize >> 10); + break; + } + default: /* not possible : lhSize is {3,4,5} */ + assert(0); + } + DEBUGLOG(5, "Compressed literals: %u -> %u", (U32)srcSize, (U32)(lhSize+cLitSize)); + return lhSize+cLitSize; +} +/**** ended inlining compress/zstd_compress_literals.c ****/ +/**** start inlining compress/zstd_compress_sequences.c ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + /*-************************************* + * Dependencies + ***************************************/ +/**** start inlining zstd_compress_sequences.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMPRESS_SEQUENCES_H +#define ZSTD_COMPRESS_SEQUENCES_H + +/**** skipping file: zstd_compress_internal.h ****/ +/**** skipping file: ../common/fse.h ****/ +/**** skipping file: ../common/zstd_internal.h ****/ + +typedef enum { + ZSTD_defaultDisallowed = 0, + ZSTD_defaultAllowed = 1 +} ZSTD_DefaultPolicy_e; + +SymbolEncodingType_e +ZSTD_selectEncodingType( + FSE_repeat* repeatMode, unsigned const* count, unsigned const max, + size_t const mostFrequent, size_t nbSeq, unsigned const FSELog, + FSE_CTable const* prevCTable, + short const* defaultNorm, U32 defaultNormLog, + ZSTD_DefaultPolicy_e const isDefaultAllowed, + ZSTD_strategy const strategy); + +size_t +ZSTD_buildCTable(void* dst, size_t dstCapacity, + FSE_CTable* nextCTable, U32 FSELog, SymbolEncodingType_e type, + unsigned* count, U32 max, + const BYTE* codeTable, size_t nbSeq, + const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax, + const FSE_CTable* prevCTable, size_t prevCTableSize, + void* entropyWorkspace, size_t entropyWorkspaceSize); + +size_t ZSTD_encodeSequences( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + SeqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2); + +size_t ZSTD_fseBitCost( + FSE_CTable const* ctable, + unsigned const* count, + unsigned const max); + +size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog, + unsigned const* count, unsigned const max); +#endif /* ZSTD_COMPRESS_SEQUENCES_H */ +/**** ended inlining zstd_compress_sequences.h ****/ + +/** + * -log2(x / 256) lookup table for x in [0, 256). + * If x == 0: Return 0 + * Else: Return floor(-log2(x / 256) * 256) + */ +static unsigned const kInverseProbabilityLog256[256] = { + 0, 2048, 1792, 1642, 1536, 1453, 1386, 1329, 1280, 1236, 1197, 1162, + 1130, 1100, 1073, 1047, 1024, 1001, 980, 960, 941, 923, 906, 889, + 874, 859, 844, 830, 817, 804, 791, 779, 768, 756, 745, 734, + 724, 714, 704, 694, 685, 676, 667, 658, 650, 642, 633, 626, + 618, 610, 603, 595, 588, 581, 574, 567, 561, 554, 548, 542, + 535, 529, 523, 517, 512, 506, 500, 495, 489, 484, 478, 473, + 468, 463, 458, 453, 448, 443, 438, 434, 429, 424, 420, 415, + 411, 407, 402, 398, 394, 390, 386, 382, 377, 373, 370, 366, + 362, 358, 354, 350, 347, 343, 339, 336, 332, 329, 325, 322, + 318, 315, 311, 308, 305, 302, 298, 295, 292, 289, 286, 282, + 279, 276, 273, 270, 267, 264, 261, 258, 256, 253, 250, 247, + 244, 241, 239, 236, 233, 230, 228, 225, 222, 220, 217, 215, + 212, 209, 207, 204, 202, 199, 197, 194, 192, 190, 187, 185, + 182, 180, 178, 175, 173, 171, 168, 166, 164, 162, 159, 157, + 155, 153, 151, 149, 146, 144, 142, 140, 138, 136, 134, 132, + 130, 128, 126, 123, 121, 119, 117, 115, 114, 112, 110, 108, + 106, 104, 102, 100, 98, 96, 94, 93, 91, 89, 87, 85, + 83, 82, 80, 78, 76, 74, 73, 71, 69, 67, 66, 64, + 62, 61, 59, 57, 55, 54, 52, 50, 49, 47, 46, 44, + 42, 41, 39, 37, 36, 34, 33, 31, 30, 28, 26, 25, + 23, 22, 20, 19, 17, 16, 14, 13, 11, 10, 8, 7, + 5, 4, 2, 1, +}; + +static unsigned ZSTD_getFSEMaxSymbolValue(FSE_CTable const* ctable) { + void const* ptr = ctable; + U16 const* u16ptr = (U16 const*)ptr; + U32 const maxSymbolValue = MEM_read16(u16ptr + 1); + return maxSymbolValue; +} + +/** + * Returns true if we should use ncount=-1 else we should + * use ncount=1 for low probability symbols instead. + */ +static unsigned ZSTD_useLowProbCount(size_t const nbSeq) +{ + /* Heuristic: This should cover most blocks <= 16K and + * start to fade out after 16K to about 32K depending on + * compressibility. + */ + return nbSeq >= 2048; +} + +/** + * Returns the cost in bytes of encoding the normalized count header. + * Returns an error if any of the helper functions return an error. + */ +static size_t ZSTD_NCountCost(unsigned const* count, unsigned const max, + size_t const nbSeq, unsigned const FSELog) +{ + BYTE wksp[FSE_NCOUNTBOUND]; + S16 norm[MaxSeq + 1]; + const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max); + FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq, max, ZSTD_useLowProbCount(nbSeq)), ""); + return FSE_writeNCount(wksp, sizeof(wksp), norm, max, tableLog); +} + +/** + * Returns the cost in bits of encoding the distribution described by count + * using the entropy bound. + */ +static size_t ZSTD_entropyCost(unsigned const* count, unsigned const max, size_t const total) +{ + unsigned cost = 0; + unsigned s; + + assert(total > 0); + for (s = 0; s <= max; ++s) { + unsigned norm = (unsigned)((256 * count[s]) / total); + if (count[s] != 0 && norm == 0) + norm = 1; + assert(count[s] < total); + cost += count[s] * kInverseProbabilityLog256[norm]; + } + return cost >> 8; +} + +/** + * Returns the cost in bits of encoding the distribution in count using ctable. + * Returns an error if ctable cannot represent all the symbols in count. + */ +size_t ZSTD_fseBitCost( + FSE_CTable const* ctable, + unsigned const* count, + unsigned const max) +{ + unsigned const kAccuracyLog = 8; + size_t cost = 0; + unsigned s; + FSE_CState_t cstate; + FSE_initCState(&cstate, ctable); + if (ZSTD_getFSEMaxSymbolValue(ctable) < max) { + DEBUGLOG(5, "Repeat FSE_CTable has maxSymbolValue %u < %u", + ZSTD_getFSEMaxSymbolValue(ctable), max); + return ERROR(GENERIC); + } + for (s = 0; s <= max; ++s) { + unsigned const tableLog = cstate.stateLog; + unsigned const badCost = (tableLog + 1) << kAccuracyLog; + unsigned const bitCost = FSE_bitCost(cstate.symbolTT, tableLog, s, kAccuracyLog); + if (count[s] == 0) + continue; + if (bitCost >= badCost) { + DEBUGLOG(5, "Repeat FSE_CTable has Prob[%u] == 0", s); + return ERROR(GENERIC); + } + cost += (size_t)count[s] * bitCost; + } + return cost >> kAccuracyLog; +} + +/** + * Returns the cost in bits of encoding the distribution in count using the + * table described by norm. The max symbol support by norm is assumed >= max. + * norm must be valid for every symbol with non-zero probability in count. + */ +size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog, + unsigned const* count, unsigned const max) +{ + unsigned const shift = 8 - accuracyLog; + size_t cost = 0; + unsigned s; + assert(accuracyLog <= 8); + for (s = 0; s <= max; ++s) { + unsigned const normAcc = (norm[s] != -1) ? (unsigned)norm[s] : 1; + unsigned const norm256 = normAcc << shift; + assert(norm256 > 0); + assert(norm256 < 256); + cost += count[s] * kInverseProbabilityLog256[norm256]; + } + return cost >> 8; +} + +SymbolEncodingType_e +ZSTD_selectEncodingType( + FSE_repeat* repeatMode, unsigned const* count, unsigned const max, + size_t const mostFrequent, size_t nbSeq, unsigned const FSELog, + FSE_CTable const* prevCTable, + short const* defaultNorm, U32 defaultNormLog, + ZSTD_DefaultPolicy_e const isDefaultAllowed, + ZSTD_strategy const strategy) +{ + ZSTD_STATIC_ASSERT(ZSTD_defaultDisallowed == 0 && ZSTD_defaultAllowed != 0); + if (mostFrequent == nbSeq) { + *repeatMode = FSE_repeat_none; + if (isDefaultAllowed && nbSeq <= 2) { + /* Prefer set_basic over set_rle when there are 2 or fewer symbols, + * since RLE uses 1 byte, but set_basic uses 5-6 bits per symbol. + * If basic encoding isn't possible, always choose RLE. + */ + DEBUGLOG(5, "Selected set_basic"); + return set_basic; + } + DEBUGLOG(5, "Selected set_rle"); + return set_rle; + } + if (strategy < ZSTD_lazy) { + if (isDefaultAllowed) { + size_t const staticFse_nbSeq_max = 1000; + size_t const mult = 10 - strategy; + size_t const baseLog = 3; + size_t const dynamicFse_nbSeq_min = (((size_t)1 << defaultNormLog) * mult) >> baseLog; /* 28-36 for offset, 56-72 for lengths */ + assert(defaultNormLog >= 5 && defaultNormLog <= 6); /* xx_DEFAULTNORMLOG */ + assert(mult <= 9 && mult >= 7); + if ( (*repeatMode == FSE_repeat_valid) + && (nbSeq < staticFse_nbSeq_max) ) { + DEBUGLOG(5, "Selected set_repeat"); + return set_repeat; + } + if ( (nbSeq < dynamicFse_nbSeq_min) + || (mostFrequent < (nbSeq >> (defaultNormLog-1))) ) { + DEBUGLOG(5, "Selected set_basic"); + /* The format allows default tables to be repeated, but it isn't useful. + * When using simple heuristics to select encoding type, we don't want + * to confuse these tables with dictionaries. When running more careful + * analysis, we don't need to waste time checking both repeating tables + * and default tables. + */ + *repeatMode = FSE_repeat_none; + return set_basic; + } + } + } else { + size_t const basicCost = isDefaultAllowed ? ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, count, max) : ERROR(GENERIC); + size_t const repeatCost = *repeatMode != FSE_repeat_none ? ZSTD_fseBitCost(prevCTable, count, max) : ERROR(GENERIC); + size_t const NCountCost = ZSTD_NCountCost(count, max, nbSeq, FSELog); + size_t const compressedCost = (NCountCost << 3) + ZSTD_entropyCost(count, max, nbSeq); + + if (isDefaultAllowed) { + assert(!ZSTD_isError(basicCost)); + assert(!(*repeatMode == FSE_repeat_valid && ZSTD_isError(repeatCost))); + } + assert(!ZSTD_isError(NCountCost)); + assert(compressedCost < ERROR(maxCode)); + DEBUGLOG(5, "Estimated bit costs: basic=%u\trepeat=%u\tcompressed=%u", + (unsigned)basicCost, (unsigned)repeatCost, (unsigned)compressedCost); + if (basicCost <= repeatCost && basicCost <= compressedCost) { + DEBUGLOG(5, "Selected set_basic"); + assert(isDefaultAllowed); + *repeatMode = FSE_repeat_none; + return set_basic; + } + if (repeatCost <= compressedCost) { + DEBUGLOG(5, "Selected set_repeat"); + assert(!ZSTD_isError(repeatCost)); + return set_repeat; + } + assert(compressedCost < basicCost && compressedCost < repeatCost); + } + DEBUGLOG(5, "Selected set_compressed"); + *repeatMode = FSE_repeat_check; + return set_compressed; +} + +typedef struct { + S16 norm[MaxSeq + 1]; + U32 wksp[FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(MaxSeq, MaxFSELog)]; +} ZSTD_BuildCTableWksp; + +size_t +ZSTD_buildCTable(void* dst, size_t dstCapacity, + FSE_CTable* nextCTable, U32 FSELog, SymbolEncodingType_e type, + unsigned* count, U32 max, + const BYTE* codeTable, size_t nbSeq, + const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax, + const FSE_CTable* prevCTable, size_t prevCTableSize, + void* entropyWorkspace, size_t entropyWorkspaceSize) +{ + BYTE* op = (BYTE*)dst; + const BYTE* const oend = op + dstCapacity; + DEBUGLOG(6, "ZSTD_buildCTable (dstCapacity=%u)", (unsigned)dstCapacity); + + switch (type) { + case set_rle: + FORWARD_IF_ERROR(FSE_buildCTable_rle(nextCTable, (BYTE)max), ""); + RETURN_ERROR_IF(dstCapacity==0, dstSize_tooSmall, "not enough space"); + *op = codeTable[0]; + return 1; + case set_repeat: + ZSTD_memcpy(nextCTable, prevCTable, prevCTableSize); + return 0; + case set_basic: + FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, entropyWorkspace, entropyWorkspaceSize), ""); /* note : could be pre-calculated */ + return 0; + case set_compressed: { + ZSTD_BuildCTableWksp* wksp = (ZSTD_BuildCTableWksp*)entropyWorkspace; + size_t nbSeq_1 = nbSeq; + const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max); + if (count[codeTable[nbSeq-1]] > 1) { + count[codeTable[nbSeq-1]]--; + nbSeq_1--; + } + assert(nbSeq_1 > 1); + assert(entropyWorkspaceSize >= sizeof(ZSTD_BuildCTableWksp)); + (void)entropyWorkspaceSize; + FORWARD_IF_ERROR(FSE_normalizeCount(wksp->norm, tableLog, count, nbSeq_1, max, ZSTD_useLowProbCount(nbSeq_1)), "FSE_normalizeCount failed"); + assert(oend >= op); + { size_t const NCountSize = FSE_writeNCount(op, (size_t)(oend - op), wksp->norm, max, tableLog); /* overflow protected */ + FORWARD_IF_ERROR(NCountSize, "FSE_writeNCount failed"); + FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, wksp->norm, max, tableLog, wksp->wksp, sizeof(wksp->wksp)), "FSE_buildCTable_wksp failed"); + return NCountSize; + } + } + default: assert(0); RETURN_ERROR(GENERIC, "impossible to reach"); + } +} + +FORCE_INLINE_TEMPLATE size_t +ZSTD_encodeSequences_body( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + SeqDef const* sequences, size_t nbSeq, int longOffsets) +{ + BIT_CStream_t blockStream; + FSE_CState_t stateMatchLength; + FSE_CState_t stateOffsetBits; + FSE_CState_t stateLitLength; + + RETURN_ERROR_IF( + ERR_isError(BIT_initCStream(&blockStream, dst, dstCapacity)), + dstSize_tooSmall, "not enough space remaining"); + DEBUGLOG(6, "available space for bitstream : %i (dstCapacity=%u)", + (int)(blockStream.endPtr - blockStream.startPtr), + (unsigned)dstCapacity); + + /* first symbols */ + FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq-1]); + FSE_initCState2(&stateOffsetBits, CTable_OffsetBits, ofCodeTable[nbSeq-1]); + FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq-1]); + BIT_addBits(&blockStream, sequences[nbSeq-1].litLength, LL_bits[llCodeTable[nbSeq-1]]); + if (MEM_32bits()) BIT_flushBits(&blockStream); + BIT_addBits(&blockStream, sequences[nbSeq-1].mlBase, ML_bits[mlCodeTable[nbSeq-1]]); + if (MEM_32bits()) BIT_flushBits(&blockStream); + if (longOffsets) { + U32 const ofBits = ofCodeTable[nbSeq-1]; + unsigned const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); + if (extraBits) { + BIT_addBits(&blockStream, sequences[nbSeq-1].offBase, extraBits); + BIT_flushBits(&blockStream); + } + BIT_addBits(&blockStream, sequences[nbSeq-1].offBase >> extraBits, + ofBits - extraBits); + } else { + BIT_addBits(&blockStream, sequences[nbSeq-1].offBase, ofCodeTable[nbSeq-1]); + } + BIT_flushBits(&blockStream); + + { size_t n; + for (n=nbSeq-2 ; n= 64-7-(LLFSELog+MLFSELog+OffFSELog))) + BIT_flushBits(&blockStream); /* (7)*/ + BIT_addBits(&blockStream, sequences[n].litLength, llBits); + if (MEM_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream); + BIT_addBits(&blockStream, sequences[n].mlBase, mlBits); + if (MEM_32bits() || (ofBits+mlBits+llBits > 56)) BIT_flushBits(&blockStream); + if (longOffsets) { + unsigned const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); + if (extraBits) { + BIT_addBits(&blockStream, sequences[n].offBase, extraBits); + BIT_flushBits(&blockStream); /* (7)*/ + } + BIT_addBits(&blockStream, sequences[n].offBase >> extraBits, + ofBits - extraBits); /* 31 */ + } else { + BIT_addBits(&blockStream, sequences[n].offBase, ofBits); /* 31 */ + } + BIT_flushBits(&blockStream); /* (7)*/ + DEBUGLOG(7, "remaining space : %i", (int)(blockStream.endPtr - blockStream.ptr)); + } } + + DEBUGLOG(6, "ZSTD_encodeSequences: flushing ML state with %u bits", stateMatchLength.stateLog); + FSE_flushCState(&blockStream, &stateMatchLength); + DEBUGLOG(6, "ZSTD_encodeSequences: flushing Off state with %u bits", stateOffsetBits.stateLog); + FSE_flushCState(&blockStream, &stateOffsetBits); + DEBUGLOG(6, "ZSTD_encodeSequences: flushing LL state with %u bits", stateLitLength.stateLog); + FSE_flushCState(&blockStream, &stateLitLength); + + { size_t const streamSize = BIT_closeCStream(&blockStream); + RETURN_ERROR_IF(streamSize==0, dstSize_tooSmall, "not enough space"); + return streamSize; + } +} + +static size_t +ZSTD_encodeSequences_default( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + SeqDef const* sequences, size_t nbSeq, int longOffsets) +{ + return ZSTD_encodeSequences_body(dst, dstCapacity, + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, longOffsets); +} + + +#if DYNAMIC_BMI2 + +static BMI2_TARGET_ATTRIBUTE size_t +ZSTD_encodeSequences_bmi2( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + SeqDef const* sequences, size_t nbSeq, int longOffsets) +{ + return ZSTD_encodeSequences_body(dst, dstCapacity, + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, longOffsets); +} + +#endif + +size_t ZSTD_encodeSequences( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + SeqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2) +{ + DEBUGLOG(5, "ZSTD_encodeSequences: dstCapacity = %u", (unsigned)dstCapacity); +#if DYNAMIC_BMI2 + if (bmi2) { + return ZSTD_encodeSequences_bmi2(dst, dstCapacity, + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, longOffsets); + } +#endif + (void)bmi2; + return ZSTD_encodeSequences_default(dst, dstCapacity, + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, longOffsets); +} +/**** ended inlining compress/zstd_compress_sequences.c ****/ +/**** start inlining compress/zstd_compress_superblock.c ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + /*-************************************* + * Dependencies + ***************************************/ +/**** start inlining zstd_compress_superblock.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMPRESS_ADVANCED_H +#define ZSTD_COMPRESS_ADVANCED_H + +/*-************************************* +* Dependencies +***************************************/ + +/**** skipping file: ../zstd.h ****/ + +/*-************************************* +* Target Compressed Block Size +***************************************/ + +/* ZSTD_compressSuperBlock() : + * Used to compress a super block when targetCBlockSize is being used. + * The given block will be compressed into multiple sub blocks that are around targetCBlockSize. */ +size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + void const* src, size_t srcSize, + unsigned lastBlock); + +#endif /* ZSTD_COMPRESS_ADVANCED_H */ +/**** ended inlining zstd_compress_superblock.h ****/ + +/**** skipping file: ../common/zstd_internal.h ****/ +/**** skipping file: hist.h ****/ +/**** skipping file: zstd_compress_internal.h ****/ +/**** skipping file: zstd_compress_sequences.h ****/ +/**** skipping file: zstd_compress_literals.h ****/ + +/** ZSTD_compressSubBlock_literal() : + * Compresses literals section for a sub-block. + * When we have to write the Huffman table we will sometimes choose a header + * size larger than necessary. This is because we have to pick the header size + * before we know the table size + compressed size, so we have a bound on the + * table size. If we guessed incorrectly, we fall back to uncompressed literals. + * + * We write the header when writeEntropy=1 and set entropyWritten=1 when we succeeded + * in writing the header, otherwise it is set to 0. + * + * hufMetadata->hType has literals block type info. + * If it is set_basic, all sub-blocks literals section will be Raw_Literals_Block. + * If it is set_rle, all sub-blocks literals section will be RLE_Literals_Block. + * If it is set_compressed, first sub-block's literals section will be Compressed_Literals_Block + * If it is set_compressed, first sub-block's literals section will be Treeless_Literals_Block + * and the following sub-blocks' literals sections will be Treeless_Literals_Block. + * @return : compressed size of literals section of a sub-block + * Or 0 if unable to compress. + * Or error code */ +static size_t +ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable, + const ZSTD_hufCTablesMetadata_t* hufMetadata, + const BYTE* literals, size_t litSize, + void* dst, size_t dstSize, + const int bmi2, int writeEntropy, int* entropyWritten) +{ + size_t const header = writeEntropy ? 200 : 0; + size_t const lhSize = 3 + (litSize >= (1 KB - header)) + (litSize >= (16 KB - header)); + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart + lhSize; + U32 const singleStream = lhSize == 3; + SymbolEncodingType_e hType = writeEntropy ? hufMetadata->hType : set_repeat; + size_t cLitSize = 0; + + DEBUGLOG(5, "ZSTD_compressSubBlock_literal (litSize=%zu, lhSize=%zu, writeEntropy=%d)", litSize, lhSize, writeEntropy); + + *entropyWritten = 0; + if (litSize == 0 || hufMetadata->hType == set_basic) { + DEBUGLOG(5, "ZSTD_compressSubBlock_literal using raw literal"); + return ZSTD_noCompressLiterals(dst, dstSize, literals, litSize); + } else if (hufMetadata->hType == set_rle) { + DEBUGLOG(5, "ZSTD_compressSubBlock_literal using rle literal"); + return ZSTD_compressRleLiteralsBlock(dst, dstSize, literals, litSize); + } + + assert(litSize > 0); + assert(hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat); + + if (writeEntropy && hufMetadata->hType == set_compressed) { + ZSTD_memcpy(op, hufMetadata->hufDesBuffer, hufMetadata->hufDesSize); + op += hufMetadata->hufDesSize; + cLitSize += hufMetadata->hufDesSize; + DEBUGLOG(5, "ZSTD_compressSubBlock_literal (hSize=%zu)", hufMetadata->hufDesSize); + } + + { int const flags = bmi2 ? HUF_flags_bmi2 : 0; + const size_t cSize = singleStream ? HUF_compress1X_usingCTable(op, (size_t)(oend-op), literals, litSize, hufTable, flags) + : HUF_compress4X_usingCTable(op, (size_t)(oend-op), literals, litSize, hufTable, flags); + op += cSize; + cLitSize += cSize; + if (cSize == 0 || ERR_isError(cSize)) { + DEBUGLOG(5, "Failed to write entropy tables %s", ZSTD_getErrorName(cSize)); + return 0; + } + /* If we expand and we aren't writing a header then emit uncompressed */ + if (!writeEntropy && cLitSize >= litSize) { + DEBUGLOG(5, "ZSTD_compressSubBlock_literal using raw literal because uncompressible"); + return ZSTD_noCompressLiterals(dst, dstSize, literals, litSize); + } + /* If we are writing headers then allow expansion that doesn't change our header size. */ + if (lhSize < (size_t)(3 + (cLitSize >= 1 KB) + (cLitSize >= 16 KB))) { + assert(cLitSize > litSize); + DEBUGLOG(5, "Literals expanded beyond allowed header size"); + return ZSTD_noCompressLiterals(dst, dstSize, literals, litSize); + } + DEBUGLOG(5, "ZSTD_compressSubBlock_literal (cSize=%zu)", cSize); + } + + /* Build header */ + switch(lhSize) + { + case 3: /* 2 - 2 - 10 - 10 */ + { U32 const lhc = hType + ((U32)(!singleStream) << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<14); + MEM_writeLE24(ostart, lhc); + break; + } + case 4: /* 2 - 2 - 14 - 14 */ + { U32 const lhc = hType + (2 << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<18); + MEM_writeLE32(ostart, lhc); + break; + } + case 5: /* 2 - 2 - 18 - 18 */ + { U32 const lhc = hType + (3 << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<22); + MEM_writeLE32(ostart, lhc); + ostart[4] = (BYTE)(cLitSize >> 10); + break; + } + default: /* not possible : lhSize is {3,4,5} */ + assert(0); + } + *entropyWritten = 1; + DEBUGLOG(5, "Compressed literals: %u -> %u", (U32)litSize, (U32)(op-ostart)); + return (size_t)(op-ostart); +} + +static size_t +ZSTD_seqDecompressedSize(SeqStore_t const* seqStore, + const SeqDef* sequences, size_t nbSeqs, + size_t litSize, int lastSubBlock) +{ + size_t matchLengthSum = 0; + size_t litLengthSum = 0; + size_t n; + for (n=0; nllType, fseMetadata->ofType, and fseMetadata->mlType have + * symbol compression modes for the super-block. + * The first successfully compressed block will have these in its header. + * We set entropyWritten=1 when we succeed in compressing the sequences. + * The following sub-blocks will always have repeat mode. + * @return : compressed size of sequences section of a sub-block + * Or 0 if it is unable to compress + * Or error code. */ +static size_t +ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables, + const ZSTD_fseCTablesMetadata_t* fseMetadata, + const SeqDef* sequences, size_t nbSeq, + const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + const int bmi2, int writeEntropy, int* entropyWritten) +{ + const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart; + BYTE* seqHead; + + DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (nbSeq=%zu, writeEntropy=%d, longOffsets=%d)", nbSeq, writeEntropy, longOffsets); + + *entropyWritten = 0; + /* Sequences Header */ + RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/, + dstSize_tooSmall, ""); + if (nbSeq < 128) + *op++ = (BYTE)nbSeq; + else if (nbSeq < LONGNBSEQ) + op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2; + else + op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3; + if (nbSeq==0) { + return (size_t)(op - ostart); + } + + /* seqHead : flags for FSE encoding type */ + seqHead = op++; + + DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (seqHeadSize=%u)", (unsigned)(op-ostart)); + + if (writeEntropy) { + const U32 LLtype = fseMetadata->llType; + const U32 Offtype = fseMetadata->ofType; + const U32 MLtype = fseMetadata->mlType; + DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (fseTablesSize=%zu)", fseMetadata->fseTablesSize); + *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2)); + ZSTD_memcpy(op, fseMetadata->fseTablesBuffer, fseMetadata->fseTablesSize); + op += fseMetadata->fseTablesSize; + } else { + const U32 repeat = set_repeat; + *seqHead = (BYTE)((repeat<<6) + (repeat<<4) + (repeat<<2)); + } + + { size_t const bitstreamSize = ZSTD_encodeSequences( + op, (size_t)(oend - op), + fseTables->matchlengthCTable, mlCode, + fseTables->offcodeCTable, ofCode, + fseTables->litlengthCTable, llCode, + sequences, nbSeq, + longOffsets, bmi2); + FORWARD_IF_ERROR(bitstreamSize, "ZSTD_encodeSequences failed"); + op += bitstreamSize; + /* zstd versions <= 1.3.4 mistakenly report corruption when + * FSE_readNCount() receives a buffer < 4 bytes. + * Fixed by https://github.com/facebook/zstd/pull/1146. + * This can happen when the last set_compressed table present is 2 + * bytes and the bitstream is only one byte. + * In this exceedingly rare case, we will simply emit an uncompressed + * block, since it isn't worth optimizing. + */ +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + if (writeEntropy && fseMetadata->lastCountSize && fseMetadata->lastCountSize + bitstreamSize < 4) { + /* NCountSize >= 2 && bitstreamSize > 0 ==> lastCountSize == 3 */ + assert(fseMetadata->lastCountSize + bitstreamSize == 3); + DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.3.4 by " + "emitting an uncompressed block."); + return 0; + } +#endif + DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (bitstreamSize=%zu)", bitstreamSize); + } + + /* zstd versions <= 1.4.0 mistakenly report error when + * sequences section body size is less than 3 bytes. + * Fixed by https://github.com/facebook/zstd/pull/1664. + * This can happen when the previous sequences section block is compressed + * with rle mode and the current block's sequences section is compressed + * with repeat mode where sequences section body size can be 1 byte. + */ +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + if (op-seqHead < 4) { + DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.4.0 by emitting " + "an uncompressed block when sequences are < 4 bytes"); + return 0; + } +#endif + + *entropyWritten = 1; + return (size_t)(op - ostart); +} + +/** ZSTD_compressSubBlock() : + * Compresses a single sub-block. + * @return : compressed size of the sub-block + * Or 0 if it failed to compress. */ +static size_t ZSTD_compressSubBlock(const ZSTD_entropyCTables_t* entropy, + const ZSTD_entropyCTablesMetadata_t* entropyMetadata, + const SeqDef* sequences, size_t nbSeq, + const BYTE* literals, size_t litSize, + const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + const int bmi2, + int writeLitEntropy, int writeSeqEntropy, + int* litEntropyWritten, int* seqEntropyWritten, + U32 lastBlock) +{ + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart + ZSTD_blockHeaderSize; + DEBUGLOG(5, "ZSTD_compressSubBlock (litSize=%zu, nbSeq=%zu, writeLitEntropy=%d, writeSeqEntropy=%d, lastBlock=%d)", + litSize, nbSeq, writeLitEntropy, writeSeqEntropy, lastBlock); + { size_t cLitSize = ZSTD_compressSubBlock_literal((const HUF_CElt*)entropy->huf.CTable, + &entropyMetadata->hufMetadata, literals, litSize, + op, (size_t)(oend-op), + bmi2, writeLitEntropy, litEntropyWritten); + FORWARD_IF_ERROR(cLitSize, "ZSTD_compressSubBlock_literal failed"); + if (cLitSize == 0) return 0; + op += cLitSize; + } + { size_t cSeqSize = ZSTD_compressSubBlock_sequences(&entropy->fse, + &entropyMetadata->fseMetadata, + sequences, nbSeq, + llCode, mlCode, ofCode, + cctxParams, + op, (size_t)(oend-op), + bmi2, writeSeqEntropy, seqEntropyWritten); + FORWARD_IF_ERROR(cSeqSize, "ZSTD_compressSubBlock_sequences failed"); + if (cSeqSize == 0) return 0; + op += cSeqSize; + } + /* Write block header */ + { size_t cSize = (size_t)(op-ostart) - ZSTD_blockHeaderSize; + U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); + MEM_writeLE24(ostart, cBlockHeader24); + } + return (size_t)(op-ostart); +} + +static size_t ZSTD_estimateSubBlockSize_literal(const BYTE* literals, size_t litSize, + const ZSTD_hufCTables_t* huf, + const ZSTD_hufCTablesMetadata_t* hufMetadata, + void* workspace, size_t wkspSize, + int writeEntropy) +{ + unsigned* const countWksp = (unsigned*)workspace; + unsigned maxSymbolValue = 255; + size_t literalSectionHeaderSize = 3; /* Use hard coded size of 3 bytes */ + + if (hufMetadata->hType == set_basic) return litSize; + else if (hufMetadata->hType == set_rle) return 1; + else if (hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat) { + size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)literals, litSize, workspace, wkspSize); + if (ZSTD_isError(largest)) return litSize; + { size_t cLitSizeEstimate = HUF_estimateCompressedSize((const HUF_CElt*)huf->CTable, countWksp, maxSymbolValue); + if (writeEntropy) cLitSizeEstimate += hufMetadata->hufDesSize; + return cLitSizeEstimate + literalSectionHeaderSize; + } } + assert(0); /* impossible */ + return 0; +} + +static size_t ZSTD_estimateSubBlockSize_symbolType(SymbolEncodingType_e type, + const BYTE* codeTable, unsigned maxCode, + size_t nbSeq, const FSE_CTable* fseCTable, + const U8* additionalBits, + short const* defaultNorm, U32 defaultNormLog, U32 defaultMax, + void* workspace, size_t wkspSize) +{ + unsigned* const countWksp = (unsigned*)workspace; + const BYTE* ctp = codeTable; + const BYTE* const ctStart = ctp; + const BYTE* const ctEnd = ctStart + nbSeq; + size_t cSymbolTypeSizeEstimateInBits = 0; + unsigned max = maxCode; + + HIST_countFast_wksp(countWksp, &max, codeTable, nbSeq, workspace, wkspSize); /* can't fail */ + if (type == set_basic) { + /* We selected this encoding type, so it must be valid. */ + assert(max <= defaultMax); + cSymbolTypeSizeEstimateInBits = max <= defaultMax + ? ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, countWksp, max) + : ERROR(GENERIC); + } else if (type == set_rle) { + cSymbolTypeSizeEstimateInBits = 0; + } else if (type == set_compressed || type == set_repeat) { + cSymbolTypeSizeEstimateInBits = ZSTD_fseBitCost(fseCTable, countWksp, max); + } + if (ZSTD_isError(cSymbolTypeSizeEstimateInBits)) return nbSeq * 10; + while (ctp < ctEnd) { + if (additionalBits) cSymbolTypeSizeEstimateInBits += additionalBits[*ctp]; + else cSymbolTypeSizeEstimateInBits += *ctp; /* for offset, offset code is also the number of additional bits */ + ctp++; + } + return cSymbolTypeSizeEstimateInBits / 8; +} + +static size_t ZSTD_estimateSubBlockSize_sequences(const BYTE* ofCodeTable, + const BYTE* llCodeTable, + const BYTE* mlCodeTable, + size_t nbSeq, + const ZSTD_fseCTables_t* fseTables, + const ZSTD_fseCTablesMetadata_t* fseMetadata, + void* workspace, size_t wkspSize, + int writeEntropy) +{ + size_t const sequencesSectionHeaderSize = 3; /* Use hard coded size of 3 bytes */ + size_t cSeqSizeEstimate = 0; + if (nbSeq == 0) return sequencesSectionHeaderSize; + cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, MaxOff, + nbSeq, fseTables->offcodeCTable, NULL, + OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, + workspace, wkspSize); + cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->llType, llCodeTable, MaxLL, + nbSeq, fseTables->litlengthCTable, LL_bits, + LL_defaultNorm, LL_defaultNormLog, MaxLL, + workspace, wkspSize); + cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->mlType, mlCodeTable, MaxML, + nbSeq, fseTables->matchlengthCTable, ML_bits, + ML_defaultNorm, ML_defaultNormLog, MaxML, + workspace, wkspSize); + if (writeEntropy) cSeqSizeEstimate += fseMetadata->fseTablesSize; + return cSeqSizeEstimate + sequencesSectionHeaderSize; +} + +typedef struct { + size_t estLitSize; + size_t estBlockSize; +} EstimatedBlockSize; +static EstimatedBlockSize ZSTD_estimateSubBlockSize(const BYTE* literals, size_t litSize, + const BYTE* ofCodeTable, + const BYTE* llCodeTable, + const BYTE* mlCodeTable, + size_t nbSeq, + const ZSTD_entropyCTables_t* entropy, + const ZSTD_entropyCTablesMetadata_t* entropyMetadata, + void* workspace, size_t wkspSize, + int writeLitEntropy, int writeSeqEntropy) +{ + EstimatedBlockSize ebs; + ebs.estLitSize = ZSTD_estimateSubBlockSize_literal(literals, litSize, + &entropy->huf, &entropyMetadata->hufMetadata, + workspace, wkspSize, writeLitEntropy); + ebs.estBlockSize = ZSTD_estimateSubBlockSize_sequences(ofCodeTable, llCodeTable, mlCodeTable, + nbSeq, &entropy->fse, &entropyMetadata->fseMetadata, + workspace, wkspSize, writeSeqEntropy); + ebs.estBlockSize += ebs.estLitSize + ZSTD_blockHeaderSize; + return ebs; +} + +static int ZSTD_needSequenceEntropyTables(ZSTD_fseCTablesMetadata_t const* fseMetadata) +{ + if (fseMetadata->llType == set_compressed || fseMetadata->llType == set_rle) + return 1; + if (fseMetadata->mlType == set_compressed || fseMetadata->mlType == set_rle) + return 1; + if (fseMetadata->ofType == set_compressed || fseMetadata->ofType == set_rle) + return 1; + return 0; +} + +static size_t countLiterals(SeqStore_t const* seqStore, const SeqDef* sp, size_t seqCount) +{ + size_t n, total = 0; + assert(sp != NULL); + for (n=0; n %zu bytes", seqCount, (const void*)sp, total); + return total; +} + +#define BYTESCALE 256 + +static size_t sizeBlockSequences(const SeqDef* sp, size_t nbSeqs, + size_t targetBudget, size_t avgLitCost, size_t avgSeqCost, + int firstSubBlock) +{ + size_t n, budget = 0, inSize=0; + /* entropy headers */ + size_t const headerSize = (size_t)firstSubBlock * 120 * BYTESCALE; /* generous estimate */ + assert(firstSubBlock==0 || firstSubBlock==1); + budget += headerSize; + + /* first sequence => at least one sequence*/ + budget += sp[0].litLength * avgLitCost + avgSeqCost; + if (budget > targetBudget) return 1; + inSize = sp[0].litLength + (sp[0].mlBase+MINMATCH); + + /* loop over sequences */ + for (n=1; n targetBudget) + /* though continue to expand until the sub-block is deemed compressible */ + && (budget < inSize * BYTESCALE) ) + break; + } + + return n; +} + +/** ZSTD_compressSubBlock_multi() : + * Breaks super-block into multiple sub-blocks and compresses them. + * Entropy will be written into the first block. + * The following blocks use repeat_mode to compress. + * Sub-blocks are all compressed, except the last one when beneficial. + * @return : compressed size of the super block (which features multiple ZSTD blocks) + * or 0 if it failed to compress. */ +static size_t ZSTD_compressSubBlock_multi(const SeqStore_t* seqStorePtr, + const ZSTD_compressedBlockState_t* prevCBlock, + ZSTD_compressedBlockState_t* nextCBlock, + const ZSTD_entropyCTablesMetadata_t* entropyMetadata, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const int bmi2, U32 lastBlock, + void* workspace, size_t wkspSize) +{ + const SeqDef* const sstart = seqStorePtr->sequencesStart; + const SeqDef* const send = seqStorePtr->sequences; + const SeqDef* sp = sstart; /* tracks progresses within seqStorePtr->sequences */ + size_t const nbSeqs = (size_t)(send - sstart); + const BYTE* const lstart = seqStorePtr->litStart; + const BYTE* const lend = seqStorePtr->lit; + const BYTE* lp = lstart; + size_t const nbLiterals = (size_t)(lend - lstart); + BYTE const* ip = (BYTE const*)src; + BYTE const* const iend = ip + srcSize; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart; + const BYTE* llCodePtr = seqStorePtr->llCode; + const BYTE* mlCodePtr = seqStorePtr->mlCode; + const BYTE* ofCodePtr = seqStorePtr->ofCode; + size_t const minTarget = ZSTD_TARGETCBLOCKSIZE_MIN; /* enforce minimum size, to reduce undesirable side effects */ + size_t const targetCBlockSize = MAX(minTarget, cctxParams->targetCBlockSize); + int writeLitEntropy = (entropyMetadata->hufMetadata.hType == set_compressed); + int writeSeqEntropy = 1; + + DEBUGLOG(5, "ZSTD_compressSubBlock_multi (srcSize=%u, litSize=%u, nbSeq=%u)", + (unsigned)srcSize, (unsigned)(lend-lstart), (unsigned)(send-sstart)); + + /* let's start by a general estimation for the full block */ + if (nbSeqs > 0) { + EstimatedBlockSize const ebs = + ZSTD_estimateSubBlockSize(lp, nbLiterals, + ofCodePtr, llCodePtr, mlCodePtr, nbSeqs, + &nextCBlock->entropy, entropyMetadata, + workspace, wkspSize, + writeLitEntropy, writeSeqEntropy); + /* quick estimation */ + size_t const avgLitCost = nbLiterals ? (ebs.estLitSize * BYTESCALE) / nbLiterals : BYTESCALE; + size_t const avgSeqCost = ((ebs.estBlockSize - ebs.estLitSize) * BYTESCALE) / nbSeqs; + const size_t nbSubBlocks = MAX((ebs.estBlockSize + (targetCBlockSize/2)) / targetCBlockSize, 1); + size_t n, avgBlockBudget, blockBudgetSupp=0; + avgBlockBudget = (ebs.estBlockSize * BYTESCALE) / nbSubBlocks; + DEBUGLOG(5, "estimated fullblock size=%u bytes ; avgLitCost=%.2f ; avgSeqCost=%.2f ; targetCBlockSize=%u, nbSubBlocks=%u ; avgBlockBudget=%.0f bytes", + (unsigned)ebs.estBlockSize, (double)avgLitCost/BYTESCALE, (double)avgSeqCost/BYTESCALE, + (unsigned)targetCBlockSize, (unsigned)nbSubBlocks, (double)avgBlockBudget/BYTESCALE); + /* simplification: if estimates states that the full superblock doesn't compress, just bail out immediately + * this will result in the production of a single uncompressed block covering @srcSize.*/ + if (ebs.estBlockSize > srcSize) return 0; + + /* compress and write sub-blocks */ + assert(nbSubBlocks>0); + for (n=0; n < nbSubBlocks-1; n++) { + /* determine nb of sequences for current sub-block + nbLiterals from next sequence */ + size_t const seqCount = sizeBlockSequences(sp, (size_t)(send-sp), + avgBlockBudget + blockBudgetSupp, avgLitCost, avgSeqCost, n==0); + /* if reached last sequence : break to last sub-block (simplification) */ + assert(seqCount <= (size_t)(send-sp)); + if (sp + seqCount == send) break; + assert(seqCount > 0); + /* compress sub-block */ + { int litEntropyWritten = 0; + int seqEntropyWritten = 0; + size_t litSize = countLiterals(seqStorePtr, sp, seqCount); + const size_t decompressedSize = + ZSTD_seqDecompressedSize(seqStorePtr, sp, seqCount, litSize, 0); + size_t const cSize = ZSTD_compressSubBlock(&nextCBlock->entropy, entropyMetadata, + sp, seqCount, + lp, litSize, + llCodePtr, mlCodePtr, ofCodePtr, + cctxParams, + op, (size_t)(oend-op), + bmi2, writeLitEntropy, writeSeqEntropy, + &litEntropyWritten, &seqEntropyWritten, + 0); + FORWARD_IF_ERROR(cSize, "ZSTD_compressSubBlock failed"); + + /* check compressibility, update state components */ + if (cSize > 0 && cSize < decompressedSize) { + DEBUGLOG(5, "Committed sub-block compressing %u bytes => %u bytes", + (unsigned)decompressedSize, (unsigned)cSize); + assert(ip + decompressedSize <= iend); + ip += decompressedSize; + lp += litSize; + op += cSize; + llCodePtr += seqCount; + mlCodePtr += seqCount; + ofCodePtr += seqCount; + /* Entropy only needs to be written once */ + if (litEntropyWritten) { + writeLitEntropy = 0; + } + if (seqEntropyWritten) { + writeSeqEntropy = 0; + } + sp += seqCount; + blockBudgetSupp = 0; + } } + /* otherwise : do not compress yet, coalesce current sub-block with following one */ + } + } /* if (nbSeqs > 0) */ + + /* write last block */ + DEBUGLOG(5, "Generate last sub-block: %u sequences remaining", (unsigned)(send - sp)); + { int litEntropyWritten = 0; + int seqEntropyWritten = 0; + size_t litSize = (size_t)(lend - lp); + size_t seqCount = (size_t)(send - sp); + const size_t decompressedSize = + ZSTD_seqDecompressedSize(seqStorePtr, sp, seqCount, litSize, 1); + size_t const cSize = ZSTD_compressSubBlock(&nextCBlock->entropy, entropyMetadata, + sp, seqCount, + lp, litSize, + llCodePtr, mlCodePtr, ofCodePtr, + cctxParams, + op, (size_t)(oend-op), + bmi2, writeLitEntropy, writeSeqEntropy, + &litEntropyWritten, &seqEntropyWritten, + lastBlock); + FORWARD_IF_ERROR(cSize, "ZSTD_compressSubBlock failed"); + + /* update pointers, the nb of literals borrowed from next sequence must be preserved */ + if (cSize > 0 && cSize < decompressedSize) { + DEBUGLOG(5, "Last sub-block compressed %u bytes => %u bytes", + (unsigned)decompressedSize, (unsigned)cSize); + assert(ip + decompressedSize <= iend); + ip += decompressedSize; + lp += litSize; + op += cSize; + llCodePtr += seqCount; + mlCodePtr += seqCount; + ofCodePtr += seqCount; + /* Entropy only needs to be written once */ + if (litEntropyWritten) { + writeLitEntropy = 0; + } + if (seqEntropyWritten) { + writeSeqEntropy = 0; + } + sp += seqCount; + } + } + + + if (writeLitEntropy) { + DEBUGLOG(5, "Literal entropy tables were never written"); + ZSTD_memcpy(&nextCBlock->entropy.huf, &prevCBlock->entropy.huf, sizeof(prevCBlock->entropy.huf)); + } + if (writeSeqEntropy && ZSTD_needSequenceEntropyTables(&entropyMetadata->fseMetadata)) { + /* If we haven't written our entropy tables, then we've violated our contract and + * must emit an uncompressed block. + */ + DEBUGLOG(5, "Sequence entropy tables were never written => cancel, emit an uncompressed block"); + return 0; + } + + if (ip < iend) { + /* some data left : last part of the block sent uncompressed */ + size_t const rSize = (size_t)((iend - ip)); + size_t const cSize = ZSTD_noCompressBlock(op, (size_t)(oend - op), ip, rSize, lastBlock); + DEBUGLOG(5, "Generate last uncompressed sub-block of %u bytes", (unsigned)(rSize)); + FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed"); + assert(cSize != 0); + op += cSize; + /* We have to regenerate the repcodes because we've skipped some sequences */ + if (sp < send) { + const SeqDef* seq; + Repcodes_t rep; + ZSTD_memcpy(&rep, prevCBlock->rep, sizeof(rep)); + for (seq = sstart; seq < sp; ++seq) { + ZSTD_updateRep(rep.rep, seq->offBase, ZSTD_getSequenceLength(seqStorePtr, seq).litLength == 0); + } + ZSTD_memcpy(nextCBlock->rep, &rep, sizeof(rep)); + } + } + + DEBUGLOG(5, "ZSTD_compressSubBlock_multi compressed all subBlocks: total compressed size = %u", + (unsigned)(op-ostart)); + return (size_t)(op-ostart); +} + +size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + unsigned lastBlock) +{ + ZSTD_entropyCTablesMetadata_t entropyMetadata; + + FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(&zc->seqStore, + &zc->blockState.prevCBlock->entropy, + &zc->blockState.nextCBlock->entropy, + &zc->appliedParams, + &entropyMetadata, + zc->tmpWorkspace, zc->tmpWkspSize /* statically allocated in resetCCtx */), ""); + + return ZSTD_compressSubBlock_multi(&zc->seqStore, + zc->blockState.prevCBlock, + zc->blockState.nextCBlock, + &entropyMetadata, + &zc->appliedParams, + dst, dstCapacity, + src, srcSize, + zc->bmi2, lastBlock, + zc->tmpWorkspace, zc->tmpWkspSize /* statically allocated in resetCCtx */); +} +/**** ended inlining compress/zstd_compress_superblock.c ****/ +/**** start inlining compress/zstd_preSplit.c ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/**** skipping file: ../common/compiler.h ****/ +/**** skipping file: ../common/mem.h ****/ +/**** skipping file: ../common/zstd_deps.h ****/ +/**** skipping file: ../common/zstd_internal.h ****/ +/**** skipping file: hist.h ****/ +/**** skipping file: zstd_preSplit.h ****/ + + +#define BLOCKSIZE_MIN 3500 +#define THRESHOLD_PENALTY_RATE 16 +#define THRESHOLD_BASE (THRESHOLD_PENALTY_RATE - 2) +#define THRESHOLD_PENALTY 3 + +#define HASHLENGTH 2 +#define HASHLOG_MAX 10 +#define HASHTABLESIZE (1 << HASHLOG_MAX) +#define HASHMASK (HASHTABLESIZE - 1) +#define KNUTH 0x9e3779b9 + +/* for hashLog > 8, hash 2 bytes. + * for hashLog == 8, just take the byte, no hashing. + * The speed of this method relies on compile-time constant propagation */ +FORCE_INLINE_TEMPLATE unsigned hash2(const void *p, unsigned hashLog) +{ + assert(hashLog >= 8); + if (hashLog == 8) return (U32)((const BYTE*)p)[0]; + assert(hashLog <= HASHLOG_MAX); + return (U32)(MEM_read16(p)) * KNUTH >> (32 - hashLog); +} + + +typedef struct { + unsigned events[HASHTABLESIZE]; + size_t nbEvents; +} Fingerprint; +typedef struct { + Fingerprint pastEvents; + Fingerprint newEvents; +} FPStats; + +static void initStats(FPStats* fpstats) +{ + ZSTD_memset(fpstats, 0, sizeof(FPStats)); +} + +FORCE_INLINE_TEMPLATE void +addEvents_generic(Fingerprint* fp, const void* src, size_t srcSize, size_t samplingRate, unsigned hashLog) +{ + const char* p = (const char*)src; + size_t limit = srcSize - HASHLENGTH + 1; + size_t n; + assert(srcSize >= HASHLENGTH); + for (n = 0; n < limit; n+=samplingRate) { + fp->events[hash2(p+n, hashLog)]++; + } + fp->nbEvents += limit/samplingRate; +} + +FORCE_INLINE_TEMPLATE void +recordFingerprint_generic(Fingerprint* fp, const void* src, size_t srcSize, size_t samplingRate, unsigned hashLog) +{ + ZSTD_memset(fp, 0, sizeof(unsigned) * ((size_t)1 << hashLog)); + fp->nbEvents = 0; + addEvents_generic(fp, src, srcSize, samplingRate, hashLog); +} + +typedef void (*RecordEvents_f)(Fingerprint* fp, const void* src, size_t srcSize); + +#define FP_RECORD(_rate) ZSTD_recordFingerprint_##_rate + +#define ZSTD_GEN_RECORD_FINGERPRINT(_rate, _hSize) \ + static void FP_RECORD(_rate)(Fingerprint* fp, const void* src, size_t srcSize) \ + { \ + recordFingerprint_generic(fp, src, srcSize, _rate, _hSize); \ + } + +ZSTD_GEN_RECORD_FINGERPRINT(1, 10) +ZSTD_GEN_RECORD_FINGERPRINT(5, 10) +ZSTD_GEN_RECORD_FINGERPRINT(11, 9) +ZSTD_GEN_RECORD_FINGERPRINT(43, 8) + + +static U64 abs64(S64 s64) { return (U64)((s64 < 0) ? -s64 : s64); } + +static U64 fpDistance(const Fingerprint* fp1, const Fingerprint* fp2, unsigned hashLog) +{ + U64 distance = 0; + size_t n; + assert(hashLog <= HASHLOG_MAX); + for (n = 0; n < ((size_t)1 << hashLog); n++) { + distance += + abs64((S64)fp1->events[n] * (S64)fp2->nbEvents - (S64)fp2->events[n] * (S64)fp1->nbEvents); + } + return distance; +} + +/* Compare newEvents with pastEvents + * return 1 when considered "too different" + */ +static int compareFingerprints(const Fingerprint* ref, + const Fingerprint* newfp, + int penalty, + unsigned hashLog) +{ + assert(ref->nbEvents > 0); + assert(newfp->nbEvents > 0); + { U64 p50 = (U64)ref->nbEvents * (U64)newfp->nbEvents; + U64 deviation = fpDistance(ref, newfp, hashLog); + U64 threshold = p50 * (U64)(THRESHOLD_BASE + penalty) / THRESHOLD_PENALTY_RATE; + return deviation >= threshold; + } +} + +static void mergeEvents(Fingerprint* acc, const Fingerprint* newfp) +{ + size_t n; + for (n = 0; n < HASHTABLESIZE; n++) { + acc->events[n] += newfp->events[n]; + } + acc->nbEvents += newfp->nbEvents; +} + +static void flushEvents(FPStats* fpstats) +{ + size_t n; + for (n = 0; n < HASHTABLESIZE; n++) { + fpstats->pastEvents.events[n] = fpstats->newEvents.events[n]; + } + fpstats->pastEvents.nbEvents = fpstats->newEvents.nbEvents; + ZSTD_memset(&fpstats->newEvents, 0, sizeof(fpstats->newEvents)); +} + +static void removeEvents(Fingerprint* acc, const Fingerprint* slice) +{ + size_t n; + for (n = 0; n < HASHTABLESIZE; n++) { + assert(acc->events[n] >= slice->events[n]); + acc->events[n] -= slice->events[n]; + } + acc->nbEvents -= slice->nbEvents; +} + +#define CHUNKSIZE (8 << 10) +static size_t ZSTD_splitBlock_byChunks(const void* blockStart, size_t blockSize, + int level, + void* workspace, size_t wkspSize) +{ + static const RecordEvents_f records_fs[] = { + FP_RECORD(43), FP_RECORD(11), FP_RECORD(5), FP_RECORD(1) + }; + static const unsigned hashParams[] = { 8, 9, 10, 10 }; + const RecordEvents_f record_f = (assert(0<=level && level<=3), records_fs[level]); + FPStats* const fpstats = (FPStats*)workspace; + const char* p = (const char*)blockStart; + int penalty = THRESHOLD_PENALTY; + size_t pos = 0; + assert(blockSize == (128 << 10)); + assert(workspace != NULL); + assert((size_t)workspace % ZSTD_ALIGNOF(FPStats) == 0); + ZSTD_STATIC_ASSERT(ZSTD_SLIPBLOCK_WORKSPACESIZE >= sizeof(FPStats)); + assert(wkspSize >= sizeof(FPStats)); (void)wkspSize; + + initStats(fpstats); + record_f(&fpstats->pastEvents, p, CHUNKSIZE); + for (pos = CHUNKSIZE; pos <= blockSize - CHUNKSIZE; pos += CHUNKSIZE) { + record_f(&fpstats->newEvents, p + pos, CHUNKSIZE); + if (compareFingerprints(&fpstats->pastEvents, &fpstats->newEvents, penalty, hashParams[level])) { + return pos; + } else { + mergeEvents(&fpstats->pastEvents, &fpstats->newEvents); + if (penalty > 0) penalty--; + } + } + assert(pos == blockSize); + return blockSize; + (void)flushEvents; (void)removeEvents; +} + +/* ZSTD_splitBlock_fromBorders(): very fast strategy : + * compare fingerprint from beginning and end of the block, + * derive from their difference if it's preferable to split in the middle, + * repeat the process a second time, for finer grained decision. + * 3 times did not brought improvements, so I stopped at 2. + * Benefits are good enough for a cheap heuristic. + * More accurate splitting saves more, but speed impact is also more perceptible. + * For better accuracy, use more elaborate variant *_byChunks. + */ +static size_t ZSTD_splitBlock_fromBorders(const void* blockStart, size_t blockSize, + void* workspace, size_t wkspSize) +{ +#define SEGMENT_SIZE 512 + FPStats* const fpstats = (FPStats*)workspace; + Fingerprint* middleEvents = (Fingerprint*)(void*)((char*)workspace + 512 * sizeof(unsigned)); + assert(blockSize == (128 << 10)); + assert(workspace != NULL); + assert((size_t)workspace % ZSTD_ALIGNOF(FPStats) == 0); + ZSTD_STATIC_ASSERT(ZSTD_SLIPBLOCK_WORKSPACESIZE >= sizeof(FPStats)); + assert(wkspSize >= sizeof(FPStats)); (void)wkspSize; + + initStats(fpstats); + HIST_add(fpstats->pastEvents.events, blockStart, SEGMENT_SIZE); + HIST_add(fpstats->newEvents.events, (const char*)blockStart + blockSize - SEGMENT_SIZE, SEGMENT_SIZE); + fpstats->pastEvents.nbEvents = fpstats->newEvents.nbEvents = SEGMENT_SIZE; + if (!compareFingerprints(&fpstats->pastEvents, &fpstats->newEvents, 0, 8)) + return blockSize; + + HIST_add(middleEvents->events, (const char*)blockStart + blockSize/2 - SEGMENT_SIZE/2, SEGMENT_SIZE); + middleEvents->nbEvents = SEGMENT_SIZE; + { U64 const distFromBegin = fpDistance(&fpstats->pastEvents, middleEvents, 8); + U64 const distFromEnd = fpDistance(&fpstats->newEvents, middleEvents, 8); + U64 const minDistance = SEGMENT_SIZE * SEGMENT_SIZE / 3; + if (abs64((S64)distFromBegin - (S64)distFromEnd) < minDistance) + return 64 KB; + return (distFromBegin > distFromEnd) ? 32 KB : 96 KB; + } +} + +size_t ZSTD_splitBlock(const void* blockStart, size_t blockSize, + int level, + void* workspace, size_t wkspSize) +{ + DEBUGLOG(6, "ZSTD_splitBlock (level=%i)", level); + assert(0<=level && level<=4); + if (level == 0) + return ZSTD_splitBlock_fromBorders(blockStart, blockSize, workspace, wkspSize); + /* level >= 1*/ + return ZSTD_splitBlock_byChunks(blockStart, blockSize, level-1, workspace, wkspSize); +} +/**** ended inlining compress/zstd_preSplit.c ****/ +/**** start inlining compress/zstd_compress.c ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/*-************************************* +* Dependencies +***************************************/ +/**** skipping file: ../common/allocations.h ****/ +/**** skipping file: ../common/zstd_deps.h ****/ +/**** skipping file: ../common/mem.h ****/ +/**** skipping file: ../common/error_private.h ****/ +/**** skipping file: hist.h ****/ +#define FSE_STATIC_LINKING_ONLY /* FSE_encodeSymbol */ +/**** skipping file: ../common/fse.h ****/ +/**** skipping file: ../common/huf.h ****/ +/**** skipping file: zstd_compress_internal.h ****/ +/**** skipping file: zstd_compress_sequences.h ****/ +/**** skipping file: zstd_compress_literals.h ****/ +/**** start inlining zstd_fast.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_FAST_H +#define ZSTD_FAST_H + +/**** skipping file: ../common/mem.h ****/ +/**** skipping file: zstd_compress_internal.h ****/ + +void ZSTD_fillHashTable(ZSTD_MatchState_t* ms, + void const* end, ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp); +size_t ZSTD_compressBlock_fast( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_fast_dictMatchState( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_fast_extDict( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +#endif /* ZSTD_FAST_H */ +/**** ended inlining zstd_fast.h ****/ +/**** start inlining zstd_double_fast.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_DOUBLE_FAST_H +#define ZSTD_DOUBLE_FAST_H + +/**** skipping file: ../common/mem.h ****/ +/**** skipping file: zstd_compress_internal.h ****/ + +#ifndef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR + +void ZSTD_fillDoubleHashTable(ZSTD_MatchState_t* ms, + void const* end, ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp); + +size_t ZSTD_compressBlock_doubleFast( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_doubleFast_dictMatchState( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_doubleFast_extDict( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +#define ZSTD_COMPRESSBLOCK_DOUBLEFAST ZSTD_compressBlock_doubleFast +#define ZSTD_COMPRESSBLOCK_DOUBLEFAST_DICTMATCHSTATE ZSTD_compressBlock_doubleFast_dictMatchState +#define ZSTD_COMPRESSBLOCK_DOUBLEFAST_EXTDICT ZSTD_compressBlock_doubleFast_extDict +#else +#define ZSTD_COMPRESSBLOCK_DOUBLEFAST NULL +#define ZSTD_COMPRESSBLOCK_DOUBLEFAST_DICTMATCHSTATE NULL +#define ZSTD_COMPRESSBLOCK_DOUBLEFAST_EXTDICT NULL +#endif /* ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR */ + +#endif /* ZSTD_DOUBLE_FAST_H */ +/**** ended inlining zstd_double_fast.h ****/ +/**** start inlining zstd_lazy.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_LAZY_H +#define ZSTD_LAZY_H + +/**** skipping file: zstd_compress_internal.h ****/ + +/** + * Dedicated Dictionary Search Structure bucket log. In the + * ZSTD_dedicatedDictSearch mode, the hashTable has + * 2 ** ZSTD_LAZY_DDSS_BUCKET_LOG entries in each bucket, rather than just + * one. + */ +#define ZSTD_LAZY_DDSS_BUCKET_LOG 2 + +#define ZSTD_ROW_HASH_TAG_BITS 8 /* nb bits to use for the tag */ + +#if !defined(ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR) +U32 ZSTD_insertAndFindFirstIndex(ZSTD_MatchState_t* ms, const BYTE* ip); +void ZSTD_row_update(ZSTD_MatchState_t* const ms, const BYTE* ip); + +void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_MatchState_t* ms, const BYTE* const ip); + +void ZSTD_preserveUnsortedMark (U32* const table, U32 const size, U32 const reducerValue); /*! used in ZSTD_reduceIndex(). preemptively increase value of ZSTD_DUBT_UNSORTED_MARK */ +#endif + +#ifndef ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_greedy( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_dictMatchState( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_dictMatchState_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_dedicatedDictSearch( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_dedicatedDictSearch_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_extDict( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_extDict_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +#define ZSTD_COMPRESSBLOCK_GREEDY ZSTD_compressBlock_greedy +#define ZSTD_COMPRESSBLOCK_GREEDY_ROW ZSTD_compressBlock_greedy_row +#define ZSTD_COMPRESSBLOCK_GREEDY_DICTMATCHSTATE ZSTD_compressBlock_greedy_dictMatchState +#define ZSTD_COMPRESSBLOCK_GREEDY_DICTMATCHSTATE_ROW ZSTD_compressBlock_greedy_dictMatchState_row +#define ZSTD_COMPRESSBLOCK_GREEDY_DEDICATEDDICTSEARCH ZSTD_compressBlock_greedy_dedicatedDictSearch +#define ZSTD_COMPRESSBLOCK_GREEDY_DEDICATEDDICTSEARCH_ROW ZSTD_compressBlock_greedy_dedicatedDictSearch_row +#define ZSTD_COMPRESSBLOCK_GREEDY_EXTDICT ZSTD_compressBlock_greedy_extDict +#define ZSTD_COMPRESSBLOCK_GREEDY_EXTDICT_ROW ZSTD_compressBlock_greedy_extDict_row +#else +#define ZSTD_COMPRESSBLOCK_GREEDY NULL +#define ZSTD_COMPRESSBLOCK_GREEDY_ROW NULL +#define ZSTD_COMPRESSBLOCK_GREEDY_DICTMATCHSTATE NULL +#define ZSTD_COMPRESSBLOCK_GREEDY_DICTMATCHSTATE_ROW NULL +#define ZSTD_COMPRESSBLOCK_GREEDY_DEDICATEDDICTSEARCH NULL +#define ZSTD_COMPRESSBLOCK_GREEDY_DEDICATEDDICTSEARCH_ROW NULL +#define ZSTD_COMPRESSBLOCK_GREEDY_EXTDICT NULL +#define ZSTD_COMPRESSBLOCK_GREEDY_EXTDICT_ROW NULL +#endif + +#ifndef ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_lazy( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_dictMatchState( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_dictMatchState_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_dedicatedDictSearch( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_dedicatedDictSearch_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_extDict( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_extDict_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +#define ZSTD_COMPRESSBLOCK_LAZY ZSTD_compressBlock_lazy +#define ZSTD_COMPRESSBLOCK_LAZY_ROW ZSTD_compressBlock_lazy_row +#define ZSTD_COMPRESSBLOCK_LAZY_DICTMATCHSTATE ZSTD_compressBlock_lazy_dictMatchState +#define ZSTD_COMPRESSBLOCK_LAZY_DICTMATCHSTATE_ROW ZSTD_compressBlock_lazy_dictMatchState_row +#define ZSTD_COMPRESSBLOCK_LAZY_DEDICATEDDICTSEARCH ZSTD_compressBlock_lazy_dedicatedDictSearch +#define ZSTD_COMPRESSBLOCK_LAZY_DEDICATEDDICTSEARCH_ROW ZSTD_compressBlock_lazy_dedicatedDictSearch_row +#define ZSTD_COMPRESSBLOCK_LAZY_EXTDICT ZSTD_compressBlock_lazy_extDict +#define ZSTD_COMPRESSBLOCK_LAZY_EXTDICT_ROW ZSTD_compressBlock_lazy_extDict_row +#else +#define ZSTD_COMPRESSBLOCK_LAZY NULL +#define ZSTD_COMPRESSBLOCK_LAZY_ROW NULL +#define ZSTD_COMPRESSBLOCK_LAZY_DICTMATCHSTATE NULL +#define ZSTD_COMPRESSBLOCK_LAZY_DICTMATCHSTATE_ROW NULL +#define ZSTD_COMPRESSBLOCK_LAZY_DEDICATEDDICTSEARCH NULL +#define ZSTD_COMPRESSBLOCK_LAZY_DEDICATEDDICTSEARCH_ROW NULL +#define ZSTD_COMPRESSBLOCK_LAZY_EXTDICT NULL +#define ZSTD_COMPRESSBLOCK_LAZY_EXTDICT_ROW NULL +#endif + +#ifndef ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_lazy2( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_dictMatchState( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_dictMatchState_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_extDict( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_extDict_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +#define ZSTD_COMPRESSBLOCK_LAZY2 ZSTD_compressBlock_lazy2 +#define ZSTD_COMPRESSBLOCK_LAZY2_ROW ZSTD_compressBlock_lazy2_row +#define ZSTD_COMPRESSBLOCK_LAZY2_DICTMATCHSTATE ZSTD_compressBlock_lazy2_dictMatchState +#define ZSTD_COMPRESSBLOCK_LAZY2_DICTMATCHSTATE_ROW ZSTD_compressBlock_lazy2_dictMatchState_row +#define ZSTD_COMPRESSBLOCK_LAZY2_DEDICATEDDICTSEARCH ZSTD_compressBlock_lazy2_dedicatedDictSearch +#define ZSTD_COMPRESSBLOCK_LAZY2_DEDICATEDDICTSEARCH_ROW ZSTD_compressBlock_lazy2_dedicatedDictSearch_row +#define ZSTD_COMPRESSBLOCK_LAZY2_EXTDICT ZSTD_compressBlock_lazy2_extDict +#define ZSTD_COMPRESSBLOCK_LAZY2_EXTDICT_ROW ZSTD_compressBlock_lazy2_extDict_row +#else +#define ZSTD_COMPRESSBLOCK_LAZY2 NULL +#define ZSTD_COMPRESSBLOCK_LAZY2_ROW NULL +#define ZSTD_COMPRESSBLOCK_LAZY2_DICTMATCHSTATE NULL +#define ZSTD_COMPRESSBLOCK_LAZY2_DICTMATCHSTATE_ROW NULL +#define ZSTD_COMPRESSBLOCK_LAZY2_DEDICATEDDICTSEARCH NULL +#define ZSTD_COMPRESSBLOCK_LAZY2_DEDICATEDDICTSEARCH_ROW NULL +#define ZSTD_COMPRESSBLOCK_LAZY2_EXTDICT NULL +#define ZSTD_COMPRESSBLOCK_LAZY2_EXTDICT_ROW NULL +#endif + +#ifndef ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_btlazy2( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btlazy2_dictMatchState( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btlazy2_extDict( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +#define ZSTD_COMPRESSBLOCK_BTLAZY2 ZSTD_compressBlock_btlazy2 +#define ZSTD_COMPRESSBLOCK_BTLAZY2_DICTMATCHSTATE ZSTD_compressBlock_btlazy2_dictMatchState +#define ZSTD_COMPRESSBLOCK_BTLAZY2_EXTDICT ZSTD_compressBlock_btlazy2_extDict +#else +#define ZSTD_COMPRESSBLOCK_BTLAZY2 NULL +#define ZSTD_COMPRESSBLOCK_BTLAZY2_DICTMATCHSTATE NULL +#define ZSTD_COMPRESSBLOCK_BTLAZY2_EXTDICT NULL +#endif + +#endif /* ZSTD_LAZY_H */ +/**** ended inlining zstd_lazy.h ****/ +/**** start inlining zstd_opt.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_OPT_H +#define ZSTD_OPT_H + +/**** skipping file: zstd_compress_internal.h ****/ + +#if !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR) +/* used in ZSTD_loadDictionaryContent() */ +void ZSTD_updateTree(ZSTD_MatchState_t* ms, const BYTE* ip, const BYTE* iend); +#endif + +#ifndef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_btopt( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btopt_dictMatchState( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btopt_extDict( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +#define ZSTD_COMPRESSBLOCK_BTOPT ZSTD_compressBlock_btopt +#define ZSTD_COMPRESSBLOCK_BTOPT_DICTMATCHSTATE ZSTD_compressBlock_btopt_dictMatchState +#define ZSTD_COMPRESSBLOCK_BTOPT_EXTDICT ZSTD_compressBlock_btopt_extDict +#else +#define ZSTD_COMPRESSBLOCK_BTOPT NULL +#define ZSTD_COMPRESSBLOCK_BTOPT_DICTMATCHSTATE NULL +#define ZSTD_COMPRESSBLOCK_BTOPT_EXTDICT NULL +#endif + +#ifndef ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_btultra( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btultra_dictMatchState( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btultra_extDict( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + + /* note : no btultra2 variant for extDict nor dictMatchState, + * because btultra2 is not meant to work with dictionaries + * and is only specific for the first block (no prefix) */ +size_t ZSTD_compressBlock_btultra2( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +#define ZSTD_COMPRESSBLOCK_BTULTRA ZSTD_compressBlock_btultra +#define ZSTD_COMPRESSBLOCK_BTULTRA_DICTMATCHSTATE ZSTD_compressBlock_btultra_dictMatchState +#define ZSTD_COMPRESSBLOCK_BTULTRA_EXTDICT ZSTD_compressBlock_btultra_extDict +#define ZSTD_COMPRESSBLOCK_BTULTRA2 ZSTD_compressBlock_btultra2 +#else +#define ZSTD_COMPRESSBLOCK_BTULTRA NULL +#define ZSTD_COMPRESSBLOCK_BTULTRA_DICTMATCHSTATE NULL +#define ZSTD_COMPRESSBLOCK_BTULTRA_EXTDICT NULL +#define ZSTD_COMPRESSBLOCK_BTULTRA2 NULL +#endif + +#endif /* ZSTD_OPT_H */ +/**** ended inlining zstd_opt.h ****/ +/**** start inlining zstd_ldm.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_LDM_H +#define ZSTD_LDM_H + +/**** skipping file: zstd_compress_internal.h ****/ +/**** skipping file: ../zstd.h ****/ + +/*-************************************* +* Long distance matching +***************************************/ + +#define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_LIMIT_DEFAULT + +void ZSTD_ldm_fillHashTable( + ldmState_t* state, const BYTE* ip, + const BYTE* iend, ldmParams_t const* params); + +/** + * ZSTD_ldm_generateSequences(): + * + * Generates the sequences using the long distance match finder. + * Generates long range matching sequences in `sequences`, which parse a prefix + * of the source. `sequences` must be large enough to store every sequence, + * which can be checked with `ZSTD_ldm_getMaxNbSeq()`. + * @returns 0 or an error code. + * + * NOTE: The user must have called ZSTD_window_update() for all of the input + * they have, even if they pass it to ZSTD_ldm_generateSequences() in chunks. + * NOTE: This function returns an error if it runs out of space to store + * sequences. + */ +size_t ZSTD_ldm_generateSequences( + ldmState_t* ldms, RawSeqStore_t* sequences, + ldmParams_t const* params, void const* src, size_t srcSize); + +/** + * ZSTD_ldm_blockCompress(): + * + * Compresses a block using the predefined sequences, along with a secondary + * block compressor. The literals section of every sequence is passed to the + * secondary block compressor, and those sequences are interspersed with the + * predefined sequences. Returns the length of the last literals. + * Updates `rawSeqStore.pos` to indicate how many sequences have been consumed. + * `rawSeqStore.seq` may also be updated to split the last sequence between two + * blocks. + * @return The length of the last literals. + * + * NOTE: The source must be at most the maximum block size, but the predefined + * sequences can be any size, and may be longer than the block. In the case that + * they are longer than the block, the last sequences may need to be split into + * two. We handle that case correctly, and update `rawSeqStore` appropriately. + * NOTE: This function does not return any errors. + */ +size_t ZSTD_ldm_blockCompress(RawSeqStore_t* rawSeqStore, + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_ParamSwitch_e useRowMatchFinder, + void const* src, size_t srcSize); + +/** + * ZSTD_ldm_skipSequences(): + * + * Skip past `srcSize` bytes worth of sequences in `rawSeqStore`. + * Avoids emitting matches less than `minMatch` bytes. + * Must be called for data that is not passed to ZSTD_ldm_blockCompress(). + */ +void ZSTD_ldm_skipSequences(RawSeqStore_t* rawSeqStore, size_t srcSize, + U32 const minMatch); + +/* ZSTD_ldm_skipRawSeqStoreBytes(): + * Moves forward in rawSeqStore by nbBytes, updating fields 'pos' and 'posInSequence'. + * Not to be used in conjunction with ZSTD_ldm_skipSequences(). + * Must be called for data with is not passed to ZSTD_ldm_blockCompress(). + */ +void ZSTD_ldm_skipRawSeqStoreBytes(RawSeqStore_t* rawSeqStore, size_t nbBytes); + +/** ZSTD_ldm_getTableSize() : + * Estimate the space needed for long distance matching tables or 0 if LDM is + * disabled. + */ +size_t ZSTD_ldm_getTableSize(ldmParams_t params); + +/** ZSTD_ldm_getSeqSpace() : + * Return an upper bound on the number of sequences that can be produced by + * the long distance matcher, or 0 if LDM is disabled. + */ +size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize); + +/** ZSTD_ldm_adjustParameters() : + * If the params->hashRateLog is not set, set it to its default value based on + * windowLog and params->hashLog. + * + * Ensures that params->bucketSizeLog is <= params->hashLog (setting it to + * params->hashLog if it is not). + * + * Ensures that the minMatchLength >= targetLength during optimal parsing. + */ +void ZSTD_ldm_adjustParameters(ldmParams_t* params, + ZSTD_compressionParameters const* cParams); + +#endif /* ZSTD_FAST_H */ +/**** ended inlining zstd_ldm.h ****/ +/**** skipping file: zstd_compress_superblock.h ****/ +/**** skipping file: ../common/bits.h ****/ + +/* *************************************************************** +* Tuning parameters +*****************************************************************/ +/*! + * COMPRESS_HEAPMODE : + * Select how default decompression function ZSTD_compress() allocates its context, + * on stack (0, default), or into heap (1). + * Note that functions with explicit context such as ZSTD_compressCCtx() are unaffected. + */ +#ifndef ZSTD_COMPRESS_HEAPMODE +# define ZSTD_COMPRESS_HEAPMODE 0 +#endif + +/*! + * ZSTD_HASHLOG3_MAX : + * Maximum size of the hash table dedicated to find 3-bytes matches, + * in log format, aka 17 => 1 << 17 == 128Ki positions. + * This structure is only used in zstd_opt. + * Since allocation is centralized for all strategies, it has to be known here. + * The actual (selected) size of the hash table is then stored in ZSTD_MatchState_t.hashLog3, + * so that zstd_opt.c doesn't need to know about this constant. + */ +#ifndef ZSTD_HASHLOG3_MAX +# define ZSTD_HASHLOG3_MAX 17 +#endif -#endif /* ZSTD_COMMON_CPU_H */ -/**** ended inlining ../common/cpu.h ****/ -/**** skipping file: ../common/mem.h ****/ -/**** skipping file: ../common/zstd_trace.h ****/ -/**** skipping file: hist.h ****/ -#define FSE_STATIC_LINKING_ONLY /* FSE_encodeSymbol */ -/**** skipping file: ../common/fse.h ****/ -#define HUF_STATIC_LINKING_ONLY -/**** skipping file: ../common/huf.h ****/ -/**** skipping file: zstd_compress_internal.h ****/ -/**** skipping file: zstd_compress_sequences.h ****/ -/**** skipping file: zstd_compress_literals.h ****/ -/**** start inlining zstd_fast.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. +/*-************************************* +* Helper functions +***************************************/ +/* ZSTD_compressBound() + * Note that the result from this function is only valid for + * the one-pass compression functions. + * When employing the streaming mode, + * if flushes are frequently altering the size of blocks, + * the overhead from block headers can make the compressed data larger + * than the return value of ZSTD_compressBound(). */ +size_t ZSTD_compressBound(size_t srcSize) { + size_t const r = ZSTD_COMPRESSBOUND(srcSize); + if (r==0) return ERROR(srcSize_wrong); + return r; +} -#ifndef ZSTD_FAST_H -#define ZSTD_FAST_H -#if defined (__cplusplus) -extern "C" { -#endif +/*-************************************* +* Context memory management +***************************************/ +struct ZSTD_CDict_s { + const void* dictContent; + size_t dictContentSize; + ZSTD_dictContentType_e dictContentType; /* The dictContentType the CDict was created with */ + U32* entropyWorkspace; /* entropy workspace of HUF_WORKSPACE_SIZE bytes */ + ZSTD_cwksp workspace; + ZSTD_MatchState_t matchState; + ZSTD_compressedBlockState_t cBlockState; + ZSTD_customMem customMem; + U32 dictID; + int compressionLevel; /* 0 indicates that advanced API was used to select CDict params */ + ZSTD_ParamSwitch_e useRowMatchFinder; /* Indicates whether the CDict was created with params that would use + * row-based matchfinder. Unless the cdict is reloaded, we will use + * the same greedy/lazy matchfinder at compression time. + */ +}; /* typedef'd to ZSTD_CDict within "zstd.h" */ -/**** skipping file: ../common/mem.h ****/ -/**** skipping file: zstd_compress_internal.h ****/ +ZSTD_CCtx* ZSTD_createCCtx(void) +{ + return ZSTD_createCCtx_advanced(ZSTD_defaultCMem); +} -void ZSTD_fillHashTable(ZSTD_matchState_t* ms, - void const* end, ZSTD_dictTableLoadMethod_e dtlm); -size_t ZSTD_compressBlock_fast( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); -size_t ZSTD_compressBlock_fast_dictMatchState( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); -size_t ZSTD_compressBlock_fast_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); +static void ZSTD_initCCtx(ZSTD_CCtx* cctx, ZSTD_customMem memManager) +{ + assert(cctx != NULL); + ZSTD_memset(cctx, 0, sizeof(*cctx)); + cctx->customMem = memManager; + cctx->bmi2 = ZSTD_cpuSupportsBmi2(); + { size_t const err = ZSTD_CCtx_reset(cctx, ZSTD_reset_parameters); + assert(!ZSTD_isError(err)); + (void)err; + } +} -#if defined (__cplusplus) +ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) +{ + ZSTD_STATIC_ASSERT(zcss_init==0); + ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN==(0ULL - 1)); + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + { ZSTD_CCtx* const cctx = (ZSTD_CCtx*)ZSTD_customMalloc(sizeof(ZSTD_CCtx), customMem); + if (!cctx) return NULL; + ZSTD_initCCtx(cctx, customMem); + return cctx; + } } -#endif -#endif /* ZSTD_FAST_H */ -/**** ended inlining zstd_fast.h ****/ -/**** start inlining zstd_double_fast.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. +ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize) +{ + ZSTD_cwksp ws; + ZSTD_CCtx* cctx; + if (workspaceSize <= sizeof(ZSTD_CCtx)) return NULL; /* minimum size */ + if ((size_t)workspace & 7) return NULL; /* must be 8-aligned */ + ZSTD_cwksp_init(&ws, workspace, workspaceSize, ZSTD_cwksp_static_alloc); + + cctx = (ZSTD_CCtx*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CCtx)); + if (cctx == NULL) return NULL; + + ZSTD_memset(cctx, 0, sizeof(ZSTD_CCtx)); + ZSTD_cwksp_move(&cctx->workspace, &ws); + cctx->staticSize = workspaceSize; + + /* statically sized space. tmpWorkspace never moves (but prev/next block swap places) */ + if (!ZSTD_cwksp_check_available(&cctx->workspace, TMP_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t))) return NULL; + cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); + cctx->blockState.nextCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); + cctx->tmpWorkspace = ZSTD_cwksp_reserve_object(&cctx->workspace, TMP_WORKSPACE_SIZE); + cctx->tmpWkspSize = TMP_WORKSPACE_SIZE; + cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); + return cctx; +} + +/** + * Clears and frees all of the dictionaries in the CCtx. */ +static void ZSTD_clearAllDicts(ZSTD_CCtx* cctx) +{ + ZSTD_customFree(cctx->localDict.dictBuffer, cctx->customMem); + ZSTD_freeCDict(cctx->localDict.cdict); + ZSTD_memset(&cctx->localDict, 0, sizeof(cctx->localDict)); + ZSTD_memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); + cctx->cdict = NULL; +} -#ifndef ZSTD_DOUBLE_FAST_H -#define ZSTD_DOUBLE_FAST_H +static size_t ZSTD_sizeof_localDict(ZSTD_localDict dict) +{ + size_t const bufferSize = dict.dictBuffer != NULL ? dict.dictSize : 0; + size_t const cdictSize = ZSTD_sizeof_CDict(dict.cdict); + return bufferSize + cdictSize; +} -#if defined (__cplusplus) -extern "C" { +static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx) +{ + assert(cctx != NULL); + assert(cctx->staticSize == 0); + ZSTD_clearAllDicts(cctx); +#ifdef ZSTD_MULTITHREAD + ZSTDMT_freeCCtx(cctx->mtctx); cctx->mtctx = NULL; #endif + ZSTD_cwksp_free(&cctx->workspace, cctx->customMem); +} -/**** skipping file: ../common/mem.h ****/ -/**** skipping file: zstd_compress_internal.h ****/ +size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) +{ + DEBUGLOG(3, "ZSTD_freeCCtx (address: %p)", (void*)cctx); + if (cctx==NULL) return 0; /* support free on NULL */ + RETURN_ERROR_IF(cctx->staticSize, memory_allocation, + "not compatible with static CCtx"); + { int cctxInWorkspace = ZSTD_cwksp_owns_buffer(&cctx->workspace, cctx); + ZSTD_freeCCtxContent(cctx); + if (!cctxInWorkspace) ZSTD_customFree(cctx, cctx->customMem); + } + return 0; +} -void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms, - void const* end, ZSTD_dictTableLoadMethod_e dtlm); -size_t ZSTD_compressBlock_doubleFast( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); -size_t ZSTD_compressBlock_doubleFast_dictMatchState( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); -size_t ZSTD_compressBlock_doubleFast_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); + +static size_t ZSTD_sizeof_mtctx(const ZSTD_CCtx* cctx) +{ +#ifdef ZSTD_MULTITHREAD + return ZSTDMT_sizeof_CCtx(cctx->mtctx); +#else + (void)cctx; + return 0; +#endif +} -#if defined (__cplusplus) +size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx) +{ + if (cctx==NULL) return 0; /* support sizeof on NULL */ + /* cctx may be in the workspace */ + return (cctx->workspace.workspace == cctx ? 0 : sizeof(*cctx)) + + ZSTD_cwksp_sizeof(&cctx->workspace) + + ZSTD_sizeof_localDict(cctx->localDict) + + ZSTD_sizeof_mtctx(cctx); } -#endif -#endif /* ZSTD_DOUBLE_FAST_H */ -/**** ended inlining zstd_double_fast.h ****/ -/**** start inlining zstd_lazy.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. +size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs) +{ + return ZSTD_sizeof_CCtx(zcs); /* same object */ +} + +/* private API call, for dictBuilder only */ +const SeqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStore); } + +/* Returns true if the strategy supports using a row based matchfinder */ +static int ZSTD_rowMatchFinderSupported(const ZSTD_strategy strategy) { + return (strategy >= ZSTD_greedy && strategy <= ZSTD_lazy2); +} + +/* Returns true if the strategy and useRowMatchFinder mode indicate that we will use the row based matchfinder + * for this compression. */ +static int ZSTD_rowMatchFinderUsed(const ZSTD_strategy strategy, const ZSTD_ParamSwitch_e mode) { + assert(mode != ZSTD_ps_auto); + return ZSTD_rowMatchFinderSupported(strategy) && (mode == ZSTD_ps_enable); +} -#ifndef ZSTD_LAZY_H -#define ZSTD_LAZY_H +/* Returns row matchfinder usage given an initial mode and cParams */ +static ZSTD_ParamSwitch_e ZSTD_resolveRowMatchFinderMode(ZSTD_ParamSwitch_e mode, + const ZSTD_compressionParameters* const cParams) { + if (mode != ZSTD_ps_auto) return mode; /* if requested enabled, but no SIMD, we still will use row matchfinder */ + mode = ZSTD_ps_disable; + if (!ZSTD_rowMatchFinderSupported(cParams->strategy)) return mode; + if (cParams->windowLog > 14) mode = ZSTD_ps_enable; + return mode; +} -#if defined (__cplusplus) -extern "C" { -#endif +/* Returns block splitter usage (generally speaking, when using slower/stronger compression modes) */ +static ZSTD_ParamSwitch_e ZSTD_resolveBlockSplitterMode(ZSTD_ParamSwitch_e mode, + const ZSTD_compressionParameters* const cParams) { + if (mode != ZSTD_ps_auto) return mode; + return (cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 17) ? ZSTD_ps_enable : ZSTD_ps_disable; +} -/**** skipping file: zstd_compress_internal.h ****/ +/* Returns 1 if the arguments indicate that we should allocate a chainTable, 0 otherwise */ +static int ZSTD_allocateChainTable(const ZSTD_strategy strategy, + const ZSTD_ParamSwitch_e useRowMatchFinder, + const U32 forDDSDict) { + assert(useRowMatchFinder != ZSTD_ps_auto); + /* We always should allocate a chaintable if we are allocating a matchstate for a DDS dictionary matchstate. + * We do not allocate a chaintable if we are using ZSTD_fast, or are using the row-based matchfinder. + */ + return forDDSDict || ((strategy != ZSTD_fast) && !ZSTD_rowMatchFinderUsed(strategy, useRowMatchFinder)); +} -/** - * Dedicated Dictionary Search Structure bucket log. In the - * ZSTD_dedicatedDictSearch mode, the hashTable has - * 2 ** ZSTD_LAZY_DDSS_BUCKET_LOG entries in each bucket, rather than just - * one. +/* Returns ZSTD_ps_enable if compression parameters are such that we should + * enable long distance matching (wlog >= 27, strategy >= btopt). + * Returns ZSTD_ps_disable otherwise. */ -#define ZSTD_LAZY_DDSS_BUCKET_LOG 2 +static ZSTD_ParamSwitch_e ZSTD_resolveEnableLdm(ZSTD_ParamSwitch_e mode, + const ZSTD_compressionParameters* const cParams) { + if (mode != ZSTD_ps_auto) return mode; + return (cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 27) ? ZSTD_ps_enable : ZSTD_ps_disable; +} -U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip); +static int ZSTD_resolveExternalSequenceValidation(int mode) { + return mode; +} -void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const BYTE* const ip); +/* Resolves maxBlockSize to the default if no value is present. */ +static size_t ZSTD_resolveMaxBlockSize(size_t maxBlockSize) { + if (maxBlockSize == 0) { + return ZSTD_BLOCKSIZE_MAX; + } else { + return maxBlockSize; + } +} -void ZSTD_preserveUnsortedMark (U32* const table, U32 const size, U32 const reducerValue); /*! used in ZSTD_reduceIndex(). preemptively increase value of ZSTD_DUBT_UNSORTED_MARK */ +static ZSTD_ParamSwitch_e ZSTD_resolveExternalRepcodeSearch(ZSTD_ParamSwitch_e value, int cLevel) { + if (value != ZSTD_ps_auto) return value; + if (cLevel < 10) { + return ZSTD_ps_disable; + } else { + return ZSTD_ps_enable; + } +} -size_t ZSTD_compressBlock_btlazy2( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); -size_t ZSTD_compressBlock_lazy2( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); -size_t ZSTD_compressBlock_lazy( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); -size_t ZSTD_compressBlock_greedy( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); +/* Returns 1 if compression parameters are such that CDict hashtable and chaintable indices are tagged. + * If so, the tags need to be removed in ZSTD_resetCCtx_byCopyingCDict. */ +static int ZSTD_CDictIndicesAreTagged(const ZSTD_compressionParameters* const cParams) { + return cParams->strategy == ZSTD_fast || cParams->strategy == ZSTD_dfast; +} + +static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams( + ZSTD_compressionParameters cParams) +{ + ZSTD_CCtx_params cctxParams; + /* should not matter, as all cParams are presumed properly defined */ + ZSTD_CCtxParams_init(&cctxParams, ZSTD_CLEVEL_DEFAULT); + cctxParams.cParams = cParams; + + /* Adjust advanced params according to cParams */ + cctxParams.ldmParams.enableLdm = ZSTD_resolveEnableLdm(cctxParams.ldmParams.enableLdm, &cParams); + if (cctxParams.ldmParams.enableLdm == ZSTD_ps_enable) { + ZSTD_ldm_adjustParameters(&cctxParams.ldmParams, &cParams); + assert(cctxParams.ldmParams.hashLog >= cctxParams.ldmParams.bucketSizeLog); + assert(cctxParams.ldmParams.hashRateLog < 32); + } + cctxParams.postBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams.postBlockSplitter, &cParams); + cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams); + cctxParams.validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams.validateSequences); + cctxParams.maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams.maxBlockSize); + cctxParams.searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(cctxParams.searchForExternalRepcodes, + cctxParams.compressionLevel); + assert(!ZSTD_checkCParams(cParams)); + return cctxParams; +} + +static ZSTD_CCtx_params* ZSTD_createCCtxParams_advanced( + ZSTD_customMem customMem) +{ + ZSTD_CCtx_params* params; + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + params = (ZSTD_CCtx_params*)ZSTD_customCalloc( + sizeof(ZSTD_CCtx_params), customMem); + if (!params) { return NULL; } + ZSTD_CCtxParams_init(params, ZSTD_CLEVEL_DEFAULT); + params->customMem = customMem; + return params; +} -size_t ZSTD_compressBlock_btlazy2_dictMatchState( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); -size_t ZSTD_compressBlock_lazy2_dictMatchState( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); -size_t ZSTD_compressBlock_lazy_dictMatchState( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); -size_t ZSTD_compressBlock_greedy_dictMatchState( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); +ZSTD_CCtx_params* ZSTD_createCCtxParams(void) +{ + return ZSTD_createCCtxParams_advanced(ZSTD_defaultCMem); +} -size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); -size_t ZSTD_compressBlock_lazy_dedicatedDictSearch( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); -size_t ZSTD_compressBlock_greedy_dedicatedDictSearch( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); +size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params) +{ + if (params == NULL) { return 0; } + ZSTD_customFree(params, params->customMem); + return 0; +} -size_t ZSTD_compressBlock_greedy_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); -size_t ZSTD_compressBlock_lazy_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); -size_t ZSTD_compressBlock_lazy2_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); -size_t ZSTD_compressBlock_btlazy2_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); +size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params) +{ + return ZSTD_CCtxParams_init(params, ZSTD_CLEVEL_DEFAULT); +} -#if defined (__cplusplus) +size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel) { + RETURN_ERROR_IF(!cctxParams, GENERIC, "NULL pointer!"); + ZSTD_memset(cctxParams, 0, sizeof(*cctxParams)); + cctxParams->compressionLevel = compressionLevel; + cctxParams->fParams.contentSizeFlag = 1; + return 0; } -#endif -#endif /* ZSTD_LAZY_H */ -/**** ended inlining zstd_lazy.h ****/ -/**** start inlining zstd_opt.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. +#define ZSTD_NO_CLEVEL 0 + +/** + * Initializes `cctxParams` from `params` and `compressionLevel`. + * @param compressionLevel If params are derived from a compression level then that compression level, otherwise ZSTD_NO_CLEVEL. */ +static void +ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams, + const ZSTD_parameters* params, + int compressionLevel) +{ + assert(!ZSTD_checkCParams(params->cParams)); + ZSTD_memset(cctxParams, 0, sizeof(*cctxParams)); + cctxParams->cParams = params->cParams; + cctxParams->fParams = params->fParams; + /* Should not matter, as all cParams are presumed properly defined. + * But, set it for tracing anyway. + */ + cctxParams->compressionLevel = compressionLevel; + cctxParams->useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams->useRowMatchFinder, ¶ms->cParams); + cctxParams->postBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams->postBlockSplitter, ¶ms->cParams); + cctxParams->ldmParams.enableLdm = ZSTD_resolveEnableLdm(cctxParams->ldmParams.enableLdm, ¶ms->cParams); + cctxParams->validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams->validateSequences); + cctxParams->maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams->maxBlockSize); + cctxParams->searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(cctxParams->searchForExternalRepcodes, compressionLevel); + DEBUGLOG(4, "ZSTD_CCtxParams_init_internal: useRowMatchFinder=%d, useBlockSplitter=%d ldm=%d", + cctxParams->useRowMatchFinder, cctxParams->postBlockSplitter, cctxParams->ldmParams.enableLdm); +} -#ifndef ZSTD_OPT_H -#define ZSTD_OPT_H +size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params) +{ + RETURN_ERROR_IF(!cctxParams, GENERIC, "NULL pointer!"); + FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) , ""); + ZSTD_CCtxParams_init_internal(cctxParams, ¶ms, ZSTD_NO_CLEVEL); + return 0; +} -#if defined (__cplusplus) -extern "C" { -#endif +/** + * Sets cctxParams' cParams and fParams from params, but otherwise leaves them alone. + * @param params Validated zstd parameters. + */ +static void ZSTD_CCtxParams_setZstdParams( + ZSTD_CCtx_params* cctxParams, const ZSTD_parameters* params) +{ + assert(!ZSTD_checkCParams(params->cParams)); + cctxParams->cParams = params->cParams; + cctxParams->fParams = params->fParams; + /* Should not matter, as all cParams are presumed properly defined. + * But, set it for tracing anyway. + */ + cctxParams->compressionLevel = ZSTD_NO_CLEVEL; +} -/**** skipping file: zstd_compress_internal.h ****/ +ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) +{ + ZSTD_bounds bounds = { 0, 0, 0 }; -/* used in ZSTD_loadDictionaryContent() */ -void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend); + switch(param) + { + case ZSTD_c_compressionLevel: + bounds.lowerBound = ZSTD_minCLevel(); + bounds.upperBound = ZSTD_maxCLevel(); + return bounds; -size_t ZSTD_compressBlock_btopt( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); -size_t ZSTD_compressBlock_btultra( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); -size_t ZSTD_compressBlock_btultra2( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); + case ZSTD_c_windowLog: + bounds.lowerBound = ZSTD_WINDOWLOG_MIN; + bounds.upperBound = ZSTD_WINDOWLOG_MAX; + return bounds; + case ZSTD_c_hashLog: + bounds.lowerBound = ZSTD_HASHLOG_MIN; + bounds.upperBound = ZSTD_HASHLOG_MAX; + return bounds; -size_t ZSTD_compressBlock_btopt_dictMatchState( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); -size_t ZSTD_compressBlock_btultra_dictMatchState( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); + case ZSTD_c_chainLog: + bounds.lowerBound = ZSTD_CHAINLOG_MIN; + bounds.upperBound = ZSTD_CHAINLOG_MAX; + return bounds; -size_t ZSTD_compressBlock_btopt_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); -size_t ZSTD_compressBlock_btultra_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); + case ZSTD_c_searchLog: + bounds.lowerBound = ZSTD_SEARCHLOG_MIN; + bounds.upperBound = ZSTD_SEARCHLOG_MAX; + return bounds; - /* note : no btultra2 variant for extDict nor dictMatchState, - * because btultra2 is not meant to work with dictionaries - * and is only specific for the first block (no prefix) */ + case ZSTD_c_minMatch: + bounds.lowerBound = ZSTD_MINMATCH_MIN; + bounds.upperBound = ZSTD_MINMATCH_MAX; + return bounds; -#if defined (__cplusplus) -} -#endif + case ZSTD_c_targetLength: + bounds.lowerBound = ZSTD_TARGETLENGTH_MIN; + bounds.upperBound = ZSTD_TARGETLENGTH_MAX; + return bounds; -#endif /* ZSTD_OPT_H */ -/**** ended inlining zstd_opt.h ****/ -/**** start inlining zstd_ldm.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ + case ZSTD_c_strategy: + bounds.lowerBound = ZSTD_STRATEGY_MIN; + bounds.upperBound = ZSTD_STRATEGY_MAX; + return bounds; -#ifndef ZSTD_LDM_H -#define ZSTD_LDM_H + case ZSTD_c_contentSizeFlag: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; -#if defined (__cplusplus) -extern "C" { -#endif + case ZSTD_c_checksumFlag: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; -/**** skipping file: zstd_compress_internal.h ****/ -/**** skipping file: ../zstd.h ****/ + case ZSTD_c_dictIDFlag: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; -/*-************************************* -* Long distance matching -***************************************/ + case ZSTD_c_nbWorkers: + bounds.lowerBound = 0; +#ifdef ZSTD_MULTITHREAD + bounds.upperBound = ZSTDMT_NBWORKERS_MAX; +#else + bounds.upperBound = 0; +#endif + return bounds; -#define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_LIMIT_DEFAULT + case ZSTD_c_jobSize: + bounds.lowerBound = 0; +#ifdef ZSTD_MULTITHREAD + bounds.upperBound = ZSTDMT_JOBSIZE_MAX; +#else + bounds.upperBound = 0; +#endif + return bounds; -void ZSTD_ldm_fillHashTable( - ldmState_t* state, const BYTE* ip, - const BYTE* iend, ldmParams_t const* params); + case ZSTD_c_overlapLog: +#ifdef ZSTD_MULTITHREAD + bounds.lowerBound = ZSTD_OVERLAPLOG_MIN; + bounds.upperBound = ZSTD_OVERLAPLOG_MAX; +#else + bounds.lowerBound = 0; + bounds.upperBound = 0; +#endif + return bounds; -/** - * ZSTD_ldm_generateSequences(): - * - * Generates the sequences using the long distance match finder. - * Generates long range matching sequences in `sequences`, which parse a prefix - * of the source. `sequences` must be large enough to store every sequence, - * which can be checked with `ZSTD_ldm_getMaxNbSeq()`. - * @returns 0 or an error code. - * - * NOTE: The user must have called ZSTD_window_update() for all of the input - * they have, even if they pass it to ZSTD_ldm_generateSequences() in chunks. - * NOTE: This function returns an error if it runs out of space to store - * sequences. - */ -size_t ZSTD_ldm_generateSequences( - ldmState_t* ldms, rawSeqStore_t* sequences, - ldmParams_t const* params, void const* src, size_t srcSize); + case ZSTD_c_enableDedicatedDictSearch: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; -/** - * ZSTD_ldm_blockCompress(): - * - * Compresses a block using the predefined sequences, along with a secondary - * block compressor. The literals section of every sequence is passed to the - * secondary block compressor, and those sequences are interspersed with the - * predefined sequences. Returns the length of the last literals. - * Updates `rawSeqStore.pos` to indicate how many sequences have been consumed. - * `rawSeqStore.seq` may also be updated to split the last sequence between two - * blocks. - * @return The length of the last literals. - * - * NOTE: The source must be at most the maximum block size, but the predefined - * sequences can be any size, and may be longer than the block. In the case that - * they are longer than the block, the last sequences may need to be split into - * two. We handle that case correctly, and update `rawSeqStore` appropriately. - * NOTE: This function does not return any errors. - */ -size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); + case ZSTD_c_enableLongDistanceMatching: + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; + return bounds; -/** - * ZSTD_ldm_skipSequences(): - * - * Skip past `srcSize` bytes worth of sequences in `rawSeqStore`. - * Avoids emitting matches less than `minMatch` bytes. - * Must be called for data that is not passed to ZSTD_ldm_blockCompress(). - */ -void ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, - U32 const minMatch); + case ZSTD_c_ldmHashLog: + bounds.lowerBound = ZSTD_LDM_HASHLOG_MIN; + bounds.upperBound = ZSTD_LDM_HASHLOG_MAX; + return bounds; -/* ZSTD_ldm_skipRawSeqStoreBytes(): - * Moves forward in rawSeqStore by nbBytes, updating fields 'pos' and 'posInSequence'. - * Not to be used in conjunction with ZSTD_ldm_skipSequences(). - * Must be called for data with is not passed to ZSTD_ldm_blockCompress(). - */ -void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes); + case ZSTD_c_ldmMinMatch: + bounds.lowerBound = ZSTD_LDM_MINMATCH_MIN; + bounds.upperBound = ZSTD_LDM_MINMATCH_MAX; + return bounds; -/** ZSTD_ldm_getTableSize() : - * Estimate the space needed for long distance matching tables or 0 if LDM is - * disabled. - */ -size_t ZSTD_ldm_getTableSize(ldmParams_t params); + case ZSTD_c_ldmBucketSizeLog: + bounds.lowerBound = ZSTD_LDM_BUCKETSIZELOG_MIN; + bounds.upperBound = ZSTD_LDM_BUCKETSIZELOG_MAX; + return bounds; -/** ZSTD_ldm_getSeqSpace() : - * Return an upper bound on the number of sequences that can be produced by - * the long distance matcher, or 0 if LDM is disabled. - */ -size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize); + case ZSTD_c_ldmHashRateLog: + bounds.lowerBound = ZSTD_LDM_HASHRATELOG_MIN; + bounds.upperBound = ZSTD_LDM_HASHRATELOG_MAX; + return bounds; -/** ZSTD_ldm_adjustParameters() : - * If the params->hashRateLog is not set, set it to its default value based on - * windowLog and params->hashLog. - * - * Ensures that params->bucketSizeLog is <= params->hashLog (setting it to - * params->hashLog if it is not). - * - * Ensures that the minMatchLength >= targetLength during optimal parsing. - */ -void ZSTD_ldm_adjustParameters(ldmParams_t* params, - ZSTD_compressionParameters const* cParams); + /* experimental parameters */ + case ZSTD_c_rsyncable: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; -#if defined (__cplusplus) -} -#endif + case ZSTD_c_forceMaxWindow : + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; -#endif /* ZSTD_FAST_H */ -/**** ended inlining zstd_ldm.h ****/ -/**** skipping file: zstd_compress_superblock.h ****/ + case ZSTD_c_format: + ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless); + bounds.lowerBound = ZSTD_f_zstd1; + bounds.upperBound = ZSTD_f_zstd1_magicless; /* note : how to ensure at compile time that this is the highest value enum ? */ + return bounds; -/* *************************************************************** -* Tuning parameters -*****************************************************************/ -/*! - * COMPRESS_HEAPMODE : - * Select how default decompression function ZSTD_compress() allocates its context, - * on stack (0, default), or into heap (1). - * Note that functions with explicit context such as ZSTD_compressCCtx() are unaffected. - */ -#ifndef ZSTD_COMPRESS_HEAPMODE -# define ZSTD_COMPRESS_HEAPMODE 0 -#endif + case ZSTD_c_forceAttachDict: + ZSTD_STATIC_ASSERT(ZSTD_dictDefaultAttach < ZSTD_dictForceLoad); + bounds.lowerBound = ZSTD_dictDefaultAttach; + bounds.upperBound = ZSTD_dictForceLoad; /* note : how to ensure at compile time that this is the highest value enum ? */ + return bounds; + case ZSTD_c_literalCompressionMode: + ZSTD_STATIC_ASSERT(ZSTD_ps_auto < ZSTD_ps_enable && ZSTD_ps_enable < ZSTD_ps_disable); + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; + return bounds; -/*-************************************* -* Helper functions -***************************************/ -/* ZSTD_compressBound() - * Note that the result from this function is only compatible with the "normal" - * full-block strategy. - * When there are a lot of small blocks due to frequent flush in streaming mode - * the overhead of headers can make the compressed data to be larger than the - * return value of ZSTD_compressBound(). - */ -size_t ZSTD_compressBound(size_t srcSize) { - return ZSTD_COMPRESSBOUND(srcSize); -} + case ZSTD_c_targetCBlockSize: + bounds.lowerBound = ZSTD_TARGETCBLOCKSIZE_MIN; + bounds.upperBound = ZSTD_TARGETCBLOCKSIZE_MAX; + return bounds; + case ZSTD_c_srcSizeHint: + bounds.lowerBound = ZSTD_SRCSIZEHINT_MIN; + bounds.upperBound = ZSTD_SRCSIZEHINT_MAX; + return bounds; -/*-************************************* -* Context memory management -***************************************/ -struct ZSTD_CDict_s { - const void* dictContent; - size_t dictContentSize; - ZSTD_dictContentType_e dictContentType; /* The dictContentType the CDict was created with */ - U32* entropyWorkspace; /* entropy workspace of HUF_WORKSPACE_SIZE bytes */ - ZSTD_cwksp workspace; - ZSTD_matchState_t matchState; - ZSTD_compressedBlockState_t cBlockState; - ZSTD_customMem customMem; - U32 dictID; - int compressionLevel; /* 0 indicates that advanced API was used to select CDict params */ -}; /* typedef'd to ZSTD_CDict within "zstd.h" */ + case ZSTD_c_stableInBuffer: + case ZSTD_c_stableOutBuffer: + bounds.lowerBound = (int)ZSTD_bm_buffered; + bounds.upperBound = (int)ZSTD_bm_stable; + return bounds; -ZSTD_CCtx* ZSTD_createCCtx(void) -{ - return ZSTD_createCCtx_advanced(ZSTD_defaultCMem); -} + case ZSTD_c_blockDelimiters: + bounds.lowerBound = (int)ZSTD_sf_noBlockDelimiters; + bounds.upperBound = (int)ZSTD_sf_explicitBlockDelimiters; + return bounds; -static void ZSTD_initCCtx(ZSTD_CCtx* cctx, ZSTD_customMem memManager) -{ - assert(cctx != NULL); - ZSTD_memset(cctx, 0, sizeof(*cctx)); - cctx->customMem = memManager; - cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); - { size_t const err = ZSTD_CCtx_reset(cctx, ZSTD_reset_parameters); - assert(!ZSTD_isError(err)); - (void)err; - } -} + case ZSTD_c_validateSequences: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; -ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) -{ - ZSTD_STATIC_ASSERT(zcss_init==0); - ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN==(0ULL - 1)); - if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; - { ZSTD_CCtx* const cctx = (ZSTD_CCtx*)ZSTD_customMalloc(sizeof(ZSTD_CCtx), customMem); - if (!cctx) return NULL; - ZSTD_initCCtx(cctx, customMem); - return cctx; - } -} + case ZSTD_c_splitAfterSequences: + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; + return bounds; -ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize) -{ - ZSTD_cwksp ws; - ZSTD_CCtx* cctx; - if (workspaceSize <= sizeof(ZSTD_CCtx)) return NULL; /* minimum size */ - if ((size_t)workspace & 7) return NULL; /* must be 8-aligned */ - ZSTD_cwksp_init(&ws, workspace, workspaceSize, ZSTD_cwksp_static_alloc); + case ZSTD_c_blockSplitterLevel: + bounds.lowerBound = 0; + bounds.upperBound = ZSTD_BLOCKSPLITTER_LEVEL_MAX; + return bounds; - cctx = (ZSTD_CCtx*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CCtx)); - if (cctx == NULL) return NULL; + case ZSTD_c_useRowMatchFinder: + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; + return bounds; - ZSTD_memset(cctx, 0, sizeof(ZSTD_CCtx)); - ZSTD_cwksp_move(&cctx->workspace, &ws); - cctx->staticSize = workspaceSize; + case ZSTD_c_deterministicRefPrefix: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; - /* statically sized space. entropyWorkspace never moves (but prev/next block swap places) */ - if (!ZSTD_cwksp_check_available(&cctx->workspace, ENTROPY_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t))) return NULL; - cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); - cctx->blockState.nextCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); - cctx->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object(&cctx->workspace, ENTROPY_WORKSPACE_SIZE); - cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); - return cctx; + case ZSTD_c_prefetchCDictTables: + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; + return bounds; + + case ZSTD_c_enableSeqProducerFallback: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_maxBlockSize: + bounds.lowerBound = ZSTD_BLOCKSIZE_MAX_MIN; + bounds.upperBound = ZSTD_BLOCKSIZE_MAX; + return bounds; + + case ZSTD_c_repcodeResolution: + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; + return bounds; + + default: + bounds.error = ERROR(parameter_unsupported); + return bounds; + } } -/** - * Clears and frees all of the dictionaries in the CCtx. +/* ZSTD_cParam_clampBounds: + * Clamps the value into the bounded range. */ -static void ZSTD_clearAllDicts(ZSTD_CCtx* cctx) +static size_t ZSTD_cParam_clampBounds(ZSTD_cParameter cParam, int* value) { - ZSTD_customFree(cctx->localDict.dictBuffer, cctx->customMem); - ZSTD_freeCDict(cctx->localDict.cdict); - ZSTD_memset(&cctx->localDict, 0, sizeof(cctx->localDict)); - ZSTD_memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); - cctx->cdict = NULL; + ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); + if (ZSTD_isError(bounds.error)) return bounds.error; + if (*value < bounds.lowerBound) *value = bounds.lowerBound; + if (*value > bounds.upperBound) *value = bounds.upperBound; + return 0; } -static size_t ZSTD_sizeof_localDict(ZSTD_localDict dict) +#define BOUNDCHECK(cParam, val) \ + do { \ + RETURN_ERROR_IF(!ZSTD_cParam_withinBounds(cParam,val), \ + parameter_outOfBound, "Param out of bounds"); \ + } while (0) + + +static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param) { - size_t const bufferSize = dict.dictBuffer != NULL ? dict.dictSize : 0; - size_t const cdictSize = ZSTD_sizeof_CDict(dict.cdict); - return bufferSize + cdictSize; + switch(param) + { + case ZSTD_c_compressionLevel: + case ZSTD_c_hashLog: + case ZSTD_c_chainLog: + case ZSTD_c_searchLog: + case ZSTD_c_minMatch: + case ZSTD_c_targetLength: + case ZSTD_c_strategy: + case ZSTD_c_blockSplitterLevel: + return 1; + + case ZSTD_c_format: + case ZSTD_c_windowLog: + case ZSTD_c_contentSizeFlag: + case ZSTD_c_checksumFlag: + case ZSTD_c_dictIDFlag: + case ZSTD_c_forceMaxWindow : + case ZSTD_c_nbWorkers: + case ZSTD_c_jobSize: + case ZSTD_c_overlapLog: + case ZSTD_c_rsyncable: + case ZSTD_c_enableDedicatedDictSearch: + case ZSTD_c_enableLongDistanceMatching: + case ZSTD_c_ldmHashLog: + case ZSTD_c_ldmMinMatch: + case ZSTD_c_ldmBucketSizeLog: + case ZSTD_c_ldmHashRateLog: + case ZSTD_c_forceAttachDict: + case ZSTD_c_literalCompressionMode: + case ZSTD_c_targetCBlockSize: + case ZSTD_c_srcSizeHint: + case ZSTD_c_stableInBuffer: + case ZSTD_c_stableOutBuffer: + case ZSTD_c_blockDelimiters: + case ZSTD_c_validateSequences: + case ZSTD_c_splitAfterSequences: + case ZSTD_c_useRowMatchFinder: + case ZSTD_c_deterministicRefPrefix: + case ZSTD_c_prefetchCDictTables: + case ZSTD_c_enableSeqProducerFallback: + case ZSTD_c_maxBlockSize: + case ZSTD_c_repcodeResolution: + default: + return 0; + } } -static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx) +size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value) { - assert(cctx != NULL); - assert(cctx->staticSize == 0); - ZSTD_clearAllDicts(cctx); -#ifdef ZSTD_MULTITHREAD - ZSTDMT_freeCCtx(cctx->mtctx); cctx->mtctx = NULL; -#endif - ZSTD_cwksp_free(&cctx->workspace, cctx->customMem); + DEBUGLOG(4, "ZSTD_CCtx_setParameter (%i, %i)", (int)param, value); + if (cctx->streamStage != zcss_init) { + if (ZSTD_isUpdateAuthorized(param)) { + cctx->cParamsChanged = 1; + } else { + RETURN_ERROR(stage_wrong, "can only set params in cctx init stage"); + } } + + switch(param) + { + case ZSTD_c_nbWorkers: + RETURN_ERROR_IF((value!=0) && cctx->staticSize, parameter_unsupported, + "MT not compatible with static alloc"); + break; + + case ZSTD_c_compressionLevel: + case ZSTD_c_windowLog: + case ZSTD_c_hashLog: + case ZSTD_c_chainLog: + case ZSTD_c_searchLog: + case ZSTD_c_minMatch: + case ZSTD_c_targetLength: + case ZSTD_c_strategy: + case ZSTD_c_ldmHashRateLog: + case ZSTD_c_format: + case ZSTD_c_contentSizeFlag: + case ZSTD_c_checksumFlag: + case ZSTD_c_dictIDFlag: + case ZSTD_c_forceMaxWindow: + case ZSTD_c_forceAttachDict: + case ZSTD_c_literalCompressionMode: + case ZSTD_c_jobSize: + case ZSTD_c_overlapLog: + case ZSTD_c_rsyncable: + case ZSTD_c_enableDedicatedDictSearch: + case ZSTD_c_enableLongDistanceMatching: + case ZSTD_c_ldmHashLog: + case ZSTD_c_ldmMinMatch: + case ZSTD_c_ldmBucketSizeLog: + case ZSTD_c_targetCBlockSize: + case ZSTD_c_srcSizeHint: + case ZSTD_c_stableInBuffer: + case ZSTD_c_stableOutBuffer: + case ZSTD_c_blockDelimiters: + case ZSTD_c_validateSequences: + case ZSTD_c_splitAfterSequences: + case ZSTD_c_blockSplitterLevel: + case ZSTD_c_useRowMatchFinder: + case ZSTD_c_deterministicRefPrefix: + case ZSTD_c_prefetchCDictTables: + case ZSTD_c_enableSeqProducerFallback: + case ZSTD_c_maxBlockSize: + case ZSTD_c_repcodeResolution: + break; + + default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); + } + return ZSTD_CCtxParams_setParameter(&cctx->requestedParams, param, value); } -size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) +size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, + ZSTD_cParameter param, int value) { - if (cctx==NULL) return 0; /* support free on NULL */ - RETURN_ERROR_IF(cctx->staticSize, memory_allocation, - "not compatible with static CCtx"); + DEBUGLOG(4, "ZSTD_CCtxParams_setParameter (%i, %i)", (int)param, value); + switch(param) { - int cctxInWorkspace = ZSTD_cwksp_owns_buffer(&cctx->workspace, cctx); - ZSTD_freeCCtxContent(cctx); - if (!cctxInWorkspace) { - ZSTD_customFree(cctx, cctx->customMem); - } + case ZSTD_c_format : + BOUNDCHECK(ZSTD_c_format, value); + CCtxParams->format = (ZSTD_format_e)value; + return (size_t)CCtxParams->format; + + case ZSTD_c_compressionLevel : { + FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), ""); + if (value == 0) + CCtxParams->compressionLevel = ZSTD_CLEVEL_DEFAULT; /* 0 == default */ + else + CCtxParams->compressionLevel = value; + if (CCtxParams->compressionLevel >= 0) return (size_t)CCtxParams->compressionLevel; + return 0; /* return type (size_t) cannot represent negative values */ } - return 0; -} + case ZSTD_c_windowLog : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_windowLog, value); + CCtxParams->cParams.windowLog = (U32)value; + return CCtxParams->cParams.windowLog; -static size_t ZSTD_sizeof_mtctx(const ZSTD_CCtx* cctx) -{ -#ifdef ZSTD_MULTITHREAD - return ZSTDMT_sizeof_CCtx(cctx->mtctx); -#else - (void)cctx; - return 0; -#endif -} + case ZSTD_c_hashLog : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_hashLog, value); + CCtxParams->cParams.hashLog = (U32)value; + return CCtxParams->cParams.hashLog; + case ZSTD_c_chainLog : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_chainLog, value); + CCtxParams->cParams.chainLog = (U32)value; + return CCtxParams->cParams.chainLog; -size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx) -{ - if (cctx==NULL) return 0; /* support sizeof on NULL */ - /* cctx may be in the workspace */ - return (cctx->workspace.workspace == cctx ? 0 : sizeof(*cctx)) - + ZSTD_cwksp_sizeof(&cctx->workspace) - + ZSTD_sizeof_localDict(cctx->localDict) - + ZSTD_sizeof_mtctx(cctx); -} + case ZSTD_c_searchLog : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_searchLog, value); + CCtxParams->cParams.searchLog = (U32)value; + return (size_t)value; -size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs) -{ - return ZSTD_sizeof_CCtx(zcs); /* same object */ -} + case ZSTD_c_minMatch : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_minMatch, value); + CCtxParams->cParams.minMatch = (U32)value; + return CCtxParams->cParams.minMatch; -/* private API call, for dictBuilder only */ -const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStore); } + case ZSTD_c_targetLength : + BOUNDCHECK(ZSTD_c_targetLength, value); + CCtxParams->cParams.targetLength = (U32)value; + return CCtxParams->cParams.targetLength; -/* Returns 1 if compression parameters are such that we should - * enable long distance matching (wlog >= 27, strategy >= btopt). - * Returns 0 otherwise. - */ -static U32 ZSTD_CParams_shouldEnableLdm(const ZSTD_compressionParameters* const cParams) { - return cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 27; -} + case ZSTD_c_strategy : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_strategy, value); + CCtxParams->cParams.strategy = (ZSTD_strategy)value; + return (size_t)CCtxParams->cParams.strategy; -static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams( - ZSTD_compressionParameters cParams) -{ - ZSTD_CCtx_params cctxParams; - /* should not matter, as all cParams are presumed properly defined */ - ZSTD_CCtxParams_init(&cctxParams, ZSTD_CLEVEL_DEFAULT); - cctxParams.cParams = cParams; + case ZSTD_c_contentSizeFlag : + /* Content size written in frame header _when known_ (default:1) */ + DEBUGLOG(4, "set content size flag = %u", (value!=0)); + CCtxParams->fParams.contentSizeFlag = value != 0; + return (size_t)CCtxParams->fParams.contentSizeFlag; - if (ZSTD_CParams_shouldEnableLdm(&cParams)) { - DEBUGLOG(4, "ZSTD_makeCCtxParamsFromCParams(): Including LDM into cctx params"); - cctxParams.ldmParams.enableLdm = 1; - /* LDM is enabled by default for optimal parser and window size >= 128MB */ - ZSTD_ldm_adjustParameters(&cctxParams.ldmParams, &cParams); - assert(cctxParams.ldmParams.hashLog >= cctxParams.ldmParams.bucketSizeLog); - assert(cctxParams.ldmParams.hashRateLog < 32); - } + case ZSTD_c_checksumFlag : + /* A 32-bits content checksum will be calculated and written at end of frame (default:0) */ + CCtxParams->fParams.checksumFlag = value != 0; + return (size_t)CCtxParams->fParams.checksumFlag; - assert(!ZSTD_checkCParams(cParams)); - return cctxParams; -} + case ZSTD_c_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */ + DEBUGLOG(4, "set dictIDFlag = %u", (value!=0)); + CCtxParams->fParams.noDictIDFlag = !value; + return !CCtxParams->fParams.noDictIDFlag; -static ZSTD_CCtx_params* ZSTD_createCCtxParams_advanced( - ZSTD_customMem customMem) -{ - ZSTD_CCtx_params* params; - if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; - params = (ZSTD_CCtx_params*)ZSTD_customCalloc( - sizeof(ZSTD_CCtx_params), customMem); - if (!params) { return NULL; } - ZSTD_CCtxParams_init(params, ZSTD_CLEVEL_DEFAULT); - params->customMem = customMem; - return params; -} + case ZSTD_c_forceMaxWindow : + CCtxParams->forceWindow = (value != 0); + return (size_t)CCtxParams->forceWindow; -ZSTD_CCtx_params* ZSTD_createCCtxParams(void) -{ - return ZSTD_createCCtxParams_advanced(ZSTD_defaultCMem); -} + case ZSTD_c_forceAttachDict : { + const ZSTD_dictAttachPref_e pref = (ZSTD_dictAttachPref_e)value; + BOUNDCHECK(ZSTD_c_forceAttachDict, (int)pref); + CCtxParams->attachDictPref = pref; + return CCtxParams->attachDictPref; + } -size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params) -{ - if (params == NULL) { return 0; } - ZSTD_customFree(params, params->customMem); - return 0; -} + case ZSTD_c_literalCompressionMode : { + const ZSTD_ParamSwitch_e lcm = (ZSTD_ParamSwitch_e)value; + BOUNDCHECK(ZSTD_c_literalCompressionMode, (int)lcm); + CCtxParams->literalCompressionMode = lcm; + return CCtxParams->literalCompressionMode; + } -size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params) -{ - return ZSTD_CCtxParams_init(params, ZSTD_CLEVEL_DEFAULT); -} + case ZSTD_c_nbWorkers : +#ifndef ZSTD_MULTITHREAD + RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); + return 0; +#else + FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), ""); + CCtxParams->nbWorkers = value; + return (size_t)(CCtxParams->nbWorkers); +#endif -size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel) { - RETURN_ERROR_IF(!cctxParams, GENERIC, "NULL pointer!"); - ZSTD_memset(cctxParams, 0, sizeof(*cctxParams)); - cctxParams->compressionLevel = compressionLevel; - cctxParams->fParams.contentSizeFlag = 1; - return 0; -} + case ZSTD_c_jobSize : +#ifndef ZSTD_MULTITHREAD + RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); + return 0; +#else + /* Adjust to the minimum non-default value. */ + if (value != 0 && value < ZSTDMT_JOBSIZE_MIN) + value = ZSTDMT_JOBSIZE_MIN; + FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), ""); + assert(value >= 0); + CCtxParams->jobSize = (size_t)value; + return CCtxParams->jobSize; +#endif -#define ZSTD_NO_CLEVEL 0 + case ZSTD_c_overlapLog : +#ifndef ZSTD_MULTITHREAD + RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); + return 0; +#else + FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(ZSTD_c_overlapLog, &value), ""); + CCtxParams->overlapLog = value; + return (size_t)CCtxParams->overlapLog; +#endif -/** - * Initializes the cctxParams from params and compressionLevel. - * @param compressionLevel If params are derived from a compression level then that compression level, otherwise ZSTD_NO_CLEVEL. - */ -static void ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams, ZSTD_parameters const* params, int compressionLevel) -{ - assert(!ZSTD_checkCParams(params->cParams)); - ZSTD_memset(cctxParams, 0, sizeof(*cctxParams)); - cctxParams->cParams = params->cParams; - cctxParams->fParams = params->fParams; - /* Should not matter, as all cParams are presumed properly defined. - * But, set it for tracing anyway. - */ - cctxParams->compressionLevel = compressionLevel; -} + case ZSTD_c_rsyncable : +#ifndef ZSTD_MULTITHREAD + RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); + return 0; +#else + FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(ZSTD_c_overlapLog, &value), ""); + CCtxParams->rsyncable = value; + return (size_t)CCtxParams->rsyncable; +#endif -size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params) -{ - RETURN_ERROR_IF(!cctxParams, GENERIC, "NULL pointer!"); - FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) , ""); - ZSTD_CCtxParams_init_internal(cctxParams, ¶ms, ZSTD_NO_CLEVEL); - return 0; -} + case ZSTD_c_enableDedicatedDictSearch : + CCtxParams->enableDedicatedDictSearch = (value!=0); + return (size_t)CCtxParams->enableDedicatedDictSearch; -/** - * Sets cctxParams' cParams and fParams from params, but otherwise leaves them alone. - * @param param Validated zstd parameters. - */ -static void ZSTD_CCtxParams_setZstdParams( - ZSTD_CCtx_params* cctxParams, const ZSTD_parameters* params) -{ - assert(!ZSTD_checkCParams(params->cParams)); - cctxParams->cParams = params->cParams; - cctxParams->fParams = params->fParams; - /* Should not matter, as all cParams are presumed properly defined. - * But, set it for tracing anyway. - */ - cctxParams->compressionLevel = ZSTD_NO_CLEVEL; -} + case ZSTD_c_enableLongDistanceMatching : + BOUNDCHECK(ZSTD_c_enableLongDistanceMatching, value); + CCtxParams->ldmParams.enableLdm = (ZSTD_ParamSwitch_e)value; + return CCtxParams->ldmParams.enableLdm; -ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) -{ - ZSTD_bounds bounds = { 0, 0, 0 }; + case ZSTD_c_ldmHashLog : + if (value!=0) /* 0 ==> auto */ + BOUNDCHECK(ZSTD_c_ldmHashLog, value); + CCtxParams->ldmParams.hashLog = (U32)value; + return CCtxParams->ldmParams.hashLog; - switch(param) - { - case ZSTD_c_compressionLevel: - bounds.lowerBound = ZSTD_minCLevel(); - bounds.upperBound = ZSTD_maxCLevel(); - return bounds; + case ZSTD_c_ldmMinMatch : + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_ldmMinMatch, value); + CCtxParams->ldmParams.minMatchLength = (U32)value; + return CCtxParams->ldmParams.minMatchLength; - case ZSTD_c_windowLog: - bounds.lowerBound = ZSTD_WINDOWLOG_MIN; - bounds.upperBound = ZSTD_WINDOWLOG_MAX; - return bounds; + case ZSTD_c_ldmBucketSizeLog : + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_ldmBucketSizeLog, value); + CCtxParams->ldmParams.bucketSizeLog = (U32)value; + return CCtxParams->ldmParams.bucketSizeLog; - case ZSTD_c_hashLog: - bounds.lowerBound = ZSTD_HASHLOG_MIN; - bounds.upperBound = ZSTD_HASHLOG_MAX; - return bounds; + case ZSTD_c_ldmHashRateLog : + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_ldmHashRateLog, value); + CCtxParams->ldmParams.hashRateLog = (U32)value; + return CCtxParams->ldmParams.hashRateLog; - case ZSTD_c_chainLog: - bounds.lowerBound = ZSTD_CHAINLOG_MIN; - bounds.upperBound = ZSTD_CHAINLOG_MAX; - return bounds; + case ZSTD_c_targetCBlockSize : + if (value!=0) { /* 0 ==> default */ + value = MAX(value, ZSTD_TARGETCBLOCKSIZE_MIN); + BOUNDCHECK(ZSTD_c_targetCBlockSize, value); + } + CCtxParams->targetCBlockSize = (U32)value; + return CCtxParams->targetCBlockSize; - case ZSTD_c_searchLog: - bounds.lowerBound = ZSTD_SEARCHLOG_MIN; - bounds.upperBound = ZSTD_SEARCHLOG_MAX; - return bounds; + case ZSTD_c_srcSizeHint : + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_srcSizeHint, value); + CCtxParams->srcSizeHint = value; + return (size_t)CCtxParams->srcSizeHint; - case ZSTD_c_minMatch: - bounds.lowerBound = ZSTD_MINMATCH_MIN; - bounds.upperBound = ZSTD_MINMATCH_MAX; - return bounds; + case ZSTD_c_stableInBuffer: + BOUNDCHECK(ZSTD_c_stableInBuffer, value); + CCtxParams->inBufferMode = (ZSTD_bufferMode_e)value; + return CCtxParams->inBufferMode; - case ZSTD_c_targetLength: - bounds.lowerBound = ZSTD_TARGETLENGTH_MIN; - bounds.upperBound = ZSTD_TARGETLENGTH_MAX; - return bounds; + case ZSTD_c_stableOutBuffer: + BOUNDCHECK(ZSTD_c_stableOutBuffer, value); + CCtxParams->outBufferMode = (ZSTD_bufferMode_e)value; + return CCtxParams->outBufferMode; - case ZSTD_c_strategy: - bounds.lowerBound = ZSTD_STRATEGY_MIN; - bounds.upperBound = ZSTD_STRATEGY_MAX; - return bounds; + case ZSTD_c_blockDelimiters: + BOUNDCHECK(ZSTD_c_blockDelimiters, value); + CCtxParams->blockDelimiters = (ZSTD_SequenceFormat_e)value; + return CCtxParams->blockDelimiters; - case ZSTD_c_contentSizeFlag: - bounds.lowerBound = 0; - bounds.upperBound = 1; - return bounds; + case ZSTD_c_validateSequences: + BOUNDCHECK(ZSTD_c_validateSequences, value); + CCtxParams->validateSequences = value; + return (size_t)CCtxParams->validateSequences; + + case ZSTD_c_splitAfterSequences: + BOUNDCHECK(ZSTD_c_splitAfterSequences, value); + CCtxParams->postBlockSplitter = (ZSTD_ParamSwitch_e)value; + return CCtxParams->postBlockSplitter; + + case ZSTD_c_blockSplitterLevel: + BOUNDCHECK(ZSTD_c_blockSplitterLevel, value); + CCtxParams->preBlockSplitter_level = value; + return (size_t)CCtxParams->preBlockSplitter_level; + + case ZSTD_c_useRowMatchFinder: + BOUNDCHECK(ZSTD_c_useRowMatchFinder, value); + CCtxParams->useRowMatchFinder = (ZSTD_ParamSwitch_e)value; + return CCtxParams->useRowMatchFinder; + + case ZSTD_c_deterministicRefPrefix: + BOUNDCHECK(ZSTD_c_deterministicRefPrefix, value); + CCtxParams->deterministicRefPrefix = !!value; + return (size_t)CCtxParams->deterministicRefPrefix; + + case ZSTD_c_prefetchCDictTables: + BOUNDCHECK(ZSTD_c_prefetchCDictTables, value); + CCtxParams->prefetchCDictTables = (ZSTD_ParamSwitch_e)value; + return CCtxParams->prefetchCDictTables; + + case ZSTD_c_enableSeqProducerFallback: + BOUNDCHECK(ZSTD_c_enableSeqProducerFallback, value); + CCtxParams->enableMatchFinderFallback = value; + return (size_t)CCtxParams->enableMatchFinderFallback; + + case ZSTD_c_maxBlockSize: + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_maxBlockSize, value); + assert(value>=0); + CCtxParams->maxBlockSize = (size_t)value; + return CCtxParams->maxBlockSize; - case ZSTD_c_checksumFlag: - bounds.lowerBound = 0; - bounds.upperBound = 1; - return bounds; + case ZSTD_c_repcodeResolution: + BOUNDCHECK(ZSTD_c_repcodeResolution, value); + CCtxParams->searchForExternalRepcodes = (ZSTD_ParamSwitch_e)value; + return CCtxParams->searchForExternalRepcodes; - case ZSTD_c_dictIDFlag: - bounds.lowerBound = 0; - bounds.upperBound = 1; - return bounds; + default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); + } +} - case ZSTD_c_nbWorkers: - bounds.lowerBound = 0; -#ifdef ZSTD_MULTITHREAD - bounds.upperBound = ZSTDMT_NBWORKERS_MAX; +size_t ZSTD_CCtx_getParameter(ZSTD_CCtx const* cctx, ZSTD_cParameter param, int* value) +{ + return ZSTD_CCtxParams_getParameter(&cctx->requestedParams, param, value); +} + +size_t ZSTD_CCtxParams_getParameter( + ZSTD_CCtx_params const* CCtxParams, ZSTD_cParameter param, int* value) +{ + switch(param) + { + case ZSTD_c_format : + *value = (int)CCtxParams->format; + break; + case ZSTD_c_compressionLevel : + *value = CCtxParams->compressionLevel; + break; + case ZSTD_c_windowLog : + *value = (int)CCtxParams->cParams.windowLog; + break; + case ZSTD_c_hashLog : + *value = (int)CCtxParams->cParams.hashLog; + break; + case ZSTD_c_chainLog : + *value = (int)CCtxParams->cParams.chainLog; + break; + case ZSTD_c_searchLog : + *value = (int)CCtxParams->cParams.searchLog; + break; + case ZSTD_c_minMatch : + *value = (int)CCtxParams->cParams.minMatch; + break; + case ZSTD_c_targetLength : + *value = (int)CCtxParams->cParams.targetLength; + break; + case ZSTD_c_strategy : + *value = (int)CCtxParams->cParams.strategy; + break; + case ZSTD_c_contentSizeFlag : + *value = CCtxParams->fParams.contentSizeFlag; + break; + case ZSTD_c_checksumFlag : + *value = CCtxParams->fParams.checksumFlag; + break; + case ZSTD_c_dictIDFlag : + *value = !CCtxParams->fParams.noDictIDFlag; + break; + case ZSTD_c_forceMaxWindow : + *value = CCtxParams->forceWindow; + break; + case ZSTD_c_forceAttachDict : + *value = (int)CCtxParams->attachDictPref; + break; + case ZSTD_c_literalCompressionMode : + *value = (int)CCtxParams->literalCompressionMode; + break; + case ZSTD_c_nbWorkers : +#ifndef ZSTD_MULTITHREAD + assert(CCtxParams->nbWorkers == 0); +#endif + *value = CCtxParams->nbWorkers; + break; + case ZSTD_c_jobSize : +#ifndef ZSTD_MULTITHREAD + RETURN_ERROR(parameter_unsupported, "not compiled with multithreading"); #else - bounds.upperBound = 0; + assert(CCtxParams->jobSize <= INT_MAX); + *value = (int)CCtxParams->jobSize; + break; #endif - return bounds; + case ZSTD_c_overlapLog : +#ifndef ZSTD_MULTITHREAD + RETURN_ERROR(parameter_unsupported, "not compiled with multithreading"); +#else + *value = CCtxParams->overlapLog; + break; +#endif + case ZSTD_c_rsyncable : +#ifndef ZSTD_MULTITHREAD + RETURN_ERROR(parameter_unsupported, "not compiled with multithreading"); +#else + *value = CCtxParams->rsyncable; + break; +#endif + case ZSTD_c_enableDedicatedDictSearch : + *value = CCtxParams->enableDedicatedDictSearch; + break; + case ZSTD_c_enableLongDistanceMatching : + *value = (int)CCtxParams->ldmParams.enableLdm; + break; + case ZSTD_c_ldmHashLog : + *value = (int)CCtxParams->ldmParams.hashLog; + break; + case ZSTD_c_ldmMinMatch : + *value = (int)CCtxParams->ldmParams.minMatchLength; + break; + case ZSTD_c_ldmBucketSizeLog : + *value = (int)CCtxParams->ldmParams.bucketSizeLog; + break; + case ZSTD_c_ldmHashRateLog : + *value = (int)CCtxParams->ldmParams.hashRateLog; + break; + case ZSTD_c_targetCBlockSize : + *value = (int)CCtxParams->targetCBlockSize; + break; + case ZSTD_c_srcSizeHint : + *value = (int)CCtxParams->srcSizeHint; + break; + case ZSTD_c_stableInBuffer : + *value = (int)CCtxParams->inBufferMode; + break; + case ZSTD_c_stableOutBuffer : + *value = (int)CCtxParams->outBufferMode; + break; + case ZSTD_c_blockDelimiters : + *value = (int)CCtxParams->blockDelimiters; + break; + case ZSTD_c_validateSequences : + *value = (int)CCtxParams->validateSequences; + break; + case ZSTD_c_splitAfterSequences : + *value = (int)CCtxParams->postBlockSplitter; + break; + case ZSTD_c_blockSplitterLevel : + *value = CCtxParams->preBlockSplitter_level; + break; + case ZSTD_c_useRowMatchFinder : + *value = (int)CCtxParams->useRowMatchFinder; + break; + case ZSTD_c_deterministicRefPrefix: + *value = (int)CCtxParams->deterministicRefPrefix; + break; + case ZSTD_c_prefetchCDictTables: + *value = (int)CCtxParams->prefetchCDictTables; + break; + case ZSTD_c_enableSeqProducerFallback: + *value = CCtxParams->enableMatchFinderFallback; + break; + case ZSTD_c_maxBlockSize: + *value = (int)CCtxParams->maxBlockSize; + break; + case ZSTD_c_repcodeResolution: + *value = (int)CCtxParams->searchForExternalRepcodes; + break; + default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); + } + return 0; +} + +/** ZSTD_CCtx_setParametersUsingCCtxParams() : + * just applies `params` into `cctx` + * no action is performed, parameters are merely stored. + * If ZSTDMT is enabled, parameters are pushed to cctx->mtctx. + * This is possible even if a compression is ongoing. + * In which case, new parameters will be applied on the fly, starting with next compression job. + */ +size_t ZSTD_CCtx_setParametersUsingCCtxParams( + ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params) +{ + DEBUGLOG(4, "ZSTD_CCtx_setParametersUsingCCtxParams"); + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "The context is in the wrong stage!"); + RETURN_ERROR_IF(cctx->cdict, stage_wrong, + "Can't override parameters with cdict attached (some must " + "be inherited from the cdict)."); - case ZSTD_c_jobSize: - bounds.lowerBound = 0; -#ifdef ZSTD_MULTITHREAD - bounds.upperBound = ZSTDMT_JOBSIZE_MAX; -#else - bounds.upperBound = 0; -#endif - return bounds; + cctx->requestedParams = *params; + return 0; +} - case ZSTD_c_overlapLog: -#ifdef ZSTD_MULTITHREAD - bounds.lowerBound = ZSTD_OVERLAPLOG_MIN; - bounds.upperBound = ZSTD_OVERLAPLOG_MAX; -#else - bounds.lowerBound = 0; - bounds.upperBound = 0; -#endif - return bounds; +size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams) +{ + ZSTD_STATIC_ASSERT(sizeof(cparams) == 7 * 4 /* all params are listed below */); + DEBUGLOG(4, "ZSTD_CCtx_setCParams"); + /* only update if all parameters are valid */ + FORWARD_IF_ERROR(ZSTD_checkCParams(cparams), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, (int)cparams.windowLog), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_chainLog, (int)cparams.chainLog), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, (int)cparams.hashLog), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_searchLog, (int)cparams.searchLog), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, (int)cparams.minMatch), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetLength, (int)cparams.targetLength), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, (int)cparams.strategy), ""); + return 0; +} - case ZSTD_c_enableDedicatedDictSearch: - bounds.lowerBound = 0; - bounds.upperBound = 1; - return bounds; +size_t ZSTD_CCtx_setFParams(ZSTD_CCtx* cctx, ZSTD_frameParameters fparams) +{ + ZSTD_STATIC_ASSERT(sizeof(fparams) == 3 * 4 /* all params are listed below */); + DEBUGLOG(4, "ZSTD_CCtx_setFParams"); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, fparams.contentSizeFlag != 0), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, fparams.checksumFlag != 0), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_dictIDFlag, fparams.noDictIDFlag == 0), ""); + return 0; +} - case ZSTD_c_enableLongDistanceMatching: - bounds.lowerBound = 0; - bounds.upperBound = 1; - return bounds; +size_t ZSTD_CCtx_setParams(ZSTD_CCtx* cctx, ZSTD_parameters params) +{ + DEBUGLOG(4, "ZSTD_CCtx_setParams"); + /* First check cParams, because we want to update all or none. */ + FORWARD_IF_ERROR(ZSTD_checkCParams(params.cParams), ""); + /* Next set fParams, because this could fail if the cctx isn't in init stage. */ + FORWARD_IF_ERROR(ZSTD_CCtx_setFParams(cctx, params.fParams), ""); + /* Finally set cParams, which should succeed. */ + FORWARD_IF_ERROR(ZSTD_CCtx_setCParams(cctx, params.cParams), ""); + return 0; +} - case ZSTD_c_ldmHashLog: - bounds.lowerBound = ZSTD_LDM_HASHLOG_MIN; - bounds.upperBound = ZSTD_LDM_HASHLOG_MAX; - return bounds; +size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %llu bytes", pledgedSrcSize); + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't set pledgedSrcSize when not in init stage."); + cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1; + return 0; +} - case ZSTD_c_ldmMinMatch: - bounds.lowerBound = ZSTD_LDM_MINMATCH_MIN; - bounds.upperBound = ZSTD_LDM_MINMATCH_MAX; - return bounds; +static ZSTD_compressionParameters ZSTD_dedicatedDictSearch_getCParams( + int const compressionLevel, + size_t const dictSize); +static int ZSTD_dedicatedDictSearch_isSupported( + const ZSTD_compressionParameters* cParams); +static void ZSTD_dedicatedDictSearch_revertCParams( + ZSTD_compressionParameters* cParams); - case ZSTD_c_ldmBucketSizeLog: - bounds.lowerBound = ZSTD_LDM_BUCKETSIZELOG_MIN; - bounds.upperBound = ZSTD_LDM_BUCKETSIZELOG_MAX; - return bounds; +/** + * Initializes the local dictionary using requested parameters. + * NOTE: Initialization does not employ the pledged src size, + * because the dictionary may be used for multiple compressions. + */ +static size_t ZSTD_initLocalDict(ZSTD_CCtx* cctx) +{ + ZSTD_localDict* const dl = &cctx->localDict; + if (dl->dict == NULL) { + /* No local dictionary. */ + assert(dl->dictBuffer == NULL); + assert(dl->cdict == NULL); + assert(dl->dictSize == 0); + return 0; + } + if (dl->cdict != NULL) { + /* Local dictionary already initialized. */ + assert(cctx->cdict == dl->cdict); + return 0; + } + assert(dl->dictSize > 0); + assert(cctx->cdict == NULL); + assert(cctx->prefixDict.dict == NULL); - case ZSTD_c_ldmHashRateLog: - bounds.lowerBound = ZSTD_LDM_HASHRATELOG_MIN; - bounds.upperBound = ZSTD_LDM_HASHRATELOG_MAX; - return bounds; + dl->cdict = ZSTD_createCDict_advanced2( + dl->dict, + dl->dictSize, + ZSTD_dlm_byRef, + dl->dictContentType, + &cctx->requestedParams, + cctx->customMem); + RETURN_ERROR_IF(!dl->cdict, memory_allocation, "ZSTD_createCDict_advanced failed"); + cctx->cdict = dl->cdict; + return 0; +} - /* experimental parameters */ - case ZSTD_c_rsyncable: - bounds.lowerBound = 0; - bounds.upperBound = 1; - return bounds; +size_t ZSTD_CCtx_loadDictionary_advanced( + ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType) +{ + DEBUGLOG(4, "ZSTD_CCtx_loadDictionary_advanced (size: %u)", (U32)dictSize); + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't load a dictionary when cctx is not in init stage."); + ZSTD_clearAllDicts(cctx); /* erase any previously set dictionary */ + if (dict == NULL || dictSize == 0) /* no dictionary */ + return 0; + if (dictLoadMethod == ZSTD_dlm_byRef) { + cctx->localDict.dict = dict; + } else { + /* copy dictionary content inside CCtx to own its lifetime */ + void* dictBuffer; + RETURN_ERROR_IF(cctx->staticSize, memory_allocation, + "static CCtx can't allocate for an internal copy of dictionary"); + dictBuffer = ZSTD_customMalloc(dictSize, cctx->customMem); + RETURN_ERROR_IF(dictBuffer==NULL, memory_allocation, + "allocation failed for dictionary content"); + ZSTD_memcpy(dictBuffer, dict, dictSize); + cctx->localDict.dictBuffer = dictBuffer; /* owned ptr to free */ + cctx->localDict.dict = dictBuffer; /* read-only reference */ + } + cctx->localDict.dictSize = dictSize; + cctx->localDict.dictContentType = dictContentType; + return 0; +} - case ZSTD_c_forceMaxWindow : - bounds.lowerBound = 0; - bounds.upperBound = 1; - return bounds; +size_t ZSTD_CCtx_loadDictionary_byReference( + ZSTD_CCtx* cctx, const void* dict, size_t dictSize) +{ + return ZSTD_CCtx_loadDictionary_advanced( + cctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto); +} - case ZSTD_c_format: - ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless); - bounds.lowerBound = ZSTD_f_zstd1; - bounds.upperBound = ZSTD_f_zstd1_magicless; /* note : how to ensure at compile time that this is the highest value enum ? */ - return bounds; +size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize) +{ + return ZSTD_CCtx_loadDictionary_advanced( + cctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto); +} - case ZSTD_c_forceAttachDict: - ZSTD_STATIC_ASSERT(ZSTD_dictDefaultAttach < ZSTD_dictForceLoad); - bounds.lowerBound = ZSTD_dictDefaultAttach; - bounds.upperBound = ZSTD_dictForceLoad; /* note : how to ensure at compile time that this is the highest value enum ? */ - return bounds; - case ZSTD_c_literalCompressionMode: - ZSTD_STATIC_ASSERT(ZSTD_lcm_auto < ZSTD_lcm_huffman && ZSTD_lcm_huffman < ZSTD_lcm_uncompressed); - bounds.lowerBound = ZSTD_lcm_auto; - bounds.upperBound = ZSTD_lcm_uncompressed; - return bounds; +size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) +{ + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't ref a dict when ctx not in init stage."); + /* Free the existing local cdict (if any) to save memory. */ + ZSTD_clearAllDicts(cctx); + cctx->cdict = cdict; + return 0; +} - case ZSTD_c_targetCBlockSize: - bounds.lowerBound = ZSTD_TARGETCBLOCKSIZE_MIN; - bounds.upperBound = ZSTD_TARGETCBLOCKSIZE_MAX; - return bounds; +size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool) +{ + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't ref a pool when ctx not in init stage."); + cctx->pool = pool; + return 0; +} - case ZSTD_c_srcSizeHint: - bounds.lowerBound = ZSTD_SRCSIZEHINT_MIN; - bounds.upperBound = ZSTD_SRCSIZEHINT_MAX; - return bounds; +size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize) +{ + return ZSTD_CCtx_refPrefix_advanced(cctx, prefix, prefixSize, ZSTD_dct_rawContent); +} - case ZSTD_c_stableInBuffer: - case ZSTD_c_stableOutBuffer: - bounds.lowerBound = (int)ZSTD_bm_buffered; - bounds.upperBound = (int)ZSTD_bm_stable; - return bounds; +size_t ZSTD_CCtx_refPrefix_advanced( + ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType) +{ + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't ref a prefix when ctx not in init stage."); + ZSTD_clearAllDicts(cctx); + if (prefix != NULL && prefixSize > 0) { + cctx->prefixDict.dict = prefix; + cctx->prefixDict.dictSize = prefixSize; + cctx->prefixDict.dictContentType = dictContentType; + } + return 0; +} - case ZSTD_c_blockDelimiters: - bounds.lowerBound = (int)ZSTD_sf_noBlockDelimiters; - bounds.upperBound = (int)ZSTD_sf_explicitBlockDelimiters; - return bounds; +/*! ZSTD_CCtx_reset() : + * Also dumps dictionary */ +size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset) +{ + if ( (reset == ZSTD_reset_session_only) + || (reset == ZSTD_reset_session_and_parameters) ) { + cctx->streamStage = zcss_init; + cctx->pledgedSrcSizePlusOne = 0; + } + if ( (reset == ZSTD_reset_parameters) + || (reset == ZSTD_reset_session_and_parameters) ) { + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Reset parameters is only possible during init stage."); + ZSTD_clearAllDicts(cctx); + return ZSTD_CCtxParams_reset(&cctx->requestedParams); + } + return 0; +} - case ZSTD_c_validateSequences: - bounds.lowerBound = 0; - bounds.upperBound = 1; - return bounds; - default: - bounds.error = ERROR(parameter_unsupported); - return bounds; - } +/** ZSTD_checkCParams() : + control CParam values remain within authorized range. + @return : 0, or an error code if one value is beyond authorized range */ +size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) +{ + BOUNDCHECK(ZSTD_c_windowLog, (int)cParams.windowLog); + BOUNDCHECK(ZSTD_c_chainLog, (int)cParams.chainLog); + BOUNDCHECK(ZSTD_c_hashLog, (int)cParams.hashLog); + BOUNDCHECK(ZSTD_c_searchLog, (int)cParams.searchLog); + BOUNDCHECK(ZSTD_c_minMatch, (int)cParams.minMatch); + BOUNDCHECK(ZSTD_c_targetLength,(int)cParams.targetLength); + BOUNDCHECK(ZSTD_c_strategy, (int)cParams.strategy); + return 0; +} + +/** ZSTD_clampCParams() : + * make CParam values within valid range. + * @return : valid CParams */ +static ZSTD_compressionParameters +ZSTD_clampCParams(ZSTD_compressionParameters cParams) +{ +# define CLAMP_TYPE(cParam, val, type) \ + do { \ + ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); \ + if ((int)valbounds.upperBound) val=(type)bounds.upperBound; \ + } while (0) +# define CLAMP(cParam, val) CLAMP_TYPE(cParam, val, unsigned) + CLAMP(ZSTD_c_windowLog, cParams.windowLog); + CLAMP(ZSTD_c_chainLog, cParams.chainLog); + CLAMP(ZSTD_c_hashLog, cParams.hashLog); + CLAMP(ZSTD_c_searchLog, cParams.searchLog); + CLAMP(ZSTD_c_minMatch, cParams.minMatch); + CLAMP(ZSTD_c_targetLength,cParams.targetLength); + CLAMP_TYPE(ZSTD_c_strategy,cParams.strategy, ZSTD_strategy); + return cParams; } -/* ZSTD_cParam_clampBounds: - * Clamps the value into the bounded range. - */ -static size_t ZSTD_cParam_clampBounds(ZSTD_cParameter cParam, int* value) +/** ZSTD_cycleLog() : + * condition for correct operation : hashLog > 1 */ +U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat) { - ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); - if (ZSTD_isError(bounds.error)) return bounds.error; - if (*value < bounds.lowerBound) *value = bounds.lowerBound; - if (*value > bounds.upperBound) *value = bounds.upperBound; - return 0; -} - -#define BOUNDCHECK(cParam, val) { \ - RETURN_ERROR_IF(!ZSTD_cParam_withinBounds(cParam,val), \ - parameter_outOfBound, "Param out of bounds"); \ + U32 const btScale = ((U32)strat >= (U32)ZSTD_btlazy2); + return hashLog - btScale; } - -static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param) +/** ZSTD_dictAndWindowLog() : + * Returns an adjusted window log that is large enough to fit the source and the dictionary. + * The zstd format says that the entire dictionary is valid if one byte of the dictionary + * is within the window. So the hashLog and chainLog should be large enough to reference both + * the dictionary and the window. So we must use this adjusted dictAndWindowLog when downsizing + * the hashLog and windowLog. + * NOTE: srcSize must not be ZSTD_CONTENTSIZE_UNKNOWN. + */ +static U32 ZSTD_dictAndWindowLog(U32 windowLog, U64 srcSize, U64 dictSize) { - switch(param) + const U64 maxWindowSize = 1ULL << ZSTD_WINDOWLOG_MAX; + /* No dictionary ==> No change */ + if (dictSize == 0) { + return windowLog; + } + assert(windowLog <= ZSTD_WINDOWLOG_MAX); + assert(srcSize != ZSTD_CONTENTSIZE_UNKNOWN); /* Handled in ZSTD_adjustCParams_internal() */ { - case ZSTD_c_compressionLevel: - case ZSTD_c_hashLog: - case ZSTD_c_chainLog: - case ZSTD_c_searchLog: - case ZSTD_c_minMatch: - case ZSTD_c_targetLength: - case ZSTD_c_strategy: - return 1; - - case ZSTD_c_format: - case ZSTD_c_windowLog: - case ZSTD_c_contentSizeFlag: - case ZSTD_c_checksumFlag: - case ZSTD_c_dictIDFlag: - case ZSTD_c_forceMaxWindow : - case ZSTD_c_nbWorkers: - case ZSTD_c_jobSize: - case ZSTD_c_overlapLog: - case ZSTD_c_rsyncable: - case ZSTD_c_enableDedicatedDictSearch: - case ZSTD_c_enableLongDistanceMatching: - case ZSTD_c_ldmHashLog: - case ZSTD_c_ldmMinMatch: - case ZSTD_c_ldmBucketSizeLog: - case ZSTD_c_ldmHashRateLog: - case ZSTD_c_forceAttachDict: - case ZSTD_c_literalCompressionMode: - case ZSTD_c_targetCBlockSize: - case ZSTD_c_srcSizeHint: - case ZSTD_c_stableInBuffer: - case ZSTD_c_stableOutBuffer: - case ZSTD_c_blockDelimiters: - case ZSTD_c_validateSequences: - default: - return 0; + U64 const windowSize = 1ULL << windowLog; + U64 const dictAndWindowSize = dictSize + windowSize; + /* If the window size is already large enough to fit both the source and the dictionary + * then just use the window size. Otherwise adjust so that it fits the dictionary and + * the window. + */ + if (windowSize >= dictSize + srcSize) { + return windowLog; /* Window size large enough already */ + } else if (dictAndWindowSize >= maxWindowSize) { + return ZSTD_WINDOWLOG_MAX; /* Larger than max window log */ + } else { + return ZSTD_highbit32((U32)dictAndWindowSize - 1) + 1; + } } } -size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value) +/** ZSTD_adjustCParams_internal() : + * optimize `cPar` for a specified input (`srcSize` and `dictSize`). + * mostly downsize to reduce memory consumption and initialization latency. + * `srcSize` can be ZSTD_CONTENTSIZE_UNKNOWN when not known. + * `mode` is the mode for parameter adjustment. See docs for `ZSTD_CParamMode_e`. + * note : `srcSize==0` means 0! + * condition : cPar is presumed validated (can be checked using ZSTD_checkCParams()). */ +static ZSTD_compressionParameters +ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, + unsigned long long srcSize, + size_t dictSize, + ZSTD_CParamMode_e mode, + ZSTD_ParamSwitch_e useRowMatchFinder) { - DEBUGLOG(4, "ZSTD_CCtx_setParameter (%i, %i)", (int)param, value); - if (cctx->streamStage != zcss_init) { - if (ZSTD_isUpdateAuthorized(param)) { - cctx->cParamsChanged = 1; - } else { - RETURN_ERROR(stage_wrong, "can only set params in ctx init stage"); - } } + const U64 minSrcSize = 513; /* (1<<9) + 1 */ + const U64 maxWindowResize = 1ULL << (ZSTD_WINDOWLOG_MAX-1); + assert(ZSTD_checkCParams(cPar)==0); - switch(param) - { - case ZSTD_c_nbWorkers: - RETURN_ERROR_IF((value!=0) && cctx->staticSize, parameter_unsupported, - "MT not compatible with static alloc"); - break; + /* Cascade the selected strategy down to the next-highest one built into + * this binary. */ +#ifdef ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR + if (cPar.strategy == ZSTD_btultra2) { + cPar.strategy = ZSTD_btultra; + } + if (cPar.strategy == ZSTD_btultra) { + cPar.strategy = ZSTD_btopt; + } +#endif +#ifdef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR + if (cPar.strategy == ZSTD_btopt) { + cPar.strategy = ZSTD_btlazy2; + } +#endif +#ifdef ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR + if (cPar.strategy == ZSTD_btlazy2) { + cPar.strategy = ZSTD_lazy2; + } +#endif +#ifdef ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR + if (cPar.strategy == ZSTD_lazy2) { + cPar.strategy = ZSTD_lazy; + } +#endif +#ifdef ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR + if (cPar.strategy == ZSTD_lazy) { + cPar.strategy = ZSTD_greedy; + } +#endif +#ifdef ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR + if (cPar.strategy == ZSTD_greedy) { + cPar.strategy = ZSTD_dfast; + } +#endif +#ifdef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR + if (cPar.strategy == ZSTD_dfast) { + cPar.strategy = ZSTD_fast; + cPar.targetLength = 0; + } +#endif - case ZSTD_c_compressionLevel: - case ZSTD_c_windowLog: - case ZSTD_c_hashLog: - case ZSTD_c_chainLog: - case ZSTD_c_searchLog: - case ZSTD_c_minMatch: - case ZSTD_c_targetLength: - case ZSTD_c_strategy: - case ZSTD_c_ldmHashRateLog: - case ZSTD_c_format: - case ZSTD_c_contentSizeFlag: - case ZSTD_c_checksumFlag: - case ZSTD_c_dictIDFlag: - case ZSTD_c_forceMaxWindow: - case ZSTD_c_forceAttachDict: - case ZSTD_c_literalCompressionMode: - case ZSTD_c_jobSize: - case ZSTD_c_overlapLog: - case ZSTD_c_rsyncable: - case ZSTD_c_enableDedicatedDictSearch: - case ZSTD_c_enableLongDistanceMatching: - case ZSTD_c_ldmHashLog: - case ZSTD_c_ldmMinMatch: - case ZSTD_c_ldmBucketSizeLog: - case ZSTD_c_targetCBlockSize: - case ZSTD_c_srcSizeHint: - case ZSTD_c_stableInBuffer: - case ZSTD_c_stableOutBuffer: - case ZSTD_c_blockDelimiters: - case ZSTD_c_validateSequences: + switch (mode) { + case ZSTD_cpm_unknown: + case ZSTD_cpm_noAttachDict: + /* If we don't know the source size, don't make any + * assumptions about it. We will already have selected + * smaller parameters if a dictionary is in use. + */ + break; + case ZSTD_cpm_createCDict: + /* Assume a small source size when creating a dictionary + * with an unknown source size. + */ + if (dictSize && srcSize == ZSTD_CONTENTSIZE_UNKNOWN) + srcSize = minSrcSize; + break; + case ZSTD_cpm_attachDict: + /* Dictionary has its own dedicated parameters which have + * already been selected. We are selecting parameters + * for only the source. + */ + dictSize = 0; + break; + default: + assert(0); break; + } - default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); + /* resize windowLog if input is small enough, to use less memory */ + if ( (srcSize <= maxWindowResize) + && (dictSize <= maxWindowResize) ) { + U32 const tSize = (U32)(srcSize + dictSize); + static U32 const hashSizeMin = 1 << ZSTD_HASHLOG_MIN; + U32 const srcLog = (tSize < hashSizeMin) ? ZSTD_HASHLOG_MIN : + ZSTD_highbit32(tSize-1) + 1; + if (cPar.windowLog > srcLog) cPar.windowLog = srcLog; + } + if (srcSize != ZSTD_CONTENTSIZE_UNKNOWN) { + U32 const dictAndWindowLog = ZSTD_dictAndWindowLog(cPar.windowLog, (U64)srcSize, (U64)dictSize); + U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy); + if (cPar.hashLog > dictAndWindowLog+1) cPar.hashLog = dictAndWindowLog+1; + if (cycleLog > dictAndWindowLog) + cPar.chainLog -= (cycleLog - dictAndWindowLog); } - return ZSTD_CCtxParams_setParameter(&cctx->requestedParams, param, value); -} -size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, - ZSTD_cParameter param, int value) -{ - DEBUGLOG(4, "ZSTD_CCtxParams_setParameter (%i, %i)", (int)param, value); - switch(param) - { - case ZSTD_c_format : - BOUNDCHECK(ZSTD_c_format, value); - CCtxParams->format = (ZSTD_format_e)value; - return (size_t)CCtxParams->format; + if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) + cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* minimum wlog required for valid frame header */ - case ZSTD_c_compressionLevel : { - FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), ""); - if (value == 0) - CCtxParams->compressionLevel = ZSTD_CLEVEL_DEFAULT; /* 0 == default */ - else - CCtxParams->compressionLevel = value; - if (CCtxParams->compressionLevel >= 0) return (size_t)CCtxParams->compressionLevel; - return 0; /* return type (size_t) cannot represent negative values */ + /* We can't use more than 32 bits of hash in total, so that means that we require: + * (hashLog + 8) <= 32 && (chainLog + 8) <= 32 + */ + if (mode == ZSTD_cpm_createCDict && ZSTD_CDictIndicesAreTagged(&cPar)) { + U32 const maxShortCacheHashLog = 32 - ZSTD_SHORT_CACHE_TAG_BITS; + if (cPar.hashLog > maxShortCacheHashLog) { + cPar.hashLog = maxShortCacheHashLog; + } + if (cPar.chainLog > maxShortCacheHashLog) { + cPar.chainLog = maxShortCacheHashLog; + } } - case ZSTD_c_windowLog : - if (value!=0) /* 0 => use default */ - BOUNDCHECK(ZSTD_c_windowLog, value); - CCtxParams->cParams.windowLog = (U32)value; - return CCtxParams->cParams.windowLog; - case ZSTD_c_hashLog : - if (value!=0) /* 0 => use default */ - BOUNDCHECK(ZSTD_c_hashLog, value); - CCtxParams->cParams.hashLog = (U32)value; - return CCtxParams->cParams.hashLog; + /* At this point, we aren't 100% sure if we are using the row match finder. + * Unless it is explicitly disabled, conservatively assume that it is enabled. + * In this case it will only be disabled for small sources, so shrinking the + * hash log a little bit shouldn't result in any ratio loss. + */ + if (useRowMatchFinder == ZSTD_ps_auto) + useRowMatchFinder = ZSTD_ps_enable; - case ZSTD_c_chainLog : - if (value!=0) /* 0 => use default */ - BOUNDCHECK(ZSTD_c_chainLog, value); - CCtxParams->cParams.chainLog = (U32)value; - return CCtxParams->cParams.chainLog; + /* We can't hash more than 32-bits in total. So that means that we require: + * (hashLog - rowLog + 8) <= 32 + */ + if (ZSTD_rowMatchFinderUsed(cPar.strategy, useRowMatchFinder)) { + /* Switch to 32-entry rows if searchLog is 5 (or more) */ + U32 const rowLog = BOUNDED(4, cPar.searchLog, 6); + U32 const maxRowHashLog = 32 - ZSTD_ROW_HASH_TAG_BITS; + U32 const maxHashLog = maxRowHashLog + rowLog; + assert(cPar.hashLog >= rowLog); + if (cPar.hashLog > maxHashLog) { + cPar.hashLog = maxHashLog; + } + } - case ZSTD_c_searchLog : - if (value!=0) /* 0 => use default */ - BOUNDCHECK(ZSTD_c_searchLog, value); - CCtxParams->cParams.searchLog = (U32)value; - return (size_t)value; + return cPar; +} - case ZSTD_c_minMatch : - if (value!=0) /* 0 => use default */ - BOUNDCHECK(ZSTD_c_minMatch, value); - CCtxParams->cParams.minMatch = value; - return CCtxParams->cParams.minMatch; +ZSTD_compressionParameters +ZSTD_adjustCParams(ZSTD_compressionParameters cPar, + unsigned long long srcSize, + size_t dictSize) +{ + cPar = ZSTD_clampCParams(cPar); /* resulting cPar is necessarily valid (all parameters within range) */ + if (srcSize == 0) srcSize = ZSTD_CONTENTSIZE_UNKNOWN; + return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize, ZSTD_cpm_unknown, ZSTD_ps_auto); +} - case ZSTD_c_targetLength : - BOUNDCHECK(ZSTD_c_targetLength, value); - CCtxParams->cParams.targetLength = value; - return CCtxParams->cParams.targetLength; +static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_CParamMode_e mode); +static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_CParamMode_e mode); - case ZSTD_c_strategy : - if (value!=0) /* 0 => use default */ - BOUNDCHECK(ZSTD_c_strategy, value); - CCtxParams->cParams.strategy = (ZSTD_strategy)value; - return (size_t)CCtxParams->cParams.strategy; +static void ZSTD_overrideCParams( + ZSTD_compressionParameters* cParams, + const ZSTD_compressionParameters* overrides) +{ + if (overrides->windowLog) cParams->windowLog = overrides->windowLog; + if (overrides->hashLog) cParams->hashLog = overrides->hashLog; + if (overrides->chainLog) cParams->chainLog = overrides->chainLog; + if (overrides->searchLog) cParams->searchLog = overrides->searchLog; + if (overrides->minMatch) cParams->minMatch = overrides->minMatch; + if (overrides->targetLength) cParams->targetLength = overrides->targetLength; + if (overrides->strategy) cParams->strategy = overrides->strategy; +} - case ZSTD_c_contentSizeFlag : - /* Content size written in frame header _when known_ (default:1) */ - DEBUGLOG(4, "set content size flag = %u", (value!=0)); - CCtxParams->fParams.contentSizeFlag = value != 0; - return CCtxParams->fParams.contentSizeFlag; +ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( + const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_CParamMode_e mode) +{ + ZSTD_compressionParameters cParams; + if (srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN && CCtxParams->srcSizeHint > 0) { + assert(CCtxParams->srcSizeHint>=0); + srcSizeHint = (U64)CCtxParams->srcSizeHint; + } + cParams = ZSTD_getCParams_internal(CCtxParams->compressionLevel, srcSizeHint, dictSize, mode); + if (CCtxParams->ldmParams.enableLdm == ZSTD_ps_enable) cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG; + ZSTD_overrideCParams(&cParams, &CCtxParams->cParams); + assert(!ZSTD_checkCParams(cParams)); + /* srcSizeHint == 0 means 0 */ + return ZSTD_adjustCParams_internal(cParams, srcSizeHint, dictSize, mode, CCtxParams->useRowMatchFinder); +} - case ZSTD_c_checksumFlag : - /* A 32-bits content checksum will be calculated and written at end of frame (default:0) */ - CCtxParams->fParams.checksumFlag = value != 0; - return CCtxParams->fParams.checksumFlag; +static size_t +ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams, + const ZSTD_ParamSwitch_e useRowMatchFinder, + const int enableDedicatedDictSearch, + const U32 forCCtx) +{ + /* chain table size should be 0 for fast or row-hash strategies */ + size_t const chainSize = ZSTD_allocateChainTable(cParams->strategy, useRowMatchFinder, enableDedicatedDictSearch && !forCCtx) + ? ((size_t)1 << cParams->chainLog) + : 0; + size_t const hSize = ((size_t)1) << cParams->hashLog; + U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; + size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0; + /* We don't use ZSTD_cwksp_alloc_size() here because the tables aren't + * surrounded by redzones in ASAN. */ + size_t const tableSpace = chainSize * sizeof(U32) + + hSize * sizeof(U32) + + h3Size * sizeof(U32); + size_t const optPotentialSpace = + ZSTD_cwksp_aligned64_alloc_size((MaxML+1) * sizeof(U32)) + + ZSTD_cwksp_aligned64_alloc_size((MaxLL+1) * sizeof(U32)) + + ZSTD_cwksp_aligned64_alloc_size((MaxOff+1) * sizeof(U32)) + + ZSTD_cwksp_aligned64_alloc_size((1<strategy, useRowMatchFinder) + ? ZSTD_cwksp_aligned64_alloc_size(hSize) + : 0; + size_t const optSpace = (forCCtx && (cParams->strategy >= ZSTD_btopt)) + ? optPotentialSpace + : 0; + size_t const slackSpace = ZSTD_cwksp_slack_space_required(); - case ZSTD_c_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */ - DEBUGLOG(4, "set dictIDFlag = %u", (value!=0)); - CCtxParams->fParams.noDictIDFlag = !value; - return !CCtxParams->fParams.noDictIDFlag; + /* tables are guaranteed to be sized in multiples of 64 bytes (or 16 uint32_t) */ + ZSTD_STATIC_ASSERT(ZSTD_HASHLOG_MIN >= 4 && ZSTD_WINDOWLOG_MIN >= 4 && ZSTD_CHAINLOG_MIN >= 4); + assert(useRowMatchFinder != ZSTD_ps_auto); - case ZSTD_c_forceMaxWindow : - CCtxParams->forceWindow = (value != 0); - return CCtxParams->forceWindow; + DEBUGLOG(4, "chainSize: %u - hSize: %u - h3Size: %u", + (U32)chainSize, (U32)hSize, (U32)h3Size); + return tableSpace + optSpace + slackSpace + lazyAdditionalSpace; +} - case ZSTD_c_forceAttachDict : { - const ZSTD_dictAttachPref_e pref = (ZSTD_dictAttachPref_e)value; - BOUNDCHECK(ZSTD_c_forceAttachDict, pref); - CCtxParams->attachDictPref = pref; - return CCtxParams->attachDictPref; - } +/* Helper function for calculating memory requirements. + * Gives a tighter bound than ZSTD_sequenceBound() by taking minMatch into account. */ +static size_t ZSTD_maxNbSeq(size_t blockSize, unsigned minMatch, int useSequenceProducer) { + U32 const divider = (minMatch==3 || useSequenceProducer) ? 3 : 4; + return blockSize / divider; +} - case ZSTD_c_literalCompressionMode : { - const ZSTD_literalCompressionMode_e lcm = (ZSTD_literalCompressionMode_e)value; - BOUNDCHECK(ZSTD_c_literalCompressionMode, lcm); - CCtxParams->literalCompressionMode = lcm; - return CCtxParams->literalCompressionMode; - } +static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal( + const ZSTD_compressionParameters* cParams, + const ldmParams_t* ldmParams, + const int isStatic, + const ZSTD_ParamSwitch_e useRowMatchFinder, + const size_t buffInSize, + const size_t buffOutSize, + const U64 pledgedSrcSize, + int useSequenceProducer, + size_t maxBlockSize) +{ + size_t const windowSize = (size_t) BOUNDED(1ULL, 1ULL << cParams->windowLog, pledgedSrcSize); + size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(maxBlockSize), windowSize); + size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, cParams->minMatch, useSequenceProducer); + size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize) + + ZSTD_cwksp_aligned64_alloc_size(maxNbSeq * sizeof(SeqDef)) + + 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE)); + size_t const tmpWorkSpace = ZSTD_cwksp_alloc_size(TMP_WORKSPACE_SIZE); + size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t)); + size_t const matchStateSize = ZSTD_sizeof_matchState(cParams, useRowMatchFinder, /* enableDedicatedDictSearch */ 0, /* forCCtx */ 1); - case ZSTD_c_nbWorkers : -#ifndef ZSTD_MULTITHREAD - RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); - return 0; -#else - FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), ""); - CCtxParams->nbWorkers = value; - return CCtxParams->nbWorkers; -#endif + size_t const ldmSpace = ZSTD_ldm_getTableSize(*ldmParams); + size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(*ldmParams, blockSize); + size_t const ldmSeqSpace = ldmParams->enableLdm == ZSTD_ps_enable ? + ZSTD_cwksp_aligned64_alloc_size(maxNbLdmSeq * sizeof(rawSeq)) : 0; - case ZSTD_c_jobSize : -#ifndef ZSTD_MULTITHREAD - RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); - return 0; -#else - /* Adjust to the minimum non-default value. */ - if (value != 0 && value < ZSTDMT_JOBSIZE_MIN) - value = ZSTDMT_JOBSIZE_MIN; - FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), ""); - assert(value >= 0); - CCtxParams->jobSize = value; - return CCtxParams->jobSize; -#endif - case ZSTD_c_overlapLog : -#ifndef ZSTD_MULTITHREAD - RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); - return 0; -#else - FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(ZSTD_c_overlapLog, &value), ""); - CCtxParams->overlapLog = value; - return CCtxParams->overlapLog; -#endif + size_t const bufferSpace = ZSTD_cwksp_alloc_size(buffInSize) + + ZSTD_cwksp_alloc_size(buffOutSize); - case ZSTD_c_rsyncable : -#ifndef ZSTD_MULTITHREAD - RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); - return 0; -#else - FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(ZSTD_c_overlapLog, &value), ""); - CCtxParams->rsyncable = value; - return CCtxParams->rsyncable; -#endif + size_t const cctxSpace = isStatic ? ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx)) : 0; - case ZSTD_c_enableDedicatedDictSearch : - CCtxParams->enableDedicatedDictSearch = (value!=0); - return CCtxParams->enableDedicatedDictSearch; + size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize); + size_t const externalSeqSpace = useSequenceProducer + ? ZSTD_cwksp_aligned64_alloc_size(maxNbExternalSeq * sizeof(ZSTD_Sequence)) + : 0; - case ZSTD_c_enableLongDistanceMatching : - CCtxParams->ldmParams.enableLdm = (value!=0); - return CCtxParams->ldmParams.enableLdm; + size_t const neededSpace = + cctxSpace + + tmpWorkSpace + + blockStateSpace + + ldmSpace + + ldmSeqSpace + + matchStateSize + + tokenSpace + + bufferSpace + + externalSeqSpace; - case ZSTD_c_ldmHashLog : - if (value!=0) /* 0 ==> auto */ - BOUNDCHECK(ZSTD_c_ldmHashLog, value); - CCtxParams->ldmParams.hashLog = value; - return CCtxParams->ldmParams.hashLog; + DEBUGLOG(5, "estimate workspace : %u", (U32)neededSpace); + return neededSpace; +} - case ZSTD_c_ldmMinMatch : - if (value!=0) /* 0 ==> default */ - BOUNDCHECK(ZSTD_c_ldmMinMatch, value); - CCtxParams->ldmParams.minMatchLength = value; - return CCtxParams->ldmParams.minMatchLength; +size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params) +{ + ZSTD_compressionParameters const cParams = + ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); + ZSTD_ParamSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params->useRowMatchFinder, + &cParams); - case ZSTD_c_ldmBucketSizeLog : - if (value!=0) /* 0 ==> default */ - BOUNDCHECK(ZSTD_c_ldmBucketSizeLog, value); - CCtxParams->ldmParams.bucketSizeLog = value; - return CCtxParams->ldmParams.bucketSizeLog; + RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only."); + /* estimateCCtxSize is for one-shot compression. So no buffers should + * be needed. However, we still allocate two 0-sized buffers, which can + * take space under ASAN. */ + return ZSTD_estimateCCtxSize_usingCCtxParams_internal( + &cParams, ¶ms->ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN, ZSTD_hasExtSeqProd(params), params->maxBlockSize); +} - case ZSTD_c_ldmHashRateLog : - RETURN_ERROR_IF(value > ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN, - parameter_outOfBound, "Param out of bounds!"); - CCtxParams->ldmParams.hashRateLog = value; - return CCtxParams->ldmParams.hashRateLog; +size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams) +{ + ZSTD_CCtx_params initialParams = ZSTD_makeCCtxParamsFromCParams(cParams); + if (ZSTD_rowMatchFinderSupported(cParams.strategy)) { + /* Pick bigger of not using and using row-based matchfinder for greedy and lazy strategies */ + size_t noRowCCtxSize; + size_t rowCCtxSize; + initialParams.useRowMatchFinder = ZSTD_ps_disable; + noRowCCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams); + initialParams.useRowMatchFinder = ZSTD_ps_enable; + rowCCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams); + return MAX(noRowCCtxSize, rowCCtxSize); + } else { + return ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams); + } +} - case ZSTD_c_targetCBlockSize : - if (value!=0) /* 0 ==> default */ - BOUNDCHECK(ZSTD_c_targetCBlockSize, value); - CCtxParams->targetCBlockSize = value; - return CCtxParams->targetCBlockSize; +static size_t ZSTD_estimateCCtxSize_internal(int compressionLevel) +{ + int tier = 0; + size_t largestSize = 0; + static const unsigned long long srcSizeTiers[4] = {16 KB, 128 KB, 256 KB, ZSTD_CONTENTSIZE_UNKNOWN}; + for (; tier < 4; ++tier) { + /* Choose the set of cParams for a given level across all srcSizes that give the largest cctxSize */ + ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, srcSizeTiers[tier], 0, ZSTD_cpm_noAttachDict); + largestSize = MAX(ZSTD_estimateCCtxSize_usingCParams(cParams), largestSize); + } + return largestSize; +} - case ZSTD_c_srcSizeHint : - if (value!=0) /* 0 ==> default */ - BOUNDCHECK(ZSTD_c_srcSizeHint, value); - CCtxParams->srcSizeHint = value; - return CCtxParams->srcSizeHint; +size_t ZSTD_estimateCCtxSize(int compressionLevel) +{ + int level; + size_t memBudget = 0; + for (level=MIN(compressionLevel, 1); level<=compressionLevel; level++) { + /* Ensure monotonically increasing memory usage as compression level increases */ + size_t const newMB = ZSTD_estimateCCtxSize_internal(level); + if (newMB > memBudget) memBudget = newMB; + } + return memBudget; +} - case ZSTD_c_stableInBuffer: - BOUNDCHECK(ZSTD_c_stableInBuffer, value); - CCtxParams->inBufferMode = (ZSTD_bufferMode_e)value; - return CCtxParams->inBufferMode; +size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params) +{ + RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only."); + { ZSTD_compressionParameters const cParams = + ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); + size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(params->maxBlockSize), (size_t)1 << cParams.windowLog); + size_t const inBuffSize = (params->inBufferMode == ZSTD_bm_buffered) + ? ((size_t)1 << cParams.windowLog) + blockSize + : 0; + size_t const outBuffSize = (params->outBufferMode == ZSTD_bm_buffered) + ? ZSTD_compressBound(blockSize) + 1 + : 0; + ZSTD_ParamSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params->useRowMatchFinder, ¶ms->cParams); - case ZSTD_c_stableOutBuffer: - BOUNDCHECK(ZSTD_c_stableOutBuffer, value); - CCtxParams->outBufferMode = (ZSTD_bufferMode_e)value; - return CCtxParams->outBufferMode; + return ZSTD_estimateCCtxSize_usingCCtxParams_internal( + &cParams, ¶ms->ldmParams, 1, useRowMatchFinder, inBuffSize, outBuffSize, + ZSTD_CONTENTSIZE_UNKNOWN, ZSTD_hasExtSeqProd(params), params->maxBlockSize); + } +} - case ZSTD_c_blockDelimiters: - BOUNDCHECK(ZSTD_c_blockDelimiters, value); - CCtxParams->blockDelimiters = (ZSTD_sequenceFormat_e)value; - return CCtxParams->blockDelimiters; +size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams) +{ + ZSTD_CCtx_params initialParams = ZSTD_makeCCtxParamsFromCParams(cParams); + if (ZSTD_rowMatchFinderSupported(cParams.strategy)) { + /* Pick bigger of not using and using row-based matchfinder for greedy and lazy strategies */ + size_t noRowCCtxSize; + size_t rowCCtxSize; + initialParams.useRowMatchFinder = ZSTD_ps_disable; + noRowCCtxSize = ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams); + initialParams.useRowMatchFinder = ZSTD_ps_enable; + rowCCtxSize = ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams); + return MAX(noRowCCtxSize, rowCCtxSize); + } else { + return ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams); + } +} - case ZSTD_c_validateSequences: - BOUNDCHECK(ZSTD_c_validateSequences, value); - CCtxParams->validateSequences = value; - return CCtxParams->validateSequences; +static size_t ZSTD_estimateCStreamSize_internal(int compressionLevel) +{ + ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); + return ZSTD_estimateCStreamSize_usingCParams(cParams); +} + +size_t ZSTD_estimateCStreamSize(int compressionLevel) +{ + int level; + size_t memBudget = 0; + for (level=MIN(compressionLevel, 1); level<=compressionLevel; level++) { + size_t const newMB = ZSTD_estimateCStreamSize_internal(level); + if (newMB > memBudget) memBudget = newMB; + } + return memBudget; +} + +/* ZSTD_getFrameProgression(): + * tells how much data has been consumed (input) and produced (output) for current frame. + * able to count progression inside worker threads (non-blocking mode). + */ +ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx) +{ +#ifdef ZSTD_MULTITHREAD + if (cctx->appliedParams.nbWorkers > 0) { + return ZSTDMT_getFrameProgression(cctx->mtctx); + } +#endif + { ZSTD_frameProgression fp; + size_t const buffered = (cctx->inBuff == NULL) ? 0 : + cctx->inBuffPos - cctx->inToCompress; + if (buffered) assert(cctx->inBuffPos >= cctx->inToCompress); + assert(buffered <= ZSTD_BLOCKSIZE_MAX); + fp.ingested = cctx->consumedSrcSize + buffered; + fp.consumed = cctx->consumedSrcSize; + fp.produced = cctx->producedCSize; + fp.flushed = cctx->producedCSize; /* simplified; some data might still be left within streaming output buffer */ + fp.currentJobID = 0; + fp.nbActiveWorkers = 0; + return fp; +} } - default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); +/*! ZSTD_toFlushNow() + * Only useful for multithreading scenarios currently (nbWorkers >= 1). + */ +size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx) +{ +#ifdef ZSTD_MULTITHREAD + if (cctx->appliedParams.nbWorkers > 0) { + return ZSTDMT_toFlushNow(cctx->mtctx); } +#endif + (void)cctx; + return 0; /* over-simplification; could also check if context is currently running in streaming mode, and in which case, report how many bytes are left to be flushed within output buffer */ } -size_t ZSTD_CCtx_getParameter(ZSTD_CCtx const* cctx, ZSTD_cParameter param, int* value) +static void ZSTD_assertEqualCParams(ZSTD_compressionParameters cParams1, + ZSTD_compressionParameters cParams2) { - return ZSTD_CCtxParams_getParameter(&cctx->requestedParams, param, value); + (void)cParams1; + (void)cParams2; + assert(cParams1.windowLog == cParams2.windowLog); + assert(cParams1.chainLog == cParams2.chainLog); + assert(cParams1.hashLog == cParams2.hashLog); + assert(cParams1.searchLog == cParams2.searchLog); + assert(cParams1.minMatch == cParams2.minMatch); + assert(cParams1.targetLength == cParams2.targetLength); + assert(cParams1.strategy == cParams2.strategy); } -size_t ZSTD_CCtxParams_getParameter( - ZSTD_CCtx_params const* CCtxParams, ZSTD_cParameter param, int* value) +void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs) { - switch(param) - { - case ZSTD_c_format : - *value = CCtxParams->format; - break; - case ZSTD_c_compressionLevel : - *value = CCtxParams->compressionLevel; - break; - case ZSTD_c_windowLog : - *value = (int)CCtxParams->cParams.windowLog; - break; - case ZSTD_c_hashLog : - *value = (int)CCtxParams->cParams.hashLog; - break; - case ZSTD_c_chainLog : - *value = (int)CCtxParams->cParams.chainLog; - break; - case ZSTD_c_searchLog : - *value = CCtxParams->cParams.searchLog; - break; - case ZSTD_c_minMatch : - *value = CCtxParams->cParams.minMatch; - break; - case ZSTD_c_targetLength : - *value = CCtxParams->cParams.targetLength; - break; - case ZSTD_c_strategy : - *value = (unsigned)CCtxParams->cParams.strategy; - break; - case ZSTD_c_contentSizeFlag : - *value = CCtxParams->fParams.contentSizeFlag; - break; - case ZSTD_c_checksumFlag : - *value = CCtxParams->fParams.checksumFlag; - break; - case ZSTD_c_dictIDFlag : - *value = !CCtxParams->fParams.noDictIDFlag; - break; - case ZSTD_c_forceMaxWindow : - *value = CCtxParams->forceWindow; - break; - case ZSTD_c_forceAttachDict : - *value = CCtxParams->attachDictPref; - break; - case ZSTD_c_literalCompressionMode : - *value = CCtxParams->literalCompressionMode; - break; - case ZSTD_c_nbWorkers : -#ifndef ZSTD_MULTITHREAD - assert(CCtxParams->nbWorkers == 0); -#endif - *value = CCtxParams->nbWorkers; - break; - case ZSTD_c_jobSize : -#ifndef ZSTD_MULTITHREAD - RETURN_ERROR(parameter_unsupported, "not compiled with multithreading"); -#else - assert(CCtxParams->jobSize <= INT_MAX); - *value = (int)CCtxParams->jobSize; - break; -#endif - case ZSTD_c_overlapLog : -#ifndef ZSTD_MULTITHREAD - RETURN_ERROR(parameter_unsupported, "not compiled with multithreading"); -#else - *value = CCtxParams->overlapLog; - break; -#endif - case ZSTD_c_rsyncable : -#ifndef ZSTD_MULTITHREAD - RETURN_ERROR(parameter_unsupported, "not compiled with multithreading"); -#else - *value = CCtxParams->rsyncable; - break; -#endif - case ZSTD_c_enableDedicatedDictSearch : - *value = CCtxParams->enableDedicatedDictSearch; - break; - case ZSTD_c_enableLongDistanceMatching : - *value = CCtxParams->ldmParams.enableLdm; - break; - case ZSTD_c_ldmHashLog : - *value = CCtxParams->ldmParams.hashLog; - break; - case ZSTD_c_ldmMinMatch : - *value = CCtxParams->ldmParams.minMatchLength; - break; - case ZSTD_c_ldmBucketSizeLog : - *value = CCtxParams->ldmParams.bucketSizeLog; - break; - case ZSTD_c_ldmHashRateLog : - *value = CCtxParams->ldmParams.hashRateLog; - break; - case ZSTD_c_targetCBlockSize : - *value = (int)CCtxParams->targetCBlockSize; - break; - case ZSTD_c_srcSizeHint : - *value = (int)CCtxParams->srcSizeHint; - break; - case ZSTD_c_stableInBuffer : - *value = (int)CCtxParams->inBufferMode; - break; - case ZSTD_c_stableOutBuffer : - *value = (int)CCtxParams->outBufferMode; - break; - case ZSTD_c_blockDelimiters : - *value = (int)CCtxParams->blockDelimiters; - break; - case ZSTD_c_validateSequences : - *value = (int)CCtxParams->validateSequences; - break; - default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); - } - return 0; + int i; + for (i = 0; i < ZSTD_REP_NUM; ++i) + bs->rep[i] = repStartValue[i]; + bs->entropy.huf.repeatMode = HUF_repeat_none; + bs->entropy.fse.offcode_repeatMode = FSE_repeat_none; + bs->entropy.fse.matchlength_repeatMode = FSE_repeat_none; + bs->entropy.fse.litlength_repeatMode = FSE_repeat_none; } -/** ZSTD_CCtx_setParametersUsingCCtxParams() : - * just applies `params` into `cctx` - * no action is performed, parameters are merely stored. - * If ZSTDMT is enabled, parameters are pushed to cctx->mtctx. - * This is possible even if a compression is ongoing. - * In which case, new parameters will be applied on the fly, starting with next compression job. +/*! ZSTD_invalidateMatchState() + * Invalidate all the matches in the match finder tables. + * Requires nextSrc and base to be set (can be NULL). */ -size_t ZSTD_CCtx_setParametersUsingCCtxParams( - ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params) +static void ZSTD_invalidateMatchState(ZSTD_MatchState_t* ms) { - DEBUGLOG(4, "ZSTD_CCtx_setParametersUsingCCtxParams"); - RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, - "The context is in the wrong stage!"); - RETURN_ERROR_IF(cctx->cdict, stage_wrong, - "Can't override parameters with cdict attached (some must " - "be inherited from the cdict)."); - - cctx->requestedParams = *params; - return 0; -} + ZSTD_window_clear(&ms->window); -ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize) -{ - DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %u bytes", (U32)pledgedSrcSize); - RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, - "Can't set pledgedSrcSize when not in init stage."); - cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1; - return 0; + ms->nextToUpdate = ms->window.dictLimit; + ms->loadedDictEnd = 0; + ms->opt.litLengthSum = 0; /* force reset of btopt stats */ + ms->dictMatchState = NULL; } -static ZSTD_compressionParameters ZSTD_dedicatedDictSearch_getCParams( - int const compressionLevel, - size_t const dictSize); -static int ZSTD_dedicatedDictSearch_isSupported( - const ZSTD_compressionParameters* cParams); -static void ZSTD_dedicatedDictSearch_revertCParams( - ZSTD_compressionParameters* cParams); +/** + * Controls, for this matchState reset, whether the tables need to be cleared / + * prepared for the coming compression (ZSTDcrp_makeClean), or whether the + * tables can be left unclean (ZSTDcrp_leaveDirty), because we know that a + * subsequent operation will overwrite the table space anyways (e.g., copying + * the matchState contents in from a CDict). + */ +typedef enum { + ZSTDcrp_makeClean, + ZSTDcrp_leaveDirty +} ZSTD_compResetPolicy_e; /** - * Initializes the local dict using the requested parameters. - * NOTE: This does not use the pledged src size, because it may be used for more - * than one compression. + * Controls, for this matchState reset, whether indexing can continue where it + * left off (ZSTDirp_continue), or whether it needs to be restarted from zero + * (ZSTDirp_reset). */ -static size_t ZSTD_initLocalDict(ZSTD_CCtx* cctx) -{ - ZSTD_localDict* const dl = &cctx->localDict; - if (dl->dict == NULL) { - /* No local dictionary. */ - assert(dl->dictBuffer == NULL); - assert(dl->cdict == NULL); - assert(dl->dictSize == 0); - return 0; - } - if (dl->cdict != NULL) { - assert(cctx->cdict == dl->cdict); - /* Local dictionary already initialized. */ - return 0; - } - assert(dl->dictSize > 0); - assert(cctx->cdict == NULL); - assert(cctx->prefixDict.dict == NULL); +typedef enum { + ZSTDirp_continue, + ZSTDirp_reset +} ZSTD_indexResetPolicy_e; - dl->cdict = ZSTD_createCDict_advanced2( - dl->dict, - dl->dictSize, - ZSTD_dlm_byRef, - dl->dictContentType, - &cctx->requestedParams, - cctx->customMem); - RETURN_ERROR_IF(!dl->cdict, memory_allocation, "ZSTD_createCDict_advanced failed"); - cctx->cdict = dl->cdict; - return 0; +typedef enum { + ZSTD_resetTarget_CDict, + ZSTD_resetTarget_CCtx +} ZSTD_resetTarget_e; + +/* Mixes bits in a 64 bits in a value, based on XXH3_rrmxmx */ +static U64 ZSTD_bitmix(U64 val, U64 len) { + val ^= ZSTD_rotateRight_U64(val, 49) ^ ZSTD_rotateRight_U64(val, 24); + val *= 0x9FB21C651E98DF25ULL; + val ^= (val >> 35) + len ; + val *= 0x9FB21C651E98DF25ULL; + return val ^ (val >> 28); } -size_t ZSTD_CCtx_loadDictionary_advanced( - ZSTD_CCtx* cctx, const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType) -{ - RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, - "Can't load a dictionary when ctx is not in init stage."); - DEBUGLOG(4, "ZSTD_CCtx_loadDictionary_advanced (size: %u)", (U32)dictSize); - ZSTD_clearAllDicts(cctx); /* in case one already exists */ - if (dict == NULL || dictSize == 0) /* no dictionary mode */ - return 0; - if (dictLoadMethod == ZSTD_dlm_byRef) { - cctx->localDict.dict = dict; - } else { - void* dictBuffer; - RETURN_ERROR_IF(cctx->staticSize, memory_allocation, - "no malloc for static CCtx"); - dictBuffer = ZSTD_customMalloc(dictSize, cctx->customMem); - RETURN_ERROR_IF(!dictBuffer, memory_allocation, "NULL pointer!"); - ZSTD_memcpy(dictBuffer, dict, dictSize); - cctx->localDict.dictBuffer = dictBuffer; - cctx->localDict.dict = dictBuffer; - } - cctx->localDict.dictSize = dictSize; - cctx->localDict.dictContentType = dictContentType; - return 0; +/* Mixes in the hashSalt and hashSaltEntropy to create a new hashSalt */ +static void ZSTD_advanceHashSalt(ZSTD_MatchState_t* ms) { + ms->hashSalt = ZSTD_bitmix(ms->hashSalt, 8) ^ ZSTD_bitmix((U64) ms->hashSaltEntropy, 4); } -ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference( - ZSTD_CCtx* cctx, const void* dict, size_t dictSize) +static size_t +ZSTD_reset_matchState(ZSTD_MatchState_t* ms, + ZSTD_cwksp* ws, + const ZSTD_compressionParameters* cParams, + const ZSTD_ParamSwitch_e useRowMatchFinder, + const ZSTD_compResetPolicy_e crp, + const ZSTD_indexResetPolicy_e forceResetIndex, + const ZSTD_resetTarget_e forWho) { - return ZSTD_CCtx_loadDictionary_advanced( - cctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto); -} + /* disable chain table allocation for fast or row-based strategies */ + size_t const chainSize = ZSTD_allocateChainTable(cParams->strategy, useRowMatchFinder, + ms->dedicatedDictSearch && (forWho == ZSTD_resetTarget_CDict)) + ? ((size_t)1 << cParams->chainLog) + : 0; + size_t const hSize = ((size_t)1) << cParams->hashLog; + U32 const hashLog3 = ((forWho == ZSTD_resetTarget_CCtx) && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; + size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0; -ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize) -{ - return ZSTD_CCtx_loadDictionary_advanced( - cctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto); -} + DEBUGLOG(4, "reset indices : %u", forceResetIndex == ZSTDirp_reset); + assert(useRowMatchFinder != ZSTD_ps_auto); + if (forceResetIndex == ZSTDirp_reset) { + ZSTD_window_init(&ms->window); + ZSTD_cwksp_mark_tables_dirty(ws); + } + ms->hashLog3 = hashLog3; + ms->lazySkipping = 0; -size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) -{ - RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, - "Can't ref a dict when ctx not in init stage."); - /* Free the existing local cdict (if any) to save memory. */ - ZSTD_clearAllDicts(cctx); - cctx->cdict = cdict; - return 0; -} + ZSTD_invalidateMatchState(ms); -size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool) -{ - RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, - "Can't ref a pool when ctx not in init stage."); - cctx->pool = pool; - return 0; -} + assert(!ZSTD_cwksp_reserve_failed(ws)); /* check that allocation hasn't already failed */ -size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize) -{ - return ZSTD_CCtx_refPrefix_advanced(cctx, prefix, prefixSize, ZSTD_dct_rawContent); -} + ZSTD_cwksp_clear_tables(ws); -size_t ZSTD_CCtx_refPrefix_advanced( - ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType) -{ - RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, - "Can't ref a prefix when ctx not in init stage."); - ZSTD_clearAllDicts(cctx); - if (prefix != NULL && prefixSize > 0) { - cctx->prefixDict.dict = prefix; - cctx->prefixDict.dictSize = prefixSize; - cctx->prefixDict.dictContentType = dictContentType; + DEBUGLOG(5, "reserving table space"); + /* table Space */ + ms->hashTable = (U32*)ZSTD_cwksp_reserve_table(ws, hSize * sizeof(U32)); + ms->chainTable = (U32*)ZSTD_cwksp_reserve_table(ws, chainSize * sizeof(U32)); + ms->hashTable3 = (U32*)ZSTD_cwksp_reserve_table(ws, h3Size * sizeof(U32)); + RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation, + "failed a workspace allocation in ZSTD_reset_matchState"); + + DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_leaveDirty); + if (crp!=ZSTDcrp_leaveDirty) { + /* reset tables only */ + ZSTD_cwksp_clean_tables(ws); } - return 0; -} -/*! ZSTD_CCtx_reset() : - * Also dumps dictionary */ -size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset) -{ - if ( (reset == ZSTD_reset_session_only) - || (reset == ZSTD_reset_session_and_parameters) ) { - cctx->streamStage = zcss_init; - cctx->pledgedSrcSizePlusOne = 0; + if (ZSTD_rowMatchFinderUsed(cParams->strategy, useRowMatchFinder)) { + /* Row match finder needs an additional table of hashes ("tags") */ + size_t const tagTableSize = hSize; + /* We want to generate a new salt in case we reset a Cctx, but we always want to use + * 0 when we reset a Cdict */ + if(forWho == ZSTD_resetTarget_CCtx) { + ms->tagTable = (BYTE*) ZSTD_cwksp_reserve_aligned_init_once(ws, tagTableSize); + ZSTD_advanceHashSalt(ms); + } else { + /* When we are not salting we want to always memset the memory */ + ms->tagTable = (BYTE*) ZSTD_cwksp_reserve_aligned64(ws, tagTableSize); + ZSTD_memset(ms->tagTable, 0, tagTableSize); + ms->hashSalt = 0; + } + { /* Switch to 32-entry rows if searchLog is 5 (or more) */ + U32 const rowLog = BOUNDED(4, cParams->searchLog, 6); + assert(cParams->hashLog >= rowLog); + ms->rowHashLog = cParams->hashLog - rowLog; + } } - if ( (reset == ZSTD_reset_parameters) - || (reset == ZSTD_reset_session_and_parameters) ) { - RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, - "Can't reset parameters only when not in init stage."); - ZSTD_clearAllDicts(cctx); - return ZSTD_CCtxParams_reset(&cctx->requestedParams); + + /* opt parser space */ + if ((forWho == ZSTD_resetTarget_CCtx) && (cParams->strategy >= ZSTD_btopt)) { + DEBUGLOG(4, "reserving optimal parser space"); + ms->opt.litFreq = (unsigned*)ZSTD_cwksp_reserve_aligned64(ws, (1<opt.litLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned64(ws, (MaxLL+1) * sizeof(unsigned)); + ms->opt.matchLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned64(ws, (MaxML+1) * sizeof(unsigned)); + ms->opt.offCodeFreq = (unsigned*)ZSTD_cwksp_reserve_aligned64(ws, (MaxOff+1) * sizeof(unsigned)); + ms->opt.matchTable = (ZSTD_match_t*)ZSTD_cwksp_reserve_aligned64(ws, ZSTD_OPT_SIZE * sizeof(ZSTD_match_t)); + ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_cwksp_reserve_aligned64(ws, ZSTD_OPT_SIZE * sizeof(ZSTD_optimal_t)); } - return 0; -} + ms->cParams = *cParams; -/** ZSTD_checkCParams() : - control CParam values remain within authorized range. - @return : 0, or an error code if one value is beyond authorized range */ -size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) -{ - BOUNDCHECK(ZSTD_c_windowLog, (int)cParams.windowLog); - BOUNDCHECK(ZSTD_c_chainLog, (int)cParams.chainLog); - BOUNDCHECK(ZSTD_c_hashLog, (int)cParams.hashLog); - BOUNDCHECK(ZSTD_c_searchLog, (int)cParams.searchLog); - BOUNDCHECK(ZSTD_c_minMatch, (int)cParams.minMatch); - BOUNDCHECK(ZSTD_c_targetLength,(int)cParams.targetLength); - BOUNDCHECK(ZSTD_c_strategy, cParams.strategy); + RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation, + "failed a workspace allocation in ZSTD_reset_matchState"); return 0; } -/** ZSTD_clampCParams() : - * make CParam values within valid range. - * @return : valid CParams */ -static ZSTD_compressionParameters -ZSTD_clampCParams(ZSTD_compressionParameters cParams) +/* ZSTD_indexTooCloseToMax() : + * minor optimization : prefer memset() rather than reduceIndex() + * which is measurably slow in some circumstances (reported for Visual Studio). + * Works when re-using a context for a lot of smallish inputs : + * if all inputs are smaller than ZSTD_INDEXOVERFLOW_MARGIN, + * memset() will be triggered before reduceIndex(). + */ +#define ZSTD_INDEXOVERFLOW_MARGIN (16 MB) +static int ZSTD_indexTooCloseToMax(ZSTD_window_t w) { -# define CLAMP_TYPE(cParam, val, type) { \ - ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); \ - if ((int)valbounds.upperBound) val=(type)bounds.upperBound; \ - } -# define CLAMP(cParam, val) CLAMP_TYPE(cParam, val, unsigned) - CLAMP(ZSTD_c_windowLog, cParams.windowLog); - CLAMP(ZSTD_c_chainLog, cParams.chainLog); - CLAMP(ZSTD_c_hashLog, cParams.hashLog); - CLAMP(ZSTD_c_searchLog, cParams.searchLog); - CLAMP(ZSTD_c_minMatch, cParams.minMatch); - CLAMP(ZSTD_c_targetLength,cParams.targetLength); - CLAMP_TYPE(ZSTD_c_strategy,cParams.strategy, ZSTD_strategy); - return cParams; + return (size_t)(w.nextSrc - w.base) > (ZSTD_CURRENT_MAX - ZSTD_INDEXOVERFLOW_MARGIN); } -/** ZSTD_cycleLog() : - * condition for correct operation : hashLog > 1 */ -U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat) +/** ZSTD_dictTooBig(): + * When dictionaries are larger than ZSTD_CHUNKSIZE_MAX they can't be loaded in + * one go generically. So we ensure that in that case we reset the tables to zero, + * so that we can load as much of the dictionary as possible. + */ +static int ZSTD_dictTooBig(size_t const loadedDictSize) { - U32 const btScale = ((U32)strat >= (U32)ZSTD_btlazy2); - return hashLog - btScale; + return loadedDictSize > ZSTD_CHUNKSIZE_MAX; } -/** ZSTD_dictAndWindowLog() : - * Returns an adjusted window log that is large enough to fit the source and the dictionary. - * The zstd format says that the entire dictionary is valid if one byte of the dictionary - * is within the window. So the hashLog and chainLog should be large enough to reference both - * the dictionary and the window. So we must use this adjusted dictAndWindowLog when downsizing - * the hashLog and windowLog. - * NOTE: srcSize must not be ZSTD_CONTENTSIZE_UNKNOWN. +/*! ZSTD_resetCCtx_internal() : + * @param loadedDictSize The size of the dictionary to be loaded + * into the context, if any. If no dictionary is used, or the + * dictionary is being attached / copied, then pass 0. + * note : `params` are assumed fully validated at this stage. */ -static U32 ZSTD_dictAndWindowLog(U32 windowLog, U64 srcSize, U64 dictSize) +static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, + ZSTD_CCtx_params const* params, + U64 const pledgedSrcSize, + size_t const loadedDictSize, + ZSTD_compResetPolicy_e const crp, + ZSTD_buffered_policy_e const zbuff) { - const U64 maxWindowSize = 1ULL << ZSTD_WINDOWLOG_MAX; - /* No dictionary ==> No change */ - if (dictSize == 0) { - return windowLog; - } - assert(windowLog <= ZSTD_WINDOWLOG_MAX); - assert(srcSize != ZSTD_CONTENTSIZE_UNKNOWN); /* Handled in ZSTD_adjustCParams_internal() */ - { - U64 const windowSize = 1ULL << windowLog; - U64 const dictAndWindowSize = dictSize + windowSize; - /* If the window size is already large enough to fit both the source and the dictionary - * then just use the window size. Otherwise adjust so that it fits the dictionary and - * the window. - */ - if (windowSize >= dictSize + srcSize) { - return windowLog; /* Window size large enough already */ - } else if (dictAndWindowSize >= maxWindowSize) { - return ZSTD_WINDOWLOG_MAX; /* Larger than max window log */ - } else { - return ZSTD_highbit32((U32)dictAndWindowSize - 1) + 1; - } - } -} + ZSTD_cwksp* const ws = &zc->workspace; + DEBUGLOG(4, "ZSTD_resetCCtx_internal: pledgedSrcSize=%u, wlog=%u, useRowMatchFinder=%d useBlockSplitter=%d", + (U32)pledgedSrcSize, params->cParams.windowLog, (int)params->useRowMatchFinder, (int)params->postBlockSplitter); + assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); -/** ZSTD_adjustCParams_internal() : - * optimize `cPar` for a specified input (`srcSize` and `dictSize`). - * mostly downsize to reduce memory consumption and initialization latency. - * `srcSize` can be ZSTD_CONTENTSIZE_UNKNOWN when not known. - * `mode` is the mode for parameter adjustment. See docs for `ZSTD_cParamMode_e`. - * note : `srcSize==0` means 0! - * condition : cPar is presumed validated (can be checked using ZSTD_checkCParams()). */ -static ZSTD_compressionParameters -ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, - unsigned long long srcSize, - size_t dictSize, - ZSTD_cParamMode_e mode) -{ - const U64 minSrcSize = 513; /* (1<<9) + 1 */ - const U64 maxWindowResize = 1ULL << (ZSTD_WINDOWLOG_MAX-1); - assert(ZSTD_checkCParams(cPar)==0); + zc->isFirstBlock = 1; - switch (mode) { - case ZSTD_cpm_unknown: - case ZSTD_cpm_noAttachDict: - /* If we don't know the source size, don't make any - * assumptions about it. We will already have selected - * smaller parameters if a dictionary is in use. - */ - break; - case ZSTD_cpm_createCDict: - /* Assume a small source size when creating a dictionary - * with an unkown source size. - */ - if (dictSize && srcSize == ZSTD_CONTENTSIZE_UNKNOWN) - srcSize = minSrcSize; - break; - case ZSTD_cpm_attachDict: - /* Dictionary has its own dedicated parameters which have - * already been selected. We are selecting parameters - * for only the source. - */ - dictSize = 0; - break; - default: - assert(0); - break; + /* Set applied params early so we can modify them for LDM, + * and point params at the applied params. + */ + zc->appliedParams = *params; + params = &zc->appliedParams; + + assert(params->useRowMatchFinder != ZSTD_ps_auto); + assert(params->postBlockSplitter != ZSTD_ps_auto); + assert(params->ldmParams.enableLdm != ZSTD_ps_auto); + assert(params->maxBlockSize != 0); + if (params->ldmParams.enableLdm == ZSTD_ps_enable) { + /* Adjust long distance matching parameters */ + ZSTD_ldm_adjustParameters(&zc->appliedParams.ldmParams, ¶ms->cParams); + assert(params->ldmParams.hashLog >= params->ldmParams.bucketSizeLog); + assert(params->ldmParams.hashRateLog < 32); } - /* resize windowLog if input is small enough, to use less memory */ - if ( (srcSize < maxWindowResize) - && (dictSize < maxWindowResize) ) { - U32 const tSize = (U32)(srcSize + dictSize); - static U32 const hashSizeMin = 1 << ZSTD_HASHLOG_MIN; - U32 const srcLog = (tSize < hashSizeMin) ? ZSTD_HASHLOG_MIN : - ZSTD_highbit32(tSize-1) + 1; - if (cPar.windowLog > srcLog) cPar.windowLog = srcLog; - } - if (srcSize != ZSTD_CONTENTSIZE_UNKNOWN) { - U32 const dictAndWindowLog = ZSTD_dictAndWindowLog(cPar.windowLog, (U64)srcSize, (U64)dictSize); - U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy); - if (cPar.hashLog > dictAndWindowLog+1) cPar.hashLog = dictAndWindowLog+1; - if (cycleLog > dictAndWindowLog) - cPar.chainLog -= (cycleLog - dictAndWindowLog); - } + { size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params->cParams.windowLog), pledgedSrcSize)); + size_t const blockSize = MIN(params->maxBlockSize, windowSize); + size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, params->cParams.minMatch, ZSTD_hasExtSeqProd(params)); + size_t const buffOutSize = (zbuff == ZSTDb_buffered && params->outBufferMode == ZSTD_bm_buffered) + ? ZSTD_compressBound(blockSize) + 1 + : 0; + size_t const buffInSize = (zbuff == ZSTDb_buffered && params->inBufferMode == ZSTD_bm_buffered) + ? windowSize + blockSize + : 0; + size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params->ldmParams, blockSize); + + int const indexTooClose = ZSTD_indexTooCloseToMax(zc->blockState.matchState.window); + int const dictTooBig = ZSTD_dictTooBig(loadedDictSize); + ZSTD_indexResetPolicy_e needsIndexReset = + (indexTooClose || dictTooBig || !zc->initialized) ? ZSTDirp_reset : ZSTDirp_continue; + + size_t const neededSpace = + ZSTD_estimateCCtxSize_usingCCtxParams_internal( + ¶ms->cParams, ¶ms->ldmParams, zc->staticSize != 0, params->useRowMatchFinder, + buffInSize, buffOutSize, pledgedSrcSize, ZSTD_hasExtSeqProd(params), params->maxBlockSize); + + FORWARD_IF_ERROR(neededSpace, "cctx size estimate failed!"); + + if (!zc->staticSize) ZSTD_cwksp_bump_oversized_duration(ws, 0); + + { /* Check if workspace is large enough, alloc a new one if needed */ + int const workspaceTooSmall = ZSTD_cwksp_sizeof(ws) < neededSpace; + int const workspaceWasteful = ZSTD_cwksp_check_wasteful(ws, neededSpace); + int resizeWorkspace = workspaceTooSmall || workspaceWasteful; + DEBUGLOG(4, "Need %zu B workspace", neededSpace); + DEBUGLOG(4, "windowSize: %zu - blockSize: %zu", windowSize, blockSize); + + if (resizeWorkspace) { + DEBUGLOG(4, "Resize workspaceSize from %zuKB to %zuKB", + ZSTD_cwksp_sizeof(ws) >> 10, + neededSpace >> 10); + + RETURN_ERROR_IF(zc->staticSize, memory_allocation, "static cctx : no resize"); + + needsIndexReset = ZSTDirp_reset; + + ZSTD_cwksp_free(ws, zc->customMem); + FORWARD_IF_ERROR(ZSTD_cwksp_create(ws, neededSpace, zc->customMem), ""); + + DEBUGLOG(5, "reserving object space"); + /* Statically sized space. + * tmpWorkspace never moves, + * though prev/next block swap places */ + assert(ZSTD_cwksp_check_available(ws, 2 * sizeof(ZSTD_compressedBlockState_t))); + zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t)); + RETURN_ERROR_IF(zc->blockState.prevCBlock == NULL, memory_allocation, "couldn't allocate prevCBlock"); + zc->blockState.nextCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t)); + RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate nextCBlock"); + zc->tmpWorkspace = ZSTD_cwksp_reserve_object(ws, TMP_WORKSPACE_SIZE); + RETURN_ERROR_IF(zc->tmpWorkspace == NULL, memory_allocation, "couldn't allocate tmpWorkspace"); + zc->tmpWkspSize = TMP_WORKSPACE_SIZE; + } } - if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) - cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* minimum wlog required for valid frame header */ + ZSTD_cwksp_clear(ws); - return cPar; -} + /* init params */ + zc->blockState.matchState.cParams = params->cParams; + zc->blockState.matchState.prefetchCDictTables = params->prefetchCDictTables == ZSTD_ps_enable; + zc->pledgedSrcSizePlusOne = pledgedSrcSize+1; + zc->consumedSrcSize = 0; + zc->producedCSize = 0; + if (pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN) + zc->appliedParams.fParams.contentSizeFlag = 0; + DEBUGLOG(4, "pledged content size : %u ; flag : %u", + (unsigned)pledgedSrcSize, zc->appliedParams.fParams.contentSizeFlag); + zc->blockSizeMax = blockSize; -ZSTD_compressionParameters -ZSTD_adjustCParams(ZSTD_compressionParameters cPar, - unsigned long long srcSize, - size_t dictSize) -{ - cPar = ZSTD_clampCParams(cPar); /* resulting cPar is necessarily valid (all parameters within range) */ - if (srcSize == 0) srcSize = ZSTD_CONTENTSIZE_UNKNOWN; - return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize, ZSTD_cpm_unknown); -} + XXH64_reset(&zc->xxhState, 0); + zc->stage = ZSTDcs_init; + zc->dictID = 0; + zc->dictContentSize = 0; -static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode); -static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode); + ZSTD_reset_compressedBlockState(zc->blockState.prevCBlock); -static void ZSTD_overrideCParams( - ZSTD_compressionParameters* cParams, - const ZSTD_compressionParameters* overrides) -{ - if (overrides->windowLog) cParams->windowLog = overrides->windowLog; - if (overrides->hashLog) cParams->hashLog = overrides->hashLog; - if (overrides->chainLog) cParams->chainLog = overrides->chainLog; - if (overrides->searchLog) cParams->searchLog = overrides->searchLog; - if (overrides->minMatch) cParams->minMatch = overrides->minMatch; - if (overrides->targetLength) cParams->targetLength = overrides->targetLength; - if (overrides->strategy) cParams->strategy = overrides->strategy; -} + FORWARD_IF_ERROR(ZSTD_reset_matchState( + &zc->blockState.matchState, + ws, + ¶ms->cParams, + params->useRowMatchFinder, + crp, + needsIndexReset, + ZSTD_resetTarget_CCtx), ""); -ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( - const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) -{ - ZSTD_compressionParameters cParams; - if (srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN && CCtxParams->srcSizeHint > 0) { - srcSizeHint = CCtxParams->srcSizeHint; - } - cParams = ZSTD_getCParams_internal(CCtxParams->compressionLevel, srcSizeHint, dictSize, mode); - if (CCtxParams->ldmParams.enableLdm) cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG; - ZSTD_overrideCParams(&cParams, &CCtxParams->cParams); - assert(!ZSTD_checkCParams(cParams)); - /* srcSizeHint == 0 means 0 */ - return ZSTD_adjustCParams_internal(cParams, srcSizeHint, dictSize, mode); -} + zc->seqStore.sequencesStart = (SeqDef*)ZSTD_cwksp_reserve_aligned64(ws, maxNbSeq * sizeof(SeqDef)); -static size_t -ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams, - const U32 forCCtx) -{ - size_t const chainSize = (cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cParams->chainLog); - size_t const hSize = ((size_t)1) << cParams->hashLog; - U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; - size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0; - /* We don't use ZSTD_cwksp_alloc_size() here because the tables aren't - * surrounded by redzones in ASAN. */ - size_t const tableSpace = chainSize * sizeof(U32) - + hSize * sizeof(U32) - + h3Size * sizeof(U32); - size_t const optPotentialSpace = - ZSTD_cwksp_alloc_size((MaxML+1) * sizeof(U32)) - + ZSTD_cwksp_alloc_size((MaxLL+1) * sizeof(U32)) - + ZSTD_cwksp_alloc_size((MaxOff+1) * sizeof(U32)) - + ZSTD_cwksp_alloc_size((1<strategy >= ZSTD_btopt)) - ? optPotentialSpace - : 0; - DEBUGLOG(4, "chainSize: %u - hSize: %u - h3Size: %u", - (U32)chainSize, (U32)hSize, (U32)h3Size); - return tableSpace + optSpace; -} + /* ldm hash table */ + if (params->ldmParams.enableLdm == ZSTD_ps_enable) { + /* TODO: avoid memset? */ + size_t const ldmHSize = ((size_t)1) << params->ldmParams.hashLog; + zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned64(ws, ldmHSize * sizeof(ldmEntry_t)); + ZSTD_memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t)); + zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned64(ws, maxNbLdmSeq * sizeof(rawSeq)); + zc->maxNbLdmSequences = maxNbLdmSeq; -static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal( - const ZSTD_compressionParameters* cParams, - const ldmParams_t* ldmParams, - const int isStatic, - const size_t buffInSize, - const size_t buffOutSize, - const U64 pledgedSrcSize) -{ - size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << cParams->windowLog), pledgedSrcSize)); - size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); - U32 const divider = (cParams->minMatch==3) ? 3 : 4; - size_t const maxNbSeq = blockSize / divider; - size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize) - + ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(seqDef)) - + 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE)); - size_t const entropySpace = ZSTD_cwksp_alloc_size(ENTROPY_WORKSPACE_SIZE); - size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t)); - size_t const matchStateSize = ZSTD_sizeof_matchState(cParams, /* forCCtx */ 1); + ZSTD_window_init(&zc->ldmState.window); + zc->ldmState.loadedDictEnd = 0; + } - size_t const ldmSpace = ZSTD_ldm_getTableSize(*ldmParams); - size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(*ldmParams, blockSize); - size_t const ldmSeqSpace = ldmParams->enableLdm ? - ZSTD_cwksp_alloc_size(maxNbLdmSeq * sizeof(rawSeq)) : 0; + /* reserve space for block-level external sequences */ + if (ZSTD_hasExtSeqProd(params)) { + size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize); + zc->extSeqBufCapacity = maxNbExternalSeq; + zc->extSeqBuf = + (ZSTD_Sequence*)ZSTD_cwksp_reserve_aligned64(ws, maxNbExternalSeq * sizeof(ZSTD_Sequence)); + } + /* buffers */ - size_t const bufferSpace = ZSTD_cwksp_alloc_size(buffInSize) - + ZSTD_cwksp_alloc_size(buffOutSize); + /* ZSTD_wildcopy() is used to copy into the literals buffer, + * so we have to oversize the buffer by WILDCOPY_OVERLENGTH bytes. + */ + zc->seqStore.litStart = ZSTD_cwksp_reserve_buffer(ws, blockSize + WILDCOPY_OVERLENGTH); + zc->seqStore.maxNbLit = blockSize; - size_t const cctxSpace = isStatic ? ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx)) : 0; + zc->bufferedPolicy = zbuff; + zc->inBuffSize = buffInSize; + zc->inBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffInSize); + zc->outBuffSize = buffOutSize; + zc->outBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffOutSize); - size_t const neededSpace = - cctxSpace + - entropySpace + - blockStateSpace + - ldmSpace + - ldmSeqSpace + - matchStateSize + - tokenSpace + - bufferSpace; + /* ldm bucketOffsets table */ + if (params->ldmParams.enableLdm == ZSTD_ps_enable) { + /* TODO: avoid memset? */ + size_t const numBuckets = + ((size_t)1) << (params->ldmParams.hashLog - + params->ldmParams.bucketSizeLog); + zc->ldmState.bucketOffsets = ZSTD_cwksp_reserve_buffer(ws, numBuckets); + ZSTD_memset(zc->ldmState.bucketOffsets, 0, numBuckets); + } - DEBUGLOG(5, "estimate workspace : %u", (U32)neededSpace); - return neededSpace; -} + /* sequences storage */ + ZSTD_referenceExternalSequences(zc, NULL, 0); + zc->seqStore.maxNbSeq = maxNbSeq; + zc->seqStore.llCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); + zc->seqStore.mlCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); + zc->seqStore.ofCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); -size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params) -{ - ZSTD_compressionParameters const cParams = - ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); + DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_cwksp_available_space(ws)); + assert(ZSTD_cwksp_estimated_space_within_bounds(ws, neededSpace)); - RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only."); - /* estimateCCtxSize is for one-shot compression. So no buffers should - * be needed. However, we still allocate two 0-sized buffers, which can - * take space under ASAN. */ - return ZSTD_estimateCCtxSize_usingCCtxParams_internal( - &cParams, ¶ms->ldmParams, 1, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN); + zc->initialized = 1; + + return 0; + } } -size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams) -{ - ZSTD_CCtx_params const params = ZSTD_makeCCtxParamsFromCParams(cParams); - return ZSTD_estimateCCtxSize_usingCCtxParams(¶ms); +/* ZSTD_invalidateRepCodes() : + * ensures next compression will not use repcodes from previous block. + * Note : only works with regular variant; + * do not use with extDict variant ! */ +void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) { + int i; + for (i=0; iblockState.prevCBlock->rep[i] = 0; + assert(!ZSTD_window_hasExtDict(cctx->blockState.matchState.window)); } -static size_t ZSTD_estimateCCtxSize_internal(int compressionLevel) +/* These are the approximate sizes for each strategy past which copying the + * dictionary tables into the working context is faster than using them + * in-place. + */ +static const size_t attachDictSizeCutoffs[ZSTD_STRATEGY_MAX+1] = { + 8 KB, /* unused */ + 8 KB, /* ZSTD_fast */ + 16 KB, /* ZSTD_dfast */ + 32 KB, /* ZSTD_greedy */ + 32 KB, /* ZSTD_lazy */ + 32 KB, /* ZSTD_lazy2 */ + 32 KB, /* ZSTD_btlazy2 */ + 32 KB, /* ZSTD_btopt */ + 8 KB, /* ZSTD_btultra */ + 8 KB /* ZSTD_btultra2 */ +}; + +static int ZSTD_shouldAttachDict(const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, + U64 pledgedSrcSize) { - ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); - return ZSTD_estimateCCtxSize_usingCParams(cParams); + size_t cutoff = attachDictSizeCutoffs[cdict->matchState.cParams.strategy]; + int const dedicatedDictSearch = cdict->matchState.dedicatedDictSearch; + return dedicatedDictSearch + || ( ( pledgedSrcSize <= cutoff + || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN + || params->attachDictPref == ZSTD_dictForceAttach ) + && params->attachDictPref != ZSTD_dictForceCopy + && !params->forceWindow ); /* dictMatchState isn't correctly + * handled in _enforceMaxDist */ } -size_t ZSTD_estimateCCtxSize(int compressionLevel) +static size_t +ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx, + const ZSTD_CDict* cdict, + ZSTD_CCtx_params params, + U64 pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) { - int level; - size_t memBudget = 0; - for (level=MIN(compressionLevel, 1); level<=compressionLevel; level++) { - size_t const newMB = ZSTD_estimateCCtxSize_internal(level); - if (newMB > memBudget) memBudget = newMB; + DEBUGLOG(4, "ZSTD_resetCCtx_byAttachingCDict() pledgedSrcSize=%llu", + (unsigned long long)pledgedSrcSize); + { + ZSTD_compressionParameters adjusted_cdict_cParams = cdict->matchState.cParams; + unsigned const windowLog = params.cParams.windowLog; + assert(windowLog != 0); + /* Resize working context table params for input only, since the dict + * has its own tables. */ + /* pledgedSrcSize == 0 means 0! */ + + if (cdict->matchState.dedicatedDictSearch) { + ZSTD_dedicatedDictSearch_revertCParams(&adjusted_cdict_cParams); + } + + params.cParams = ZSTD_adjustCParams_internal(adjusted_cdict_cParams, pledgedSrcSize, + cdict->dictContentSize, ZSTD_cpm_attachDict, + params.useRowMatchFinder); + params.cParams.windowLog = windowLog; + params.useRowMatchFinder = cdict->useRowMatchFinder; /* cdict overrides */ + FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, ¶ms, pledgedSrcSize, + /* loadedDictSize */ 0, + ZSTDcrp_makeClean, zbuff), ""); + assert(cctx->appliedParams.cParams.strategy == adjusted_cdict_cParams.strategy); } - return memBudget; -} -size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params) -{ - RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only."); - { ZSTD_compressionParameters const cParams = - ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); - size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); - size_t const inBuffSize = (params->inBufferMode == ZSTD_bm_buffered) - ? ((size_t)1 << cParams.windowLog) + blockSize - : 0; - size_t const outBuffSize = (params->outBufferMode == ZSTD_bm_buffered) - ? ZSTD_compressBound(blockSize) + 1 - : 0; + { const U32 cdictEnd = (U32)( cdict->matchState.window.nextSrc + - cdict->matchState.window.base); + const U32 cdictLen = cdictEnd - cdict->matchState.window.dictLimit; + if (cdictLen == 0) { + /* don't even attach dictionaries with no contents */ + DEBUGLOG(4, "skipping attaching empty dictionary"); + } else { + DEBUGLOG(4, "attaching dictionary into context"); + cctx->blockState.matchState.dictMatchState = &cdict->matchState; + + /* prep working match state so dict matches never have negative indices + * when they are translated to the working context's index space. */ + if (cctx->blockState.matchState.window.dictLimit < cdictEnd) { + cctx->blockState.matchState.window.nextSrc = + cctx->blockState.matchState.window.base + cdictEnd; + ZSTD_window_clear(&cctx->blockState.matchState.window); + } + /* loadedDictEnd is expressed within the referential of the active context */ + cctx->blockState.matchState.loadedDictEnd = cctx->blockState.matchState.window.dictLimit; + } } + + cctx->dictID = cdict->dictID; + cctx->dictContentSize = cdict->dictContentSize; + + /* copy block state */ + ZSTD_memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, sizeof(cdict->cBlockState)); + + return 0; +} - return ZSTD_estimateCCtxSize_usingCCtxParams_internal( - &cParams, ¶ms->ldmParams, 1, inBuffSize, outBuffSize, - ZSTD_CONTENTSIZE_UNKNOWN); +static void ZSTD_copyCDictTableIntoCCtx(U32* dst, U32 const* src, size_t tableSize, + ZSTD_compressionParameters const* cParams) { + if (ZSTD_CDictIndicesAreTagged(cParams)){ + /* Remove tags from the CDict table if they are present. + * See docs on "short cache" in zstd_compress_internal.h for context. */ + size_t i; + for (i = 0; i < tableSize; i++) { + U32 const taggedIndex = src[i]; + U32 const index = taggedIndex >> ZSTD_SHORT_CACHE_TAG_BITS; + dst[i] = index; + } + } else { + ZSTD_memcpy(dst, src, tableSize * sizeof(U32)); } } -size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams) +static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, + const ZSTD_CDict* cdict, + ZSTD_CCtx_params params, + U64 pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) { - ZSTD_CCtx_params const params = ZSTD_makeCCtxParamsFromCParams(cParams); - return ZSTD_estimateCStreamSize_usingCCtxParams(¶ms); -} + const ZSTD_compressionParameters *cdict_cParams = &cdict->matchState.cParams; -static size_t ZSTD_estimateCStreamSize_internal(int compressionLevel) -{ - ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); - return ZSTD_estimateCStreamSize_usingCParams(cParams); -} + assert(!cdict->matchState.dedicatedDictSearch); + DEBUGLOG(4, "ZSTD_resetCCtx_byCopyingCDict() pledgedSrcSize=%llu", + (unsigned long long)pledgedSrcSize); -size_t ZSTD_estimateCStreamSize(int compressionLevel) -{ - int level; - size_t memBudget = 0; - for (level=MIN(compressionLevel, 1); level<=compressionLevel; level++) { - size_t const newMB = ZSTD_estimateCStreamSize_internal(level); - if (newMB > memBudget) memBudget = newMB; + { unsigned const windowLog = params.cParams.windowLog; + assert(windowLog != 0); + /* Copy only compression parameters related to tables. */ + params.cParams = *cdict_cParams; + params.cParams.windowLog = windowLog; + params.useRowMatchFinder = cdict->useRowMatchFinder; + FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, ¶ms, pledgedSrcSize, + /* loadedDictSize */ 0, + ZSTDcrp_leaveDirty, zbuff), ""); + assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy); + assert(cctx->appliedParams.cParams.hashLog == cdict_cParams->hashLog); + assert(cctx->appliedParams.cParams.chainLog == cdict_cParams->chainLog); } - return memBudget; -} -/* ZSTD_getFrameProgression(): - * tells how much data has been consumed (input) and produced (output) for current frame. - * able to count progression inside worker threads (non-blocking mode). - */ -ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx) -{ -#ifdef ZSTD_MULTITHREAD - if (cctx->appliedParams.nbWorkers > 0) { - return ZSTDMT_getFrameProgression(cctx->mtctx); + ZSTD_cwksp_mark_tables_dirty(&cctx->workspace); + assert(params.useRowMatchFinder != ZSTD_ps_auto); + + /* copy tables */ + { size_t const chainSize = ZSTD_allocateChainTable(cdict_cParams->strategy, cdict->useRowMatchFinder, 0 /* DDS guaranteed disabled */) + ? ((size_t)1 << cdict_cParams->chainLog) + : 0; + size_t const hSize = (size_t)1 << cdict_cParams->hashLog; + + ZSTD_copyCDictTableIntoCCtx(cctx->blockState.matchState.hashTable, + cdict->matchState.hashTable, + hSize, cdict_cParams); + + /* Do not copy cdict's chainTable if cctx has parameters such that it would not use chainTable */ + if (ZSTD_allocateChainTable(cctx->appliedParams.cParams.strategy, cctx->appliedParams.useRowMatchFinder, 0 /* forDDSDict */)) { + ZSTD_copyCDictTableIntoCCtx(cctx->blockState.matchState.chainTable, + cdict->matchState.chainTable, + chainSize, cdict_cParams); + } + /* copy tag table */ + if (ZSTD_rowMatchFinderUsed(cdict_cParams->strategy, cdict->useRowMatchFinder)) { + size_t const tagTableSize = hSize; + ZSTD_memcpy(cctx->blockState.matchState.tagTable, + cdict->matchState.tagTable, + tagTableSize); + cctx->blockState.matchState.hashSalt = cdict->matchState.hashSalt; + } } -#endif - { ZSTD_frameProgression fp; - size_t const buffered = (cctx->inBuff == NULL) ? 0 : - cctx->inBuffPos - cctx->inToCompress; - if (buffered) assert(cctx->inBuffPos >= cctx->inToCompress); - assert(buffered <= ZSTD_BLOCKSIZE_MAX); - fp.ingested = cctx->consumedSrcSize + buffered; - fp.consumed = cctx->consumedSrcSize; - fp.produced = cctx->producedCSize; - fp.flushed = cctx->producedCSize; /* simplified; some data might still be left within streaming output buffer */ - fp.currentJobID = 0; - fp.nbActiveWorkers = 0; - return fp; -} } -/*! ZSTD_toFlushNow() - * Only useful for multithreading scenarios currently (nbWorkers >= 1). - */ -size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx) -{ -#ifdef ZSTD_MULTITHREAD - if (cctx->appliedParams.nbWorkers > 0) { - return ZSTDMT_toFlushNow(cctx->mtctx); + /* Zero the hashTable3, since the cdict never fills it */ + assert(cctx->blockState.matchState.hashLog3 <= 31); + { U32 const h3log = cctx->blockState.matchState.hashLog3; + size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0; + assert(cdict->matchState.hashLog3 == 0); + ZSTD_memset(cctx->blockState.matchState.hashTable3, 0, h3Size * sizeof(U32)); } -#endif - (void)cctx; - return 0; /* over-simplification; could also check if context is currently running in streaming mode, and in which case, report how many bytes are left to be flushed within output buffer */ -} -static void ZSTD_assertEqualCParams(ZSTD_compressionParameters cParams1, - ZSTD_compressionParameters cParams2) -{ - (void)cParams1; - (void)cParams2; - assert(cParams1.windowLog == cParams2.windowLog); - assert(cParams1.chainLog == cParams2.chainLog); - assert(cParams1.hashLog == cParams2.hashLog); - assert(cParams1.searchLog == cParams2.searchLog); - assert(cParams1.minMatch == cParams2.minMatch); - assert(cParams1.targetLength == cParams2.targetLength); - assert(cParams1.strategy == cParams2.strategy); -} + ZSTD_cwksp_mark_tables_clean(&cctx->workspace); -void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs) -{ - int i; - for (i = 0; i < ZSTD_REP_NUM; ++i) - bs->rep[i] = repStartValue[i]; - bs->entropy.huf.repeatMode = HUF_repeat_none; - bs->entropy.fse.offcode_repeatMode = FSE_repeat_none; - bs->entropy.fse.matchlength_repeatMode = FSE_repeat_none; - bs->entropy.fse.litlength_repeatMode = FSE_repeat_none; -} + /* copy dictionary offsets */ + { ZSTD_MatchState_t const* srcMatchState = &cdict->matchState; + ZSTD_MatchState_t* dstMatchState = &cctx->blockState.matchState; + dstMatchState->window = srcMatchState->window; + dstMatchState->nextToUpdate = srcMatchState->nextToUpdate; + dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd; + } -/*! ZSTD_invalidateMatchState() - * Invalidate all the matches in the match finder tables. - * Requires nextSrc and base to be set (can be NULL). - */ -static void ZSTD_invalidateMatchState(ZSTD_matchState_t* ms) -{ - ZSTD_window_clear(&ms->window); + cctx->dictID = cdict->dictID; + cctx->dictContentSize = cdict->dictContentSize; - ms->nextToUpdate = ms->window.dictLimit; - ms->loadedDictEnd = 0; - ms->opt.litLengthSum = 0; /* force reset of btopt stats */ - ms->dictMatchState = NULL; + /* copy block state */ + ZSTD_memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, sizeof(cdict->cBlockState)); + + return 0; } -/** - * Controls, for this matchState reset, whether the tables need to be cleared / - * prepared for the coming compression (ZSTDcrp_makeClean), or whether the - * tables can be left unclean (ZSTDcrp_leaveDirty), because we know that a - * subsequent operation will overwrite the table space anyways (e.g., copying - * the matchState contents in from a CDict). - */ -typedef enum { - ZSTDcrp_makeClean, - ZSTDcrp_leaveDirty -} ZSTD_compResetPolicy_e; +/* We have a choice between copying the dictionary context into the working + * context, or referencing the dictionary context from the working context + * in-place. We decide here which strategy to use. */ +static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx, + const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, + U64 pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) +{ -/** - * Controls, for this matchState reset, whether indexing can continue where it - * left off (ZSTDirp_continue), or whether it needs to be restarted from zero - * (ZSTDirp_reset). - */ -typedef enum { - ZSTDirp_continue, - ZSTDirp_reset -} ZSTD_indexResetPolicy_e; + DEBUGLOG(4, "ZSTD_resetCCtx_usingCDict (pledgedSrcSize=%u)", + (unsigned)pledgedSrcSize); -typedef enum { - ZSTD_resetTarget_CDict, - ZSTD_resetTarget_CCtx -} ZSTD_resetTarget_e; + if (ZSTD_shouldAttachDict(cdict, params, pledgedSrcSize)) { + return ZSTD_resetCCtx_byAttachingCDict( + cctx, cdict, *params, pledgedSrcSize, zbuff); + } else { + return ZSTD_resetCCtx_byCopyingCDict( + cctx, cdict, *params, pledgedSrcSize, zbuff); + } +} -static size_t -ZSTD_reset_matchState(ZSTD_matchState_t* ms, - ZSTD_cwksp* ws, - const ZSTD_compressionParameters* cParams, - const ZSTD_compResetPolicy_e crp, - const ZSTD_indexResetPolicy_e forceResetIndex, - const ZSTD_resetTarget_e forWho) +/*! ZSTD_copyCCtx_internal() : + * Duplicate an existing context `srcCCtx` into another one `dstCCtx`. + * Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). + * The "context", in this case, refers to the hash and chain tables, + * entropy tables, and dictionary references. + * `windowLog` value is enforced if != 0, otherwise value is copied from srcCCtx. + * @return : 0, or an error code */ +static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, + const ZSTD_CCtx* srcCCtx, + ZSTD_frameParameters fParams, + U64 pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) { - size_t const chainSize = (cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cParams->chainLog); - size_t const hSize = ((size_t)1) << cParams->hashLog; - U32 const hashLog3 = ((forWho == ZSTD_resetTarget_CCtx) && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; - size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0; - - DEBUGLOG(4, "reset indices : %u", forceResetIndex == ZSTDirp_reset); - if (forceResetIndex == ZSTDirp_reset) { - ZSTD_window_init(&ms->window); - ZSTD_cwksp_mark_tables_dirty(ws); + RETURN_ERROR_IF(srcCCtx->stage!=ZSTDcs_init, stage_wrong, + "Can't copy a ctx that's not in init stage."); + DEBUGLOG(5, "ZSTD_copyCCtx_internal"); + ZSTD_memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); + { ZSTD_CCtx_params params = dstCCtx->requestedParams; + /* Copy only compression parameters related to tables. */ + params.cParams = srcCCtx->appliedParams.cParams; + assert(srcCCtx->appliedParams.useRowMatchFinder != ZSTD_ps_auto); + assert(srcCCtx->appliedParams.postBlockSplitter != ZSTD_ps_auto); + assert(srcCCtx->appliedParams.ldmParams.enableLdm != ZSTD_ps_auto); + params.useRowMatchFinder = srcCCtx->appliedParams.useRowMatchFinder; + params.postBlockSplitter = srcCCtx->appliedParams.postBlockSplitter; + params.ldmParams = srcCCtx->appliedParams.ldmParams; + params.fParams = fParams; + params.maxBlockSize = srcCCtx->appliedParams.maxBlockSize; + ZSTD_resetCCtx_internal(dstCCtx, ¶ms, pledgedSrcSize, + /* loadedDictSize */ 0, + ZSTDcrp_leaveDirty, zbuff); + assert(dstCCtx->appliedParams.cParams.windowLog == srcCCtx->appliedParams.cParams.windowLog); + assert(dstCCtx->appliedParams.cParams.strategy == srcCCtx->appliedParams.cParams.strategy); + assert(dstCCtx->appliedParams.cParams.hashLog == srcCCtx->appliedParams.cParams.hashLog); + assert(dstCCtx->appliedParams.cParams.chainLog == srcCCtx->appliedParams.cParams.chainLog); + assert(dstCCtx->blockState.matchState.hashLog3 == srcCCtx->blockState.matchState.hashLog3); } - ms->hashLog3 = hashLog3; - - ZSTD_invalidateMatchState(ms); - - assert(!ZSTD_cwksp_reserve_failed(ws)); /* check that allocation hasn't already failed */ - - ZSTD_cwksp_clear_tables(ws); + ZSTD_cwksp_mark_tables_dirty(&dstCCtx->workspace); - DEBUGLOG(5, "reserving table space"); - /* table Space */ - ms->hashTable = (U32*)ZSTD_cwksp_reserve_table(ws, hSize * sizeof(U32)); - ms->chainTable = (U32*)ZSTD_cwksp_reserve_table(ws, chainSize * sizeof(U32)); - ms->hashTable3 = (U32*)ZSTD_cwksp_reserve_table(ws, h3Size * sizeof(U32)); - RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation, - "failed a workspace allocation in ZSTD_reset_matchState"); + /* copy tables */ + { size_t const chainSize = ZSTD_allocateChainTable(srcCCtx->appliedParams.cParams.strategy, + srcCCtx->appliedParams.useRowMatchFinder, + 0 /* forDDSDict */) + ? ((size_t)1 << srcCCtx->appliedParams.cParams.chainLog) + : 0; + size_t const hSize = (size_t)1 << srcCCtx->appliedParams.cParams.hashLog; + U32 const h3log = srcCCtx->blockState.matchState.hashLog3; + size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0; - DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_leaveDirty); - if (crp!=ZSTDcrp_leaveDirty) { - /* reset tables only */ - ZSTD_cwksp_clean_tables(ws); + ZSTD_memcpy(dstCCtx->blockState.matchState.hashTable, + srcCCtx->blockState.matchState.hashTable, + hSize * sizeof(U32)); + ZSTD_memcpy(dstCCtx->blockState.matchState.chainTable, + srcCCtx->blockState.matchState.chainTable, + chainSize * sizeof(U32)); + ZSTD_memcpy(dstCCtx->blockState.matchState.hashTable3, + srcCCtx->blockState.matchState.hashTable3, + h3Size * sizeof(U32)); } - /* opt parser space */ - if ((forWho == ZSTD_resetTarget_CCtx) && (cParams->strategy >= ZSTD_btopt)) { - DEBUGLOG(4, "reserving optimal parser space"); - ms->opt.litFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (1<opt.litLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxLL+1) * sizeof(unsigned)); - ms->opt.matchLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxML+1) * sizeof(unsigned)); - ms->opt.offCodeFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxOff+1) * sizeof(unsigned)); - ms->opt.matchTable = (ZSTD_match_t*)ZSTD_cwksp_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_match_t)); - ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_cwksp_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t)); - } + ZSTD_cwksp_mark_tables_clean(&dstCCtx->workspace); - ms->cParams = *cParams; + /* copy dictionary offsets */ + { + const ZSTD_MatchState_t* srcMatchState = &srcCCtx->blockState.matchState; + ZSTD_MatchState_t* dstMatchState = &dstCCtx->blockState.matchState; + dstMatchState->window = srcMatchState->window; + dstMatchState->nextToUpdate = srcMatchState->nextToUpdate; + dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd; + } + dstCCtx->dictID = srcCCtx->dictID; + dstCCtx->dictContentSize = srcCCtx->dictContentSize; - RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation, - "failed a workspace allocation in ZSTD_reset_matchState"); + /* copy block state */ + ZSTD_memcpy(dstCCtx->blockState.prevCBlock, srcCCtx->blockState.prevCBlock, sizeof(*srcCCtx->blockState.prevCBlock)); return 0; } -/* ZSTD_indexTooCloseToMax() : - * minor optimization : prefer memset() rather than reduceIndex() - * which is measurably slow in some circumstances (reported for Visual Studio). - * Works when re-using a context for a lot of smallish inputs : - * if all inputs are smaller than ZSTD_INDEXOVERFLOW_MARGIN, - * memset() will be triggered before reduceIndex(). - */ -#define ZSTD_INDEXOVERFLOW_MARGIN (16 MB) -static int ZSTD_indexTooCloseToMax(ZSTD_window_t w) -{ - return (size_t)(w.nextSrc - w.base) > (ZSTD_CURRENT_MAX - ZSTD_INDEXOVERFLOW_MARGIN); -} - -/*! ZSTD_resetCCtx_internal() : - note : `params` are assumed fully validated at this stage */ -static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, - ZSTD_CCtx_params params, - U64 const pledgedSrcSize, - ZSTD_compResetPolicy_e const crp, - ZSTD_buffered_policy_e const zbuff) +/*! ZSTD_copyCCtx() : + * Duplicate an existing context `srcCCtx` into another one `dstCCtx`. + * Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). + * pledgedSrcSize==0 means "unknown". +* @return : 0, or an error code */ +size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long long pledgedSrcSize) { - ZSTD_cwksp* const ws = &zc->workspace; - DEBUGLOG(4, "ZSTD_resetCCtx_internal: pledgedSrcSize=%u, wlog=%u", - (U32)pledgedSrcSize, params.cParams.windowLog); - assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); - - zc->isFirstBlock = 1; - - if (params.ldmParams.enableLdm) { - /* Adjust long distance matching parameters */ - ZSTD_ldm_adjustParameters(¶ms.ldmParams, ¶ms.cParams); - assert(params.ldmParams.hashLog >= params.ldmParams.bucketSizeLog); - assert(params.ldmParams.hashRateLog < 32); - } - - { size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params.cParams.windowLog), pledgedSrcSize)); - size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); - U32 const divider = (params.cParams.minMatch==3) ? 3 : 4; - size_t const maxNbSeq = blockSize / divider; - size_t const buffOutSize = (zbuff == ZSTDb_buffered && params.outBufferMode == ZSTD_bm_buffered) - ? ZSTD_compressBound(blockSize) + 1 - : 0; - size_t const buffInSize = (zbuff == ZSTDb_buffered && params.inBufferMode == ZSTD_bm_buffered) - ? windowSize + blockSize - : 0; - size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params.ldmParams, blockSize); - - int const indexTooClose = ZSTD_indexTooCloseToMax(zc->blockState.matchState.window); - ZSTD_indexResetPolicy_e needsIndexReset = - (!indexTooClose && zc->initialized) ? ZSTDirp_continue : ZSTDirp_reset; + ZSTD_frameParameters fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; + ZSTD_buffered_policy_e const zbuff = srcCCtx->bufferedPolicy; + ZSTD_STATIC_ASSERT((U32)ZSTDb_buffered==1); + if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; + fParams.contentSizeFlag = (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN); - size_t const neededSpace = - ZSTD_estimateCCtxSize_usingCCtxParams_internal( - ¶ms.cParams, ¶ms.ldmParams, zc->staticSize != 0, - buffInSize, buffOutSize, pledgedSrcSize); - FORWARD_IF_ERROR(neededSpace, "cctx size estimate failed!"); + return ZSTD_copyCCtx_internal(dstCCtx, srcCCtx, + fParams, pledgedSrcSize, + zbuff); +} - if (!zc->staticSize) ZSTD_cwksp_bump_oversized_duration(ws, 0); - /* Check if workspace is large enough, alloc a new one if needed */ - { - int const workspaceTooSmall = ZSTD_cwksp_sizeof(ws) < neededSpace; - int const workspaceWasteful = ZSTD_cwksp_check_wasteful(ws, neededSpace); +#define ZSTD_ROWSIZE 16 +/*! ZSTD_reduceTable() : + * reduce table indexes by `reducerValue`, or squash to zero. + * PreserveMark preserves "unsorted mark" for btlazy2 strategy. + * It must be set to a clear 0/1 value, to remove branch during inlining. + * Presume table size is a multiple of ZSTD_ROWSIZE + * to help auto-vectorization */ +FORCE_INLINE_TEMPLATE void +ZSTD_reduceTable_internal (U32* const table, U32 const size, U32 const reducerValue, int const preserveMark) +{ + int const nbRows = (int)size / ZSTD_ROWSIZE; + int cellNb = 0; + int rowNb; + /* Protect special index values < ZSTD_WINDOW_START_INDEX. */ + U32 const reducerThreshold = reducerValue + ZSTD_WINDOW_START_INDEX; + assert((size & (ZSTD_ROWSIZE-1)) == 0); /* multiple of ZSTD_ROWSIZE */ + assert(size < (1U<<31)); /* can be cast to int */ - DEBUGLOG(4, "Need %zu B workspace", neededSpace); - DEBUGLOG(4, "windowSize: %zu - blockSize: %zu", windowSize, blockSize); +#if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) + /* To validate that the table reuse logic is sound, and that we don't + * access table space that we haven't cleaned, we re-"poison" the table + * space every time we mark it dirty. + * + * This function however is intended to operate on those dirty tables and + * re-clean them. So when this function is used correctly, we can unpoison + * the memory it operated on. This introduces a blind spot though, since + * if we now try to operate on __actually__ poisoned memory, we will not + * detect that. */ + __msan_unpoison(table, size * sizeof(U32)); +#endif - if (workspaceTooSmall || workspaceWasteful) { - DEBUGLOG(4, "Resize workspaceSize from %zuKB to %zuKB", - ZSTD_cwksp_sizeof(ws) >> 10, - neededSpace >> 10); + for (rowNb=0 ; rowNb < nbRows ; rowNb++) { + int column; + for (column=0; columnstaticSize, memory_allocation, "static cctx : no resize"); +static void ZSTD_reduceTable(U32* const table, U32 const size, U32 const reducerValue) +{ + ZSTD_reduceTable_internal(table, size, reducerValue, 0); +} - needsIndexReset = ZSTDirp_reset; +static void ZSTD_reduceTable_btlazy2(U32* const table, U32 const size, U32 const reducerValue) +{ + ZSTD_reduceTable_internal(table, size, reducerValue, 1); +} - ZSTD_cwksp_free(ws, zc->customMem); - FORWARD_IF_ERROR(ZSTD_cwksp_create(ws, neededSpace, zc->customMem), ""); +/*! ZSTD_reduceIndex() : +* rescale all indexes to avoid future overflow (indexes are U32) */ +static void ZSTD_reduceIndex (ZSTD_MatchState_t* ms, ZSTD_CCtx_params const* params, const U32 reducerValue) +{ + { U32 const hSize = (U32)1 << params->cParams.hashLog; + ZSTD_reduceTable(ms->hashTable, hSize, reducerValue); + } - DEBUGLOG(5, "reserving object space"); - /* Statically sized space. - * entropyWorkspace never moves, - * though prev/next block swap places */ - assert(ZSTD_cwksp_check_available(ws, 2 * sizeof(ZSTD_compressedBlockState_t))); - zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t)); - RETURN_ERROR_IF(zc->blockState.prevCBlock == NULL, memory_allocation, "couldn't allocate prevCBlock"); - zc->blockState.nextCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t)); - RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate nextCBlock"); - zc->entropyWorkspace = (U32*) ZSTD_cwksp_reserve_object(ws, ENTROPY_WORKSPACE_SIZE); - RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate entropyWorkspace"); - } } + if (ZSTD_allocateChainTable(params->cParams.strategy, params->useRowMatchFinder, (U32)ms->dedicatedDictSearch)) { + U32 const chainSize = (U32)1 << params->cParams.chainLog; + if (params->cParams.strategy == ZSTD_btlazy2) + ZSTD_reduceTable_btlazy2(ms->chainTable, chainSize, reducerValue); + else + ZSTD_reduceTable(ms->chainTable, chainSize, reducerValue); + } - ZSTD_cwksp_clear(ws); + if (ms->hashLog3) { + U32 const h3Size = (U32)1 << ms->hashLog3; + ZSTD_reduceTable(ms->hashTable3, h3Size, reducerValue); + } +} - /* init params */ - zc->appliedParams = params; - zc->blockState.matchState.cParams = params.cParams; - zc->pledgedSrcSizePlusOne = pledgedSrcSize+1; - zc->consumedSrcSize = 0; - zc->producedCSize = 0; - if (pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN) - zc->appliedParams.fParams.contentSizeFlag = 0; - DEBUGLOG(4, "pledged content size : %u ; flag : %u", - (unsigned)pledgedSrcSize, zc->appliedParams.fParams.contentSizeFlag); - zc->blockSize = blockSize; - XXH64_reset(&zc->xxhState, 0); - zc->stage = ZSTDcs_init; - zc->dictID = 0; - zc->dictContentSize = 0; +/*-******************************************************* +* Block entropic compression +*********************************************************/ - ZSTD_reset_compressedBlockState(zc->blockState.prevCBlock); +/* See doc/zstd_compression_format.md for detailed format description */ - /* ZSTD_wildcopy() is used to copy into the literals buffer, - * so we have to oversize the buffer by WILDCOPY_OVERLENGTH bytes. - */ - zc->seqStore.litStart = ZSTD_cwksp_reserve_buffer(ws, blockSize + WILDCOPY_OVERLENGTH); - zc->seqStore.maxNbLit = blockSize; +int ZSTD_seqToCodes(const SeqStore_t* seqStorePtr) +{ + const SeqDef* const sequences = seqStorePtr->sequencesStart; + BYTE* const llCodeTable = seqStorePtr->llCode; + BYTE* const ofCodeTable = seqStorePtr->ofCode; + BYTE* const mlCodeTable = seqStorePtr->mlCode; + U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + U32 u; + int longOffsets = 0; + assert(nbSeq <= seqStorePtr->maxNbSeq); + for (u=0; u= STREAM_ACCUMULATOR_MIN)); + if (MEM_32bits() && ofCode >= STREAM_ACCUMULATOR_MIN) + longOffsets = 1; + } + if (seqStorePtr->longLengthType==ZSTD_llt_literalLength) + llCodeTable[seqStorePtr->longLengthPos] = MaxLL; + if (seqStorePtr->longLengthType==ZSTD_llt_matchLength) + mlCodeTable[seqStorePtr->longLengthPos] = MaxML; + return longOffsets; +} - /* buffers */ - zc->bufferedPolicy = zbuff; - zc->inBuffSize = buffInSize; - zc->inBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffInSize); - zc->outBuffSize = buffOutSize; - zc->outBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffOutSize); +/* ZSTD_useTargetCBlockSize(): + * Returns if target compressed block size param is being used. + * If used, compression will do best effort to make a compressed block size to be around targetCBlockSize. + * Returns 1 if true, 0 otherwise. */ +static int ZSTD_useTargetCBlockSize(const ZSTD_CCtx_params* cctxParams) +{ + DEBUGLOG(5, "ZSTD_useTargetCBlockSize (targetCBlockSize=%zu)", cctxParams->targetCBlockSize); + return (cctxParams->targetCBlockSize != 0); +} - /* ldm bucketOffsets table */ - if (params.ldmParams.enableLdm) { - /* TODO: avoid memset? */ - size_t const numBuckets = - ((size_t)1) << (params.ldmParams.hashLog - - params.ldmParams.bucketSizeLog); - zc->ldmState.bucketOffsets = ZSTD_cwksp_reserve_buffer(ws, numBuckets); - ZSTD_memset(zc->ldmState.bucketOffsets, 0, numBuckets); - } +/* ZSTD_blockSplitterEnabled(): + * Returns if block splitting param is being used + * If used, compression will do best effort to split a block in order to improve compression ratio. + * At the time this function is called, the parameter must be finalized. + * Returns 1 if true, 0 otherwise. */ +static int ZSTD_blockSplitterEnabled(ZSTD_CCtx_params* cctxParams) +{ + DEBUGLOG(5, "ZSTD_blockSplitterEnabled (postBlockSplitter=%d)", cctxParams->postBlockSplitter); + assert(cctxParams->postBlockSplitter != ZSTD_ps_auto); + return (cctxParams->postBlockSplitter == ZSTD_ps_enable); +} - /* sequences storage */ - ZSTD_referenceExternalSequences(zc, NULL, 0); - zc->seqStore.maxNbSeq = maxNbSeq; - zc->seqStore.llCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); - zc->seqStore.mlCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); - zc->seqStore.ofCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); - zc->seqStore.sequencesStart = (seqDef*)ZSTD_cwksp_reserve_aligned(ws, maxNbSeq * sizeof(seqDef)); +/* Type returned by ZSTD_buildSequencesStatistics containing finalized symbol encoding types + * and size of the sequences statistics + */ +typedef struct { + U32 LLtype; + U32 Offtype; + U32 MLtype; + size_t size; + size_t lastCountSize; /* Accounts for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */ + int longOffsets; +} ZSTD_symbolEncodingTypeStats_t; + +/* ZSTD_buildSequencesStatistics(): + * Returns a ZSTD_symbolEncodingTypeStats_t, or a zstd error code in the `size` field. + * Modifies `nextEntropy` to have the appropriate values as a side effect. + * nbSeq must be greater than 0. + * + * entropyWkspSize must be of size at least ENTROPY_WORKSPACE_SIZE - (MaxSeq + 1)*sizeof(U32) + */ +static ZSTD_symbolEncodingTypeStats_t +ZSTD_buildSequencesStatistics( + const SeqStore_t* seqStorePtr, size_t nbSeq, + const ZSTD_fseCTables_t* prevEntropy, ZSTD_fseCTables_t* nextEntropy, + BYTE* dst, const BYTE* const dstEnd, + ZSTD_strategy strategy, unsigned* countWorkspace, + void* entropyWorkspace, size_t entropyWkspSize) +{ + BYTE* const ostart = dst; + const BYTE* const oend = dstEnd; + BYTE* op = ostart; + FSE_CTable* CTable_LitLength = nextEntropy->litlengthCTable; + FSE_CTable* CTable_OffsetBits = nextEntropy->offcodeCTable; + FSE_CTable* CTable_MatchLength = nextEntropy->matchlengthCTable; + const BYTE* const ofCodeTable = seqStorePtr->ofCode; + const BYTE* const llCodeTable = seqStorePtr->llCode; + const BYTE* const mlCodeTable = seqStorePtr->mlCode; + ZSTD_symbolEncodingTypeStats_t stats; - FORWARD_IF_ERROR(ZSTD_reset_matchState( - &zc->blockState.matchState, - ws, - ¶ms.cParams, - crp, - needsIndexReset, - ZSTD_resetTarget_CCtx), ""); + stats.lastCountSize = 0; + /* convert length/distances into codes */ + stats.longOffsets = ZSTD_seqToCodes(seqStorePtr); + assert(op <= oend); + assert(nbSeq != 0); /* ZSTD_selectEncodingType() divides by nbSeq */ + /* build CTable for Literal Lengths */ + { unsigned max = MaxLL; + size_t const mostFrequent = HIST_countFast_wksp(countWorkspace, &max, llCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ + DEBUGLOG(5, "Building LL table"); + nextEntropy->litlength_repeatMode = prevEntropy->litlength_repeatMode; + stats.LLtype = ZSTD_selectEncodingType(&nextEntropy->litlength_repeatMode, + countWorkspace, max, mostFrequent, nbSeq, + LLFSELog, prevEntropy->litlengthCTable, + LL_defaultNorm, LL_defaultNormLog, + ZSTD_defaultAllowed, strategy); + assert(set_basic < set_compressed && set_rle < set_compressed); + assert(!(stats.LLtype < set_compressed && nextEntropy->litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ + { size_t const countSize = ZSTD_buildCTable( + op, (size_t)(oend - op), + CTable_LitLength, LLFSELog, (SymbolEncodingType_e)stats.LLtype, + countWorkspace, max, llCodeTable, nbSeq, + LL_defaultNorm, LL_defaultNormLog, MaxLL, + prevEntropy->litlengthCTable, + sizeof(prevEntropy->litlengthCTable), + entropyWorkspace, entropyWkspSize); + if (ZSTD_isError(countSize)) { + DEBUGLOG(3, "ZSTD_buildCTable for LitLens failed"); + stats.size = countSize; + return stats; + } + if (stats.LLtype == set_compressed) + stats.lastCountSize = countSize; + op += countSize; + assert(op <= oend); + } } + /* build CTable for Offsets */ + { unsigned max = MaxOff; + size_t const mostFrequent = HIST_countFast_wksp( + countWorkspace, &max, ofCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ + /* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */ + ZSTD_DefaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed; + DEBUGLOG(5, "Building OF table"); + nextEntropy->offcode_repeatMode = prevEntropy->offcode_repeatMode; + stats.Offtype = ZSTD_selectEncodingType(&nextEntropy->offcode_repeatMode, + countWorkspace, max, mostFrequent, nbSeq, + OffFSELog, prevEntropy->offcodeCTable, + OF_defaultNorm, OF_defaultNormLog, + defaultPolicy, strategy); + assert(!(stats.Offtype < set_compressed && nextEntropy->offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */ + { size_t const countSize = ZSTD_buildCTable( + op, (size_t)(oend - op), + CTable_OffsetBits, OffFSELog, (SymbolEncodingType_e)stats.Offtype, + countWorkspace, max, ofCodeTable, nbSeq, + OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, + prevEntropy->offcodeCTable, + sizeof(prevEntropy->offcodeCTable), + entropyWorkspace, entropyWkspSize); + if (ZSTD_isError(countSize)) { + DEBUGLOG(3, "ZSTD_buildCTable for Offsets failed"); + stats.size = countSize; + return stats; + } + if (stats.Offtype == set_compressed) + stats.lastCountSize = countSize; + op += countSize; + assert(op <= oend); + } } + /* build CTable for MatchLengths */ + { unsigned max = MaxML; + size_t const mostFrequent = HIST_countFast_wksp( + countWorkspace, &max, mlCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ + DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op)); + nextEntropy->matchlength_repeatMode = prevEntropy->matchlength_repeatMode; + stats.MLtype = ZSTD_selectEncodingType(&nextEntropy->matchlength_repeatMode, + countWorkspace, max, mostFrequent, nbSeq, + MLFSELog, prevEntropy->matchlengthCTable, + ML_defaultNorm, ML_defaultNormLog, + ZSTD_defaultAllowed, strategy); + assert(!(stats.MLtype < set_compressed && nextEntropy->matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ + { size_t const countSize = ZSTD_buildCTable( + op, (size_t)(oend - op), + CTable_MatchLength, MLFSELog, (SymbolEncodingType_e)stats.MLtype, + countWorkspace, max, mlCodeTable, nbSeq, + ML_defaultNorm, ML_defaultNormLog, MaxML, + prevEntropy->matchlengthCTable, + sizeof(prevEntropy->matchlengthCTable), + entropyWorkspace, entropyWkspSize); + if (ZSTD_isError(countSize)) { + DEBUGLOG(3, "ZSTD_buildCTable for MatchLengths failed"); + stats.size = countSize; + return stats; + } + if (stats.MLtype == set_compressed) + stats.lastCountSize = countSize; + op += countSize; + assert(op <= oend); + } } + stats.size = (size_t)(op-ostart); + return stats; +} - /* ldm hash table */ - if (params.ldmParams.enableLdm) { - /* TODO: avoid memset? */ - size_t const ldmHSize = ((size_t)1) << params.ldmParams.hashLog; - zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned(ws, ldmHSize * sizeof(ldmEntry_t)); - ZSTD_memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t)); - zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(ws, maxNbLdmSeq * sizeof(rawSeq)); - zc->maxNbLdmSequences = maxNbLdmSeq; +/* ZSTD_entropyCompressSeqStore_internal(): + * compresses both literals and sequences + * Returns compressed size of block, or a zstd error. + */ +#define SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO 20 +MEM_STATIC size_t +ZSTD_entropyCompressSeqStore_internal( + void* dst, size_t dstCapacity, + const void* literals, size_t litSize, + const SeqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + void* entropyWorkspace, size_t entropyWkspSize, + const int bmi2) +{ + ZSTD_strategy const strategy = cctxParams->cParams.strategy; + unsigned* count = (unsigned*)entropyWorkspace; + FSE_CTable* CTable_LitLength = nextEntropy->fse.litlengthCTable; + FSE_CTable* CTable_OffsetBits = nextEntropy->fse.offcodeCTable; + FSE_CTable* CTable_MatchLength = nextEntropy->fse.matchlengthCTable; + const SeqDef* const sequences = seqStorePtr->sequencesStart; + const size_t nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + const BYTE* const ofCodeTable = seqStorePtr->ofCode; + const BYTE* const llCodeTable = seqStorePtr->llCode; + const BYTE* const mlCodeTable = seqStorePtr->mlCode; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart; + size_t lastCountSize; + int longOffsets = 0; - ZSTD_window_init(&zc->ldmState.window); - ZSTD_window_clear(&zc->ldmState.window); - zc->ldmState.loadedDictEnd = 0; - } + entropyWorkspace = count + (MaxSeq + 1); + entropyWkspSize -= (MaxSeq + 1) * sizeof(*count); - /* Due to alignment, when reusing a workspace, we can actually consume - * up to 3 extra bytes for alignment. See the comments in zstd_cwksp.h - */ - assert(ZSTD_cwksp_used(ws) >= neededSpace && - ZSTD_cwksp_used(ws) <= neededSpace + 3); + DEBUGLOG(5, "ZSTD_entropyCompressSeqStore_internal (nbSeq=%zu, dstCapacity=%zu)", nbSeq, dstCapacity); + ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<= HUF_WORKSPACE_SIZE); - DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_cwksp_available_space(ws)); - zc->initialized = 1; + /* Compress literals */ + { size_t const numSequences = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + /* Base suspicion of uncompressibility on ratio of literals to sequences */ + int const suspectUncompressible = (numSequences == 0) || (litSize / numSequences >= SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO); - return 0; + size_t const cSize = ZSTD_compressLiterals( + op, dstCapacity, + literals, litSize, + entropyWorkspace, entropyWkspSize, + &prevEntropy->huf, &nextEntropy->huf, + cctxParams->cParams.strategy, + ZSTD_literalsCompressionIsDisabled(cctxParams), + suspectUncompressible, bmi2); + FORWARD_IF_ERROR(cSize, "ZSTD_compressLiterals failed"); + assert(cSize <= dstCapacity); + op += cSize; } -} -/* ZSTD_invalidateRepCodes() : - * ensures next compression will not use repcodes from previous block. - * Note : only works with regular variant; - * do not use with extDict variant ! */ -void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) { - int i; - for (i=0; iblockState.prevCBlock->rep[i] = 0; - assert(!ZSTD_window_hasExtDict(cctx->blockState.matchState.window)); -} + /* Sequences Header */ + RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/, + dstSize_tooSmall, "Can't fit seq hdr in output buf!"); + if (nbSeq < 128) { + *op++ = (BYTE)nbSeq; + } else if (nbSeq < LONGNBSEQ) { + op[0] = (BYTE)((nbSeq>>8) + 0x80); + op[1] = (BYTE)nbSeq; + op+=2; + } else { + op[0]=0xFF; + MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)); + op+=3; + } + assert(op <= oend); + if (nbSeq==0) { + /* Copy the old tables over as if we repeated them */ + ZSTD_memcpy(&nextEntropy->fse, &prevEntropy->fse, sizeof(prevEntropy->fse)); + return (size_t)(op - ostart); + } + { BYTE* const seqHead = op++; + /* build stats for sequences */ + const ZSTD_symbolEncodingTypeStats_t stats = + ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq, + &prevEntropy->fse, &nextEntropy->fse, + op, oend, + strategy, count, + entropyWorkspace, entropyWkspSize); + FORWARD_IF_ERROR(stats.size, "ZSTD_buildSequencesStatistics failed!"); + *seqHead = (BYTE)((stats.LLtype<<6) + (stats.Offtype<<4) + (stats.MLtype<<2)); + lastCountSize = stats.lastCountSize; + op += stats.size; + longOffsets = stats.longOffsets; + } -/* These are the approximate sizes for each strategy past which copying the - * dictionary tables into the working context is faster than using them - * in-place. - */ -static const size_t attachDictSizeCutoffs[ZSTD_STRATEGY_MAX+1] = { - 8 KB, /* unused */ - 8 KB, /* ZSTD_fast */ - 16 KB, /* ZSTD_dfast */ - 32 KB, /* ZSTD_greedy */ - 32 KB, /* ZSTD_lazy */ - 32 KB, /* ZSTD_lazy2 */ - 32 KB, /* ZSTD_btlazy2 */ - 32 KB, /* ZSTD_btopt */ - 8 KB, /* ZSTD_btultra */ - 8 KB /* ZSTD_btultra2 */ -}; + { size_t const bitstreamSize = ZSTD_encodeSequences( + op, (size_t)(oend - op), + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, + longOffsets, bmi2); + FORWARD_IF_ERROR(bitstreamSize, "ZSTD_encodeSequences failed"); + op += bitstreamSize; + assert(op <= oend); + /* zstd versions <= 1.3.4 mistakenly report corruption when + * FSE_readNCount() receives a buffer < 4 bytes. + * Fixed by https://github.com/facebook/zstd/pull/1146. + * This can happen when the last set_compressed table present is 2 + * bytes and the bitstream is only one byte. + * In this exceedingly rare case, we will simply emit an uncompressed + * block, since it isn't worth optimizing. + */ + if (lastCountSize && (lastCountSize + bitstreamSize) < 4) { + /* lastCountSize >= 2 && bitstreamSize > 0 ==> lastCountSize == 3 */ + assert(lastCountSize + bitstreamSize == 3); + DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.3.4 by " + "emitting an uncompressed block."); + return 0; + } + } -static int ZSTD_shouldAttachDict(const ZSTD_CDict* cdict, - const ZSTD_CCtx_params* params, - U64 pledgedSrcSize) -{ - size_t cutoff = attachDictSizeCutoffs[cdict->matchState.cParams.strategy]; - int const dedicatedDictSearch = cdict->matchState.dedicatedDictSearch; - return dedicatedDictSearch - || ( ( pledgedSrcSize <= cutoff - || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN - || params->attachDictPref == ZSTD_dictForceAttach ) - && params->attachDictPref != ZSTD_dictForceCopy - && !params->forceWindow ); /* dictMatchState isn't correctly - * handled in _enforceMaxDist */ + DEBUGLOG(5, "compressed block size : %u", (unsigned)(op - ostart)); + return (size_t)(op - ostart); } static size_t -ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx, - const ZSTD_CDict* cdict, - ZSTD_CCtx_params params, - U64 pledgedSrcSize, - ZSTD_buffered_policy_e zbuff) -{ - { - ZSTD_compressionParameters adjusted_cdict_cParams = cdict->matchState.cParams; - unsigned const windowLog = params.cParams.windowLog; - assert(windowLog != 0); - /* Resize working context table params for input only, since the dict - * has its own tables. */ - /* pledgedSrcSize == 0 means 0! */ - - if (cdict->matchState.dedicatedDictSearch) { - ZSTD_dedicatedDictSearch_revertCParams(&adjusted_cdict_cParams); - } - - params.cParams = ZSTD_adjustCParams_internal(adjusted_cdict_cParams, pledgedSrcSize, - cdict->dictContentSize, ZSTD_cpm_attachDict); - params.cParams.windowLog = windowLog; - FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, - ZSTDcrp_makeClean, zbuff), ""); - assert(cctx->appliedParams.cParams.strategy == adjusted_cdict_cParams.strategy); +ZSTD_entropyCompressSeqStore_wExtLitBuffer( + void* dst, size_t dstCapacity, + const void* literals, size_t litSize, + size_t blockSize, + const SeqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + void* entropyWorkspace, size_t entropyWkspSize, + int bmi2) +{ + size_t const cSize = ZSTD_entropyCompressSeqStore_internal( + dst, dstCapacity, + literals, litSize, + seqStorePtr, prevEntropy, nextEntropy, cctxParams, + entropyWorkspace, entropyWkspSize, bmi2); + if (cSize == 0) return 0; + /* When srcSize <= dstCapacity, there is enough space to write a raw uncompressed block. + * Since we ran out of space, block must be not compressible, so fall back to raw uncompressed block. + */ + if ((cSize == ERROR(dstSize_tooSmall)) & (blockSize <= dstCapacity)) { + DEBUGLOG(4, "not enough dstCapacity (%zu) for ZSTD_entropyCompressSeqStore_internal()=> do not compress block", dstCapacity); + return 0; /* block not compressed */ } + FORWARD_IF_ERROR(cSize, "ZSTD_entropyCompressSeqStore_internal failed"); - { const U32 cdictEnd = (U32)( cdict->matchState.window.nextSrc - - cdict->matchState.window.base); - const U32 cdictLen = cdictEnd - cdict->matchState.window.dictLimit; - if (cdictLen == 0) { - /* don't even attach dictionaries with no contents */ - DEBUGLOG(4, "skipping attaching empty dictionary"); - } else { - DEBUGLOG(4, "attaching dictionary into context"); - cctx->blockState.matchState.dictMatchState = &cdict->matchState; - - /* prep working match state so dict matches never have negative indices - * when they are translated to the working context's index space. */ - if (cctx->blockState.matchState.window.dictLimit < cdictEnd) { - cctx->blockState.matchState.window.nextSrc = - cctx->blockState.matchState.window.base + cdictEnd; - ZSTD_window_clear(&cctx->blockState.matchState.window); - } - /* loadedDictEnd is expressed within the referential of the active context */ - cctx->blockState.matchState.loadedDictEnd = cctx->blockState.matchState.window.dictLimit; - } } - - cctx->dictID = cdict->dictID; - cctx->dictContentSize = cdict->dictContentSize; - - /* copy block state */ - ZSTD_memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, sizeof(cdict->cBlockState)); + /* Check compressibility */ + { size_t const maxCSize = blockSize - ZSTD_minGain(blockSize, cctxParams->cParams.strategy); + if (cSize >= maxCSize) return 0; /* block not compressed */ + } + DEBUGLOG(5, "ZSTD_entropyCompressSeqStore() cSize: %zu", cSize); + /* libzstd decoder before > v1.5.4 is not compatible with compressed blocks of size ZSTD_BLOCKSIZE_MAX exactly. + * This restriction is indirectly already fulfilled by respecting ZSTD_minGain() condition above. + */ + assert(cSize < ZSTD_BLOCKSIZE_MAX); + return cSize; +} - return 0; +static size_t +ZSTD_entropyCompressSeqStore( + const SeqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + size_t srcSize, + void* entropyWorkspace, size_t entropyWkspSize, + int bmi2) +{ + return ZSTD_entropyCompressSeqStore_wExtLitBuffer( + dst, dstCapacity, + seqStorePtr->litStart, (size_t)(seqStorePtr->lit - seqStorePtr->litStart), + srcSize, + seqStorePtr, + prevEntropy, nextEntropy, + cctxParams, + entropyWorkspace, entropyWkspSize, + bmi2); } -static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, - const ZSTD_CDict* cdict, - ZSTD_CCtx_params params, - U64 pledgedSrcSize, - ZSTD_buffered_policy_e zbuff) +/* ZSTD_selectBlockCompressor() : + * Not static, but internal use only (used by long distance matcher) + * assumption : strat is a valid strategy */ +ZSTD_BlockCompressor_f ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_ParamSwitch_e useRowMatchFinder, ZSTD_dictMode_e dictMode) { - const ZSTD_compressionParameters *cdict_cParams = &cdict->matchState.cParams; - - assert(!cdict->matchState.dedicatedDictSearch); - - DEBUGLOG(4, "copying dictionary into context"); + static const ZSTD_BlockCompressor_f blockCompressor[4][ZSTD_STRATEGY_MAX+1] = { + { ZSTD_compressBlock_fast /* default for 0 */, + ZSTD_compressBlock_fast, + ZSTD_COMPRESSBLOCK_DOUBLEFAST, + ZSTD_COMPRESSBLOCK_GREEDY, + ZSTD_COMPRESSBLOCK_LAZY, + ZSTD_COMPRESSBLOCK_LAZY2, + ZSTD_COMPRESSBLOCK_BTLAZY2, + ZSTD_COMPRESSBLOCK_BTOPT, + ZSTD_COMPRESSBLOCK_BTULTRA, + ZSTD_COMPRESSBLOCK_BTULTRA2 + }, + { ZSTD_compressBlock_fast_extDict /* default for 0 */, + ZSTD_compressBlock_fast_extDict, + ZSTD_COMPRESSBLOCK_DOUBLEFAST_EXTDICT, + ZSTD_COMPRESSBLOCK_GREEDY_EXTDICT, + ZSTD_COMPRESSBLOCK_LAZY_EXTDICT, + ZSTD_COMPRESSBLOCK_LAZY2_EXTDICT, + ZSTD_COMPRESSBLOCK_BTLAZY2_EXTDICT, + ZSTD_COMPRESSBLOCK_BTOPT_EXTDICT, + ZSTD_COMPRESSBLOCK_BTULTRA_EXTDICT, + ZSTD_COMPRESSBLOCK_BTULTRA_EXTDICT + }, + { ZSTD_compressBlock_fast_dictMatchState /* default for 0 */, + ZSTD_compressBlock_fast_dictMatchState, + ZSTD_COMPRESSBLOCK_DOUBLEFAST_DICTMATCHSTATE, + ZSTD_COMPRESSBLOCK_GREEDY_DICTMATCHSTATE, + ZSTD_COMPRESSBLOCK_LAZY_DICTMATCHSTATE, + ZSTD_COMPRESSBLOCK_LAZY2_DICTMATCHSTATE, + ZSTD_COMPRESSBLOCK_BTLAZY2_DICTMATCHSTATE, + ZSTD_COMPRESSBLOCK_BTOPT_DICTMATCHSTATE, + ZSTD_COMPRESSBLOCK_BTULTRA_DICTMATCHSTATE, + ZSTD_COMPRESSBLOCK_BTULTRA_DICTMATCHSTATE + }, + { NULL /* default for 0 */, + NULL, + NULL, + ZSTD_COMPRESSBLOCK_GREEDY_DEDICATEDDICTSEARCH, + ZSTD_COMPRESSBLOCK_LAZY_DEDICATEDDICTSEARCH, + ZSTD_COMPRESSBLOCK_LAZY2_DEDICATEDDICTSEARCH, + NULL, + NULL, + NULL, + NULL } + }; + ZSTD_BlockCompressor_f selectedCompressor; + ZSTD_STATIC_ASSERT((unsigned)ZSTD_fast == 1); - { unsigned const windowLog = params.cParams.windowLog; - assert(windowLog != 0); - /* Copy only compression parameters related to tables. */ - params.cParams = *cdict_cParams; - params.cParams.windowLog = windowLog; - FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, - ZSTDcrp_leaveDirty, zbuff), ""); - assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy); - assert(cctx->appliedParams.cParams.hashLog == cdict_cParams->hashLog); - assert(cctx->appliedParams.cParams.chainLog == cdict_cParams->chainLog); + assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, (int)strat)); + DEBUGLOG(5, "Selected block compressor: dictMode=%d strat=%d rowMatchfinder=%d", (int)dictMode, (int)strat, (int)useRowMatchFinder); + if (ZSTD_rowMatchFinderUsed(strat, useRowMatchFinder)) { + static const ZSTD_BlockCompressor_f rowBasedBlockCompressors[4][3] = { + { + ZSTD_COMPRESSBLOCK_GREEDY_ROW, + ZSTD_COMPRESSBLOCK_LAZY_ROW, + ZSTD_COMPRESSBLOCK_LAZY2_ROW + }, + { + ZSTD_COMPRESSBLOCK_GREEDY_EXTDICT_ROW, + ZSTD_COMPRESSBLOCK_LAZY_EXTDICT_ROW, + ZSTD_COMPRESSBLOCK_LAZY2_EXTDICT_ROW + }, + { + ZSTD_COMPRESSBLOCK_GREEDY_DICTMATCHSTATE_ROW, + ZSTD_COMPRESSBLOCK_LAZY_DICTMATCHSTATE_ROW, + ZSTD_COMPRESSBLOCK_LAZY2_DICTMATCHSTATE_ROW + }, + { + ZSTD_COMPRESSBLOCK_GREEDY_DEDICATEDDICTSEARCH_ROW, + ZSTD_COMPRESSBLOCK_LAZY_DEDICATEDDICTSEARCH_ROW, + ZSTD_COMPRESSBLOCK_LAZY2_DEDICATEDDICTSEARCH_ROW + } + }; + DEBUGLOG(5, "Selecting a row-based matchfinder"); + assert(useRowMatchFinder != ZSTD_ps_auto); + selectedCompressor = rowBasedBlockCompressors[(int)dictMode][(int)strat - (int)ZSTD_greedy]; + } else { + selectedCompressor = blockCompressor[(int)dictMode][(int)strat]; } + assert(selectedCompressor != NULL); + return selectedCompressor; +} - ZSTD_cwksp_mark_tables_dirty(&cctx->workspace); +static void ZSTD_storeLastLiterals(SeqStore_t* seqStorePtr, + const BYTE* anchor, size_t lastLLSize) +{ + ZSTD_memcpy(seqStorePtr->lit, anchor, lastLLSize); + seqStorePtr->lit += lastLLSize; +} - /* copy tables */ - { size_t const chainSize = (cdict_cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cdict_cParams->chainLog); - size_t const hSize = (size_t)1 << cdict_cParams->hashLog; +void ZSTD_resetSeqStore(SeqStore_t* ssPtr) +{ + ssPtr->lit = ssPtr->litStart; + ssPtr->sequences = ssPtr->sequencesStart; + ssPtr->longLengthType = ZSTD_llt_none; +} + +/* ZSTD_postProcessSequenceProducerResult() : + * Validates and post-processes sequences obtained through the external matchfinder API: + * - Checks whether nbExternalSeqs represents an error condition. + * - Appends a block delimiter to outSeqs if one is not already present. + * See zstd.h for context regarding block delimiters. + * Returns the number of sequences after post-processing, or an error code. */ +static size_t ZSTD_postProcessSequenceProducerResult( + ZSTD_Sequence* outSeqs, size_t nbExternalSeqs, size_t outSeqsCapacity, size_t srcSize +) { + RETURN_ERROR_IF( + nbExternalSeqs > outSeqsCapacity, + sequenceProducer_failed, + "External sequence producer returned error code %lu", + (unsigned long)nbExternalSeqs + ); - ZSTD_memcpy(cctx->blockState.matchState.hashTable, - cdict->matchState.hashTable, - hSize * sizeof(U32)); - ZSTD_memcpy(cctx->blockState.matchState.chainTable, - cdict->matchState.chainTable, - chainSize * sizeof(U32)); - } + RETURN_ERROR_IF( + nbExternalSeqs == 0 && srcSize > 0, + sequenceProducer_failed, + "Got zero sequences from external sequence producer for a non-empty src buffer!" + ); - /* Zero the hashTable3, since the cdict never fills it */ - { int const h3log = cctx->blockState.matchState.hashLog3; - size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0; - assert(cdict->matchState.hashLog3 == 0); - ZSTD_memset(cctx->blockState.matchState.hashTable3, 0, h3Size * sizeof(U32)); + if (srcSize == 0) { + ZSTD_memset(&outSeqs[0], 0, sizeof(ZSTD_Sequence)); + return 1; } - ZSTD_cwksp_mark_tables_clean(&cctx->workspace); + { + ZSTD_Sequence const lastSeq = outSeqs[nbExternalSeqs - 1]; - /* copy dictionary offsets */ - { ZSTD_matchState_t const* srcMatchState = &cdict->matchState; - ZSTD_matchState_t* dstMatchState = &cctx->blockState.matchState; - dstMatchState->window = srcMatchState->window; - dstMatchState->nextToUpdate = srcMatchState->nextToUpdate; - dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd; - } + /* We can return early if lastSeq is already a block delimiter. */ + if (lastSeq.offset == 0 && lastSeq.matchLength == 0) { + return nbExternalSeqs; + } - cctx->dictID = cdict->dictID; - cctx->dictContentSize = cdict->dictContentSize; + /* This error condition is only possible if the external matchfinder + * produced an invalid parse, by definition of ZSTD_sequenceBound(). */ + RETURN_ERROR_IF( + nbExternalSeqs == outSeqsCapacity, + sequenceProducer_failed, + "nbExternalSeqs == outSeqsCapacity but lastSeq is not a block delimiter!" + ); - /* copy block state */ - ZSTD_memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, sizeof(cdict->cBlockState)); + /* lastSeq is not a block delimiter, so we need to append one. */ + ZSTD_memset(&outSeqs[nbExternalSeqs], 0, sizeof(ZSTD_Sequence)); + return nbExternalSeqs + 1; + } +} - return 0; +/* ZSTD_fastSequenceLengthSum() : + * Returns sum(litLen) + sum(matchLen) + lastLits for *seqBuf*. + * Similar to another function in zstd_compress.c (determine_blockSize), + * except it doesn't check for a block delimiter to end summation. + * Removing the early exit allows the compiler to auto-vectorize (https://godbolt.org/z/cY1cajz9P). + * This function can be deleted and replaced by determine_blockSize after we resolve issue #3456. */ +static size_t ZSTD_fastSequenceLengthSum(ZSTD_Sequence const* seqBuf, size_t seqBufSize) { + size_t matchLenSum, litLenSum, i; + matchLenSum = 0; + litLenSum = 0; + for (i = 0; i < seqBufSize; i++) { + litLenSum += seqBuf[i].litLength; + matchLenSum += seqBuf[i].matchLength; + } + return litLenSum + matchLenSum; } -/* We have a choice between copying the dictionary context into the working - * context, or referencing the dictionary context from the working context - * in-place. We decide here which strategy to use. */ -static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx, - const ZSTD_CDict* cdict, - const ZSTD_CCtx_params* params, - U64 pledgedSrcSize, - ZSTD_buffered_policy_e zbuff) +/** + * Function to validate sequences produced by a block compressor. + */ +static void ZSTD_validateSeqStore(const SeqStore_t* seqStore, const ZSTD_compressionParameters* cParams) { - - DEBUGLOG(4, "ZSTD_resetCCtx_usingCDict (pledgedSrcSize=%u)", - (unsigned)pledgedSrcSize); - - if (ZSTD_shouldAttachDict(cdict, params, pledgedSrcSize)) { - return ZSTD_resetCCtx_byAttachingCDict( - cctx, cdict, *params, pledgedSrcSize, zbuff); - } else { - return ZSTD_resetCCtx_byCopyingCDict( - cctx, cdict, *params, pledgedSrcSize, zbuff); +#if DEBUGLEVEL >= 1 + const SeqDef* seq = seqStore->sequencesStart; + const SeqDef* const seqEnd = seqStore->sequences; + size_t const matchLenLowerBound = cParams->minMatch == 3 ? 3 : 4; + for (; seq < seqEnd; ++seq) { + const ZSTD_SequenceLength seqLength = ZSTD_getSequenceLength(seqStore, seq); + assert(seqLength.matchLength >= matchLenLowerBound); + (void)seqLength; + (void)matchLenLowerBound; } +#else + (void)seqStore; + (void)cParams; +#endif } -/*! ZSTD_copyCCtx_internal() : - * Duplicate an existing context `srcCCtx` into another one `dstCCtx`. - * Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). - * The "context", in this case, refers to the hash and chain tables, - * entropy tables, and dictionary references. - * `windowLog` value is enforced if != 0, otherwise value is copied from srcCCtx. - * @return : 0, or an error code */ -static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, - const ZSTD_CCtx* srcCCtx, - ZSTD_frameParameters fParams, - U64 pledgedSrcSize, - ZSTD_buffered_policy_e zbuff) +static size_t +ZSTD_transferSequences_wBlockDelim(ZSTD_CCtx* cctx, + ZSTD_SequencePosition* seqPos, + const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, + const void* src, size_t blockSize, + ZSTD_ParamSwitch_e externalRepSearch); + +typedef enum { ZSTDbss_compress, ZSTDbss_noCompress } ZSTD_BuildSeqStore_e; + +static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) { - DEBUGLOG(5, "ZSTD_copyCCtx_internal"); - RETURN_ERROR_IF(srcCCtx->stage!=ZSTDcs_init, stage_wrong, - "Can't copy a ctx that's not in init stage."); + ZSTD_MatchState_t* const ms = &zc->blockState.matchState; + DEBUGLOG(5, "ZSTD_buildSeqStore (srcSize=%zu)", srcSize); + assert(srcSize <= ZSTD_BLOCKSIZE_MAX); + /* Assert that we have correctly flushed the ctx params into the ms's copy */ + ZSTD_assertEqualCParams(zc->appliedParams.cParams, ms->cParams); + /* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding + * additional 1. We need to revisit and change this logic to be more consistent */ + if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1+1) { + if (zc->appliedParams.cParams.strategy >= ZSTD_btopt) { + ZSTD_ldm_skipRawSeqStoreBytes(&zc->externSeqStore, srcSize); + } else { + ZSTD_ldm_skipSequences(&zc->externSeqStore, srcSize, zc->appliedParams.cParams.minMatch); + } + return ZSTDbss_noCompress; /* don't even attempt compression below a certain srcSize */ + } + ZSTD_resetSeqStore(&(zc->seqStore)); + /* required for optimal parser to read stats from dictionary */ + ms->opt.symbolCosts = &zc->blockState.prevCBlock->entropy; + /* tell the optimal parser how we expect to compress literals */ + ms->opt.literalCompressionMode = zc->appliedParams.literalCompressionMode; + /* a gap between an attached dict and the current window is not safe, + * they must remain adjacent, + * and when that stops being the case, the dict must be unset */ + assert(ms->dictMatchState == NULL || ms->loadedDictEnd == ms->window.dictLimit); - ZSTD_memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); - { ZSTD_CCtx_params params = dstCCtx->requestedParams; - /* Copy only compression parameters related to tables. */ - params.cParams = srcCCtx->appliedParams.cParams; - params.fParams = fParams; - ZSTD_resetCCtx_internal(dstCCtx, params, pledgedSrcSize, - ZSTDcrp_leaveDirty, zbuff); - assert(dstCCtx->appliedParams.cParams.windowLog == srcCCtx->appliedParams.cParams.windowLog); - assert(dstCCtx->appliedParams.cParams.strategy == srcCCtx->appliedParams.cParams.strategy); - assert(dstCCtx->appliedParams.cParams.hashLog == srcCCtx->appliedParams.cParams.hashLog); - assert(dstCCtx->appliedParams.cParams.chainLog == srcCCtx->appliedParams.cParams.chainLog); - assert(dstCCtx->blockState.matchState.hashLog3 == srcCCtx->blockState.matchState.hashLog3); + /* limited update after a very long match */ + { const BYTE* const base = ms->window.base; + const BYTE* const istart = (const BYTE*)src; + const U32 curr = (U32)(istart-base); + if (sizeof(ptrdiff_t)==8) assert(istart - base < (ptrdiff_t)(U32)(-1)); /* ensure no overflow */ + if (curr > ms->nextToUpdate + 384) + ms->nextToUpdate = curr - MIN(192, (U32)(curr - ms->nextToUpdate - 384)); } - ZSTD_cwksp_mark_tables_dirty(&dstCCtx->workspace); + /* select and store sequences */ + { ZSTD_dictMode_e const dictMode = ZSTD_matchState_dictMode(ms); + size_t lastLLSize; + { int i; + for (i = 0; i < ZSTD_REP_NUM; ++i) + zc->blockState.nextCBlock->rep[i] = zc->blockState.prevCBlock->rep[i]; + } + if (zc->externSeqStore.pos < zc->externSeqStore.size) { + assert(zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_disable); - /* copy tables */ - { size_t const chainSize = (srcCCtx->appliedParams.cParams.strategy == ZSTD_fast) ? 0 : ((size_t)1 << srcCCtx->appliedParams.cParams.chainLog); - size_t const hSize = (size_t)1 << srcCCtx->appliedParams.cParams.hashLog; - int const h3log = srcCCtx->blockState.matchState.hashLog3; - size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0; + /* External matchfinder + LDM is technically possible, just not implemented yet. + * We need to revisit soon and implement it. */ + RETURN_ERROR_IF( + ZSTD_hasExtSeqProd(&zc->appliedParams), + parameter_combination_unsupported, + "Long-distance matching with external sequence producer enabled is not currently supported." + ); - ZSTD_memcpy(dstCCtx->blockState.matchState.hashTable, - srcCCtx->blockState.matchState.hashTable, - hSize * sizeof(U32)); - ZSTD_memcpy(dstCCtx->blockState.matchState.chainTable, - srcCCtx->blockState.matchState.chainTable, - chainSize * sizeof(U32)); - ZSTD_memcpy(dstCCtx->blockState.matchState.hashTable3, - srcCCtx->blockState.matchState.hashTable3, - h3Size * sizeof(U32)); - } + /* Updates ldmSeqStore.pos */ + lastLLSize = + ZSTD_ldm_blockCompress(&zc->externSeqStore, + ms, &zc->seqStore, + zc->blockState.nextCBlock->rep, + zc->appliedParams.useRowMatchFinder, + src, srcSize); + assert(zc->externSeqStore.pos <= zc->externSeqStore.size); + } else if (zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_enable) { + RawSeqStore_t ldmSeqStore = kNullRawSeqStore; - ZSTD_cwksp_mark_tables_clean(&dstCCtx->workspace); + /* External matchfinder + LDM is technically possible, just not implemented yet. + * We need to revisit soon and implement it. */ + RETURN_ERROR_IF( + ZSTD_hasExtSeqProd(&zc->appliedParams), + parameter_combination_unsupported, + "Long-distance matching with external sequence producer enabled is not currently supported." + ); - /* copy dictionary offsets */ - { - const ZSTD_matchState_t* srcMatchState = &srcCCtx->blockState.matchState; - ZSTD_matchState_t* dstMatchState = &dstCCtx->blockState.matchState; - dstMatchState->window = srcMatchState->window; - dstMatchState->nextToUpdate = srcMatchState->nextToUpdate; - dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd; - } - dstCCtx->dictID = srcCCtx->dictID; - dstCCtx->dictContentSize = srcCCtx->dictContentSize; + ldmSeqStore.seq = zc->ldmSequences; + ldmSeqStore.capacity = zc->maxNbLdmSequences; + /* Updates ldmSeqStore.size */ + FORWARD_IF_ERROR(ZSTD_ldm_generateSequences(&zc->ldmState, &ldmSeqStore, + &zc->appliedParams.ldmParams, + src, srcSize), ""); + /* Updates ldmSeqStore.pos */ + lastLLSize = + ZSTD_ldm_blockCompress(&ldmSeqStore, + ms, &zc->seqStore, + zc->blockState.nextCBlock->rep, + zc->appliedParams.useRowMatchFinder, + src, srcSize); + assert(ldmSeqStore.pos == ldmSeqStore.size); + } else if (ZSTD_hasExtSeqProd(&zc->appliedParams)) { + assert( + zc->extSeqBufCapacity >= ZSTD_sequenceBound(srcSize) + ); + assert(zc->appliedParams.extSeqProdFunc != NULL); + + { U32 const windowSize = (U32)1 << zc->appliedParams.cParams.windowLog; + + size_t const nbExternalSeqs = (zc->appliedParams.extSeqProdFunc)( + zc->appliedParams.extSeqProdState, + zc->extSeqBuf, + zc->extSeqBufCapacity, + src, srcSize, + NULL, 0, /* dict and dictSize, currently not supported */ + zc->appliedParams.compressionLevel, + windowSize + ); + + size_t const nbPostProcessedSeqs = ZSTD_postProcessSequenceProducerResult( + zc->extSeqBuf, + nbExternalSeqs, + zc->extSeqBufCapacity, + srcSize + ); + + /* Return early if there is no error, since we don't need to worry about last literals */ + if (!ZSTD_isError(nbPostProcessedSeqs)) { + ZSTD_SequencePosition seqPos = {0,0,0}; + size_t const seqLenSum = ZSTD_fastSequenceLengthSum(zc->extSeqBuf, nbPostProcessedSeqs); + RETURN_ERROR_IF(seqLenSum > srcSize, externalSequences_invalid, "External sequences imply too large a block!"); + FORWARD_IF_ERROR( + ZSTD_transferSequences_wBlockDelim( + zc, &seqPos, + zc->extSeqBuf, nbPostProcessedSeqs, + src, srcSize, + zc->appliedParams.searchForExternalRepcodes + ), + "Failed to copy external sequences to seqStore!" + ); + ms->ldmSeqStore = NULL; + DEBUGLOG(5, "Copied %lu sequences from external sequence producer to internal seqStore.", (unsigned long)nbExternalSeqs); + return ZSTDbss_compress; + } - /* copy block state */ - ZSTD_memcpy(dstCCtx->blockState.prevCBlock, srcCCtx->blockState.prevCBlock, sizeof(*srcCCtx->blockState.prevCBlock)); + /* Propagate the error if fallback is disabled */ + if (!zc->appliedParams.enableMatchFinderFallback) { + return nbPostProcessedSeqs; + } - return 0; + /* Fallback to software matchfinder */ + { ZSTD_BlockCompressor_f const blockCompressor = + ZSTD_selectBlockCompressor( + zc->appliedParams.cParams.strategy, + zc->appliedParams.useRowMatchFinder, + dictMode); + ms->ldmSeqStore = NULL; + DEBUGLOG( + 5, + "External sequence producer returned error code %lu. Falling back to internal parser.", + (unsigned long)nbExternalSeqs + ); + lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize); + } } + } else { /* not long range mode and no external matchfinder */ + ZSTD_BlockCompressor_f const blockCompressor = ZSTD_selectBlockCompressor( + zc->appliedParams.cParams.strategy, + zc->appliedParams.useRowMatchFinder, + dictMode); + ms->ldmSeqStore = NULL; + lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize); + } + { const BYTE* const lastLiterals = (const BYTE*)src + srcSize - lastLLSize; + ZSTD_storeLastLiterals(&zc->seqStore, lastLiterals, lastLLSize); + } } + ZSTD_validateSeqStore(&zc->seqStore, &zc->appliedParams.cParams); + return ZSTDbss_compress; } -/*! ZSTD_copyCCtx() : - * Duplicate an existing context `srcCCtx` into another one `dstCCtx`. - * Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). - * pledgedSrcSize==0 means "unknown". -* @return : 0, or an error code */ -size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long long pledgedSrcSize) +static size_t ZSTD_copyBlockSequences(SeqCollector* seqCollector, const SeqStore_t* seqStore, const U32 prevRepcodes[ZSTD_REP_NUM]) { - ZSTD_frameParameters fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; - ZSTD_buffered_policy_e const zbuff = srcCCtx->bufferedPolicy; - ZSTD_STATIC_ASSERT((U32)ZSTDb_buffered==1); - if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; - fParams.contentSizeFlag = (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN); + const SeqDef* inSeqs = seqStore->sequencesStart; + const size_t nbInSequences = (size_t)(seqStore->sequences - inSeqs); + const size_t nbInLiterals = (size_t)(seqStore->lit - seqStore->litStart); - return ZSTD_copyCCtx_internal(dstCCtx, srcCCtx, - fParams, pledgedSrcSize, - zbuff); -} + ZSTD_Sequence* outSeqs = seqCollector->seqIndex == 0 ? seqCollector->seqStart : seqCollector->seqStart + seqCollector->seqIndex; + const size_t nbOutSequences = nbInSequences + 1; + size_t nbOutLiterals = 0; + Repcodes_t repcodes; + size_t i; + /* Bounds check that we have enough space for every input sequence + * and the block delimiter + */ + assert(seqCollector->seqIndex <= seqCollector->maxSequences); + RETURN_ERROR_IF( + nbOutSequences > (size_t)(seqCollector->maxSequences - seqCollector->seqIndex), + dstSize_tooSmall, + "Not enough space to copy sequences"); -#define ZSTD_ROWSIZE 16 -/*! ZSTD_reduceTable() : - * reduce table indexes by `reducerValue`, or squash to zero. - * PreserveMark preserves "unsorted mark" for btlazy2 strategy. - * It must be set to a clear 0/1 value, to remove branch during inlining. - * Presume table size is a multiple of ZSTD_ROWSIZE - * to help auto-vectorization */ -FORCE_INLINE_TEMPLATE void -ZSTD_reduceTable_internal (U32* const table, U32 const size, U32 const reducerValue, int const preserveMark) -{ - int const nbRows = (int)size / ZSTD_ROWSIZE; - int cellNb = 0; - int rowNb; - assert((size & (ZSTD_ROWSIZE-1)) == 0); /* multiple of ZSTD_ROWSIZE */ - assert(size < (1U<<31)); /* can be casted to int */ + ZSTD_memcpy(&repcodes, prevRepcodes, sizeof(repcodes)); + for (i = 0; i < nbInSequences; ++i) { + U32 rawOffset; + outSeqs[i].litLength = inSeqs[i].litLength; + outSeqs[i].matchLength = inSeqs[i].mlBase + MINMATCH; + outSeqs[i].rep = 0; -#if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) - /* To validate that the table re-use logic is sound, and that we don't - * access table space that we haven't cleaned, we re-"poison" the table - * space every time we mark it dirty. - * - * This function however is intended to operate on those dirty tables and - * re-clean them. So when this function is used correctly, we can unpoison - * the memory it operated on. This introduces a blind spot though, since - * if we now try to operate on __actually__ poisoned memory, we will not - * detect that. */ - __msan_unpoison(table, size * sizeof(U32)); -#endif + /* Handle the possible single length >= 64K + * There can only be one because we add MINMATCH to every match length, + * and blocks are at most 128K. + */ + if (i == seqStore->longLengthPos) { + if (seqStore->longLengthType == ZSTD_llt_literalLength) { + outSeqs[i].litLength += 0x10000; + } else if (seqStore->longLengthType == ZSTD_llt_matchLength) { + outSeqs[i].matchLength += 0x10000; + } + } - for (rowNb=0 ; rowNb < nbRows ; rowNb++) { - int column; - for (column=0; column 0); + outSeqs[i].rep = repcode; + if (outSeqs[i].litLength != 0) { + rawOffset = repcodes.rep[repcode - 1]; + } else { + if (repcode == 3) { + assert(repcodes.rep[0] > 1); + rawOffset = repcodes.rep[0] - 1; + } else { + rawOffset = repcodes.rep[repcode]; + } } - if (table[cellNb] < reducerValue) table[cellNb] = 0; - else table[cellNb] -= reducerValue; - cellNb++; - } } -} + } else { + rawOffset = OFFBASE_TO_OFFSET(inSeqs[i].offBase); + } + outSeqs[i].offset = rawOffset; -static void ZSTD_reduceTable(U32* const table, U32 const size, U32 const reducerValue) -{ - ZSTD_reduceTable_internal(table, size, reducerValue, 0); + /* Update repcode history for the sequence */ + ZSTD_updateRep(repcodes.rep, + inSeqs[i].offBase, + inSeqs[i].litLength == 0); + + nbOutLiterals += outSeqs[i].litLength; + } + /* Insert last literals (if any exist) in the block as a sequence with ml == off == 0. + * If there are no last literals, then we'll emit (of: 0, ml: 0, ll: 0), which is a marker + * for the block boundary, according to the API. + */ + assert(nbInLiterals >= nbOutLiterals); + { + const size_t lastLLSize = nbInLiterals - nbOutLiterals; + outSeqs[nbInSequences].litLength = (U32)lastLLSize; + outSeqs[nbInSequences].matchLength = 0; + outSeqs[nbInSequences].offset = 0; + assert(nbOutSequences == nbInSequences + 1); + } + seqCollector->seqIndex += nbOutSequences; + assert(seqCollector->seqIndex <= seqCollector->maxSequences); + + return 0; } -static void ZSTD_reduceTable_btlazy2(U32* const table, U32 const size, U32 const reducerValue) -{ - ZSTD_reduceTable_internal(table, size, reducerValue, 1); +size_t ZSTD_sequenceBound(size_t srcSize) { + const size_t maxNbSeq = (srcSize / ZSTD_MINMATCH_MIN) + 1; + const size_t maxNbDelims = (srcSize / ZSTD_BLOCKSIZE_MAX_MIN) + 1; + return maxNbSeq + maxNbDelims; } -/*! ZSTD_reduceIndex() : -* rescale all indexes to avoid future overflow (indexes are U32) */ -static void ZSTD_reduceIndex (ZSTD_matchState_t* ms, ZSTD_CCtx_params const* params, const U32 reducerValue) +size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, + size_t outSeqsSize, const void* src, size_t srcSize) { - { U32 const hSize = (U32)1 << params->cParams.hashLog; - ZSTD_reduceTable(ms->hashTable, hSize, reducerValue); + const size_t dstCapacity = ZSTD_compressBound(srcSize); + void* dst; /* Make C90 happy. */ + SeqCollector seqCollector; + { + int targetCBlockSize; + FORWARD_IF_ERROR(ZSTD_CCtx_getParameter(zc, ZSTD_c_targetCBlockSize, &targetCBlockSize), ""); + RETURN_ERROR_IF(targetCBlockSize != 0, parameter_unsupported, "targetCBlockSize != 0"); + } + { + int nbWorkers; + FORWARD_IF_ERROR(ZSTD_CCtx_getParameter(zc, ZSTD_c_nbWorkers, &nbWorkers), ""); + RETURN_ERROR_IF(nbWorkers != 0, parameter_unsupported, "nbWorkers != 0"); } - if (params->cParams.strategy != ZSTD_fast) { - U32 const chainSize = (U32)1 << params->cParams.chainLog; - if (params->cParams.strategy == ZSTD_btlazy2) - ZSTD_reduceTable_btlazy2(ms->chainTable, chainSize, reducerValue); - else - ZSTD_reduceTable(ms->chainTable, chainSize, reducerValue); + dst = ZSTD_customMalloc(dstCapacity, ZSTD_defaultCMem); + RETURN_ERROR_IF(dst == NULL, memory_allocation, "NULL pointer!"); + + seqCollector.collectSequences = 1; + seqCollector.seqStart = outSeqs; + seqCollector.seqIndex = 0; + seqCollector.maxSequences = outSeqsSize; + zc->seqCollector = seqCollector; + + { + const size_t ret = ZSTD_compress2(zc, dst, dstCapacity, src, srcSize); + ZSTD_customFree(dst, ZSTD_defaultCMem); + FORWARD_IF_ERROR(ret, "ZSTD_compress2 failed"); } + assert(zc->seqCollector.seqIndex <= ZSTD_sequenceBound(srcSize)); + return zc->seqCollector.seqIndex; +} - if (ms->hashLog3) { - U32 const h3Size = (U32)1 << ms->hashLog3; - ZSTD_reduceTable(ms->hashTable3, h3Size, reducerValue); +size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize) { + size_t in = 0; + size_t out = 0; + for (; in < seqsSize; ++in) { + if (sequences[in].offset == 0 && sequences[in].matchLength == 0) { + if (in != seqsSize - 1) { + sequences[in+1].litLength += sequences[in].litLength; + } + } else { + sequences[out] = sequences[in]; + ++out; + } } + return out; } +/* Unrolled loop to read four size_ts of input at a time. Returns 1 if is RLE, 0 if not. */ +static int ZSTD_isRLE(const BYTE* src, size_t length) { + const BYTE* ip = src; + const BYTE value = ip[0]; + const size_t valueST = (size_t)((U64)value * 0x0101010101010101ULL); + const size_t unrollSize = sizeof(size_t) * 4; + const size_t unrollMask = unrollSize - 1; + const size_t prefixLength = length & unrollMask; + size_t i; + if (length == 1) return 1; + /* Check if prefix is RLE first before using unrolled loop */ + if (prefixLength && ZSTD_count(ip+1, ip, ip+prefixLength) != prefixLength-1) { + return 0; + } + for (i = prefixLength; i != length; i += unrollSize) { + size_t u; + for (u = 0; u < unrollSize; u += sizeof(size_t)) { + if (MEM_readST(ip + i + u) != valueST) { + return 0; + } } } + return 1; +} -/*-******************************************************* -* Block entropic compression -*********************************************************/ +/* Returns true if the given block may be RLE. + * This is just a heuristic based on the compressibility. + * It may return both false positives and false negatives. + */ +static int ZSTD_maybeRLE(SeqStore_t const* seqStore) +{ + size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart); + size_t const nbLits = (size_t)(seqStore->lit - seqStore->litStart); -/* See doc/zstd_compression_format.md for detailed format description */ + return nbSeqs < 4 && nbLits < 10; +} -void ZSTD_seqToCodes(const seqStore_t* seqStorePtr) +static void +ZSTD_blockState_confirmRepcodesAndEntropyTables(ZSTD_blockState_t* const bs) { - const seqDef* const sequences = seqStorePtr->sequencesStart; - BYTE* const llCodeTable = seqStorePtr->llCode; - BYTE* const ofCodeTable = seqStorePtr->ofCode; - BYTE* const mlCodeTable = seqStorePtr->mlCode; - U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); - U32 u; - assert(nbSeq <= seqStorePtr->maxNbSeq); - for (u=0; ulongLengthID==1) - llCodeTable[seqStorePtr->longLengthPos] = MaxLL; - if (seqStorePtr->longLengthID==2) - mlCodeTable[seqStorePtr->longLengthPos] = MaxML; + ZSTD_compressedBlockState_t* const tmp = bs->prevCBlock; + bs->prevCBlock = bs->nextCBlock; + bs->nextCBlock = tmp; } -/* ZSTD_useTargetCBlockSize(): - * Returns if target compressed block size param is being used. - * If used, compression will do best effort to make a compressed block size to be around targetCBlockSize. - * Returns 1 if true, 0 otherwise. */ -static int ZSTD_useTargetCBlockSize(const ZSTD_CCtx_params* cctxParams) +/* Writes the block header */ +static void +writeBlockHeader(void* op, size_t cSize, size_t blockSize, U32 lastBlock) { - DEBUGLOG(5, "ZSTD_useTargetCBlockSize (targetCBlockSize=%zu)", cctxParams->targetCBlockSize); - return (cctxParams->targetCBlockSize != 0); + U32 const cBlockHeader = cSize == 1 ? + lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) : + lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); + MEM_writeLE24(op, cBlockHeader); + DEBUGLOG(5, "writeBlockHeader: cSize: %zu blockSize: %zu lastBlock: %u", cSize, blockSize, lastBlock); } -/* ZSTD_entropyCompressSequences_internal(): - * actually compresses both literals and sequences */ -MEM_STATIC size_t -ZSTD_entropyCompressSequences_internal(seqStore_t* seqStorePtr, - const ZSTD_entropyCTables_t* prevEntropy, - ZSTD_entropyCTables_t* nextEntropy, - const ZSTD_CCtx_params* cctxParams, - void* dst, size_t dstCapacity, - void* entropyWorkspace, size_t entropyWkspSize, - const int bmi2) +/** ZSTD_buildBlockEntropyStats_literals() : + * Builds entropy for the literals. + * Stores literals block type (raw, rle, compressed, repeat) and + * huffman description table to hufMetadata. + * Requires ENTROPY_WORKSPACE_SIZE workspace + * @return : size of huffman description table, or an error code + */ +static size_t +ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSize, + const ZSTD_hufCTables_t* prevHuf, + ZSTD_hufCTables_t* nextHuf, + ZSTD_hufCTablesMetadata_t* hufMetadata, + const int literalsCompressionIsDisabled, + void* workspace, size_t wkspSize, + int hufFlags) { - const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN; - ZSTD_strategy const strategy = cctxParams->cParams.strategy; - unsigned* count = (unsigned*)entropyWorkspace; - FSE_CTable* CTable_LitLength = nextEntropy->fse.litlengthCTable; - FSE_CTable* CTable_OffsetBits = nextEntropy->fse.offcodeCTable; - FSE_CTable* CTable_MatchLength = nextEntropy->fse.matchlengthCTable; - U32 LLtype, Offtype, MLtype; /* compressed, raw or rle */ - const seqDef* const sequences = seqStorePtr->sequencesStart; - const BYTE* const ofCodeTable = seqStorePtr->ofCode; - const BYTE* const llCodeTable = seqStorePtr->llCode; - const BYTE* const mlCodeTable = seqStorePtr->mlCode; - BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = ostart + dstCapacity; - BYTE* op = ostart; - size_t const nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart); - BYTE* seqHead; - BYTE* lastNCount = NULL; - - entropyWorkspace = count + (MaxSeq + 1); - entropyWkspSize -= (MaxSeq + 1) * sizeof(*count); - - DEBUGLOG(4, "ZSTD_entropyCompressSequences_internal (nbSeq=%zu)", nbSeq); - ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<= HUF_WORKSPACE_SIZE); + BYTE* const wkspStart = (BYTE*)workspace; + BYTE* const wkspEnd = wkspStart + wkspSize; + BYTE* const countWkspStart = wkspStart; + unsigned* const countWksp = (unsigned*)workspace; + const size_t countWkspSize = (HUF_SYMBOLVALUE_MAX + 1) * sizeof(unsigned); + BYTE* const nodeWksp = countWkspStart + countWkspSize; + const size_t nodeWkspSize = (size_t)(wkspEnd - nodeWksp); + unsigned maxSymbolValue = HUF_SYMBOLVALUE_MAX; + unsigned huffLog = LitHufLog; + HUF_repeat repeat = prevHuf->repeatMode; + DEBUGLOG(5, "ZSTD_buildBlockEntropyStats_literals (srcSize=%zu)", srcSize); - /* Compress literals */ - { const BYTE* const literals = seqStorePtr->litStart; - size_t const litSize = (size_t)(seqStorePtr->lit - literals); - size_t const cSize = ZSTD_compressLiterals( - &prevEntropy->huf, &nextEntropy->huf, - cctxParams->cParams.strategy, - ZSTD_disableLiteralsCompression(cctxParams), - op, dstCapacity, - literals, litSize, - entropyWorkspace, entropyWkspSize, - bmi2); - FORWARD_IF_ERROR(cSize, "ZSTD_compressLiterals failed"); - assert(cSize <= dstCapacity); - op += cSize; - } + /* Prepare nextEntropy assuming reusing the existing table */ + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); - /* Sequences Header */ - RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/, - dstSize_tooSmall, "Can't fit seq hdr in output buf!"); - if (nbSeq < 128) { - *op++ = (BYTE)nbSeq; - } else if (nbSeq < LONGNBSEQ) { - op[0] = (BYTE)((nbSeq>>8) + 0x80); - op[1] = (BYTE)nbSeq; - op+=2; - } else { - op[0]=0xFF; - MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)); - op+=3; - } - assert(op <= oend); - if (nbSeq==0) { - /* Copy the old tables over as if we repeated them */ - ZSTD_memcpy(&nextEntropy->fse, &prevEntropy->fse, sizeof(prevEntropy->fse)); - return (size_t)(op - ostart); + if (literalsCompressionIsDisabled) { + DEBUGLOG(5, "set_basic - disabled"); + hufMetadata->hType = set_basic; + return 0; } - /* seqHead : flags for FSE encoding type */ - seqHead = op++; - assert(op <= oend); - - /* convert length/distances into codes */ - ZSTD_seqToCodes(seqStorePtr); - /* build CTable for Literal Lengths */ - { unsigned max = MaxLL; - size_t const mostFrequent = HIST_countFast_wksp(count, &max, llCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ - DEBUGLOG(5, "Building LL table"); - nextEntropy->fse.litlength_repeatMode = prevEntropy->fse.litlength_repeatMode; - LLtype = ZSTD_selectEncodingType(&nextEntropy->fse.litlength_repeatMode, - count, max, mostFrequent, nbSeq, - LLFSELog, prevEntropy->fse.litlengthCTable, - LL_defaultNorm, LL_defaultNormLog, - ZSTD_defaultAllowed, strategy); - assert(set_basic < set_compressed && set_rle < set_compressed); - assert(!(LLtype < set_compressed && nextEntropy->fse.litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ - { size_t const countSize = ZSTD_buildCTable( - op, (size_t)(oend - op), - CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype, - count, max, llCodeTable, nbSeq, - LL_defaultNorm, LL_defaultNormLog, MaxLL, - prevEntropy->fse.litlengthCTable, - sizeof(prevEntropy->fse.litlengthCTable), - entropyWorkspace, entropyWkspSize); - FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for LitLens failed"); - if (LLtype == set_compressed) - lastNCount = op; - op += countSize; - assert(op <= oend); - } } - /* build CTable for Offsets */ - { unsigned max = MaxOff; - size_t const mostFrequent = HIST_countFast_wksp( - count, &max, ofCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ - /* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */ - ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed; - DEBUGLOG(5, "Building OF table"); - nextEntropy->fse.offcode_repeatMode = prevEntropy->fse.offcode_repeatMode; - Offtype = ZSTD_selectEncodingType(&nextEntropy->fse.offcode_repeatMode, - count, max, mostFrequent, nbSeq, - OffFSELog, prevEntropy->fse.offcodeCTable, - OF_defaultNorm, OF_defaultNormLog, - defaultPolicy, strategy); - assert(!(Offtype < set_compressed && nextEntropy->fse.offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */ - { size_t const countSize = ZSTD_buildCTable( - op, (size_t)(oend - op), - CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype, - count, max, ofCodeTable, nbSeq, - OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, - prevEntropy->fse.offcodeCTable, - sizeof(prevEntropy->fse.offcodeCTable), - entropyWorkspace, entropyWkspSize); - FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for Offsets failed"); - if (Offtype == set_compressed) - lastNCount = op; - op += countSize; - assert(op <= oend); - } } - /* build CTable for MatchLengths */ - { unsigned max = MaxML; - size_t const mostFrequent = HIST_countFast_wksp( - count, &max, mlCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ - DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op)); - nextEntropy->fse.matchlength_repeatMode = prevEntropy->fse.matchlength_repeatMode; - MLtype = ZSTD_selectEncodingType(&nextEntropy->fse.matchlength_repeatMode, - count, max, mostFrequent, nbSeq, - MLFSELog, prevEntropy->fse.matchlengthCTable, - ML_defaultNorm, ML_defaultNormLog, - ZSTD_defaultAllowed, strategy); - assert(!(MLtype < set_compressed && nextEntropy->fse.matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ - { size_t const countSize = ZSTD_buildCTable( - op, (size_t)(oend - op), - CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype, - count, max, mlCodeTable, nbSeq, - ML_defaultNorm, ML_defaultNormLog, MaxML, - prevEntropy->fse.matchlengthCTable, - sizeof(prevEntropy->fse.matchlengthCTable), - entropyWorkspace, entropyWkspSize); - FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for MatchLengths failed"); - if (MLtype == set_compressed) - lastNCount = op; - op += countSize; - assert(op <= oend); + /* small ? don't even attempt compression (speed opt) */ +#ifndef COMPRESS_LITERALS_SIZE_MIN +# define COMPRESS_LITERALS_SIZE_MIN 63 /* heuristic */ +#endif + { size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN; + if (srcSize <= minLitSize) { + DEBUGLOG(5, "set_basic - too small"); + hufMetadata->hType = set_basic; + return 0; } } - *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2)); - - { size_t const bitstreamSize = ZSTD_encodeSequences( - op, (size_t)(oend - op), - CTable_MatchLength, mlCodeTable, - CTable_OffsetBits, ofCodeTable, - CTable_LitLength, llCodeTable, - sequences, nbSeq, - longOffsets, bmi2); - FORWARD_IF_ERROR(bitstreamSize, "ZSTD_encodeSequences failed"); - op += bitstreamSize; - assert(op <= oend); - /* zstd versions <= 1.3.4 mistakenly report corruption when - * FSE_readNCount() receives a buffer < 4 bytes. - * Fixed by https://github.com/facebook/zstd/pull/1146. - * This can happen when the last set_compressed table present is 2 - * bytes and the bitstream is only one byte. - * In this exceedingly rare case, we will simply emit an uncompressed - * block, since it isn't worth optimizing. - */ - if (lastNCount && (op - lastNCount) < 4) { - /* NCountSize >= 2 && bitstreamSize > 0 ==> lastCountSize == 3 */ - assert(op - lastNCount == 3); - DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.3.4 by " - "emitting an uncompressed block."); + /* Scan input and build symbol stats */ + { size_t const largest = + HIST_count_wksp (countWksp, &maxSymbolValue, + (const BYTE*)src, srcSize, + workspace, wkspSize); + FORWARD_IF_ERROR(largest, "HIST_count_wksp failed"); + if (largest == srcSize) { + /* only one literal symbol */ + DEBUGLOG(5, "set_rle"); + hufMetadata->hType = set_rle; + return 0; + } + if (largest <= (srcSize >> 7)+4) { + /* heuristic: likely not compressible */ + DEBUGLOG(5, "set_basic - no gain"); + hufMetadata->hType = set_basic; return 0; - } + } } + + /* Validate the previous Huffman table */ + if (repeat == HUF_repeat_check + && !HUF_validateCTable((HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue)) { + repeat = HUF_repeat_none; } - DEBUGLOG(5, "compressed block size : %u", (unsigned)(op - ostart)); - return (size_t)(op - ostart); + /* Build Huffman Tree */ + ZSTD_memset(nextHuf->CTable, 0, sizeof(nextHuf->CTable)); + huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, nodeWksp, nodeWkspSize, nextHuf->CTable, countWksp, hufFlags); + assert(huffLog <= LitHufLog); + { size_t const maxBits = HUF_buildCTable_wksp((HUF_CElt*)nextHuf->CTable, countWksp, + maxSymbolValue, huffLog, + nodeWksp, nodeWkspSize); + FORWARD_IF_ERROR(maxBits, "HUF_buildCTable_wksp"); + huffLog = (U32)maxBits; + } + { /* Build and write the CTable */ + size_t const newCSize = HUF_estimateCompressedSize( + (HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue); + size_t const hSize = HUF_writeCTable_wksp( + hufMetadata->hufDesBuffer, sizeof(hufMetadata->hufDesBuffer), + (HUF_CElt*)nextHuf->CTable, maxSymbolValue, huffLog, + nodeWksp, nodeWkspSize); + /* Check against repeating the previous CTable */ + if (repeat != HUF_repeat_none) { + size_t const oldCSize = HUF_estimateCompressedSize( + (HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue); + if (oldCSize < srcSize && (oldCSize <= hSize + newCSize || hSize + 12 >= srcSize)) { + DEBUGLOG(5, "set_repeat - smaller"); + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + hufMetadata->hType = set_repeat; + return 0; + } } + if (newCSize + hSize >= srcSize) { + DEBUGLOG(5, "set_basic - no gains"); + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + hufMetadata->hType = set_basic; + return 0; + } + DEBUGLOG(5, "set_compressed (hSize=%u)", (U32)hSize); + hufMetadata->hType = set_compressed; + nextHuf->repeatMode = HUF_repeat_check; + return hSize; + } } -MEM_STATIC size_t -ZSTD_entropyCompressSequences(seqStore_t* seqStorePtr, - const ZSTD_entropyCTables_t* prevEntropy, - ZSTD_entropyCTables_t* nextEntropy, - const ZSTD_CCtx_params* cctxParams, - void* dst, size_t dstCapacity, - size_t srcSize, - void* entropyWorkspace, size_t entropyWkspSize, - int bmi2) -{ - size_t const cSize = ZSTD_entropyCompressSequences_internal( - seqStorePtr, prevEntropy, nextEntropy, cctxParams, - dst, dstCapacity, - entropyWorkspace, entropyWkspSize, bmi2); - if (cSize == 0) return 0; - /* When srcSize <= dstCapacity, there is enough space to write a raw uncompressed block. - * Since we ran out of space, block must be not compressible, so fall back to raw uncompressed block. - */ - if ((cSize == ERROR(dstSize_tooSmall)) & (srcSize <= dstCapacity)) - return 0; /* block not compressed */ - FORWARD_IF_ERROR(cSize, "ZSTD_entropyCompressSequences_internal failed"); - /* Check compressibility */ - { size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, cctxParams->cParams.strategy); - if (cSize >= maxCSize) return 0; /* block not compressed */ - } - DEBUGLOG(4, "ZSTD_entropyCompressSequences() cSize: %zu\n", cSize); - return cSize; +/* ZSTD_buildDummySequencesStatistics(): + * Returns a ZSTD_symbolEncodingTypeStats_t with all encoding types as set_basic, + * and updates nextEntropy to the appropriate repeatMode. + */ +static ZSTD_symbolEncodingTypeStats_t +ZSTD_buildDummySequencesStatistics(ZSTD_fseCTables_t* nextEntropy) +{ + ZSTD_symbolEncodingTypeStats_t stats = {set_basic, set_basic, set_basic, 0, 0, 0}; + nextEntropy->litlength_repeatMode = FSE_repeat_none; + nextEntropy->offcode_repeatMode = FSE_repeat_none; + nextEntropy->matchlength_repeatMode = FSE_repeat_none; + return stats; } -/* ZSTD_selectBlockCompressor() : - * Not static, but internal use only (used by long distance matcher) - * assumption : strat is a valid strategy */ -ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_dictMode_e dictMode) +/** ZSTD_buildBlockEntropyStats_sequences() : + * Builds entropy for the sequences. + * Stores symbol compression modes and fse table to fseMetadata. + * Requires ENTROPY_WORKSPACE_SIZE wksp. + * @return : size of fse tables or error code */ +static size_t +ZSTD_buildBlockEntropyStats_sequences( + const SeqStore_t* seqStorePtr, + const ZSTD_fseCTables_t* prevEntropy, + ZSTD_fseCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + ZSTD_fseCTablesMetadata_t* fseMetadata, + void* workspace, size_t wkspSize) { - static const ZSTD_blockCompressor blockCompressor[4][ZSTD_STRATEGY_MAX+1] = { - { ZSTD_compressBlock_fast /* default for 0 */, - ZSTD_compressBlock_fast, - ZSTD_compressBlock_doubleFast, - ZSTD_compressBlock_greedy, - ZSTD_compressBlock_lazy, - ZSTD_compressBlock_lazy2, - ZSTD_compressBlock_btlazy2, - ZSTD_compressBlock_btopt, - ZSTD_compressBlock_btultra, - ZSTD_compressBlock_btultra2 }, - { ZSTD_compressBlock_fast_extDict /* default for 0 */, - ZSTD_compressBlock_fast_extDict, - ZSTD_compressBlock_doubleFast_extDict, - ZSTD_compressBlock_greedy_extDict, - ZSTD_compressBlock_lazy_extDict, - ZSTD_compressBlock_lazy2_extDict, - ZSTD_compressBlock_btlazy2_extDict, - ZSTD_compressBlock_btopt_extDict, - ZSTD_compressBlock_btultra_extDict, - ZSTD_compressBlock_btultra_extDict }, - { ZSTD_compressBlock_fast_dictMatchState /* default for 0 */, - ZSTD_compressBlock_fast_dictMatchState, - ZSTD_compressBlock_doubleFast_dictMatchState, - ZSTD_compressBlock_greedy_dictMatchState, - ZSTD_compressBlock_lazy_dictMatchState, - ZSTD_compressBlock_lazy2_dictMatchState, - ZSTD_compressBlock_btlazy2_dictMatchState, - ZSTD_compressBlock_btopt_dictMatchState, - ZSTD_compressBlock_btultra_dictMatchState, - ZSTD_compressBlock_btultra_dictMatchState }, - { NULL /* default for 0 */, - NULL, - NULL, - ZSTD_compressBlock_greedy_dedicatedDictSearch, - ZSTD_compressBlock_lazy_dedicatedDictSearch, - ZSTD_compressBlock_lazy2_dedicatedDictSearch, - NULL, - NULL, - NULL, - NULL } - }; - ZSTD_blockCompressor selectedCompressor; - ZSTD_STATIC_ASSERT((unsigned)ZSTD_fast == 1); + ZSTD_strategy const strategy = cctxParams->cParams.strategy; + size_t const nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + BYTE* const ostart = fseMetadata->fseTablesBuffer; + BYTE* const oend = ostart + sizeof(fseMetadata->fseTablesBuffer); + BYTE* op = ostart; + unsigned* countWorkspace = (unsigned*)workspace; + unsigned* entropyWorkspace = countWorkspace + (MaxSeq + 1); + size_t entropyWorkspaceSize = wkspSize - (MaxSeq + 1) * sizeof(*countWorkspace); + ZSTD_symbolEncodingTypeStats_t stats; + + DEBUGLOG(5, "ZSTD_buildBlockEntropyStats_sequences (nbSeq=%zu)", nbSeq); + stats = nbSeq != 0 ? ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq, + prevEntropy, nextEntropy, op, oend, + strategy, countWorkspace, + entropyWorkspace, entropyWorkspaceSize) + : ZSTD_buildDummySequencesStatistics(nextEntropy); + FORWARD_IF_ERROR(stats.size, "ZSTD_buildSequencesStatistics failed!"); + fseMetadata->llType = (SymbolEncodingType_e) stats.LLtype; + fseMetadata->ofType = (SymbolEncodingType_e) stats.Offtype; + fseMetadata->mlType = (SymbolEncodingType_e) stats.MLtype; + fseMetadata->lastCountSize = stats.lastCountSize; + return stats.size; +} + + +/** ZSTD_buildBlockEntropyStats() : + * Builds entropy for the block. + * Requires workspace size ENTROPY_WORKSPACE_SIZE + * @return : 0 on success, or an error code + * Note : also employed in superblock + */ +size_t ZSTD_buildBlockEntropyStats( + const SeqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + ZSTD_entropyCTablesMetadata_t* entropyMetadata, + void* workspace, size_t wkspSize) +{ + size_t const litSize = (size_t)(seqStorePtr->lit - seqStorePtr->litStart); + int const huf_useOptDepth = (cctxParams->cParams.strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD); + int const hufFlags = huf_useOptDepth ? HUF_flags_optimalDepth : 0; - assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, strat)); - selectedCompressor = blockCompressor[(int)dictMode][(int)strat]; - assert(selectedCompressor != NULL); - return selectedCompressor; -} + entropyMetadata->hufMetadata.hufDesSize = + ZSTD_buildBlockEntropyStats_literals(seqStorePtr->litStart, litSize, + &prevEntropy->huf, &nextEntropy->huf, + &entropyMetadata->hufMetadata, + ZSTD_literalsCompressionIsDisabled(cctxParams), + workspace, wkspSize, hufFlags); -static void ZSTD_storeLastLiterals(seqStore_t* seqStorePtr, - const BYTE* anchor, size_t lastLLSize) -{ - ZSTD_memcpy(seqStorePtr->lit, anchor, lastLLSize); - seqStorePtr->lit += lastLLSize; + FORWARD_IF_ERROR(entropyMetadata->hufMetadata.hufDesSize, "ZSTD_buildBlockEntropyStats_literals failed"); + entropyMetadata->fseMetadata.fseTablesSize = + ZSTD_buildBlockEntropyStats_sequences(seqStorePtr, + &prevEntropy->fse, &nextEntropy->fse, + cctxParams, + &entropyMetadata->fseMetadata, + workspace, wkspSize); + FORWARD_IF_ERROR(entropyMetadata->fseMetadata.fseTablesSize, "ZSTD_buildBlockEntropyStats_sequences failed"); + return 0; } -void ZSTD_resetSeqStore(seqStore_t* ssPtr) +/* Returns the size estimate for the literals section (header + content) of a block */ +static size_t +ZSTD_estimateBlockSize_literal(const BYTE* literals, size_t litSize, + const ZSTD_hufCTables_t* huf, + const ZSTD_hufCTablesMetadata_t* hufMetadata, + void* workspace, size_t wkspSize, + int writeEntropy) { - ssPtr->lit = ssPtr->litStart; - ssPtr->sequences = ssPtr->sequencesStart; - ssPtr->longLengthID = 0; -} + unsigned* const countWksp = (unsigned*)workspace; + unsigned maxSymbolValue = HUF_SYMBOLVALUE_MAX; + size_t literalSectionHeaderSize = 3 + (litSize >= 1 KB) + (litSize >= 16 KB); + U32 singleStream = litSize < 256; -typedef enum { ZSTDbss_compress, ZSTDbss_noCompress } ZSTD_buildSeqStore_e; + if (hufMetadata->hType == set_basic) return litSize; + else if (hufMetadata->hType == set_rle) return 1; + else if (hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat) { + size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)literals, litSize, workspace, wkspSize); + if (ZSTD_isError(largest)) return litSize; + { size_t cLitSizeEstimate = HUF_estimateCompressedSize((const HUF_CElt*)huf->CTable, countWksp, maxSymbolValue); + if (writeEntropy) cLitSizeEstimate += hufMetadata->hufDesSize; + if (!singleStream) cLitSizeEstimate += 6; /* multi-stream huffman uses 6-byte jump table */ + return cLitSizeEstimate + literalSectionHeaderSize; + } } + assert(0); /* impossible */ + return 0; +} -static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) +/* Returns the size estimate for the FSE-compressed symbols (of, ml, ll) of a block */ +static size_t +ZSTD_estimateBlockSize_symbolType(SymbolEncodingType_e type, + const BYTE* codeTable, size_t nbSeq, unsigned maxCode, + const FSE_CTable* fseCTable, + const U8* additionalBits, + short const* defaultNorm, U32 defaultNormLog, U32 defaultMax, + void* workspace, size_t wkspSize) { - ZSTD_matchState_t* const ms = &zc->blockState.matchState; - DEBUGLOG(5, "ZSTD_buildSeqStore (srcSize=%zu)", srcSize); - assert(srcSize <= ZSTD_BLOCKSIZE_MAX); - /* Assert that we have correctly flushed the ctx params into the ms's copy */ - ZSTD_assertEqualCParams(zc->appliedParams.cParams, ms->cParams); - if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) { - if (zc->appliedParams.cParams.strategy >= ZSTD_btopt) { - ZSTD_ldm_skipRawSeqStoreBytes(&zc->externSeqStore, srcSize); - } else { - ZSTD_ldm_skipSequences(&zc->externSeqStore, srcSize, zc->appliedParams.cParams.minMatch); - } - return ZSTDbss_noCompress; /* don't even attempt compression below a certain srcSize */ - } - ZSTD_resetSeqStore(&(zc->seqStore)); - /* required for optimal parser to read stats from dictionary */ - ms->opt.symbolCosts = &zc->blockState.prevCBlock->entropy; - /* tell the optimal parser how we expect to compress literals */ - ms->opt.literalCompressionMode = zc->appliedParams.literalCompressionMode; - /* a gap between an attached dict and the current window is not safe, - * they must remain adjacent, - * and when that stops being the case, the dict must be unset */ - assert(ms->dictMatchState == NULL || ms->loadedDictEnd == ms->window.dictLimit); + unsigned* const countWksp = (unsigned*)workspace; + const BYTE* ctp = codeTable; + const BYTE* const ctStart = ctp; + const BYTE* const ctEnd = ctStart + nbSeq; + size_t cSymbolTypeSizeEstimateInBits = 0; + unsigned max = maxCode; - /* limited update after a very long match */ - { const BYTE* const base = ms->window.base; - const BYTE* const istart = (const BYTE*)src; - const U32 curr = (U32)(istart-base); - if (sizeof(ptrdiff_t)==8) assert(istart - base < (ptrdiff_t)(U32)(-1)); /* ensure no overflow */ - if (curr > ms->nextToUpdate + 384) - ms->nextToUpdate = curr - MIN(192, (U32)(curr - ms->nextToUpdate - 384)); + HIST_countFast_wksp(countWksp, &max, codeTable, nbSeq, workspace, wkspSize); /* can't fail */ + if (type == set_basic) { + /* We selected this encoding type, so it must be valid. */ + assert(max <= defaultMax); + (void)defaultMax; + cSymbolTypeSizeEstimateInBits = ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, countWksp, max); + } else if (type == set_rle) { + cSymbolTypeSizeEstimateInBits = 0; + } else if (type == set_compressed || type == set_repeat) { + cSymbolTypeSizeEstimateInBits = ZSTD_fseBitCost(fseCTable, countWksp, max); + } + if (ZSTD_isError(cSymbolTypeSizeEstimateInBits)) { + return nbSeq * 10; } + while (ctp < ctEnd) { + if (additionalBits) cSymbolTypeSizeEstimateInBits += additionalBits[*ctp]; + else cSymbolTypeSizeEstimateInBits += *ctp; /* for offset, offset code is also the number of additional bits */ + ctp++; + } + return cSymbolTypeSizeEstimateInBits >> 3; +} - /* select and store sequences */ - { ZSTD_dictMode_e const dictMode = ZSTD_matchState_dictMode(ms); - size_t lastLLSize; - { int i; - for (i = 0; i < ZSTD_REP_NUM; ++i) - zc->blockState.nextCBlock->rep[i] = zc->blockState.prevCBlock->rep[i]; - } - if (zc->externSeqStore.pos < zc->externSeqStore.size) { - assert(!zc->appliedParams.ldmParams.enableLdm); - /* Updates ldmSeqStore.pos */ - lastLLSize = - ZSTD_ldm_blockCompress(&zc->externSeqStore, - ms, &zc->seqStore, - zc->blockState.nextCBlock->rep, - src, srcSize); - assert(zc->externSeqStore.pos <= zc->externSeqStore.size); - } else if (zc->appliedParams.ldmParams.enableLdm) { - rawSeqStore_t ldmSeqStore = kNullRawSeqStore; +/* Returns the size estimate for the sequences section (header + content) of a block */ +static size_t +ZSTD_estimateBlockSize_sequences(const BYTE* ofCodeTable, + const BYTE* llCodeTable, + const BYTE* mlCodeTable, + size_t nbSeq, + const ZSTD_fseCTables_t* fseTables, + const ZSTD_fseCTablesMetadata_t* fseMetadata, + void* workspace, size_t wkspSize, + int writeEntropy) +{ + size_t sequencesSectionHeaderSize = 1 /* seqHead */ + 1 /* min seqSize size */ + (nbSeq >= 128) + (nbSeq >= LONGNBSEQ); + size_t cSeqSizeEstimate = 0; + cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, nbSeq, MaxOff, + fseTables->offcodeCTable, NULL, + OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, + workspace, wkspSize); + cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->llType, llCodeTable, nbSeq, MaxLL, + fseTables->litlengthCTable, LL_bits, + LL_defaultNorm, LL_defaultNormLog, MaxLL, + workspace, wkspSize); + cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->mlType, mlCodeTable, nbSeq, MaxML, + fseTables->matchlengthCTable, ML_bits, + ML_defaultNorm, ML_defaultNormLog, MaxML, + workspace, wkspSize); + if (writeEntropy) cSeqSizeEstimate += fseMetadata->fseTablesSize; + return cSeqSizeEstimate + sequencesSectionHeaderSize; +} - ldmSeqStore.seq = zc->ldmSequences; - ldmSeqStore.capacity = zc->maxNbLdmSequences; - /* Updates ldmSeqStore.size */ - FORWARD_IF_ERROR(ZSTD_ldm_generateSequences(&zc->ldmState, &ldmSeqStore, - &zc->appliedParams.ldmParams, - src, srcSize), ""); - /* Updates ldmSeqStore.pos */ - lastLLSize = - ZSTD_ldm_blockCompress(&ldmSeqStore, - ms, &zc->seqStore, - zc->blockState.nextCBlock->rep, - src, srcSize); - assert(ldmSeqStore.pos == ldmSeqStore.size); - } else { /* not long range mode */ - ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy, dictMode); - ms->ldmSeqStore = NULL; - lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize); - } - { const BYTE* const lastLiterals = (const BYTE*)src + srcSize - lastLLSize; - ZSTD_storeLastLiterals(&zc->seqStore, lastLiterals, lastLLSize); +/* Returns the size estimate for a given stream of literals, of, ll, ml */ +static size_t +ZSTD_estimateBlockSize(const BYTE* literals, size_t litSize, + const BYTE* ofCodeTable, + const BYTE* llCodeTable, + const BYTE* mlCodeTable, + size_t nbSeq, + const ZSTD_entropyCTables_t* entropy, + const ZSTD_entropyCTablesMetadata_t* entropyMetadata, + void* workspace, size_t wkspSize, + int writeLitEntropy, int writeSeqEntropy) +{ + size_t const literalsSize = ZSTD_estimateBlockSize_literal(literals, litSize, + &entropy->huf, &entropyMetadata->hufMetadata, + workspace, wkspSize, writeLitEntropy); + size_t const seqSize = ZSTD_estimateBlockSize_sequences(ofCodeTable, llCodeTable, mlCodeTable, + nbSeq, &entropy->fse, &entropyMetadata->fseMetadata, + workspace, wkspSize, writeSeqEntropy); + return seqSize + literalsSize + ZSTD_blockHeaderSize; +} + +/* Builds entropy statistics and uses them for blocksize estimation. + * + * @return: estimated compressed size of the seqStore, or a zstd error. + */ +static size_t +ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(SeqStore_t* seqStore, ZSTD_CCtx* zc) +{ + ZSTD_entropyCTablesMetadata_t* const entropyMetadata = &zc->blockSplitCtx.entropyMetadata; + DEBUGLOG(6, "ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize()"); + FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(seqStore, + &zc->blockState.prevCBlock->entropy, + &zc->blockState.nextCBlock->entropy, + &zc->appliedParams, + entropyMetadata, + zc->tmpWorkspace, zc->tmpWkspSize), ""); + return ZSTD_estimateBlockSize( + seqStore->litStart, (size_t)(seqStore->lit - seqStore->litStart), + seqStore->ofCode, seqStore->llCode, seqStore->mlCode, + (size_t)(seqStore->sequences - seqStore->sequencesStart), + &zc->blockState.nextCBlock->entropy, + entropyMetadata, + zc->tmpWorkspace, zc->tmpWkspSize, + (int)(entropyMetadata->hufMetadata.hType == set_compressed), 1); +} + +/* Returns literals bytes represented in a seqStore */ +static size_t ZSTD_countSeqStoreLiteralsBytes(const SeqStore_t* const seqStore) +{ + size_t literalsBytes = 0; + size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart); + size_t i; + for (i = 0; i < nbSeqs; ++i) { + SeqDef const seq = seqStore->sequencesStart[i]; + literalsBytes += seq.litLength; + if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_literalLength) { + literalsBytes += 0x10000; } } - return ZSTDbss_compress; + return literalsBytes; } -static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) +/* Returns match bytes represented in a seqStore */ +static size_t ZSTD_countSeqStoreMatchBytes(const SeqStore_t* const seqStore) { - const seqStore_t* seqStore = ZSTD_getSeqStore(zc); - const seqDef* seqStoreSeqs = seqStore->sequencesStart; - size_t seqStoreSeqSize = seqStore->sequences - seqStoreSeqs; - size_t seqStoreLiteralsSize = (size_t)(seqStore->lit - seqStore->litStart); - size_t literalsRead = 0; - size_t lastLLSize; - - ZSTD_Sequence* outSeqs = &zc->seqCollector.seqStart[zc->seqCollector.seqIndex]; + size_t matchBytes = 0; + size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart); size_t i; - repcodes_t updatedRepcodes; - - assert(zc->seqCollector.seqIndex + 1 < zc->seqCollector.maxSequences); - /* Ensure we have enough space for last literals "sequence" */ - assert(zc->seqCollector.maxSequences >= seqStoreSeqSize + 1); - ZSTD_memcpy(updatedRepcodes.rep, zc->blockState.prevCBlock->rep, sizeof(repcodes_t)); - for (i = 0; i < seqStoreSeqSize; ++i) { - U32 rawOffset = seqStoreSeqs[i].offset - ZSTD_REP_NUM; - outSeqs[i].litLength = seqStoreSeqs[i].litLength; - outSeqs[i].matchLength = seqStoreSeqs[i].matchLength + MINMATCH; - outSeqs[i].rep = 0; + for (i = 0; i < nbSeqs; ++i) { + SeqDef seq = seqStore->sequencesStart[i]; + matchBytes += seq.mlBase + MINMATCH; + if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_matchLength) { + matchBytes += 0x10000; + } } + return matchBytes; +} - if (i == seqStore->longLengthPos) { - if (seqStore->longLengthID == 1) { - outSeqs[i].litLength += 0x10000; - } else if (seqStore->longLengthID == 2) { - outSeqs[i].matchLength += 0x10000; - } +/* Derives the seqStore that is a chunk of the originalSeqStore from [startIdx, endIdx). + * Stores the result in resultSeqStore. + */ +static void ZSTD_deriveSeqStoreChunk(SeqStore_t* resultSeqStore, + const SeqStore_t* originalSeqStore, + size_t startIdx, size_t endIdx) +{ + *resultSeqStore = *originalSeqStore; + if (startIdx > 0) { + resultSeqStore->sequences = originalSeqStore->sequencesStart + startIdx; + resultSeqStore->litStart += ZSTD_countSeqStoreLiteralsBytes(resultSeqStore); + } + + /* Move longLengthPos into the correct position if necessary */ + if (originalSeqStore->longLengthType != ZSTD_llt_none) { + if (originalSeqStore->longLengthPos < startIdx || originalSeqStore->longLengthPos > endIdx) { + resultSeqStore->longLengthType = ZSTD_llt_none; + } else { + resultSeqStore->longLengthPos -= (U32)startIdx; } + } + resultSeqStore->sequencesStart = originalSeqStore->sequencesStart + startIdx; + resultSeqStore->sequences = originalSeqStore->sequencesStart + endIdx; + if (endIdx == (size_t)(originalSeqStore->sequences - originalSeqStore->sequencesStart)) { + /* This accounts for possible last literals if the derived chunk reaches the end of the block */ + assert(resultSeqStore->lit == originalSeqStore->lit); + } else { + size_t const literalsBytes = ZSTD_countSeqStoreLiteralsBytes(resultSeqStore); + resultSeqStore->lit = resultSeqStore->litStart + literalsBytes; + } + resultSeqStore->llCode += startIdx; + resultSeqStore->mlCode += startIdx; + resultSeqStore->ofCode += startIdx; +} - if (seqStoreSeqs[i].offset <= ZSTD_REP_NUM) { - /* Derive the correct offset corresponding to a repcode */ - outSeqs[i].rep = seqStoreSeqs[i].offset; - if (outSeqs[i].litLength != 0) { - rawOffset = updatedRepcodes.rep[outSeqs[i].rep - 1]; - } else { - if (outSeqs[i].rep == 3) { - rawOffset = updatedRepcodes.rep[0] - 1; - } else { - rawOffset = updatedRepcodes.rep[outSeqs[i].rep]; - } +/** + * Returns the raw offset represented by the combination of offBase, ll0, and repcode history. + * offBase must represent a repcode in the numeric representation of ZSTD_storeSeq(). + */ +static U32 +ZSTD_resolveRepcodeToRawOffset(const U32 rep[ZSTD_REP_NUM], const U32 offBase, const U32 ll0) +{ + U32 const adjustedRepCode = OFFBASE_TO_REPCODE(offBase) - 1 + ll0; /* [ 0 - 3 ] */ + assert(OFFBASE_IS_REPCODE(offBase)); + if (adjustedRepCode == ZSTD_REP_NUM) { + assert(ll0); + /* litlength == 0 and offCode == 2 implies selection of first repcode - 1 + * This is only valid if it results in a valid offset value, aka > 0. + * Note : it may happen that `rep[0]==1` in exceptional circumstances. + * In which case this function will return 0, which is an invalid offset. + * It's not an issue though, since this value will be + * compared and discarded within ZSTD_seqStore_resolveOffCodes(). + */ + return rep[0] - 1; + } + return rep[adjustedRepCode]; +} + +/** + * ZSTD_seqStore_resolveOffCodes() reconciles any possible divergences in offset history that may arise + * due to emission of RLE/raw blocks that disturb the offset history, + * and replaces any repcodes within the seqStore that may be invalid. + * + * dRepcodes are updated as would be on the decompression side. + * cRepcodes are updated exactly in accordance with the seqStore. + * + * Note : this function assumes seq->offBase respects the following numbering scheme : + * 0 : invalid + * 1-3 : repcode 1-3 + * 4+ : real_offset+3 + */ +static void +ZSTD_seqStore_resolveOffCodes(Repcodes_t* const dRepcodes, Repcodes_t* const cRepcodes, + const SeqStore_t* const seqStore, U32 const nbSeq) +{ + U32 idx = 0; + U32 const longLitLenIdx = seqStore->longLengthType == ZSTD_llt_literalLength ? seqStore->longLengthPos : nbSeq; + for (; idx < nbSeq; ++idx) { + SeqDef* const seq = seqStore->sequencesStart + idx; + U32 const ll0 = (seq->litLength == 0) && (idx != longLitLenIdx); + U32 const offBase = seq->offBase; + assert(offBase > 0); + if (OFFBASE_IS_REPCODE(offBase)) { + U32 const dRawOffset = ZSTD_resolveRepcodeToRawOffset(dRepcodes->rep, offBase, ll0); + U32 const cRawOffset = ZSTD_resolveRepcodeToRawOffset(cRepcodes->rep, offBase, ll0); + /* Adjust simulated decompression repcode history if we come across a mismatch. Replace + * the repcode with the offset it actually references, determined by the compression + * repcode history. + */ + if (dRawOffset != cRawOffset) { + seq->offBase = OFFSET_TO_OFFBASE(cRawOffset); } } - outSeqs[i].offset = rawOffset; - /* seqStoreSeqs[i].offset == offCode+1, and ZSTD_updateRep() expects offCode - so we provide seqStoreSeqs[i].offset - 1 */ - updatedRepcodes = ZSTD_updateRep(updatedRepcodes.rep, - seqStoreSeqs[i].offset - 1, - seqStoreSeqs[i].litLength == 0); - literalsRead += outSeqs[i].litLength; + /* Compression repcode history is always updated with values directly from the unmodified seqStore. + * Decompression repcode history may use modified seq->offset value taken from compression repcode history. + */ + ZSTD_updateRep(dRepcodes->rep, seq->offBase, ll0); + ZSTD_updateRep(cRepcodes->rep, offBase, ll0); } - /* Insert last literals (if any exist) in the block as a sequence with ml == off == 0. - * If there are no last literals, then we'll emit (of: 0, ml: 0, ll: 0), which is a marker - * for the block boundary, according to the API. - */ - assert(seqStoreLiteralsSize >= literalsRead); - lastLLSize = seqStoreLiteralsSize - literalsRead; - outSeqs[i].litLength = (U32)lastLLSize; - outSeqs[i].matchLength = outSeqs[i].offset = outSeqs[i].rep = 0; - seqStoreSeqSize++; - zc->seqCollector.seqIndex += seqStoreSeqSize; } -size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, - size_t outSeqsSize, const void* src, size_t srcSize) +/* ZSTD_compressSeqStore_singleBlock(): + * Compresses a seqStore into a block with a block header, into the buffer dst. + * + * Returns the total size of that block (including header) or a ZSTD error code. + */ +static size_t +ZSTD_compressSeqStore_singleBlock(ZSTD_CCtx* zc, + const SeqStore_t* const seqStore, + Repcodes_t* const dRep, Repcodes_t* const cRep, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + U32 lastBlock, U32 isPartition) { - const size_t dstCapacity = ZSTD_compressBound(srcSize); - void* dst = ZSTD_customMalloc(dstCapacity, ZSTD_defaultCMem); - SeqCollector seqCollector; + const U32 rleMaxLength = 25; + BYTE* op = (BYTE*)dst; + const BYTE* ip = (const BYTE*)src; + size_t cSize; + size_t cSeqsSize; + + /* In case of an RLE or raw block, the simulated decompression repcode history must be reset */ + Repcodes_t const dRepOriginal = *dRep; + DEBUGLOG(5, "ZSTD_compressSeqStore_singleBlock"); + if (isPartition) + ZSTD_seqStore_resolveOffCodes(dRep, cRep, seqStore, (U32)(seqStore->sequences - seqStore->sequencesStart)); + + RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize, dstSize_tooSmall, "Block header doesn't fit"); + cSeqsSize = ZSTD_entropyCompressSeqStore(seqStore, + &zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy, + &zc->appliedParams, + op + ZSTD_blockHeaderSize, dstCapacity - ZSTD_blockHeaderSize, + srcSize, + zc->tmpWorkspace, zc->tmpWkspSize /* statically allocated in resetCCtx */, + zc->bmi2); + FORWARD_IF_ERROR(cSeqsSize, "ZSTD_entropyCompressSeqStore failed!"); + + if (!zc->isFirstBlock && + cSeqsSize < rleMaxLength && + ZSTD_isRLE((BYTE const*)src, srcSize)) { + /* We don't want to emit our first block as a RLE even if it qualifies because + * doing so will cause the decoder (cli only) to throw a "should consume all input error." + * This is only an issue for zstd <= v1.4.3 + */ + cSeqsSize = 1; + } + + /* Sequence collection not supported when block splitting */ + if (zc->seqCollector.collectSequences) { + FORWARD_IF_ERROR(ZSTD_copyBlockSequences(&zc->seqCollector, seqStore, dRepOriginal.rep), "copyBlockSequences failed"); + ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); + return 0; + } - RETURN_ERROR_IF(dst == NULL, memory_allocation, "NULL pointer!"); + if (cSeqsSize == 0) { + cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, srcSize, lastBlock); + FORWARD_IF_ERROR(cSize, "Nocompress block failed"); + DEBUGLOG(5, "Writing out nocompress block, size: %zu", cSize); + *dRep = dRepOriginal; /* reset simulated decompression repcode history */ + } else if (cSeqsSize == 1) { + cSize = ZSTD_rleCompressBlock(op, dstCapacity, *ip, srcSize, lastBlock); + FORWARD_IF_ERROR(cSize, "RLE compress block failed"); + DEBUGLOG(5, "Writing out RLE block, size: %zu", cSize); + *dRep = dRepOriginal; /* reset simulated decompression repcode history */ + } else { + ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); + writeBlockHeader(op, cSeqsSize, srcSize, lastBlock); + cSize = ZSTD_blockHeaderSize + cSeqsSize; + DEBUGLOG(5, "Writing out compressed block, size: %zu", cSize); + } - seqCollector.collectSequences = 1; - seqCollector.seqStart = outSeqs; - seqCollector.seqIndex = 0; - seqCollector.maxSequences = outSeqsSize; - zc->seqCollector = seqCollector; + if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) + zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; - ZSTD_compress2(zc, dst, dstCapacity, src, srcSize); - ZSTD_customFree(dst, ZSTD_defaultCMem); - return zc->seqCollector.seqIndex; + return cSize; } -size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize) { - size_t in = 0; - size_t out = 0; - for (; in < seqsSize; ++in) { - if (sequences[in].offset == 0 && sequences[in].matchLength == 0) { - if (in != seqsSize - 1) { - sequences[in+1].litLength += sequences[in].litLength; - } - } else { - sequences[out] = sequences[in]; - ++out; - } +/* Struct to keep track of where we are in our recursive calls. */ +typedef struct { + U32* splitLocations; /* Array of split indices */ + size_t idx; /* The current index within splitLocations being worked on */ +} seqStoreSplits; + +#define MIN_SEQUENCES_BLOCK_SPLITTING 300 + +/* Helper function to perform the recursive search for block splits. + * Estimates the cost of seqStore prior to split, and estimates the cost of splitting the sequences in half. + * If advantageous to split, then we recurse down the two sub-blocks. + * If not, or if an error occurred in estimation, then we do not recurse. + * + * Note: The recursion depth is capped by a heuristic minimum number of sequences, + * defined by MIN_SEQUENCES_BLOCK_SPLITTING. + * In theory, this means the absolute largest recursion depth is 10 == log2(maxNbSeqInBlock/MIN_SEQUENCES_BLOCK_SPLITTING). + * In practice, recursion depth usually doesn't go beyond 4. + * + * Furthermore, the number of splits is capped by ZSTD_MAX_NB_BLOCK_SPLITS. + * At ZSTD_MAX_NB_BLOCK_SPLITS == 196 with the current existing blockSize + * maximum of 128 KB, this value is actually impossible to reach. + */ +static void +ZSTD_deriveBlockSplitsHelper(seqStoreSplits* splits, size_t startIdx, size_t endIdx, + ZSTD_CCtx* zc, const SeqStore_t* origSeqStore) +{ + SeqStore_t* const fullSeqStoreChunk = &zc->blockSplitCtx.fullSeqStoreChunk; + SeqStore_t* const firstHalfSeqStore = &zc->blockSplitCtx.firstHalfSeqStore; + SeqStore_t* const secondHalfSeqStore = &zc->blockSplitCtx.secondHalfSeqStore; + size_t estimatedOriginalSize; + size_t estimatedFirstHalfSize; + size_t estimatedSecondHalfSize; + size_t midIdx = (startIdx + endIdx)/2; + + DEBUGLOG(5, "ZSTD_deriveBlockSplitsHelper: startIdx=%zu endIdx=%zu", startIdx, endIdx); + assert(endIdx >= startIdx); + if (endIdx - startIdx < MIN_SEQUENCES_BLOCK_SPLITTING || splits->idx >= ZSTD_MAX_NB_BLOCK_SPLITS) { + DEBUGLOG(6, "ZSTD_deriveBlockSplitsHelper: Too few sequences (%zu)", endIdx - startIdx); + return; + } + ZSTD_deriveSeqStoreChunk(fullSeqStoreChunk, origSeqStore, startIdx, endIdx); + ZSTD_deriveSeqStoreChunk(firstHalfSeqStore, origSeqStore, startIdx, midIdx); + ZSTD_deriveSeqStoreChunk(secondHalfSeqStore, origSeqStore, midIdx, endIdx); + estimatedOriginalSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(fullSeqStoreChunk, zc); + estimatedFirstHalfSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(firstHalfSeqStore, zc); + estimatedSecondHalfSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(secondHalfSeqStore, zc); + DEBUGLOG(5, "Estimated original block size: %zu -- First half split: %zu -- Second half split: %zu", + estimatedOriginalSize, estimatedFirstHalfSize, estimatedSecondHalfSize); + if (ZSTD_isError(estimatedOriginalSize) || ZSTD_isError(estimatedFirstHalfSize) || ZSTD_isError(estimatedSecondHalfSize)) { + return; + } + if (estimatedFirstHalfSize + estimatedSecondHalfSize < estimatedOriginalSize) { + DEBUGLOG(5, "split decided at seqNb:%zu", midIdx); + ZSTD_deriveBlockSplitsHelper(splits, startIdx, midIdx, zc, origSeqStore); + splits->splitLocations[splits->idx] = (U32)midIdx; + splits->idx++; + ZSTD_deriveBlockSplitsHelper(splits, midIdx, endIdx, zc, origSeqStore); } - return out; } -/* Unrolled loop to read four size_ts of input at a time. Returns 1 if is RLE, 0 if not. */ -static int ZSTD_isRLE(const BYTE* src, size_t length) { - const BYTE* ip = src; - const BYTE value = ip[0]; - const size_t valueST = (size_t)((U64)value * 0x0101010101010101ULL); - const size_t unrollSize = sizeof(size_t) * 4; - const size_t unrollMask = unrollSize - 1; - const size_t prefixLength = length & unrollMask; - size_t i; - size_t u; - if (length == 1) return 1; - /* Check if prefix is RLE first before using unrolled loop */ - if (prefixLength && ZSTD_count(ip+1, ip, ip+prefixLength) != prefixLength-1) { +/* Base recursive function. + * Populates a table with intra-block partition indices that can improve compression ratio. + * + * @return: number of splits made (which equals the size of the partition table - 1). + */ +static size_t ZSTD_deriveBlockSplits(ZSTD_CCtx* zc, U32 partitions[], U32 nbSeq) +{ + seqStoreSplits splits; + splits.splitLocations = partitions; + splits.idx = 0; + if (nbSeq <= 4) { + DEBUGLOG(5, "ZSTD_deriveBlockSplits: Too few sequences to split (%u <= 4)", nbSeq); + /* Refuse to try and split anything with less than 4 sequences */ return 0; } - for (i = prefixLength; i != length; i += unrollSize) { - for (u = 0; u < unrollSize; u += sizeof(size_t)) { - if (MEM_readST(ip + i + u) != valueST) { - return 0; - } - } - } - return 1; + ZSTD_deriveBlockSplitsHelper(&splits, 0, nbSeq, zc, &zc->seqStore); + splits.splitLocations[splits.idx] = nbSeq; + DEBUGLOG(5, "ZSTD_deriveBlockSplits: final nb partitions: %zu", splits.idx+1); + return splits.idx; } -/* Returns true if the given block may be RLE. - * This is just a heuristic based on the compressibility. - * It may return both false positives and false negatives. +/* ZSTD_compressBlock_splitBlock(): + * Attempts to split a given block into multiple blocks to improve compression ratio. + * + * Returns combined size of all blocks (which includes headers), or a ZSTD error code. */ -static int ZSTD_maybeRLE(seqStore_t const* seqStore) +static size_t +ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t blockSize, + U32 lastBlock, U32 nbSeq) { - size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart); - size_t const nbLits = (size_t)(seqStore->lit - seqStore->litStart); + size_t cSize = 0; + const BYTE* ip = (const BYTE*)src; + BYTE* op = (BYTE*)dst; + size_t i = 0; + size_t srcBytesTotal = 0; + U32* const partitions = zc->blockSplitCtx.partitions; /* size == ZSTD_MAX_NB_BLOCK_SPLITS */ + SeqStore_t* const nextSeqStore = &zc->blockSplitCtx.nextSeqStore; + SeqStore_t* const currSeqStore = &zc->blockSplitCtx.currSeqStore; + size_t const numSplits = ZSTD_deriveBlockSplits(zc, partitions, nbSeq); + + /* If a block is split and some partitions are emitted as RLE/uncompressed, then repcode history + * may become invalid. In order to reconcile potentially invalid repcodes, we keep track of two + * separate repcode histories that simulate repcode history on compression and decompression side, + * and use the histories to determine whether we must replace a particular repcode with its raw offset. + * + * 1) cRep gets updated for each partition, regardless of whether the block was emitted as uncompressed + * or RLE. This allows us to retrieve the offset value that an invalid repcode references within + * a nocompress/RLE block. + * 2) dRep gets updated only for compressed partitions, and when a repcode gets replaced, will use + * the replacement offset value rather than the original repcode to update the repcode history. + * dRep also will be the final repcode history sent to the next block. + * + * See ZSTD_seqStore_resolveOffCodes() for more details. + */ + Repcodes_t dRep; + Repcodes_t cRep; + ZSTD_memcpy(dRep.rep, zc->blockState.prevCBlock->rep, sizeof(Repcodes_t)); + ZSTD_memcpy(cRep.rep, zc->blockState.prevCBlock->rep, sizeof(Repcodes_t)); + ZSTD_memset(nextSeqStore, 0, sizeof(SeqStore_t)); - return nbSeqs < 4 && nbLits < 10; + DEBUGLOG(5, "ZSTD_compressBlock_splitBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)", + (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, + (unsigned)zc->blockState.matchState.nextToUpdate); + + if (numSplits == 0) { + size_t cSizeSingleBlock = + ZSTD_compressSeqStore_singleBlock(zc, &zc->seqStore, + &dRep, &cRep, + op, dstCapacity, + ip, blockSize, + lastBlock, 0 /* isPartition */); + FORWARD_IF_ERROR(cSizeSingleBlock, "Compressing single block from splitBlock_internal() failed!"); + DEBUGLOG(5, "ZSTD_compressBlock_splitBlock_internal: No splits"); + assert(zc->blockSizeMax <= ZSTD_BLOCKSIZE_MAX); + assert(cSizeSingleBlock <= zc->blockSizeMax + ZSTD_blockHeaderSize); + return cSizeSingleBlock; + } + + ZSTD_deriveSeqStoreChunk(currSeqStore, &zc->seqStore, 0, partitions[0]); + for (i = 0; i <= numSplits; ++i) { + size_t cSizeChunk; + U32 const lastPartition = (i == numSplits); + U32 lastBlockEntireSrc = 0; + + size_t srcBytes = ZSTD_countSeqStoreLiteralsBytes(currSeqStore) + ZSTD_countSeqStoreMatchBytes(currSeqStore); + srcBytesTotal += srcBytes; + if (lastPartition) { + /* This is the final partition, need to account for possible last literals */ + srcBytes += blockSize - srcBytesTotal; + lastBlockEntireSrc = lastBlock; + } else { + ZSTD_deriveSeqStoreChunk(nextSeqStore, &zc->seqStore, partitions[i], partitions[i+1]); + } + + cSizeChunk = ZSTD_compressSeqStore_singleBlock(zc, currSeqStore, + &dRep, &cRep, + op, dstCapacity, + ip, srcBytes, + lastBlockEntireSrc, 1 /* isPartition */); + DEBUGLOG(5, "Estimated size: %zu vs %zu : actual size", + ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(currSeqStore, zc), cSizeChunk); + FORWARD_IF_ERROR(cSizeChunk, "Compressing chunk failed!"); + + ip += srcBytes; + op += cSizeChunk; + dstCapacity -= cSizeChunk; + cSize += cSizeChunk; + *currSeqStore = *nextSeqStore; + assert(cSizeChunk <= zc->blockSizeMax + ZSTD_blockHeaderSize); + } + /* cRep and dRep may have diverged during the compression. + * If so, we use the dRep repcodes for the next block. + */ + ZSTD_memcpy(zc->blockState.prevCBlock->rep, dRep.rep, sizeof(Repcodes_t)); + return cSize; } -static void ZSTD_confirmRepcodesAndEntropyTables(ZSTD_CCtx* zc) +static size_t +ZSTD_compressBlock_splitBlock(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, U32 lastBlock) { - ZSTD_compressedBlockState_t* const tmp = zc->blockState.prevCBlock; - zc->blockState.prevCBlock = zc->blockState.nextCBlock; - zc->blockState.nextCBlock = tmp; + U32 nbSeq; + size_t cSize; + DEBUGLOG(5, "ZSTD_compressBlock_splitBlock"); + assert(zc->appliedParams.postBlockSplitter == ZSTD_ps_enable); + + { const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize); + FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed"); + if (bss == ZSTDbss_noCompress) { + if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) + zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; + RETURN_ERROR_IF(zc->seqCollector.collectSequences, sequenceProducer_failed, "Uncompressible block"); + cSize = ZSTD_noCompressBlock(dst, dstCapacity, src, srcSize, lastBlock); + FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed"); + DEBUGLOG(5, "ZSTD_compressBlock_splitBlock: Nocompress block"); + return cSize; + } + nbSeq = (U32)(zc->seqStore.sequences - zc->seqStore.sequencesStart); + } + + cSize = ZSTD_compressBlock_splitBlock_internal(zc, dst, dstCapacity, src, srcSize, lastBlock, nbSeq); + FORWARD_IF_ERROR(cSize, "Splitting blocks failed!"); + return cSize; } -static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, U32 frame) +static size_t +ZSTD_compressBlock_internal(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, U32 frame) { - /* This the upper bound for the length of an rle block. - * This isn't the actual upper bound. Finding the real threshold - * needs further investigation. + /* This is an estimated upper bound for the length of an rle block. + * This isn't the actual upper bound. + * Finding the real threshold needs further investigation. */ const U32 rleMaxLength = 25; size_t cSize; @@ -17162,30 +24154,28 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, { const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize); FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed"); - if (bss == ZSTDbss_noCompress) { cSize = 0; goto out; } + if (bss == ZSTDbss_noCompress) { + RETURN_ERROR_IF(zc->seqCollector.collectSequences, sequenceProducer_failed, "Uncompressible block"); + cSize = 0; + goto out; + } } if (zc->seqCollector.collectSequences) { - ZSTD_copyBlockSequences(zc); - ZSTD_confirmRepcodesAndEntropyTables(zc); + FORWARD_IF_ERROR(ZSTD_copyBlockSequences(&zc->seqCollector, ZSTD_getSeqStore(zc), zc->blockState.prevCBlock->rep), "copyBlockSequences failed"); + ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); return 0; } /* encode sequences and literals */ - cSize = ZSTD_entropyCompressSequences(&zc->seqStore, + cSize = ZSTD_entropyCompressSeqStore(&zc->seqStore, &zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy, &zc->appliedParams, dst, dstCapacity, srcSize, - zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, + zc->tmpWorkspace, zc->tmpWkspSize /* statically allocated in resetCCtx */, zc->bmi2); - if (zc->seqCollector.collectSequences) { - ZSTD_copyBlockSequences(zc); - return 0; - } - - if (frame && /* We don't want to emit our first block as a RLE even if it qualifies because * doing so will cause the decoder (cli only) to throw a "should consume all input error." @@ -17201,7 +24191,7 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, out: if (!ZSTD_isError(cSize) && cSize > 1) { - ZSTD_confirmRepcodesAndEntropyTables(zc); + ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); } /* We check that dictionaries have offset codes available for the first * block. After the first block, the offcode table might not have large @@ -17248,18 +24238,19 @@ static size_t ZSTD_compressBlock_targetCBlockSize_body(ZSTD_CCtx* zc, * * cSize >= blockBound(srcSize): We have expanded the block too much so * emit an uncompressed block. */ - { - size_t const cSize = ZSTD_compressSuperBlock(zc, dst, dstCapacity, src, srcSize, lastBlock); + { size_t const cSize = + ZSTD_compressSuperBlock(zc, dst, dstCapacity, src, srcSize, lastBlock); if (cSize != ERROR(dstSize_tooSmall)) { - size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, zc->appliedParams.cParams.strategy); + size_t const maxCSize = + srcSize - ZSTD_minGain(srcSize, zc->appliedParams.cParams.strategy); FORWARD_IF_ERROR(cSize, "ZSTD_compressSuperBlock failed"); if (cSize != 0 && cSize < maxCSize + ZSTD_blockHeaderSize) { - ZSTD_confirmRepcodesAndEntropyTables(zc); + ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); return cSize; } } } - } + } /* if (bss == ZSTDbss_compress)*/ DEBUGLOG(6, "Resorting to ZSTD_noCompressBlock()"); /* Superblock compression failed, attempt to emit a single no compress block. @@ -17288,15 +24279,15 @@ static size_t ZSTD_compressBlock_targetCBlockSize(ZSTD_CCtx* zc, return cSize; } -static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, +static void ZSTD_overflowCorrectIfNeeded(ZSTD_MatchState_t* ms, ZSTD_cwksp* ws, ZSTD_CCtx_params const* params, void const* ip, void const* iend) { - if (ZSTD_window_needOverflowCorrection(ms->window, iend)) { - U32 const maxDist = (U32)1 << params->cParams.windowLog; - U32 const cycleLog = ZSTD_cycleLog(params->cParams.chainLog, params->cParams.strategy); + U32 const cycleLog = ZSTD_cycleLog(params->cParams.chainLog, params->cParams.strategy); + U32 const maxDist = (U32)1 << params->cParams.windowLog; + if (ZSTD_window_needOverflowCorrection(ms->window, cycleLog, maxDist, ms->loadedDictEnd, ip, iend)) { U32 const correction = ZSTD_window_correctOverflow(&ms->window, cycleLog, maxDist, ip); ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30); ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30); @@ -17312,43 +24303,87 @@ static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, } } +/**** skipping file: zstd_preSplit.h ****/ + +static size_t ZSTD_optimalBlockSize(ZSTD_CCtx* cctx, const void* src, size_t srcSize, size_t blockSizeMax, int splitLevel, ZSTD_strategy strat, S64 savings) +{ + /* split level based on compression strategy, from `fast` to `btultra2` */ + static const int splitLevels[] = { 0, 0, 1, 2, 2, 3, 3, 4, 4, 4 }; + /* note: conservatively only split full blocks (128 KB) currently. + * While it's possible to go lower, let's keep it simple for a first implementation. + * Besides, benefits of splitting are reduced when blocks are already small. + */ + if (srcSize < 128 KB || blockSizeMax < 128 KB) + return MIN(srcSize, blockSizeMax); + /* do not split incompressible data though: + * require verified savings to allow pre-splitting. + * Note: as a consequence, the first full block is not split. + */ + if (savings < 3) { + DEBUGLOG(6, "don't attempt splitting: savings (%i) too low", (int)savings); + return 128 KB; + } + /* apply @splitLevel, or use default value (which depends on @strat). + * note that splitting heuristic is still conditioned by @savings >= 3, + * so the first block will not reach this code path */ + if (splitLevel == 1) return 128 KB; + if (splitLevel == 0) { + assert(ZSTD_fast <= strat && strat <= ZSTD_btultra2); + splitLevel = splitLevels[strat]; + } else { + assert(2 <= splitLevel && splitLevel <= 6); + splitLevel -= 2; + } + return ZSTD_splitBlock(src, blockSizeMax, splitLevel, cctx->tmpWorkspace, cctx->tmpWkspSize); +} + /*! ZSTD_compress_frameChunk() : * Compress a chunk of data into one or multiple blocks. * All blocks will be terminated, all input will be consumed. * Function will issue an error if there is not enough `dstCapacity` to hold the compressed content. * Frame is supposed already started (header already produced) -* @return : compressed size, or an error code +* @return : compressed size, or an error code */ -static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, +static size_t ZSTD_compress_frameChunk(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 lastFrameChunk) { - size_t blockSize = cctx->blockSize; + size_t blockSizeMax = cctx->blockSizeMax; size_t remaining = srcSize; const BYTE* ip = (const BYTE*)src; BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; U32 const maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog; + S64 savings = (S64)cctx->consumedSrcSize - (S64)cctx->producedCSize; assert(cctx->appliedParams.cParams.windowLog <= ZSTD_WINDOWLOG_MAX); - DEBUGLOG(4, "ZSTD_compress_frameChunk (blockSize=%u)", (unsigned)blockSize); + DEBUGLOG(5, "ZSTD_compress_frameChunk (srcSize=%u, blockSizeMax=%u)", (unsigned)srcSize, (unsigned)blockSizeMax); if (cctx->appliedParams.fParams.checksumFlag && srcSize) XXH64_update(&cctx->xxhState, src, srcSize); while (remaining) { - ZSTD_matchState_t* const ms = &cctx->blockState.matchState; - U32 const lastBlock = lastFrameChunk & (blockSize >= remaining); - - RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE, + ZSTD_MatchState_t* const ms = &cctx->blockState.matchState; + size_t const blockSize = ZSTD_optimalBlockSize(cctx, + ip, remaining, + blockSizeMax, + cctx->appliedParams.preBlockSplitter_level, + cctx->appliedParams.cParams.strategy, + savings); + U32 const lastBlock = lastFrameChunk & (blockSize == remaining); + assert(blockSize <= remaining); + + /* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding + * additional 1. We need to revisit and change this logic to be more consistent */ + RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE + 1, dstSize_tooSmall, "not enough space to store compressed block"); - if (remaining < blockSize) blockSize = remaining; ZSTD_overflowCorrectIfNeeded( ms, &cctx->workspace, &cctx->appliedParams, ip, ip + blockSize); ZSTD_checkDictValidity(&ms->window, ip + blockSize, maxDist, &ms->loadedDictEnd, &ms->dictMatchState); + ZSTD_window_enforceMaxDist(&ms->window, ip, maxDist, &ms->loadedDictEnd, &ms->dictMatchState); /* Ensure hash/chain table insertion resumes no sooner than lowlimit */ if (ms->nextToUpdate < ms->window.lowLimit) ms->nextToUpdate = ms->window.lowLimit; @@ -17359,6 +24394,10 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_targetCBlockSize failed"); assert(cSize > 0); assert(cSize <= blockSize + ZSTD_blockHeaderSize); + } else if (ZSTD_blockSplitterEnabled(&cctx->appliedParams)) { + cSize = ZSTD_compressBlock_splitBlock(cctx, op, dstCapacity, ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_splitBlock failed"); + assert(cSize > 0 || cctx->seqCollector.collectSequences == 1); } else { cSize = ZSTD_compressBlock_internal(cctx, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, @@ -17375,8 +24414,23 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, MEM_writeLE24(op, cBlockHeader); cSize += ZSTD_blockHeaderSize; } - } - + } /* if (ZSTD_useTargetCBlockSize(&cctx->appliedParams))*/ + + /* @savings is employed to ensure that splitting doesn't worsen expansion of incompressible data. + * Without splitting, the maximum expansion is 3 bytes per full block. + * An adversarial input could attempt to fudge the split detector, + * and make it split incompressible data, resulting in more block headers. + * Note that, since ZSTD_COMPRESSBOUND() assumes a worst case scenario of 1KB per block, + * and the splitter never creates blocks that small (current lower limit is 8 KB), + * there is already no risk to expand beyond ZSTD_COMPRESSBOUND() limit. + * But if the goal is to not expand by more than 3-bytes per 128 KB full block, + * then yes, it becomes possible to make the block splitter oversplit incompressible data. + * Using @savings, we enforce an even more conservative condition, + * requiring the presence of enough savings (at least 3 bytes) to authorize splitting, + * otherwise only full blocks are used. + * But being conservative is fine, + * since splitting barely compressible blocks is not fruitful anyway */ + savings += (S64)blockSize - (S64)cSize; ip += blockSize; assert(remaining >= blockSize); @@ -17395,8 +24449,10 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, - const ZSTD_CCtx_params* params, U64 pledgedSrcSize, U32 dictID) -{ BYTE* const op = (BYTE*)dst; + const ZSTD_CCtx_params* params, + U64 pledgedSrcSize, U32 dictID) +{ + BYTE* const op = (BYTE*)dst; U32 const dictIDSizeCodeLength = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */ U32 const dictIDSizeCode = params->fParams.noDictIDFlag ? 0 : dictIDSizeCodeLength; /* 0-3 */ U32 const checksumFlag = params->fParams.checksumFlag>0; @@ -17421,7 +24477,9 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, if (!singleSegment) op[pos++] = windowLogByte; switch(dictIDSizeCode) { - default: assert(0); /* impossible */ + default: + assert(0); /* impossible */ + ZSTD_FALLTHROUGH; case 0 : break; case 1 : op[pos] = (BYTE)(dictID); pos++; break; case 2 : MEM_writeLE16(op+pos, (U16)dictID); pos+=2; break; @@ -17429,7 +24487,9 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, } switch(fcsCode) { - default: assert(0); /* impossible */ + default: + assert(0); /* impossible */ + ZSTD_FALLTHROUGH; case 0 : if (singleSegment) op[pos++] = (BYTE)(pledgedSrcSize); break; case 1 : MEM_writeLE16(op+pos, (U16)(pledgedSrcSize-256)); pos+=2; break; case 2 : MEM_writeLE32(op+pos, (U32)(pledgedSrcSize)); pos+=4; break; @@ -17473,19 +24533,15 @@ size_t ZSTD_writeLastEmptyBlock(void* dst, size_t dstCapacity) } } -size_t ZSTD_referenceExternalSequences(ZSTD_CCtx* cctx, rawSeq* seq, size_t nbSeq) +void ZSTD_referenceExternalSequences(ZSTD_CCtx* cctx, rawSeq* seq, size_t nbSeq) { - RETURN_ERROR_IF(cctx->stage != ZSTDcs_init, stage_wrong, - "wrong cctx stage"); - RETURN_ERROR_IF(cctx->appliedParams.ldmParams.enableLdm, - parameter_unsupported, - "incompatible with ldm"); + assert(cctx->stage == ZSTDcs_init); + assert(nbSeq == 0 || cctx->appliedParams.ldmParams.enableLdm != ZSTD_ps_enable); cctx->externSeqStore.seq = seq; cctx->externSeqStore.size = nbSeq; cctx->externSeqStore.capacity = nbSeq; cctx->externSeqStore.pos = 0; cctx->externSeqStore.posInSequence = 0; - return 0; } @@ -17494,7 +24550,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, const void* src, size_t srcSize, U32 frame, U32 lastFrameChunk) { - ZSTD_matchState_t* const ms = &cctx->blockState.matchState; + ZSTD_MatchState_t* const ms = &cctx->blockState.matchState; size_t fhSize = 0; DEBUGLOG(5, "ZSTD_compressContinue_internal, stage: %u, srcSize: %u", @@ -17514,11 +24570,12 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, if (!srcSize) return fhSize; /* do not generate an empty block if no input */ - if (!ZSTD_window_update(&ms->window, src, srcSize)) { + if (!ZSTD_window_update(&ms->window, src, srcSize, ms->forceNonContiguous)) { + ms->forceNonContiguous = 0; ms->nextToUpdate = ms->window.dictLimit; } - if (cctx->appliedParams.ldmParams.enableLdm) { - ZSTD_window_update(&cctx->ldmState.window, src, srcSize); + if (cctx->appliedParams.ldmParams.enableLdm == ZSTD_ps_enable) { + ZSTD_window_update(&cctx->ldmState.window, src, srcSize, /* forceNonContiguous */ 0); } if (!frame) { @@ -17528,7 +24585,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, src, (BYTE const*)src + srcSize); } - DEBUGLOG(5, "ZSTD_compressContinue_internal (blockSize=%u)", (unsigned)cctx->blockSize); + DEBUGLOG(5, "ZSTD_compressContinue_internal (blockSize=%u)", (unsigned)cctx->blockSizeMax); { size_t const cSize = frame ? ZSTD_compress_frameChunk (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) : ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize, 0 /* frame */); @@ -17549,100 +24606,189 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, } } -size_t ZSTD_compressContinue (ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize) +size_t ZSTD_compressContinue_public(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) { DEBUGLOG(5, "ZSTD_compressContinue (srcSize=%u)", (unsigned)srcSize); return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1 /* frame mode */, 0 /* last chunk */); } +/* NOTE: Must just wrap ZSTD_compressContinue_public() */ +size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + return ZSTD_compressContinue_public(cctx, dst, dstCapacity, src, srcSize); +} -size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx) +static size_t ZSTD_getBlockSize_deprecated(const ZSTD_CCtx* cctx) { ZSTD_compressionParameters const cParams = cctx->appliedParams.cParams; assert(!ZSTD_checkCParams(cParams)); - return MIN (ZSTD_BLOCKSIZE_MAX, (U32)1 << cParams.windowLog); + return MIN(cctx->appliedParams.maxBlockSize, (size_t)1 << cParams.windowLog); } -size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +/* NOTE: Must just wrap ZSTD_getBlockSize_deprecated() */ +size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx) +{ + return ZSTD_getBlockSize_deprecated(cctx); +} + +/* NOTE: Must just wrap ZSTD_compressBlock_deprecated() */ +size_t ZSTD_compressBlock_deprecated(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { DEBUGLOG(5, "ZSTD_compressBlock: srcSize = %u", (unsigned)srcSize); - { size_t const blockSizeMax = ZSTD_getBlockSize(cctx); + { size_t const blockSizeMax = ZSTD_getBlockSize_deprecated(cctx); RETURN_ERROR_IF(srcSize > blockSizeMax, srcSize_wrong, "input is larger than a block"); } return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0 /* frame mode */, 0 /* last chunk */); } +/* NOTE: Must just wrap ZSTD_compressBlock_deprecated() */ +size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_deprecated(cctx, dst, dstCapacity, src, srcSize); +} + /*! ZSTD_loadDictionaryContent() : * @return : 0, or an error code */ -static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, - ldmState_t* ls, - ZSTD_cwksp* ws, - ZSTD_CCtx_params const* params, - const void* src, size_t srcSize, - ZSTD_dictTableLoadMethod_e dtlm) +static size_t +ZSTD_loadDictionaryContent(ZSTD_MatchState_t* ms, + ldmState_t* ls, + ZSTD_cwksp* ws, + ZSTD_CCtx_params const* params, + const void* src, size_t srcSize, + ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp) { const BYTE* ip = (const BYTE*) src; const BYTE* const iend = ip + srcSize; + int const loadLdmDict = params->ldmParams.enableLdm == ZSTD_ps_enable && ls != NULL; - ZSTD_window_update(&ms->window, src, srcSize); - ms->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ms->window.base); + /* Assert that the ms params match the params we're being given */ + ZSTD_assertEqualCParams(params->cParams, ms->cParams); - if (params->ldmParams.enableLdm && ls != NULL) { - ZSTD_window_update(&ls->window, src, srcSize); - ls->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ls->window.base); + { /* Ensure large dictionaries can't cause index overflow */ + + /* Allow the dictionary to set indices up to exactly ZSTD_CURRENT_MAX. + * Dictionaries right at the edge will immediately trigger overflow + * correction, but I don't want to insert extra constraints here. + */ + U32 maxDictSize = ZSTD_CURRENT_MAX - ZSTD_WINDOW_START_INDEX; + + int const CDictTaggedIndices = ZSTD_CDictIndicesAreTagged(¶ms->cParams); + if (CDictTaggedIndices && tfp == ZSTD_tfp_forCDict) { + /* Some dictionary matchfinders in zstd use "short cache", + * which treats the lower ZSTD_SHORT_CACHE_TAG_BITS of each + * CDict hashtable entry as a tag rather than as part of an index. + * When short cache is used, we need to truncate the dictionary + * so that its indices don't overlap with the tag. */ + U32 const shortCacheMaxDictSize = (1u << (32 - ZSTD_SHORT_CACHE_TAG_BITS)) - ZSTD_WINDOW_START_INDEX; + maxDictSize = MIN(maxDictSize, shortCacheMaxDictSize); + assert(!loadLdmDict); + } + + /* If the dictionary is too large, only load the suffix of the dictionary. */ + if (srcSize > maxDictSize) { + ip = iend - maxDictSize; + src = ip; + srcSize = maxDictSize; + } } - /* Assert that we the ms params match the params we're being given */ - ZSTD_assertEqualCParams(params->cParams, ms->cParams); + if (srcSize > ZSTD_CHUNKSIZE_MAX) { + /* We must have cleared our windows when our source is this large. */ + assert(ZSTD_window_isEmpty(ms->window)); + if (loadLdmDict) assert(ZSTD_window_isEmpty(ls->window)); + } + ZSTD_window_update(&ms->window, src, srcSize, /* forceNonContiguous */ 0); - if (srcSize <= HASH_READ_SIZE) return 0; + DEBUGLOG(4, "ZSTD_loadDictionaryContent: useRowMatchFinder=%d", (int)params->useRowMatchFinder); - while (iend - ip > HASH_READ_SIZE) { - size_t const remaining = (size_t)(iend - ip); - size_t const chunk = MIN(remaining, ZSTD_CHUNKSIZE_MAX); - const BYTE* const ichunk = ip + chunk; + if (loadLdmDict) { /* Load the entire dict into LDM matchfinders. */ + DEBUGLOG(4, "ZSTD_loadDictionaryContent: Trigger loadLdmDict"); + ZSTD_window_update(&ls->window, src, srcSize, /* forceNonContiguous */ 0); + ls->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ls->window.base); + ZSTD_ldm_fillHashTable(ls, ip, iend, ¶ms->ldmParams); + DEBUGLOG(4, "ZSTD_loadDictionaryContent: ZSTD_ldm_fillHashTable completes"); + } - ZSTD_overflowCorrectIfNeeded(ms, ws, params, ip, ichunk); + /* If the dict is larger than we can reasonably index in our tables, only load the suffix. */ + { U32 maxDictSize = 1U << MIN(MAX(params->cParams.hashLog + 3, params->cParams.chainLog + 1), 31); + if (srcSize > maxDictSize) { + ip = iend - maxDictSize; + src = ip; + srcSize = maxDictSize; + } + } - if (params->ldmParams.enableLdm && ls != NULL) - ZSTD_ldm_fillHashTable(ls, (const BYTE*)src, (const BYTE*)src + srcSize, ¶ms->ldmParams); + ms->nextToUpdate = (U32)(ip - ms->window.base); + ms->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ms->window.base); + ms->forceNonContiguous = params->deterministicRefPrefix; - switch(params->cParams.strategy) - { - case ZSTD_fast: - ZSTD_fillHashTable(ms, ichunk, dtlm); - break; - case ZSTD_dfast: - ZSTD_fillDoubleHashTable(ms, ichunk, dtlm); - break; + if (srcSize <= HASH_READ_SIZE) return 0; - case ZSTD_greedy: - case ZSTD_lazy: - case ZSTD_lazy2: - if (chunk >= HASH_READ_SIZE && ms->dedicatedDictSearch) { - assert(chunk == remaining); /* must load everything in one go */ - ZSTD_dedicatedDictSearch_lazy_loadDictionary(ms, ichunk-HASH_READ_SIZE); - } else if (chunk >= HASH_READ_SIZE) { - ZSTD_insertAndFindFirstIndex(ms, ichunk-HASH_READ_SIZE); - } - break; + ZSTD_overflowCorrectIfNeeded(ms, ws, params, ip, iend); - case ZSTD_btlazy2: /* we want the dictionary table fully sorted */ - case ZSTD_btopt: - case ZSTD_btultra: - case ZSTD_btultra2: - if (chunk >= HASH_READ_SIZE) - ZSTD_updateTree(ms, ichunk-HASH_READ_SIZE, ichunk); - break; + switch(params->cParams.strategy) + { + case ZSTD_fast: + ZSTD_fillHashTable(ms, iend, dtlm, tfp); + break; + case ZSTD_dfast: +#ifndef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR + ZSTD_fillDoubleHashTable(ms, iend, dtlm, tfp); +#else + assert(0); /* shouldn't be called: cparams should've been adjusted. */ +#endif + break; - default: - assert(0); /* not possible : not a valid strategy id */ + case ZSTD_greedy: + case ZSTD_lazy: + case ZSTD_lazy2: +#if !defined(ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR) + assert(srcSize >= HASH_READ_SIZE); + if (ms->dedicatedDictSearch) { + assert(ms->chainTable != NULL); + ZSTD_dedicatedDictSearch_lazy_loadDictionary(ms, iend-HASH_READ_SIZE); + } else { + assert(params->useRowMatchFinder != ZSTD_ps_auto); + if (params->useRowMatchFinder == ZSTD_ps_enable) { + size_t const tagTableSize = ((size_t)1 << params->cParams.hashLog); + ZSTD_memset(ms->tagTable, 0, tagTableSize); + ZSTD_row_update(ms, iend-HASH_READ_SIZE); + DEBUGLOG(4, "Using row-based hash table for lazy dict"); + } else { + ZSTD_insertAndFindFirstIndex(ms, iend-HASH_READ_SIZE); + DEBUGLOG(4, "Using chain-based hash table for lazy dict"); + } } +#else + assert(0); /* shouldn't be called: cparams should've been adjusted. */ +#endif + break; + + case ZSTD_btlazy2: /* we want the dictionary table fully sorted */ + case ZSTD_btopt: + case ZSTD_btultra: + case ZSTD_btultra2: +#if !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR) + assert(srcSize >= HASH_READ_SIZE); + DEBUGLOG(4, "Fill %u bytes into the Binary Tree", (unsigned)srcSize); + ZSTD_updateTree(ms, iend-HASH_READ_SIZE, iend); +#else + assert(0); /* shouldn't be called: cparams should've been adjusted. */ +#endif + break; - ip = ichunk; + default: + assert(0); /* not possible : not a valid strategy id */ } ms->nextToUpdate = (U32)(iend - ms->window.base); @@ -17681,20 +24827,19 @@ size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, { unsigned maxSymbolValue = 255; unsigned hasZeroWeights = 1; size_t const hufHeaderSize = HUF_readCTable((HUF_CElt*)bs->entropy.huf.CTable, &maxSymbolValue, dictPtr, - dictEnd-dictPtr, &hasZeroWeights); + (size_t)(dictEnd-dictPtr), &hasZeroWeights); /* We only set the loaded table as valid if it contains all non-zero * weights. Otherwise, we set it to check */ - if (!hasZeroWeights) + if (!hasZeroWeights && maxSymbolValue == 255) bs->entropy.huf.repeatMode = HUF_repeat_valid; RETURN_ERROR_IF(HUF_isError(hufHeaderSize), dictionary_corrupted, ""); - RETURN_ERROR_IF(maxSymbolValue < 255, dictionary_corrupted, ""); dictPtr += hufHeaderSize; } { unsigned offcodeLog; - size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); + size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, (size_t)(dictEnd-dictPtr)); RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted, ""); RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted, ""); /* fill all offset symbols to avoid garbage at end of table */ @@ -17709,7 +24854,7 @@ size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, { short matchlengthNCount[MaxML+1]; unsigned matchlengthMaxValue = MaxML, matchlengthLog; - size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); + size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, (size_t)(dictEnd-dictPtr)); RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted, ""); RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted, ""); RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp( @@ -17723,7 +24868,7 @@ size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, { short litlengthNCount[MaxLL+1]; unsigned litlengthMaxValue = MaxLL, litlengthLog; - size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); + size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, (size_t)(dictEnd-dictPtr)); RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted, ""); RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted, ""); RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp( @@ -17757,7 +24902,7 @@ size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, RETURN_ERROR_IF(bs->rep[u] > dictContentSize, dictionary_corrupted, ""); } } } - return dictPtr - (const BYTE*)dict; + return (size_t)(dictPtr - (const BYTE*)dict); } /* Dictionary format : @@ -17770,18 +24915,18 @@ size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, * dictSize supposed >= 8 */ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, - ZSTD_matchState_t* ms, + ZSTD_MatchState_t* ms, ZSTD_cwksp* ws, ZSTD_CCtx_params const* params, const void* dict, size_t dictSize, ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp, void* workspace) { const BYTE* dictPtr = (const BYTE*)dict; const BYTE* const dictEnd = dictPtr + dictSize; size_t dictID; size_t eSize; - ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<= 8); assert(MEM_readLE32(dictPtr) == ZSTD_MAGIC_DICTIONARY); @@ -17794,7 +24939,7 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, { size_t const dictContentSize = (size_t)(dictEnd - dictPtr); FORWARD_IF_ERROR(ZSTD_loadDictionaryContent( - ms, NULL, ws, params, dictPtr, dictContentSize, dtlm), ""); + ms, NULL, ws, params, dictPtr, dictContentSize, dtlm, tfp), ""); } return dictID; } @@ -17803,13 +24948,14 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, * @return : dictID, or an error code */ static size_t ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, - ZSTD_matchState_t* ms, + ZSTD_MatchState_t* ms, ldmState_t* ls, ZSTD_cwksp* ws, const ZSTD_CCtx_params* params, const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType, ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp, void* workspace) { DEBUGLOG(4, "ZSTD_compress_insertDictionary (dictSize=%u)", (U32)dictSize); @@ -17822,13 +24968,13 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, /* dict restricted modes */ if (dictContentType == ZSTD_dct_rawContent) - return ZSTD_loadDictionaryContent(ms, ls, ws, params, dict, dictSize, dtlm); + return ZSTD_loadDictionaryContent(ms, ls, ws, params, dict, dictSize, dtlm, tfp); if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) { if (dictContentType == ZSTD_dct_auto) { DEBUGLOG(4, "raw content dictionary detected"); return ZSTD_loadDictionaryContent( - ms, ls, ws, params, dict, dictSize, dtlm); + ms, ls, ws, params, dict, dictSize, dtlm, tfp); } RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong, ""); assert(0); /* impossible */ @@ -17836,13 +24982,14 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, /* dict as full zstd dictionary */ return ZSTD_loadZstdDictionary( - bs, ms, ws, params, dict, dictSize, dtlm, workspace); + bs, ms, ws, params, dict, dictSize, dtlm, tfp, workspace); } #define ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF (128 KB) #define ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER (6ULL) /*! ZSTD_compressBegin_internal() : + * Assumption : either @dict OR @cdict (or none) is non-NULL, never both * @return : 0, or an error code */ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, @@ -17852,8 +24999,9 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params, U64 pledgedSrcSize, ZSTD_buffered_policy_e zbuff) { + size_t const dictContentSize = cdict ? cdict->dictContentSize : dictSize; #if ZSTD_TRACE - cctx->traceCtx = ZSTD_trace_compress_begin(cctx); + cctx->traceCtx = (ZSTD_trace_compress_begin != NULL) ? ZSTD_trace_compress_begin(cctx) : 0; #endif DEBUGLOG(4, "ZSTD_compressBegin_internal: wlog=%u", params->cParams.windowLog); /* params are supposed to be fully validated at this point */ @@ -17869,22 +25017,23 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff); } - FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, *params, pledgedSrcSize, + FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, + dictContentSize, ZSTDcrp_makeClean, zbuff) , ""); { size_t const dictID = cdict ? ZSTD_compress_insertDictionary( cctx->blockState.prevCBlock, &cctx->blockState.matchState, &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, cdict->dictContent, cdict->dictContentSize, cdict->dictContentType, dtlm, - cctx->entropyWorkspace) + ZSTD_tfp_forCCtx, cctx->tmpWorkspace) : ZSTD_compress_insertDictionary( cctx->blockState.prevCBlock, &cctx->blockState.matchState, &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, dict, dictSize, - dictContentType, dtlm, cctx->entropyWorkspace); + dictContentType, dtlm, ZSTD_tfp_forCCtx, cctx->tmpWorkspace); FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed"); assert(dictID <= UINT_MAX); cctx->dictID = (U32)dictID; - cctx->dictContentSize = cdict ? cdict->dictContentSize : dictSize; + cctx->dictContentSize = dictContentSize; } return 0; } @@ -17921,11 +25070,11 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, &cctxParams, pledgedSrcSize); } -size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) +static size_t +ZSTD_compressBegin_usingDict_deprecated(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) { ZSTD_CCtx_params cctxParams; - { - ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_noAttachDict); + { ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_noAttachDict); ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel); } DEBUGLOG(4, "ZSTD_compressBegin_usingDict (dictSize=%u)", (unsigned)dictSize); @@ -17933,9 +25082,15 @@ size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t di &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered); } +size_t +ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) +{ + return ZSTD_compressBegin_usingDict_deprecated(cctx, dict, dictSize, compressionLevel); +} + size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel) { - return ZSTD_compressBegin_usingDict(cctx, NULL, 0, compressionLevel); + return ZSTD_compressBegin_usingDict_deprecated(cctx, NULL, 0, compressionLevel); } @@ -17946,14 +25101,13 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) { BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; - size_t fhSize = 0; DEBUGLOG(4, "ZSTD_writeEpilogue"); RETURN_ERROR_IF(cctx->stage == ZSTDcs_created, stage_wrong, "init missing"); /* special case : empty frame */ if (cctx->stage == ZSTDcs_init) { - fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, &cctx->appliedParams, 0, 0); + size_t fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, &cctx->appliedParams, 0, 0); FORWARD_IF_ERROR(fhSize, "ZSTD_writeFrameHeader failed"); dstCapacity -= fhSize; op += fhSize; @@ -17963,8 +25117,9 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) if (cctx->stage != ZSTDcs_ending) { /* write one last empty block, make it the "last" block */ U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1) + 0; - RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "no room for epilogue"); - MEM_writeLE32(op, cBlockHeader24); + ZSTD_STATIC_ASSERT(ZSTD_BLOCKHEADERSIZE == 3); + RETURN_ERROR_IF(dstCapacity<3, dstSize_tooSmall, "no room for epilogue"); + MEM_writeLE24(op, cBlockHeader24); op += ZSTD_blockHeaderSize; dstCapacity -= ZSTD_blockHeaderSize; } @@ -17978,13 +25133,13 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) } cctx->stage = ZSTDcs_created; /* return to "created but no init" status */ - return op-ostart; + return (size_t)(op-ostart); } void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize) { #if ZSTD_TRACE - if (cctx->traceCtx) { + if (cctx->traceCtx && ZSTD_trace_compress_end != NULL) { int const streaming = cctx->inBuffSize > 0 || cctx->outBuffSize > 0 || cctx->appliedParams.nbWorkers > 0; ZSTD_Trace trace; ZSTD_memset(&trace, 0, sizeof(trace)); @@ -18005,9 +25160,9 @@ void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize) #endif } -size_t ZSTD_compressEnd (ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize) +size_t ZSTD_compressEnd_public(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) { size_t endResult; size_t const cSize = ZSTD_compressContinue_internal(cctx, @@ -18031,21 +25186,28 @@ size_t ZSTD_compressEnd (ZSTD_CCtx* cctx, return cSize + endResult; } +/* NOTE: Must just wrap ZSTD_compressEnd_public() */ +size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + return ZSTD_compressEnd_public(cctx, dst, dstCapacity, src, srcSize); +} + size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict,size_t dictSize, ZSTD_parameters params) { - ZSTD_CCtx_params cctxParams; DEBUGLOG(4, "ZSTD_compress_advanced"); FORWARD_IF_ERROR(ZSTD_checkCParams(params.cParams), ""); - ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, ZSTD_NO_CLEVEL); + ZSTD_CCtxParams_init_internal(&cctx->simpleApiParams, ¶ms, ZSTD_NO_CLEVEL); return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, - &cctxParams); + &cctx->simpleApiParams); } /* Internal */ @@ -18060,7 +25222,7 @@ size_t ZSTD_compress_advanced_internal( FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL, params, srcSize, ZSTDb_not_buffered) , ""); - return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); + return ZSTD_compressEnd_public(cctx, dst, dstCapacity, src, srcSize); } size_t ZSTD_compress_usingDict(ZSTD_CCtx* cctx, @@ -18069,14 +25231,13 @@ size_t ZSTD_compress_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) { - ZSTD_CCtx_params cctxParams; { ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, srcSize, dict ? dictSize : 0, ZSTD_cpm_noAttachDict); assert(params.fParams.contentSizeFlag == 1); - ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT: compressionLevel); + ZSTD_CCtxParams_init_internal(&cctx->simpleApiParams, ¶ms, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT: compressionLevel); } DEBUGLOG(4, "ZSTD_compress_usingDict (srcSize=%u)", (unsigned)srcSize); - return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, &cctxParams); + return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, &cctx->simpleApiParams); } size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, @@ -18120,7 +25281,10 @@ size_t ZSTD_estimateCDictSize_advanced( DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (unsigned)sizeof(ZSTD_CDict)); return ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) - + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) + /* enableDedicatedDictSearch == 1 ensures that CDict estimation will not be too small + * in case we are using DDS with row-hash. */ + + ZSTD_sizeof_matchState(&cParams, ZSTD_resolveRowMatchFinderMode(ZSTD_ps_auto, &cParams), + /* enableDedicatedDictSearch */ 1, /* forCCtx */ 0) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void *)))); } @@ -18151,9 +25315,6 @@ static size_t ZSTD_initCDict_internal( assert(!ZSTD_checkCParams(params.cParams)); cdict->matchState.cParams = params.cParams; cdict->matchState.dedicatedDictSearch = params.enableDedicatedDictSearch; - if (cdict->matchState.dedicatedDictSearch && dictSize > ZSTD_CHUNKSIZE_MAX) { - cdict->matchState.dedicatedDictSearch = 0; - } if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dictBuffer) || (!dictSize)) { cdict->dictContent = dictBuffer; } else { @@ -18174,6 +25335,7 @@ static size_t ZSTD_initCDict_internal( &cdict->matchState, &cdict->workspace, ¶ms.cParams, + params.useRowMatchFinder, ZSTDcrp_makeClean, ZSTDirp_reset, ZSTD_resetTarget_CDict), ""); @@ -18185,7 +25347,7 @@ static size_t ZSTD_initCDict_internal( { size_t const dictID = ZSTD_compress_insertDictionary( &cdict->cBlockState, &cdict->matchState, NULL, &cdict->workspace, ¶ms, cdict->dictContent, cdict->dictContentSize, - dictContentType, ZSTD_dtlm_full, cdict->entropyWorkspace); + dictContentType, ZSTD_dtlm_full, ZSTD_tfp_forCDict, cdict->entropyWorkspace); FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed"); assert(dictID <= (size_t)(U32)-1); cdict->dictID = (U32)dictID; @@ -18195,16 +25357,21 @@ static size_t ZSTD_initCDict_internal( return 0; } -static ZSTD_CDict* ZSTD_createCDict_advanced_internal(size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, - ZSTD_compressionParameters cParams, ZSTD_customMem customMem) +static ZSTD_CDict* +ZSTD_createCDict_advanced_internal(size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_compressionParameters cParams, + ZSTD_ParamSwitch_e useRowMatchFinder, + int enableDedicatedDictSearch, + ZSTD_customMem customMem) { if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + DEBUGLOG(3, "ZSTD_createCDict_advanced_internal (dictSize=%u)", (unsigned)dictSize); { size_t const workspaceSize = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) + - ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) + + ZSTD_sizeof_matchState(&cParams, useRowMatchFinder, enableDedicatedDictSearch, /* forCCtx */ 0) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*)))); void* const workspace = ZSTD_customMalloc(workspaceSize, customMem); @@ -18223,7 +25390,7 @@ static ZSTD_CDict* ZSTD_createCDict_advanced_internal(size_t dictSize, ZSTD_cwksp_move(&cdict->workspace, &ws); cdict->customMem = customMem; cdict->compressionLevel = ZSTD_NO_CLEVEL; /* signals advanced API usage */ - + cdict->useRowMatchFinder = useRowMatchFinder; return cdict; } } @@ -18236,6 +25403,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, { ZSTD_CCtx_params cctxParams; ZSTD_memset(&cctxParams, 0, sizeof(cctxParams)); + DEBUGLOG(3, "ZSTD_createCDict_advanced, dictSize=%u, mode=%u", (unsigned)dictSize, (unsigned)dictContentType); ZSTD_CCtxParams_init(&cctxParams, 0); cctxParams.cParams = cParams; cctxParams.customMem = customMem; @@ -18245,7 +25413,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, &cctxParams, customMem); } -ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced2( +ZSTD_CDict* ZSTD_createCDict_advanced2( const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType, @@ -18256,7 +25424,7 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced2( ZSTD_compressionParameters cParams; ZSTD_CDict* cdict; - DEBUGLOG(3, "ZSTD_createCDict_advanced2, mode %u", (unsigned)dictContentType); + DEBUGLOG(3, "ZSTD_createCDict_advanced2, dictSize=%u, mode=%u", (unsigned)dictSize, (unsigned)dictContentType); if (!customMem.customAlloc ^ !customMem.customFree) return NULL; if (cctxParams.enableDedicatedDictSearch) { @@ -18275,13 +25443,16 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced2( &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); } + DEBUGLOG(3, "ZSTD_createCDict_advanced2: DedicatedDictSearch=%u", cctxParams.enableDedicatedDictSearch); cctxParams.cParams = cParams; + cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams); cdict = ZSTD_createCDict_advanced_internal(dictSize, dictLoadMethod, cctxParams.cParams, + cctxParams.useRowMatchFinder, cctxParams.enableDedicatedDictSearch, customMem); - if (ZSTD_isError( ZSTD_initCDict_internal(cdict, + if (!cdict || ZSTD_isError( ZSTD_initCDict_internal(cdict, dict, dictSize, dictLoadMethod, dictContentType, cctxParams) )) { @@ -18335,7 +25506,7 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict) * workspaceSize: Use ZSTD_estimateCDictSize() * to determine how large workspace must be. * cParams : use ZSTD_getCParams() to transform a compression level - * into its relevants cParams. + * into its relevant cParams. * @return : pointer to ZSTD_CDict*, or NULL if error (size too small) * Note : there is no corresponding "free" function. * Since workspace was allocated externally, it must be freed externally. @@ -18347,7 +25518,9 @@ const ZSTD_CDict* ZSTD_initStaticCDict( ZSTD_dictContentType_e dictContentType, ZSTD_compressionParameters cParams) { - size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0); + ZSTD_ParamSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(ZSTD_ps_auto, &cParams); + /* enableDedicatedDictSearch == 1 ensures matchstate is not too small in case this CDict will be used for DDS + row hash */ + size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, useRowMatchFinder, /* enableDedicatedDictSearch */ 1, /* forCCtx */ 0); size_t const neededSize = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*)))) @@ -18356,6 +25529,7 @@ const ZSTD_CDict* ZSTD_initStaticCDict( ZSTD_CDict* cdict; ZSTD_CCtx_params params; + DEBUGLOG(4, "ZSTD_initStaticCDict (dictSize==%u)", (unsigned)dictSize); if ((size_t)workspace & 7) return NULL; /* 8-aligned */ { @@ -18366,12 +25540,13 @@ const ZSTD_CDict* ZSTD_initStaticCDict( ZSTD_cwksp_move(&cdict->workspace, &ws); } - DEBUGLOG(4, "(workspaceSize < neededSize) : (%u < %u) => %u", - (unsigned)workspaceSize, (unsigned)neededSize, (unsigned)(workspaceSize < neededSize)); if (workspaceSize < neededSize) return NULL; ZSTD_CCtxParams_init(¶ms, 0); params.cParams = cParams; + params.useRowMatchFinder = useRowMatchFinder; + cdict->useRowMatchFinder = useRowMatchFinder; + cdict->compressionLevel = ZSTD_NO_CLEVEL; if (ZSTD_isError( ZSTD_initCDict_internal(cdict, dict, dictSize, @@ -18398,15 +25573,15 @@ unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict) return cdict->dictID; } - -/* ZSTD_compressBegin_usingCDict_advanced() : - * cdict must be != NULL */ -size_t ZSTD_compressBegin_usingCDict_advanced( +/* ZSTD_compressBegin_usingCDict_internal() : + * Implementation of various ZSTD_compressBegin_usingCDict* functions. + */ +static size_t ZSTD_compressBegin_usingCDict_internal( ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize) { ZSTD_CCtx_params cctxParams; - DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced"); + DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_internal"); RETURN_ERROR_IF(cdict==NULL, dictionary_wrong, "NULL pointer!"); /* Initialize the cctxParams from the cdict */ { @@ -18438,23 +25613,51 @@ size_t ZSTD_compressBegin_usingCDict_advanced( ZSTDb_not_buffered); } + +/* ZSTD_compressBegin_usingCDict_advanced() : + * This function is DEPRECATED. + * cdict must be != NULL */ +size_t ZSTD_compressBegin_usingCDict_advanced( + ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, + ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize) +{ + return ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, pledgedSrcSize); +} + /* ZSTD_compressBegin_usingCDict() : - * pledgedSrcSize=0 means "unknown" - * if pledgedSrcSize>0, it will enable contentSizeFlag */ -size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) + * cdict must be != NULL */ +size_t ZSTD_compressBegin_usingCDict_deprecated(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) { ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; - DEBUGLOG(4, "ZSTD_compressBegin_usingCDict : dictIDFlag == %u", !fParams.noDictIDFlag); - return ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN); + return ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN); } +size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) +{ + return ZSTD_compressBegin_usingCDict_deprecated(cctx, cdict); +} + +/*! ZSTD_compress_usingCDict_internal(): + * Implementation of various ZSTD_compress_usingCDict* functions. + */ +static size_t ZSTD_compress_usingCDict_internal(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict, ZSTD_frameParameters fParams) +{ + FORWARD_IF_ERROR(ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, srcSize), ""); /* will check if cdict != NULL */ + return ZSTD_compressEnd_public(cctx, dst, dstCapacity, src, srcSize); +} + +/*! ZSTD_compress_usingCDict_advanced(): + * This function is DEPRECATED. + */ size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams) { - FORWARD_IF_ERROR(ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, srcSize), ""); /* will check if cdict != NULL */ - return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); + return ZSTD_compress_usingCDict_internal(cctx, dst, dstCapacity, src, srcSize, cdict, fParams); } /*! ZSTD_compress_usingCDict() : @@ -18468,7 +25671,7 @@ size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) { ZSTD_frameParameters const fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; - return ZSTD_compress_usingCDict_advanced(cctx, dst, dstCapacity, src, srcSize, cdict, fParams); + return ZSTD_compress_usingCDict_internal(cctx, dst, dstCapacity, src, srcSize, cdict, fParams); } @@ -18509,7 +25712,7 @@ size_t ZSTD_CStreamOutSize(void) return ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; } -static ZSTD_cParamMode_e ZSTD_getCParamMode(ZSTD_CDict const* cdict, ZSTD_CCtx_params const* params, U64 pledgedSrcSize) +static ZSTD_CParamMode_e ZSTD_getCParamMode(ZSTD_CDict const* cdict, ZSTD_CCtx_params const* params, U64 pledgedSrcSize) { if (cdict != NULL && ZSTD_shouldAttachDict(cdict, params, pledgedSrcSize)) return ZSTD_cpm_attachDict; @@ -18640,30 +25843,41 @@ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) static size_t ZSTD_nextInputSizeHint(const ZSTD_CCtx* cctx) { - size_t hintInSize = cctx->inBuffTarget - cctx->inBuffPos; - if (hintInSize==0) hintInSize = cctx->blockSize; - return hintInSize; + if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) { + return cctx->blockSizeMax - cctx->stableIn_notConsumed; + } + assert(cctx->appliedParams.inBufferMode == ZSTD_bm_buffered); + { size_t hintInSize = cctx->inBuffTarget - cctx->inBuffPos; + if (hintInSize==0) hintInSize = cctx->blockSizeMax; + return hintInSize; + } } /** ZSTD_compressStream_generic(): * internal function for all *compressStream*() variants - * non-static, because can be called from zstdmt_compress.c - * @return : hint size for next input */ + * @return : hint size for next input to complete ongoing block */ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input, ZSTD_EndDirective const flushMode) { - const char* const istart = (const char*)input->src; - const char* const iend = input->size != 0 ? istart + input->size : istart; - const char* ip = input->pos != 0 ? istart + input->pos : istart; - char* const ostart = (char*)output->dst; - char* const oend = output->size != 0 ? ostart + output->size : ostart; - char* op = output->pos != 0 ? ostart + output->pos : ostart; + const char* const istart = (assert(input != NULL), (const char*)input->src); + const char* const iend = (istart != NULL) ? istart + input->size : istart; + const char* ip = (istart != NULL) ? istart + input->pos : istart; + char* const ostart = (assert(output != NULL), (char*)output->dst); + char* const oend = (ostart != NULL) ? ostart + output->size : ostart; + char* op = (ostart != NULL) ? ostart + output->pos : ostart; U32 someMoreWork = 1; /* check expectations */ - DEBUGLOG(5, "ZSTD_compressStream_generic, flush=%u", (unsigned)flushMode); + DEBUGLOG(5, "ZSTD_compressStream_generic, flush=%i, srcSize = %zu", (int)flushMode, input->size - input->pos); + assert(zcs != NULL); + if (zcs->appliedParams.inBufferMode == ZSTD_bm_stable) { + assert(input->pos >= zcs->stableIn_notConsumed); + input->pos -= zcs->stableIn_notConsumed; + if (ip) ip -= zcs->stableIn_notConsumed; + zcs->stableIn_notConsumed = 0; + } if (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered) { assert(zcs->inBuff != NULL); assert(zcs->inBuffSize > 0); @@ -18672,8 +25886,10 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, assert(zcs->outBuff != NULL); assert(zcs->outBuffSize > 0); } - assert(output->pos <= output->size); + if (input->src == NULL) assert(input->size == 0); assert(input->pos <= input->size); + if (output->dst == NULL) assert(output->size == 0); + assert(output->pos <= output->size); assert((U32)flushMode <= (U32)ZSTD_e_end); while (someMoreWork) { @@ -18684,12 +25900,13 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, case zcss_load: if ( (flushMode == ZSTD_e_end) - && ( (size_t)(oend-op) >= ZSTD_compressBound(iend-ip) /* Enough output space */ + && ( (size_t)(oend-op) >= ZSTD_compressBound((size_t)(iend-ip)) /* Enough output space */ || zcs->appliedParams.outBufferMode == ZSTD_bm_stable) /* OR we are allowed to return dstSizeTooSmall */ && (zcs->inBuffPos == 0) ) { /* shortcut to compression pass directly into output buffer */ - size_t const cSize = ZSTD_compressEnd(zcs, - op, oend-op, ip, iend-ip); + size_t const cSize = ZSTD_compressEnd_public(zcs, + op, (size_t)(oend-op), + ip, (size_t)(iend-ip)); DEBUGLOG(4, "ZSTD_compressEnd : cSize=%u", (unsigned)cSize); FORWARD_IF_ERROR(cSize, "ZSTD_compressEnd failed"); ip = iend; @@ -18703,10 +25920,9 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos; size_t const loaded = ZSTD_limitCopy( zcs->inBuff + zcs->inBuffPos, toLoad, - ip, iend-ip); + ip, (size_t)(iend-ip)); zcs->inBuffPos += loaded; - if (loaded != 0) - ip += loaded; + if (ip) ip += loaded; if ( (flushMode == ZSTD_e_continue) && (zcs->inBuffPos < zcs->inBuffTarget) ) { /* not enough input to fill full block : stop here */ @@ -18717,16 +25933,29 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, /* empty */ someMoreWork = 0; break; } + } else { + assert(zcs->appliedParams.inBufferMode == ZSTD_bm_stable); + if ( (flushMode == ZSTD_e_continue) + && ( (size_t)(iend - ip) < zcs->blockSizeMax) ) { + /* can't compress a full block : stop here */ + zcs->stableIn_notConsumed = (size_t)(iend - ip); + ip = iend; /* pretend to have consumed input */ + someMoreWork = 0; break; + } + if ( (flushMode == ZSTD_e_flush) + && (ip == iend) ) { + /* empty */ + someMoreWork = 0; break; + } } /* compress current block (note : this stage cannot be stopped in the middle) */ DEBUGLOG(5, "stream compression stage (flushMode==%u)", flushMode); { int const inputBuffered = (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered); void* cDst; size_t cSize; - size_t oSize = oend-op; - size_t const iSize = inputBuffered - ? zcs->inBuffPos - zcs->inToCompress - : MIN((size_t)(iend - ip), zcs->blockSize); + size_t oSize = (size_t)(oend-op); + size_t const iSize = inputBuffered ? zcs->inBuffPos - zcs->inToCompress + : MIN((size_t)(iend - ip), zcs->blockSizeMax); if (oSize >= ZSTD_compressBound(iSize) || zcs->appliedParams.outBufferMode == ZSTD_bm_stable) cDst = op; /* compress into output buffer, to skip flush stage */ else @@ -18734,34 +25963,31 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, if (inputBuffered) { unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip==iend); cSize = lastBlock ? - ZSTD_compressEnd(zcs, cDst, oSize, + ZSTD_compressEnd_public(zcs, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize) : - ZSTD_compressContinue(zcs, cDst, oSize, + ZSTD_compressContinue_public(zcs, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize); FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed"); zcs->frameEnded = lastBlock; /* prepare next block */ - zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize; + zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSizeMax; if (zcs->inBuffTarget > zcs->inBuffSize) - zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize; + zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSizeMax; DEBUGLOG(5, "inBuffTarget:%u / inBuffSize:%u", (unsigned)zcs->inBuffTarget, (unsigned)zcs->inBuffSize); if (!lastBlock) assert(zcs->inBuffTarget <= zcs->inBuffSize); zcs->inToCompress = zcs->inBuffPos; - } else { - unsigned const lastBlock = (ip + iSize == iend); - assert(flushMode == ZSTD_e_end /* Already validated */); + } else { /* !inputBuffered, hence ZSTD_bm_stable */ + unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip + iSize == iend); cSize = lastBlock ? - ZSTD_compressEnd(zcs, cDst, oSize, ip, iSize) : - ZSTD_compressContinue(zcs, cDst, oSize, ip, iSize); + ZSTD_compressEnd_public(zcs, cDst, oSize, ip, iSize) : + ZSTD_compressContinue_public(zcs, cDst, oSize, ip, iSize); /* Consume the input prior to error checking to mirror buffered mode. */ - if (iSize > 0) - ip += iSize; + if (ip) ip += iSize; FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed"); zcs->frameEnded = lastBlock; - if (lastBlock) - assert(ip == iend); + if (lastBlock) assert(ip == iend); } if (cDst == op) { /* no need to flush */ op += cSize; @@ -18776,7 +26002,7 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, zcs->outBuffFlushedSize = 0; zcs->streamStage = zcss_flush; /* pass-through to flush stage */ } - /* fall-through */ + ZSTD_FALLTHROUGH; case zcss_flush: DEBUGLOG(5, "flush stage"); assert(zcs->appliedParams.outBufferMode == ZSTD_bm_buffered); @@ -18810,8 +26036,8 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, } } - input->pos = ip - istart; - output->pos = op - ostart; + input->pos = (size_t)(ip - istart); + output->pos = (size_t)(op - ostart); if (zcs->frameEnded) return 0; return ZSTD_nextInputSizeHint(zcs); } @@ -18837,8 +26063,10 @@ size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuf /* After a compression call set the expected input/output buffer. * This is validated at the start of the next compression call. */ -static void ZSTD_setBufferExpectations(ZSTD_CCtx* cctx, ZSTD_outBuffer const* output, ZSTD_inBuffer const* input) +static void +ZSTD_setBufferExpectations(ZSTD_CCtx* cctx, const ZSTD_outBuffer* output, const ZSTD_inBuffer* input) { + DEBUGLOG(5, "ZSTD_setBufferExpectations (for advanced stable in/out modes)"); if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) { cctx->expectedInBuffer = *input; } @@ -18857,55 +26085,73 @@ static size_t ZSTD_checkBufferStability(ZSTD_CCtx const* cctx, { if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) { ZSTD_inBuffer const expect = cctx->expectedInBuffer; - if (expect.src != input->src || expect.pos != input->pos || expect.size != input->size) - RETURN_ERROR(srcBuffer_wrong, "ZSTD_c_stableInBuffer enabled but input differs!"); - if (endOp != ZSTD_e_end) - RETURN_ERROR(srcBuffer_wrong, "ZSTD_c_stableInBuffer can only be used with ZSTD_e_end!"); + if (expect.src != input->src || expect.pos != input->pos) + RETURN_ERROR(stabilityCondition_notRespected, "ZSTD_c_stableInBuffer enabled but input differs!"); } + (void)endOp; if (cctx->appliedParams.outBufferMode == ZSTD_bm_stable) { size_t const outBufferSize = output->size - output->pos; if (cctx->expectedOutBufferSize != outBufferSize) - RETURN_ERROR(dstBuffer_wrong, "ZSTD_c_stableOutBuffer enabled but output size differs!"); + RETURN_ERROR(stabilityCondition_notRespected, "ZSTD_c_stableOutBuffer enabled but output size differs!"); } return 0; } +/* + * If @endOp == ZSTD_e_end, @inSize becomes pledgedSrcSize. + * Otherwise, it's ignored. + * @return: 0 on success, or a ZSTD_error code otherwise. + */ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx, ZSTD_EndDirective endOp, - size_t inSize) { + size_t inSize) +{ ZSTD_CCtx_params params = cctx->requestedParams; ZSTD_prefixDict const prefixDict = cctx->prefixDict; FORWARD_IF_ERROR( ZSTD_initLocalDict(cctx) , ""); /* Init the local dict if present. */ ZSTD_memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); /* single usage */ assert(prefixDict.dict==NULL || cctx->cdict==NULL); /* only one can be set */ - if (cctx->cdict) - params.compressionLevel = cctx->cdict->compressionLevel; /* let cdict take priority in terms of compression level */ - DEBUGLOG(4, "ZSTD_compressStream2 : transparent init stage"); - if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = inSize + 1; /* auto-fix pledgedSrcSize */ - { - size_t const dictSize = prefixDict.dict + if (cctx->cdict && !cctx->localDict.cdict) { + /* Let the cdict's compression level take priority over the requested params. + * But do not take the cdict's compression level if the "cdict" is actually a localDict + * generated from ZSTD_initLocalDict(). + */ + params.compressionLevel = cctx->cdict->compressionLevel; + } + DEBUGLOG(4, "ZSTD_CCtx_init_compressStream2 : transparent init stage"); + if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = inSize + 1; /* auto-determine pledgedSrcSize */ + + { size_t const dictSize = prefixDict.dict ? prefixDict.dictSize : (cctx->cdict ? cctx->cdict->dictContentSize : 0); - ZSTD_cParamMode_e const mode = ZSTD_getCParamMode(cctx->cdict, ¶ms, cctx->pledgedSrcSizePlusOne - 1); + ZSTD_CParamMode_e const mode = ZSTD_getCParamMode(cctx->cdict, ¶ms, cctx->pledgedSrcSizePlusOne - 1); params.cParams = ZSTD_getCParamsFromCCtxParams( ¶ms, cctx->pledgedSrcSizePlusOne-1, dictSize, mode); } - if (ZSTD_CParams_shouldEnableLdm(¶ms.cParams)) { - /* Enable LDM by default for optimal parser and window size >= 128MB */ - DEBUGLOG(4, "LDM enabled by default (window size >= 128MB, strategy >= btopt)"); - params.ldmParams.enableLdm = 1; - } + params.postBlockSplitter = ZSTD_resolveBlockSplitterMode(params.postBlockSplitter, ¶ms.cParams); + params.ldmParams.enableLdm = ZSTD_resolveEnableLdm(params.ldmParams.enableLdm, ¶ms.cParams); + params.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params.useRowMatchFinder, ¶ms.cParams); + params.validateSequences = ZSTD_resolveExternalSequenceValidation(params.validateSequences); + params.maxBlockSize = ZSTD_resolveMaxBlockSize(params.maxBlockSize); + params.searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(params.searchForExternalRepcodes, params.compressionLevel); #ifdef ZSTD_MULTITHREAD + /* If external matchfinder is enabled, make sure to fail before checking job size (for consistency) */ + RETURN_ERROR_IF( + ZSTD_hasExtSeqProd(¶ms) && params.nbWorkers >= 1, + parameter_combination_unsupported, + "External sequence producer isn't supported with nbWorkers >= 1" + ); + if ((cctx->pledgedSrcSizePlusOne-1) <= ZSTDMT_JOBSIZE_MIN) { params.nbWorkers = 0; /* do not invoke multi-threading when src size is too small */ } if (params.nbWorkers > 0) { -#if ZSTD_TRACE - cctx->traceCtx = ZSTD_trace_compress_begin(cctx); -#endif +# if ZSTD_TRACE + cctx->traceCtx = (ZSTD_trace_compress_begin != NULL) ? ZSTD_trace_compress_begin(cctx) : 0; +# endif /* mt context creation */ if (cctx->mtctx == NULL) { DEBUGLOG(4, "ZSTD_compressStream2: creating new mtctx for nbWorkers=%u", @@ -18926,7 +26172,7 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx, cctx->streamStage = zcss_load; cctx->appliedParams = params; } else -#endif +#endif /* ZSTD_MULTITHREAD */ { U64 const pledgedSrcSize = cctx->pledgedSrcSizePlusOne - 1; assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx, @@ -18941,7 +26187,7 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx, /* for small input: avoid automatic flush on reaching end of block, since * it would require to add a 3-bytes null block to end frame */ - cctx->inBuffTarget = cctx->blockSize + (cctx->blockSize == pledgedSrcSize); + cctx->inBuffTarget = cctx->blockSizeMax + (cctx->blockSizeMax == pledgedSrcSize); } else { cctx->inBuffTarget = 0; } @@ -18952,6 +26198,8 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx, return 0; } +/* @return provides a minimum amount of data remaining to be flushed from internal buffers + */ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input, @@ -18966,8 +26214,27 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, /* transparent initialization stage */ if (cctx->streamStage == zcss_init) { - FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, endOp, input->size), "CompressStream2 initialization failed"); - ZSTD_setBufferExpectations(cctx, output, input); /* Set initial buffer expectations now that we've initialized */ + size_t const inputSize = input->size - input->pos; /* no obligation to start from pos==0 */ + size_t const totalInputSize = inputSize + cctx->stableIn_notConsumed; + if ( (cctx->requestedParams.inBufferMode == ZSTD_bm_stable) /* input is presumed stable, across invocations */ + && (endOp == ZSTD_e_continue) /* no flush requested, more input to come */ + && (totalInputSize < ZSTD_BLOCKSIZE_MAX) ) { /* not even reached one block yet */ + if (cctx->stableIn_notConsumed) { /* not the first time */ + /* check stable source guarantees */ + RETURN_ERROR_IF(input->src != cctx->expectedInBuffer.src, stabilityCondition_notRespected, "stableInBuffer condition not respected: wrong src pointer"); + RETURN_ERROR_IF(input->pos != cctx->expectedInBuffer.size, stabilityCondition_notRespected, "stableInBuffer condition not respected: externally modified pos"); + } + /* pretend input was consumed, to give a sense forward progress */ + input->pos = input->size; + /* save stable inBuffer, for later control, and flush/end */ + cctx->expectedInBuffer = *input; + /* but actually input wasn't consumed, so keep track of position from where compression shall resume */ + cctx->stableIn_notConsumed += inputSize; + /* don't initialize yet, wait for the first block of flush() order, for better parameters adaptation */ + return ZSTD_FRAMEHEADERSIZE_MIN(cctx->requestedParams.format); /* at least some header to produce */ + } + FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, endOp, totalInputSize), "compressStream2 initialization failed"); + ZSTD_setBufferExpectations(cctx, output, input); /* Set initial buffer expectations now that we've initialized */ } /* end of transparent initialization stage */ @@ -18980,6 +26247,13 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, ZSTDMT_updateCParams_whileCompressing(cctx->mtctx, &cctx->requestedParams); cctx->cParamsChanged = 0; } + if (cctx->stableIn_notConsumed) { + assert(cctx->appliedParams.inBufferMode == ZSTD_bm_stable); + /* some early data was skipped - make it available for consumption */ + assert(input->pos >= cctx->stableIn_notConsumed); + input->pos -= cctx->stableIn_notConsumed; + cctx->stableIn_notConsumed = 0; + } for (;;) { size_t const ipos = input->pos; size_t const opos = output->pos; @@ -19018,7 +26292,7 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, ZSTD_setBufferExpectations(cctx, output, input); return flushMin; } -#endif +#endif /* ZSTD_MULTITHREAD */ FORWARD_IF_ERROR( ZSTD_compressStream_generic(cctx, output, input, endOp) , ""); DEBUGLOG(5, "completed ZSTD_compressStream2"); ZSTD_setBufferExpectations(cctx, output, input); @@ -19031,13 +26305,20 @@ size_t ZSTD_compressStream2_simpleArgs ( const void* src, size_t srcSize, size_t* srcPos, ZSTD_EndDirective endOp) { - ZSTD_outBuffer output = { dst, dstCapacity, *dstPos }; - ZSTD_inBuffer input = { src, srcSize, *srcPos }; + ZSTD_outBuffer output; + ZSTD_inBuffer input; + output.dst = dst; + output.size = dstCapacity; + output.pos = *dstPos; + input.src = src; + input.size = srcSize; + input.pos = *srcPos; /* ZSTD_compressStream2() will check validity of dstPos and srcPos */ - size_t const cErr = ZSTD_compressStream2(cctx, &output, &input, endOp); - *dstPos = output.pos; - *srcPos = input.pos; - return cErr; + { size_t const cErr = ZSTD_compressStream2(cctx, &output, &input, endOp); + *dstPos = output.pos; + *srcPos = input.pos; + return cErr; + } } size_t ZSTD_compress2(ZSTD_CCtx* cctx, @@ -19060,6 +26341,7 @@ size_t ZSTD_compress2(ZSTD_CCtx* cctx, /* Reset to the original values. */ cctx->requestedParams.inBufferMode = originalInBufferMode; cctx->requestedParams.outBufferMode = originalOutBufferMode; + FORWARD_IF_ERROR(result, "ZSTD_compressStream2_simpleArgs failed"); if (result != 0) { /* compression not completed, due to lack of output space */ assert(oPos == dstCapacity); @@ -19070,64 +26352,66 @@ size_t ZSTD_compress2(ZSTD_CCtx* cctx, } } -typedef struct { - U32 idx; /* Index in array of ZSTD_Sequence */ - U32 posInSequence; /* Position within sequence at idx */ - size_t posInSrc; /* Number of bytes given by sequences provided so far */ -} ZSTD_sequencePosition; - -/* Returns a ZSTD error code if sequence is not valid */ -static size_t ZSTD_validateSequence(U32 offCode, U32 matchLength, - size_t posInSrc, U32 windowLog, size_t dictSize, U32 minMatch) { - size_t offsetBound; - U32 windowSize = 1 << windowLog; - /* posInSrc represents the amount of data the the decoder would decode up to this point. +/* ZSTD_validateSequence() : + * @offBase : must use the format required by ZSTD_storeSeq() + * @returns a ZSTD error code if sequence is not valid + */ +static size_t +ZSTD_validateSequence(U32 offBase, U32 matchLength, U32 minMatch, + size_t posInSrc, U32 windowLog, size_t dictSize, int useSequenceProducer) +{ + U32 const windowSize = 1u << windowLog; + /* posInSrc represents the amount of data the decoder would decode up to this point. * As long as the amount of data decoded is less than or equal to window size, offsets may be * larger than the total length of output decoded in order to reference the dict, even larger than * window size. After output surpasses windowSize, we're limited to windowSize offsets again. */ - offsetBound = posInSrc > windowSize ? (size_t)windowSize : posInSrc + (size_t)dictSize; - RETURN_ERROR_IF(offCode > offsetBound + ZSTD_REP_MOVE, corruption_detected, "Offset too large!"); - RETURN_ERROR_IF(matchLength < minMatch, corruption_detected, "Matchlength too small"); + size_t const offsetBound = posInSrc > windowSize ? (size_t)windowSize : posInSrc + (size_t)dictSize; + size_t const matchLenLowerBound = (minMatch == 3 || useSequenceProducer) ? 3 : 4; + RETURN_ERROR_IF(offBase > OFFSET_TO_OFFBASE(offsetBound), externalSequences_invalid, "Offset too large!"); + /* Validate maxNbSeq is large enough for the given matchLength and minMatch */ + RETURN_ERROR_IF(matchLength < matchLenLowerBound, externalSequences_invalid, "Matchlength too small for the minMatch"); return 0; } /* Returns an offset code, given a sequence's raw offset, the ongoing repcode array, and whether litLength == 0 */ -static U32 ZSTD_finalizeOffCode(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32 ll0) { - U32 offCode = rawOffset + ZSTD_REP_MOVE; - U32 repCode = 0; +static U32 ZSTD_finalizeOffBase(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32 ll0) +{ + U32 offBase = OFFSET_TO_OFFBASE(rawOffset); if (!ll0 && rawOffset == rep[0]) { - repCode = 1; + offBase = REPCODE1_TO_OFFBASE; } else if (rawOffset == rep[1]) { - repCode = 2 - ll0; + offBase = REPCODE_TO_OFFBASE(2 - ll0); } else if (rawOffset == rep[2]) { - repCode = 3 - ll0; + offBase = REPCODE_TO_OFFBASE(3 - ll0); } else if (ll0 && rawOffset == rep[0] - 1) { - repCode = 3; + offBase = REPCODE3_TO_OFFBASE; } - if (repCode) { - /* ZSTD_storeSeq expects a number in the range [0, 2] to represent a repcode */ - offCode = repCode - 1; - } - return offCode; + return offBase; } -/* Returns 0 on success, and a ZSTD_error otherwise. This function scans through an array of - * ZSTD_Sequence, storing the sequences it finds, until it reaches a block delimiter. +/* This function scans through an array of ZSTD_Sequence, + * storing the sequences it reads, until it reaches a block delimiter. + * Note that the block delimiter includes the last literals of the block. + * @blockSize must be == sum(sequence_lengths). + * @returns @blockSize on success, and a ZSTD_error otherwise. */ -static size_t ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos, - const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, - const void* src, size_t blockSize) { +static size_t +ZSTD_transferSequences_wBlockDelim(ZSTD_CCtx* cctx, + ZSTD_SequencePosition* seqPos, + const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, + const void* src, size_t blockSize, + ZSTD_ParamSwitch_e externalRepSearch) +{ U32 idx = seqPos->idx; + U32 const startIdx = idx; BYTE const* ip = (BYTE const*)(src); const BYTE* const iend = ip + blockSize; - repcodes_t updatedRepcodes; + Repcodes_t updatedRepcodes; U32 dictSize; - U32 litLength; - U32 matchLength; - U32 ll0; - U32 offCode; + + DEBUGLOG(5, "ZSTD_transferSequences_wBlockDelim (blockSize = %zu)", blockSize); if (cctx->cdict) { dictSize = (U32)cctx->cdict->dictContentSize; @@ -19136,28 +26420,60 @@ static size_t ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx, ZS } else { dictSize = 0; } - ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t)); - for (; (inSeqs[idx].matchLength != 0 || inSeqs[idx].offset != 0) && idx < inSeqsSize; ++idx) { - litLength = inSeqs[idx].litLength; - matchLength = inSeqs[idx].matchLength; - ll0 = litLength == 0; - offCode = ZSTD_finalizeOffCode(inSeqs[idx].offset, updatedRepcodes.rep, ll0); - updatedRepcodes = ZSTD_updateRep(updatedRepcodes.rep, offCode, ll0); + ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(Repcodes_t)); + for (; idx < inSeqsSize && (inSeqs[idx].matchLength != 0 || inSeqs[idx].offset != 0); ++idx) { + U32 const litLength = inSeqs[idx].litLength; + U32 const matchLength = inSeqs[idx].matchLength; + U32 offBase; + + if (externalRepSearch == ZSTD_ps_disable) { + offBase = OFFSET_TO_OFFBASE(inSeqs[idx].offset); + } else { + U32 const ll0 = (litLength == 0); + offBase = ZSTD_finalizeOffBase(inSeqs[idx].offset, updatedRepcodes.rep, ll0); + ZSTD_updateRep(updatedRepcodes.rep, offBase, ll0); + } - DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offCode, matchLength, litLength); + DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength); if (cctx->appliedParams.validateSequences) { seqPos->posInSrc += litLength + matchLength; - FORWARD_IF_ERROR(ZSTD_validateSequence(offCode, matchLength, seqPos->posInSrc, + FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, cctx->appliedParams.cParams.minMatch, + seqPos->posInSrc, cctx->appliedParams.cParams.windowLog, dictSize, - cctx->appliedParams.cParams.minMatch), + ZSTD_hasExtSeqProd(&cctx->appliedParams)), "Sequence validation failed"); } - RETURN_ERROR_IF(idx - seqPos->idx > cctx->seqStore.maxNbSeq, memory_allocation, + RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, externalSequences_invalid, "Not enough memory allocated. Try adjusting ZSTD_c_minMatch."); - ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offCode, matchLength - MINMATCH); + ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength); ip += matchLength + litLength; } - ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(repcodes_t)); + RETURN_ERROR_IF(idx == inSeqsSize, externalSequences_invalid, "Block delimiter not found."); + + /* If we skipped repcode search while parsing, we need to update repcodes now */ + assert(externalRepSearch != ZSTD_ps_auto); + assert(idx >= startIdx); + if (externalRepSearch == ZSTD_ps_disable && idx != startIdx) { + U32* const rep = updatedRepcodes.rep; + U32 lastSeqIdx = idx - 1; /* index of last non-block-delimiter sequence */ + + if (lastSeqIdx >= startIdx + 2) { + rep[2] = inSeqs[lastSeqIdx - 2].offset; + rep[1] = inSeqs[lastSeqIdx - 1].offset; + rep[0] = inSeqs[lastSeqIdx].offset; + } else if (lastSeqIdx == startIdx + 1) { + rep[2] = rep[0]; + rep[1] = inSeqs[lastSeqIdx - 1].offset; + rep[0] = inSeqs[lastSeqIdx].offset; + } else { + assert(lastSeqIdx == startIdx); + rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = inSeqs[lastSeqIdx].offset; + } + } + + ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(Repcodes_t)); if (inSeqs[idx].litLength) { DEBUGLOG(6, "Storing last literals of size: %u", inSeqs[idx].litLength); @@ -19165,38 +26481,42 @@ static size_t ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx, ZS ip += inSeqs[idx].litLength; seqPos->posInSrc += inSeqs[idx].litLength; } - RETURN_ERROR_IF(ip != iend, corruption_detected, "Blocksize doesn't agree with block delimiter!"); + RETURN_ERROR_IF(ip != iend, externalSequences_invalid, "Blocksize doesn't agree with block delimiter!"); seqPos->idx = idx+1; - return 0; + return blockSize; } -/* Returns the number of bytes to move the current read position back by. Only non-zero - * if we ended up splitting a sequence. Otherwise, it may return a ZSTD error if something - * went wrong. +/* + * This function attempts to scan through @blockSize bytes in @src + * represented by the sequences in @inSeqs, + * storing any (partial) sequences. * - * This function will attempt to scan through blockSize bytes represented by the sequences - * in inSeqs, storing any (partial) sequences. + * Occasionally, we may want to reduce the actual number of bytes consumed from @src + * to avoid splitting a match, notably if it would produce a match smaller than MINMATCH. * - * Occasionally, we may want to change the actual number of bytes we consumed from inSeqs to - * avoid splitting a match, or to avoid splitting a match such that it would produce a match - * smaller than MINMATCH. In this case, we return the number of bytes that we didn't read from this block. + * @returns the number of bytes consumed from @src, necessarily <= @blockSize. + * Otherwise, it may return a ZSTD error if something went wrong. */ -static size_t ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos, - const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, - const void* src, size_t blockSize) { +static size_t +ZSTD_transferSequences_noDelim(ZSTD_CCtx* cctx, + ZSTD_SequencePosition* seqPos, + const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, + const void* src, size_t blockSize, + ZSTD_ParamSwitch_e externalRepSearch) +{ U32 idx = seqPos->idx; U32 startPosInSequence = seqPos->posInSequence; U32 endPosInSequence = seqPos->posInSequence + (U32)blockSize; size_t dictSize; - BYTE const* ip = (BYTE const*)(src); - BYTE const* iend = ip + blockSize; /* May be adjusted if we decide to process fewer than blockSize bytes */ - repcodes_t updatedRepcodes; + const BYTE* const istart = (const BYTE*)(src); + const BYTE* ip = istart; + const BYTE* iend = istart + blockSize; /* May be adjusted if we decide to process fewer than blockSize bytes */ + Repcodes_t updatedRepcodes; U32 bytesAdjustment = 0; U32 finalMatchSplit = 0; - U32 litLength; - U32 matchLength; - U32 rawOffset; - U32 offCode; + + /* TODO(embg) support fast parsing mode in noBlockDelim mode */ + (void)externalRepSearch; if (cctx->cdict) { dictSize = cctx->cdict->dictContentSize; @@ -19205,14 +26525,15 @@ static size_t ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_seq } else { dictSize = 0; } - DEBUGLOG(5, "ZSTD_copySequencesToSeqStore: idx: %u PIS: %u blockSize: %zu", idx, startPosInSequence, blockSize); + DEBUGLOG(5, "ZSTD_transferSequences_noDelim: idx: %u PIS: %u blockSize: %zu", idx, startPosInSequence, blockSize); DEBUGLOG(5, "Start seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength); - ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t)); + ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(Repcodes_t)); while (endPosInSequence && idx < inSeqsSize && !finalMatchSplit) { const ZSTD_Sequence currSeq = inSeqs[idx]; - litLength = currSeq.litLength; - matchLength = currSeq.matchLength; - rawOffset = currSeq.offset; + U32 litLength = currSeq.litLength; + U32 matchLength = currSeq.matchLength; + U32 const rawOffset = currSeq.offset; + U32 offBase; /* Modify the sequence depending on where endPosInSequence lies */ if (endPosInSequence >= currSeq.litLength + currSeq.matchLength) { @@ -19226,7 +26547,6 @@ static size_t ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_seq /* Move to the next sequence */ endPosInSequence -= currSeq.litLength + currSeq.matchLength; startPosInSequence = 0; - idx++; } else { /* This is the final (partial) sequence we're adding from inSeqs, and endPosInSequence does not reach the end of the match. So, we have to split the sequence */ @@ -19265,146 +26585,730 @@ static size_t ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_seq } } /* Check if this offset can be represented with a repcode */ - { U32 ll0 = (litLength == 0); - offCode = ZSTD_finalizeOffCode(rawOffset, updatedRepcodes.rep, ll0); - updatedRepcodes = ZSTD_updateRep(updatedRepcodes.rep, offCode, ll0); + { U32 const ll0 = (litLength == 0); + offBase = ZSTD_finalizeOffBase(rawOffset, updatedRepcodes.rep, ll0); + ZSTD_updateRep(updatedRepcodes.rep, offBase, ll0); } if (cctx->appliedParams.validateSequences) { seqPos->posInSrc += litLength + matchLength; - FORWARD_IF_ERROR(ZSTD_validateSequence(offCode, matchLength, seqPos->posInSrc, - cctx->appliedParams.cParams.windowLog, dictSize, - cctx->appliedParams.cParams.minMatch), + FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, cctx->appliedParams.cParams.minMatch, seqPos->posInSrc, + cctx->appliedParams.cParams.windowLog, dictSize, ZSTD_hasExtSeqProd(&cctx->appliedParams)), "Sequence validation failed"); } - DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offCode, matchLength, litLength); - RETURN_ERROR_IF(idx - seqPos->idx > cctx->seqStore.maxNbSeq, memory_allocation, + DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength); + RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, externalSequences_invalid, "Not enough memory allocated. Try adjusting ZSTD_c_minMatch."); - ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offCode, matchLength - MINMATCH); + ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength); ip += matchLength + litLength; + if (!finalMatchSplit) + idx++; /* Next Sequence */ } DEBUGLOG(5, "Ending seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength); assert(idx == inSeqsSize || endPosInSequence <= inSeqs[idx].litLength + inSeqs[idx].matchLength); seqPos->idx = idx; seqPos->posInSequence = endPosInSequence; - ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(repcodes_t)); + ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(Repcodes_t)); iend -= bytesAdjustment; if (ip != iend) { /* Store any last literals */ - U32 lastLLSize = (U32)(iend - ip); + U32 const lastLLSize = (U32)(iend - ip); assert(ip <= iend); DEBUGLOG(6, "Storing last literals of size: %u", lastLLSize); ZSTD_storeLastLiterals(&cctx->seqStore, ip, lastLLSize); seqPos->posInSrc += lastLLSize; } - return bytesAdjustment; + return (size_t)(iend-istart); +} + +/* @seqPos represents a position within @inSeqs, + * it is read and updated by this function, + * once the goal to produce a block of size @blockSize is reached. + * @return: nb of bytes consumed from @src, necessarily <= @blockSize. + */ +typedef size_t (*ZSTD_SequenceCopier_f)(ZSTD_CCtx* cctx, + ZSTD_SequencePosition* seqPos, + const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, + const void* src, size_t blockSize, + ZSTD_ParamSwitch_e externalRepSearch); + +static ZSTD_SequenceCopier_f ZSTD_selectSequenceCopier(ZSTD_SequenceFormat_e mode) +{ + assert(ZSTD_cParam_withinBounds(ZSTD_c_blockDelimiters, (int)mode)); + if (mode == ZSTD_sf_explicitBlockDelimiters) { + return ZSTD_transferSequences_wBlockDelim; + } + assert(mode == ZSTD_sf_noBlockDelimiters); + return ZSTD_transferSequences_noDelim; +} + +/* Discover the size of next block by searching for the delimiter. + * Note that a block delimiter **must** exist in this mode, + * otherwise it's an input error. + * The block size retrieved will be later compared to ensure it remains within bounds */ +static size_t +blockSize_explicitDelimiter(const ZSTD_Sequence* inSeqs, size_t inSeqsSize, ZSTD_SequencePosition seqPos) +{ + int end = 0; + size_t blockSize = 0; + size_t spos = seqPos.idx; + DEBUGLOG(6, "blockSize_explicitDelimiter : seq %zu / %zu", spos, inSeqsSize); + assert(spos <= inSeqsSize); + while (spos < inSeqsSize) { + end = (inSeqs[spos].offset == 0); + blockSize += inSeqs[spos].litLength + inSeqs[spos].matchLength; + if (end) { + if (inSeqs[spos].matchLength != 0) + RETURN_ERROR(externalSequences_invalid, "delimiter format error : both matchlength and offset must be == 0"); + break; + } + spos++; + } + if (!end) + RETURN_ERROR(externalSequences_invalid, "Reached end of sequences without finding a block delimiter"); + return blockSize; +} + +static size_t determine_blockSize(ZSTD_SequenceFormat_e mode, + size_t blockSize, size_t remaining, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + ZSTD_SequencePosition seqPos) +{ + DEBUGLOG(6, "determine_blockSize : remainingSize = %zu", remaining); + if (mode == ZSTD_sf_noBlockDelimiters) { + /* Note: more a "target" block size */ + return MIN(remaining, blockSize); + } + assert(mode == ZSTD_sf_explicitBlockDelimiters); + { size_t const explicitBlockSize = blockSize_explicitDelimiter(inSeqs, inSeqsSize, seqPos); + FORWARD_IF_ERROR(explicitBlockSize, "Error while determining block size with explicit delimiters"); + if (explicitBlockSize > blockSize) + RETURN_ERROR(externalSequences_invalid, "sequences incorrectly define a too large block"); + if (explicitBlockSize > remaining) + RETURN_ERROR(externalSequences_invalid, "sequences define a frame longer than source"); + return explicitBlockSize; + } +} + +/* Compress all provided sequences, block-by-block. + * + * Returns the cumulative size of all compressed blocks (including their headers), + * otherwise a ZSTD error. + */ +static size_t +ZSTD_compressSequences_internal(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + const void* src, size_t srcSize) +{ + size_t cSize = 0; + size_t remaining = srcSize; + ZSTD_SequencePosition seqPos = {0, 0, 0}; + + const BYTE* ip = (BYTE const*)src; + BYTE* op = (BYTE*)dst; + ZSTD_SequenceCopier_f const sequenceCopier = ZSTD_selectSequenceCopier(cctx->appliedParams.blockDelimiters); + + DEBUGLOG(4, "ZSTD_compressSequences_internal srcSize: %zu, inSeqsSize: %zu", srcSize, inSeqsSize); + /* Special case: empty frame */ + if (remaining == 0) { + U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1); + RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "No room for empty frame block header"); + MEM_writeLE32(op, cBlockHeader24); + op += ZSTD_blockHeaderSize; + dstCapacity -= ZSTD_blockHeaderSize; + cSize += ZSTD_blockHeaderSize; + } + + while (remaining) { + size_t compressedSeqsSize; + size_t cBlockSize; + size_t blockSize = determine_blockSize(cctx->appliedParams.blockDelimiters, + cctx->blockSizeMax, remaining, + inSeqs, inSeqsSize, seqPos); + U32 const lastBlock = (blockSize == remaining); + FORWARD_IF_ERROR(blockSize, "Error while trying to determine block size"); + assert(blockSize <= remaining); + ZSTD_resetSeqStore(&cctx->seqStore); + + blockSize = sequenceCopier(cctx, + &seqPos, inSeqs, inSeqsSize, + ip, blockSize, + cctx->appliedParams.searchForExternalRepcodes); + FORWARD_IF_ERROR(blockSize, "Bad sequence copy"); + + /* If blocks are too small, emit as a nocompress block */ + /* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding + * additional 1. We need to revisit and change this logic to be more consistent */ + if (blockSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1+1) { + cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cBlockSize, "Nocompress block failed"); + DEBUGLOG(5, "Block too small (%zu): data remains uncompressed: cSize=%zu", blockSize, cBlockSize); + cSize += cBlockSize; + ip += blockSize; + op += cBlockSize; + remaining -= blockSize; + dstCapacity -= cBlockSize; + continue; + } + + RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize, dstSize_tooSmall, "not enough dstCapacity to write a new compressed block"); + compressedSeqsSize = ZSTD_entropyCompressSeqStore(&cctx->seqStore, + &cctx->blockState.prevCBlock->entropy, &cctx->blockState.nextCBlock->entropy, + &cctx->appliedParams, + op + ZSTD_blockHeaderSize /* Leave space for block header */, dstCapacity - ZSTD_blockHeaderSize, + blockSize, + cctx->tmpWorkspace, cctx->tmpWkspSize /* statically allocated in resetCCtx */, + cctx->bmi2); + FORWARD_IF_ERROR(compressedSeqsSize, "Compressing sequences of block failed"); + DEBUGLOG(5, "Compressed sequences size: %zu", compressedSeqsSize); + + if (!cctx->isFirstBlock && + ZSTD_maybeRLE(&cctx->seqStore) && + ZSTD_isRLE(ip, blockSize)) { + /* Note: don't emit the first block as RLE even if it qualifies because + * doing so will cause the decoder (cli <= v1.4.3 only) to throw an (invalid) error + * "should consume all input error." + */ + compressedSeqsSize = 1; + } + + if (compressedSeqsSize == 0) { + /* ZSTD_noCompressBlock writes the block header as well */ + cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cBlockSize, "ZSTD_noCompressBlock failed"); + DEBUGLOG(5, "Writing out nocompress block, size: %zu", cBlockSize); + } else if (compressedSeqsSize == 1) { + cBlockSize = ZSTD_rleCompressBlock(op, dstCapacity, *ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cBlockSize, "ZSTD_rleCompressBlock failed"); + DEBUGLOG(5, "Writing out RLE block, size: %zu", cBlockSize); + } else { + U32 cBlockHeader; + /* Error checking and repcodes update */ + ZSTD_blockState_confirmRepcodesAndEntropyTables(&cctx->blockState); + if (cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) + cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; + + /* Write block header into beginning of block*/ + cBlockHeader = lastBlock + (((U32)bt_compressed)<<1) + (U32)(compressedSeqsSize << 3); + MEM_writeLE24(op, cBlockHeader); + cBlockSize = ZSTD_blockHeaderSize + compressedSeqsSize; + DEBUGLOG(5, "Writing out compressed block, size: %zu", cBlockSize); + } + + cSize += cBlockSize; + + if (lastBlock) { + break; + } else { + ip += blockSize; + op += cBlockSize; + remaining -= blockSize; + dstCapacity -= cBlockSize; + cctx->isFirstBlock = 0; + } + DEBUGLOG(5, "cSize running total: %zu (remaining dstCapacity=%zu)", cSize, dstCapacity); + } + + DEBUGLOG(4, "cSize final total: %zu", cSize); + return cSize; +} + +size_t ZSTD_compressSequences(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + const void* src, size_t srcSize) +{ + BYTE* op = (BYTE*)dst; + size_t cSize = 0; + + /* Transparent initialization stage, same as compressStream2() */ + DEBUGLOG(4, "ZSTD_compressSequences (nbSeqs=%zu,dstCapacity=%zu)", inSeqsSize, dstCapacity); + assert(cctx != NULL); + FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, ZSTD_e_end, srcSize), "CCtx initialization failed"); + + /* Begin writing output, starting with frame header */ + { size_t const frameHeaderSize = ZSTD_writeFrameHeader(op, dstCapacity, + &cctx->appliedParams, srcSize, cctx->dictID); + op += frameHeaderSize; + assert(frameHeaderSize <= dstCapacity); + dstCapacity -= frameHeaderSize; + cSize += frameHeaderSize; + } + if (cctx->appliedParams.fParams.checksumFlag && srcSize) { + XXH64_update(&cctx->xxhState, src, srcSize); + } + + /* Now generate compressed blocks */ + { size_t const cBlocksSize = ZSTD_compressSequences_internal(cctx, + op, dstCapacity, + inSeqs, inSeqsSize, + src, srcSize); + FORWARD_IF_ERROR(cBlocksSize, "Compressing blocks failed!"); + cSize += cBlocksSize; + assert(cBlocksSize <= dstCapacity); + dstCapacity -= cBlocksSize; + } + + /* Complete with frame checksum, if needed */ + if (cctx->appliedParams.fParams.checksumFlag) { + U32 const checksum = (U32) XXH64_digest(&cctx->xxhState); + RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "no room for checksum"); + DEBUGLOG(4, "Write checksum : %08X", (unsigned)checksum); + MEM_writeLE32((char*)dst + cSize, checksum); + cSize += 4; + } + + DEBUGLOG(4, "Final compressed size: %zu", cSize); + return cSize; +} + + +#if defined(__AVX2__) + +#include /* AVX2 intrinsics */ + +/* + * Convert 2 sequences per iteration, using AVX2 intrinsics: + * - offset -> offBase = offset + 2 + * - litLength -> (U16) litLength + * - matchLength -> (U16)(matchLength - 3) + * - rep is ignored + * Store only 8 bytes per SeqDef (offBase[4], litLength[2], mlBase[2]). + * + * At the end, instead of extracting two __m128i, + * we use _mm256_permute4x64_epi64(..., 0xE8) to move lane2 into lane1, + * then store the lower 16 bytes in one go. + * + * @returns 0 on succes, with no long length detected + * @returns > 0 if there is one long length (> 65535), + * indicating the position, and type. + */ +static size_t convertSequences_noRepcodes( + SeqDef* dstSeqs, + const ZSTD_Sequence* inSeqs, + size_t nbSequences) +{ + /* + * addition: + * For each 128-bit half: (offset+2, litLength+0, matchLength-3, rep+0) + */ + const __m256i addition = _mm256_setr_epi32( + ZSTD_REP_NUM, 0, -MINMATCH, 0, /* for sequence i */ + ZSTD_REP_NUM, 0, -MINMATCH, 0 /* for sequence i+1 */ + ); + + /* limit: check if there is a long length */ + const __m256i limit = _mm256_set1_epi32(65535); + + /* + * shuffle mask for byte-level rearrangement in each 128-bit half: + * + * Input layout (after addition) per 128-bit half: + * [ offset+2 (4 bytes) | litLength (4 bytes) | matchLength (4 bytes) | rep (4 bytes) ] + * We only need: + * offBase (4 bytes) = offset+2 + * litLength (2 bytes) = low 2 bytes of litLength + * mlBase (2 bytes) = low 2 bytes of (matchLength) + * => Bytes [0..3, 4..5, 8..9], zero the rest. + */ + const __m256i mask = _mm256_setr_epi8( + /* For the lower 128 bits => sequence i */ + 0, 1, 2, 3, /* offset+2 */ + 4, 5, /* litLength (16 bits) */ + 8, 9, /* matchLength (16 bits) */ + (BYTE)0x80, (BYTE)0x80, (BYTE)0x80, (BYTE)0x80, + (BYTE)0x80, (BYTE)0x80, (BYTE)0x80, (BYTE)0x80, + + /* For the upper 128 bits => sequence i+1 */ + 16,17,18,19, /* offset+2 */ + 20,21, /* litLength */ + 24,25, /* matchLength */ + (BYTE)0x80, (BYTE)0x80, (BYTE)0x80, (BYTE)0x80, + (BYTE)0x80, (BYTE)0x80, (BYTE)0x80, (BYTE)0x80 + ); + + /* + * Next, we'll use _mm256_permute4x64_epi64(vshf, 0xE8). + * Explanation of 0xE8 = 11101000b => [lane0, lane2, lane2, lane3]. + * So the lower 128 bits become [lane0, lane2] => combining seq0 and seq1. + */ +#define PERM_LANE_0X_E8 0xE8 /* [0,2,2,3] in lane indices */ + + size_t longLen = 0, i = 0; + + /* AVX permutation depends on the specific definition of target structures */ + ZSTD_STATIC_ASSERT(sizeof(ZSTD_Sequence) == 16); + ZSTD_STATIC_ASSERT(offsetof(ZSTD_Sequence, offset) == 0); + ZSTD_STATIC_ASSERT(offsetof(ZSTD_Sequence, litLength) == 4); + ZSTD_STATIC_ASSERT(offsetof(ZSTD_Sequence, matchLength) == 8); + ZSTD_STATIC_ASSERT(sizeof(SeqDef) == 8); + ZSTD_STATIC_ASSERT(offsetof(SeqDef, offBase) == 0); + ZSTD_STATIC_ASSERT(offsetof(SeqDef, litLength) == 4); + ZSTD_STATIC_ASSERT(offsetof(SeqDef, mlBase) == 6); + + /* Process 2 sequences per loop iteration */ + for (; i + 1 < nbSequences; i += 2) { + /* Load 2 ZSTD_Sequence (32 bytes) */ + __m256i vin = _mm256_loadu_si256((const __m256i*)(const void*)&inSeqs[i]); + + /* Add {2, 0, -3, 0} in each 128-bit half */ + __m256i vadd = _mm256_add_epi32(vin, addition); + + /* Check for long length */ + __m256i ll_cmp = _mm256_cmpgt_epi32(vadd, limit); /* 0xFFFFFFFF for element > 65535 */ + int ll_res = _mm256_movemask_epi8(ll_cmp); + + /* Shuffle bytes so each half gives us the 8 bytes we need */ + __m256i vshf = _mm256_shuffle_epi8(vadd, mask); + /* + * Now: + * Lane0 = seq0's 8 bytes + * Lane1 = 0 + * Lane2 = seq1's 8 bytes + * Lane3 = 0 + */ + + /* Permute 64-bit lanes => move Lane2 down into Lane1. */ + __m256i vperm = _mm256_permute4x64_epi64(vshf, PERM_LANE_0X_E8); + /* + * Now the lower 16 bytes (Lane0+Lane1) = [seq0, seq1]. + * The upper 16 bytes are [Lane2, Lane3] = [seq1, 0], but we won't use them. + */ + + /* Store only the lower 16 bytes => 2 SeqDef (8 bytes each) */ + _mm_storeu_si128((__m128i *)(void*)&dstSeqs[i], _mm256_castsi256_si128(vperm)); + /* + * This writes out 16 bytes total: + * - offset 0..7 => seq0 (offBase, litLength, mlBase) + * - offset 8..15 => seq1 (offBase, litLength, mlBase) + */ + + /* check (unlikely) long lengths > 65535 + * indices for lengths correspond to bits [4..7], [8..11], [20..23], [24..27] + * => combined mask = 0x0FF00FF0 + */ + if (UNLIKELY((ll_res & 0x0FF00FF0) != 0)) { + /* long length detected: let's figure out which one*/ + if (inSeqs[i].matchLength > 65535+MINMATCH) { + assert(longLen == 0); + longLen = i + 1; + } + if (inSeqs[i].litLength > 65535) { + assert(longLen == 0); + longLen = i + nbSequences + 1; + } + if (inSeqs[i+1].matchLength > 65535+MINMATCH) { + assert(longLen == 0); + longLen = i + 1 + 1; + } + if (inSeqs[i+1].litLength > 65535) { + assert(longLen == 0); + longLen = i + 1 + nbSequences + 1; + } + } + } + + /* Handle leftover if @nbSequences is odd */ + if (i < nbSequences) { + /* process last sequence */ + assert(i == nbSequences - 1); + dstSeqs[i].offBase = OFFSET_TO_OFFBASE(inSeqs[i].offset); + dstSeqs[i].litLength = (U16)inSeqs[i].litLength; + dstSeqs[i].mlBase = (U16)(inSeqs[i].matchLength - MINMATCH); + /* check (unlikely) long lengths > 65535 */ + if (UNLIKELY(inSeqs[i].matchLength > 65535+MINMATCH)) { + assert(longLen == 0); + longLen = i + 1; + } + if (UNLIKELY(inSeqs[i].litLength > 65535)) { + assert(longLen == 0); + longLen = i + nbSequences + 1; + } + } + + return longLen; +} + +/* the vector implementation could also be ported to SSSE3, + * but since this implementation is targeting modern systems (>= Sapphire Rapid), + * it's not useful to develop and maintain code for older pre-AVX2 platforms */ + +#else /* no AVX2 */ + +static size_t convertSequences_noRepcodes( + SeqDef* dstSeqs, + const ZSTD_Sequence* inSeqs, + size_t nbSequences) +{ + size_t longLen = 0; + size_t n; + for (n=0; n 65535 */ + if (UNLIKELY(inSeqs[n].matchLength > 65535+MINMATCH)) { + assert(longLen == 0); + longLen = n + 1; + } + if (UNLIKELY(inSeqs[n].litLength > 65535)) { + assert(longLen == 0); + longLen = n + nbSequences + 1; + } + } + return longLen; +} + +#endif + +/* + * Precondition: Sequences must end on an explicit Block Delimiter + * @return: 0 on success, or an error code. + * Note: Sequence validation functionality has been disabled (removed). + * This is helpful to generate a lean main pipeline, improving performance. + * It may be re-inserted later. + */ +size_t ZSTD_convertBlockSequences(ZSTD_CCtx* cctx, + const ZSTD_Sequence* const inSeqs, size_t nbSequences, + int repcodeResolution) +{ + Repcodes_t updatedRepcodes; + size_t seqNb = 0; + + DEBUGLOG(5, "ZSTD_convertBlockSequences (nbSequences = %zu)", nbSequences); + + RETURN_ERROR_IF(nbSequences >= cctx->seqStore.maxNbSeq, externalSequences_invalid, + "Not enough memory allocated. Try adjusting ZSTD_c_minMatch."); + + ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(Repcodes_t)); + + /* check end condition */ + assert(nbSequences >= 1); + assert(inSeqs[nbSequences-1].matchLength == 0); + assert(inSeqs[nbSequences-1].offset == 0); + + /* Convert Sequences from public format to internal format */ + if (!repcodeResolution) { + size_t const longl = convertSequences_noRepcodes(cctx->seqStore.sequencesStart, inSeqs, nbSequences-1); + cctx->seqStore.sequences = cctx->seqStore.sequencesStart + nbSequences-1; + if (longl) { + DEBUGLOG(5, "long length"); + assert(cctx->seqStore.longLengthType == ZSTD_llt_none); + if (longl <= nbSequences-1) { + DEBUGLOG(5, "long match length detected at pos %zu", longl-1); + cctx->seqStore.longLengthType = ZSTD_llt_matchLength; + cctx->seqStore.longLengthPos = (U32)(longl-1); + } else { + DEBUGLOG(5, "long literals length detected at pos %zu", longl-nbSequences); + assert(longl <= 2* (nbSequences-1)); + cctx->seqStore.longLengthType = ZSTD_llt_literalLength; + cctx->seqStore.longLengthPos = (U32)(longl-(nbSequences-1)-1); + } + } + } else { + for (seqNb = 0; seqNb < nbSequences - 1 ; seqNb++) { + U32 const litLength = inSeqs[seqNb].litLength; + U32 const matchLength = inSeqs[seqNb].matchLength; + U32 const ll0 = (litLength == 0); + U32 const offBase = ZSTD_finalizeOffBase(inSeqs[seqNb].offset, updatedRepcodes.rep, ll0); + + DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength); + ZSTD_storeSeqOnly(&cctx->seqStore, litLength, offBase, matchLength); + ZSTD_updateRep(updatedRepcodes.rep, offBase, ll0); + } + } + + /* If we skipped repcode search while parsing, we need to update repcodes now */ + if (!repcodeResolution && nbSequences > 1) { + U32* const rep = updatedRepcodes.rep; + + if (nbSequences >= 4) { + U32 lastSeqIdx = (U32)nbSequences - 2; /* index of last full sequence */ + rep[2] = inSeqs[lastSeqIdx - 2].offset; + rep[1] = inSeqs[lastSeqIdx - 1].offset; + rep[0] = inSeqs[lastSeqIdx].offset; + } else if (nbSequences == 3) { + rep[2] = rep[0]; + rep[1] = inSeqs[0].offset; + rep[0] = inSeqs[1].offset; + } else { + assert(nbSequences == 2); + rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = inSeqs[0].offset; + } + } + + ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(Repcodes_t)); + + return 0; +} + +#if defined(ZSTD_ARCH_X86_AVX2) + +BlockSummary ZSTD_get1BlockSummary(const ZSTD_Sequence* seqs, size_t nbSeqs) +{ + size_t i; + __m256i const zeroVec = _mm256_setzero_si256(); + __m256i sumVec = zeroVec; /* accumulates match+lit in 32-bit lanes */ + ZSTD_ALIGNED(32) U32 tmp[8]; /* temporary buffer for reduction */ + size_t mSum = 0, lSum = 0; + ZSTD_STATIC_ASSERT(sizeof(ZSTD_Sequence) == 16); + + /* Process 2 structs (32 bytes) at a time */ + for (i = 0; i + 2 <= nbSeqs; i += 2) { + /* Load two consecutive ZSTD_Sequence (8×4 = 32 bytes) */ + __m256i data = _mm256_loadu_si256((const __m256i*)(const void*)&seqs[i]); + /* check end of block signal */ + __m256i cmp = _mm256_cmpeq_epi32(data, zeroVec); + int cmp_res = _mm256_movemask_epi8(cmp); + /* indices for match lengths correspond to bits [8..11], [24..27] + * => combined mask = 0x0F000F00 */ + ZSTD_STATIC_ASSERT(offsetof(ZSTD_Sequence, matchLength) == 8); + if (cmp_res & 0x0F000F00) break; + /* Accumulate in sumVec */ + sumVec = _mm256_add_epi32(sumVec, data); + } + + /* Horizontal reduction */ + _mm256_store_si256((__m256i*)tmp, sumVec); + lSum = tmp[1] + tmp[5]; + mSum = tmp[2] + tmp[6]; + + /* Handle the leftover */ + for (; i < nbSeqs; i++) { + lSum += seqs[i].litLength; + mSum += seqs[i].matchLength; + if (seqs[i].matchLength == 0) break; /* end of block */ + } + + if (i==nbSeqs) { + /* reaching end of sequences: end of block signal was not present */ + BlockSummary bs; + bs.nbSequences = ERROR(externalSequences_invalid); + return bs; + } + { BlockSummary bs; + bs.nbSequences = i+1; + bs.blockSize = lSum + mSum; + bs.litSize = lSum; + return bs; + } } -typedef size_t (*ZSTD_sequenceCopier) (ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos, - const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, - const void* src, size_t blockSize); -static ZSTD_sequenceCopier ZSTD_selectSequenceCopier(ZSTD_sequenceFormat_e mode) { - ZSTD_sequenceCopier sequenceCopier = NULL; - assert(ZSTD_cParam_withinBounds(ZSTD_c_blockDelimiters, mode)); - if (mode == ZSTD_sf_explicitBlockDelimiters) { - return ZSTD_copySequencesToSeqStoreExplicitBlockDelim; - } else if (mode == ZSTD_sf_noBlockDelimiters) { - return ZSTD_copySequencesToSeqStoreNoBlockDelim; +#else + +BlockSummary ZSTD_get1BlockSummary(const ZSTD_Sequence* seqs, size_t nbSeqs) +{ + size_t totalMatchSize = 0; + size_t litSize = 0; + size_t n; + assert(seqs); + for (n=0; nappliedParams.blockDelimiters); + int const repcodeResolution = (cctx->appliedParams.searchForExternalRepcodes == ZSTD_ps_enable); + assert(cctx->appliedParams.searchForExternalRepcodes != ZSTD_ps_auto); + + DEBUGLOG(4, "ZSTD_compressSequencesAndLiterals_internal: nbSeqs=%zu, litSize=%zu", nbSequences, litSize); + RETURN_ERROR_IF(nbSequences == 0, externalSequences_invalid, "Requires at least 1 end-of-block"); - DEBUGLOG(4, "ZSTD_compressSequences_internal srcSize: %zu, inSeqsSize: %zu", srcSize, inSeqsSize); /* Special case: empty frame */ - if (remaining == 0) { + if ((nbSequences == 1) && (inSeqs[0].litLength == 0)) { U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1); - RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "No room for empty frame block header"); - MEM_writeLE32(op, cBlockHeader24); + RETURN_ERROR_IF(dstCapacity<3, dstSize_tooSmall, "No room for empty frame block header"); + MEM_writeLE24(op, cBlockHeader24); op += ZSTD_blockHeaderSize; dstCapacity -= ZSTD_blockHeaderSize; cSize += ZSTD_blockHeaderSize; } - while (remaining) { - size_t cBlockSize; - size_t additionalByteAdjustment; - lastBlock = remaining <= cctx->blockSize; - blockSize = lastBlock ? (U32)remaining : (U32)cctx->blockSize; + while (nbSequences) { + size_t compressedSeqsSize, cBlockSize, conversionStatus; + BlockSummary const block = ZSTD_get1BlockSummary(inSeqs, nbSequences); + U32 const lastBlock = (block.nbSequences == nbSequences); + FORWARD_IF_ERROR(block.nbSequences, "Error while trying to determine nb of sequences for a block"); + assert(block.nbSequences <= nbSequences); + RETURN_ERROR_IF(block.litSize > litSize, externalSequences_invalid, "discrepancy: Sequences require more literals than present in buffer"); ZSTD_resetSeqStore(&cctx->seqStore); - DEBUGLOG(4, "Working on new block. Blocksize: %zu", blockSize); - additionalByteAdjustment = sequenceCopier(cctx, &seqPos, inSeqs, inSeqsSize, ip, blockSize); - FORWARD_IF_ERROR(additionalByteAdjustment, "Bad sequence copy"); - blockSize -= additionalByteAdjustment; + conversionStatus = ZSTD_convertBlockSequences(cctx, + inSeqs, block.nbSequences, + repcodeResolution); + FORWARD_IF_ERROR(conversionStatus, "Bad sequence conversion"); + inSeqs += block.nbSequences; + nbSequences -= block.nbSequences; + remaining -= block.blockSize; - /* If blocks are too small, emit as a nocompress block */ - if (blockSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) { - cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); - FORWARD_IF_ERROR(cBlockSize, "Nocompress block failed"); - DEBUGLOG(4, "Block too small, writing out nocompress block: cSize: %zu", cBlockSize); - cSize += cBlockSize; - ip += blockSize; - op += cBlockSize; - remaining -= blockSize; - dstCapacity -= cBlockSize; - continue; - } + /* Note: when blockSize is very small, other variant send it uncompressed. + * Here, we still send the sequences, because we don't have the original source to send it uncompressed. + * One could imagine in theory reproducing the source from the sequences, + * but that's complex and costly memory intensive, and goes against the objectives of this variant. */ - compressedSeqsSize = ZSTD_entropyCompressSequences(&cctx->seqStore, + RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize, dstSize_tooSmall, "not enough dstCapacity to write a new compressed block"); + + compressedSeqsSize = ZSTD_entropyCompressSeqStore_internal( + op + ZSTD_blockHeaderSize /* Leave space for block header */, dstCapacity - ZSTD_blockHeaderSize, + literals, block.litSize, + &cctx->seqStore, &cctx->blockState.prevCBlock->entropy, &cctx->blockState.nextCBlock->entropy, &cctx->appliedParams, - op + ZSTD_blockHeaderSize /* Leave space for block header */, dstCapacity - ZSTD_blockHeaderSize, - blockSize, - cctx->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, + cctx->tmpWorkspace, cctx->tmpWkspSize /* statically allocated in resetCCtx */, cctx->bmi2); FORWARD_IF_ERROR(compressedSeqsSize, "Compressing sequences of block failed"); - DEBUGLOG(4, "Compressed sequences size: %zu", compressedSeqsSize); + /* note: the spec forbids for any compressed block to be larger than maximum block size */ + if (compressedSeqsSize > cctx->blockSizeMax) compressedSeqsSize = 0; + DEBUGLOG(5, "Compressed sequences size: %zu", compressedSeqsSize); + litSize -= block.litSize; + literals = (const char*)literals + block.litSize; - if (!cctx->isFirstBlock && - ZSTD_maybeRLE(&cctx->seqStore) && - ZSTD_isRLE((BYTE const*)src, srcSize)) { - /* We don't want to emit our first block as a RLE even if it qualifies because - * doing so will cause the decoder (cli only) to throw a "should consume all input error." - * This is only an issue for zstd <= v1.4.3 - */ - compressedSeqsSize = 1; - } + /* Note: difficult to check source for RLE block when only Literals are provided, + * but it could be considered from analyzing the sequence directly */ if (compressedSeqsSize == 0) { - /* ZSTD_noCompressBlock writes the block header as well */ - cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); - FORWARD_IF_ERROR(cBlockSize, "Nocompress block failed"); - DEBUGLOG(4, "Writing out nocompress block, size: %zu", cBlockSize); - } else if (compressedSeqsSize == 1) { - cBlockSize = ZSTD_rleCompressBlock(op, dstCapacity, *ip, blockSize, lastBlock); - FORWARD_IF_ERROR(cBlockSize, "RLE compress block failed"); - DEBUGLOG(4, "Writing out RLE block, size: %zu", cBlockSize); + /* Sending uncompressed blocks is out of reach, because the source is not provided. + * In theory, one could use the sequences to regenerate the source, like a decompressor, + * but it's complex, and memory hungry, killing the purpose of this variant. + * Current outcome: generate an error code. + */ + RETURN_ERROR(cannotProduce_uncompressedBlock, "ZSTD_compressSequencesAndLiterals cannot generate an uncompressed block"); } else { U32 cBlockHeader; + assert(compressedSeqsSize > 1); /* no RLE */ /* Error checking and repcodes update */ - ZSTD_confirmRepcodesAndEntropyTables(cctx); + ZSTD_blockState_confirmRepcodesAndEntropyTables(&cctx->blockState); if (cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; @@ -19412,83 +27316,102 @@ static size_t ZSTD_compressSequences_internal(ZSTD_CCtx* cctx, cBlockHeader = lastBlock + (((U32)bt_compressed)<<1) + (U32)(compressedSeqsSize << 3); MEM_writeLE24(op, cBlockHeader); cBlockSize = ZSTD_blockHeaderSize + compressedSeqsSize; - DEBUGLOG(4, "Writing out compressed block, size: %zu", cBlockSize); + DEBUGLOG(5, "Writing out compressed block, size: %zu", cBlockSize); } cSize += cBlockSize; - DEBUGLOG(4, "cSize running total: %zu", cSize); + op += cBlockSize; + dstCapacity -= cBlockSize; + cctx->isFirstBlock = 0; + DEBUGLOG(5, "cSize running total: %zu (remaining dstCapacity=%zu)", cSize, dstCapacity); if (lastBlock) { + assert(nbSequences == 0); break; - } else { - ip += blockSize; - op += cBlockSize; - remaining -= blockSize; - dstCapacity -= cBlockSize; - cctx->isFirstBlock = 0; } } + RETURN_ERROR_IF(litSize != 0, externalSequences_invalid, "literals must be entirely and exactly consumed"); + RETURN_ERROR_IF(remaining != 0, externalSequences_invalid, "Sequences must represent a total of exactly srcSize=%zu", srcSize); + DEBUGLOG(4, "cSize final total: %zu", cSize); return cSize; } -size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstCapacity, - const ZSTD_Sequence* inSeqs, size_t inSeqsSize, - const void* src, size_t srcSize) { +size_t +ZSTD_compressSequencesAndLiterals(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + const void* literals, size_t litSize, size_t litCapacity, + size_t decompressedSize) +{ BYTE* op = (BYTE*)dst; size_t cSize = 0; - size_t compressedBlocksSize = 0; - size_t frameHeaderSize = 0; /* Transparent initialization stage, same as compressStream2() */ - DEBUGLOG(3, "ZSTD_compressSequences()"); + DEBUGLOG(4, "ZSTD_compressSequencesAndLiterals (dstCapacity=%zu)", dstCapacity); assert(cctx != NULL); - FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, ZSTD_e_end, srcSize), "CCtx initialization failed"); - /* Begin writing output, starting with frame header */ - frameHeaderSize = ZSTD_writeFrameHeader(op, dstCapacity, &cctx->appliedParams, srcSize, cctx->dictID); - op += frameHeaderSize; - dstCapacity -= frameHeaderSize; - cSize += frameHeaderSize; - if (cctx->appliedParams.fParams.checksumFlag && srcSize) { - XXH64_update(&cctx->xxhState, src, srcSize); + if (litCapacity < litSize) { + RETURN_ERROR(workSpace_tooSmall, "literals buffer is not large enough: must be at least 8 bytes larger than litSize (risk of read out-of-bound)"); } - /* cSize includes block header size and compressed sequences size */ - compressedBlocksSize = ZSTD_compressSequences_internal(cctx, - op, dstCapacity, - inSeqs, inSeqsSize, - src, srcSize); - FORWARD_IF_ERROR(compressedBlocksSize, "Compressing blocks failed!"); - cSize += compressedBlocksSize; - dstCapacity -= compressedBlocksSize; + FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, ZSTD_e_end, decompressedSize), "CCtx initialization failed"); + if (cctx->appliedParams.blockDelimiters == ZSTD_sf_noBlockDelimiters) { + RETURN_ERROR(frameParameter_unsupported, "This mode is only compatible with explicit delimiters"); + } + if (cctx->appliedParams.validateSequences) { + RETURN_ERROR(parameter_unsupported, "This mode is not compatible with Sequence validation"); + } if (cctx->appliedParams.fParams.checksumFlag) { - U32 const checksum = (U32) XXH64_digest(&cctx->xxhState); - RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "no room for checksum"); - DEBUGLOG(4, "Write checksum : %08X", (unsigned)checksum); - MEM_writeLE32((char*)dst + cSize, checksum); - cSize += 4; + RETURN_ERROR(frameParameter_unsupported, "this mode is not compatible with frame checksum"); } - DEBUGLOG(3, "Final compressed size: %zu", cSize); + /* Begin writing output, starting with frame header */ + { size_t const frameHeaderSize = ZSTD_writeFrameHeader(op, dstCapacity, + &cctx->appliedParams, decompressedSize, cctx->dictID); + op += frameHeaderSize; + assert(frameHeaderSize <= dstCapacity); + dstCapacity -= frameHeaderSize; + cSize += frameHeaderSize; + } + + /* Now generate compressed blocks */ + { size_t const cBlocksSize = ZSTD_compressSequencesAndLiterals_internal(cctx, + op, dstCapacity, + inSeqs, inSeqsSize, + literals, litSize, decompressedSize); + FORWARD_IF_ERROR(cBlocksSize, "Compressing blocks failed!"); + cSize += cBlocksSize; + assert(cBlocksSize <= dstCapacity); + dstCapacity -= cBlocksSize; + } + + DEBUGLOG(4, "Final compressed size: %zu", cSize); return cSize; } /*====== Finalize ======*/ +static ZSTD_inBuffer inBuffer_forEndFlush(const ZSTD_CStream* zcs) +{ + const ZSTD_inBuffer nullInput = { NULL, 0, 0 }; + const int stableInput = (zcs->appliedParams.inBufferMode == ZSTD_bm_stable); + return stableInput ? zcs->expectedInBuffer : nullInput; +} + /*! ZSTD_flushStream() : * @return : amount of data remaining to flush */ size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) { - ZSTD_inBuffer input = { NULL, 0, 0 }; + ZSTD_inBuffer input = inBuffer_forEndFlush(zcs); + input.size = input.pos; /* do not ingest more input during flush */ return ZSTD_compressStream2(zcs, output, &input, ZSTD_e_flush); } - size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) { - ZSTD_inBuffer input = { NULL, 0, 0 }; + ZSTD_inBuffer input = inBuffer_forEndFlush(zcs); size_t const remainingToFlush = ZSTD_compressStream2(zcs, output, &input, ZSTD_e_end); - FORWARD_IF_ERROR( remainingToFlush , "ZSTD_compressStream2 failed"); + FORWARD_IF_ERROR(remainingToFlush , "ZSTD_compressStream2(,,ZSTD_e_end) failed"); if (zcs->appliedParams.nbWorkers > 0) return remainingToFlush; /* minimal estimation */ /* single thread mode : attempt to calculate remaining to flush more precisely */ { size_t const lastBlockSize = zcs->frameEnded ? 0 : ZSTD_BLOCKHEADERSIZE; @@ -19500,11 +27423,31 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) } +/*-===== Pre-defined compression levels =====-*/ +/**** start inlining clevels.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_CLEVELS_H +#define ZSTD_CLEVELS_H + +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */ +/**** skipping file: ../zstd.h ****/ + /*-===== Pre-defined compression levels =====-*/ #define ZSTD_MAX_CLEVEL 22 -int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } -int ZSTD_minCLevel(void) { return (int)-ZSTD_TARGETLENGTH_MAX; } + +#ifdef __GNUC__ +__attribute__((__unused__)) +#endif static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = { { /* "default" - for any srcSize > 256 KB */ @@ -19514,15 +27457,15 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV { 20, 15, 16, 1, 6, 0, ZSTD_fast }, /* level 2 */ { 21, 16, 17, 1, 5, 0, ZSTD_dfast }, /* level 3 */ { 21, 18, 18, 1, 5, 0, ZSTD_dfast }, /* level 4 */ - { 21, 18, 19, 2, 5, 2, ZSTD_greedy }, /* level 5 */ - { 21, 19, 19, 3, 5, 4, ZSTD_greedy }, /* level 6 */ - { 21, 19, 19, 3, 5, 8, ZSTD_lazy }, /* level 7 */ - { 21, 19, 19, 3, 5, 16, ZSTD_lazy2 }, /* level 8 */ - { 21, 19, 20, 4, 5, 16, ZSTD_lazy2 }, /* level 9 */ - { 22, 20, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 10 */ - { 22, 21, 22, 4, 5, 16, ZSTD_lazy2 }, /* level 11 */ - { 22, 21, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 12 */ - { 22, 21, 22, 5, 5, 32, ZSTD_btlazy2 }, /* level 13 */ + { 21, 18, 19, 3, 5, 2, ZSTD_greedy }, /* level 5 */ + { 21, 18, 19, 3, 5, 4, ZSTD_lazy }, /* level 6 */ + { 21, 19, 20, 4, 5, 8, ZSTD_lazy }, /* level 7 */ + { 21, 19, 20, 4, 5, 16, ZSTD_lazy2 }, /* level 8 */ + { 22, 20, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 9 */ + { 22, 21, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 10 */ + { 22, 21, 22, 6, 5, 16, ZSTD_lazy2 }, /* level 11 */ + { 22, 22, 23, 6, 5, 32, ZSTD_lazy2 }, /* level 12 */ + { 22, 22, 22, 4, 5, 32, ZSTD_btlazy2 }, /* level 13 */ { 22, 22, 23, 5, 5, 32, ZSTD_btlazy2 }, /* level 14 */ { 22, 23, 23, 6, 5, 32, ZSTD_btlazy2 }, /* level 15 */ { 22, 22, 22, 5, 5, 48, ZSTD_btopt }, /* level 16 */ @@ -19539,8 +27482,8 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV { 18, 13, 14, 1, 6, 0, ZSTD_fast }, /* level 1 */ { 18, 14, 14, 1, 5, 0, ZSTD_dfast }, /* level 2 */ { 18, 16, 16, 1, 4, 0, ZSTD_dfast }, /* level 3 */ - { 18, 16, 17, 2, 5, 2, ZSTD_greedy }, /* level 4.*/ - { 18, 18, 18, 3, 5, 2, ZSTD_greedy }, /* level 5.*/ + { 18, 16, 17, 3, 5, 2, ZSTD_greedy }, /* level 4.*/ + { 18, 17, 18, 5, 5, 2, ZSTD_greedy }, /* level 5.*/ { 18, 18, 19, 3, 5, 4, ZSTD_lazy }, /* level 6.*/ { 18, 18, 19, 4, 4, 4, ZSTD_lazy }, /* level 7 */ { 18, 18, 19, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ @@ -19567,11 +27510,11 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV { 17, 15, 16, 2, 5, 0, ZSTD_dfast }, /* level 3 */ { 17, 17, 17, 2, 4, 0, ZSTD_dfast }, /* level 4 */ { 17, 16, 17, 3, 4, 2, ZSTD_greedy }, /* level 5 */ - { 17, 17, 17, 3, 4, 4, ZSTD_lazy }, /* level 6 */ - { 17, 17, 17, 3, 4, 8, ZSTD_lazy2 }, /* level 7 */ - { 17, 17, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ - { 17, 17, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ - { 17, 17, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ + { 17, 16, 17, 3, 4, 4, ZSTD_lazy }, /* level 6 */ + { 17, 16, 17, 3, 4, 8, ZSTD_lazy2 }, /* level 7 */ + { 17, 16, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ + { 17, 16, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ + { 17, 16, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ { 17, 17, 17, 5, 4, 8, ZSTD_btlazy2 }, /* level 11 */ { 17, 18, 17, 7, 4, 12, ZSTD_btlazy2 }, /* level 12 */ { 17, 18, 17, 3, 4, 12, ZSTD_btopt }, /* level 13.*/ @@ -19613,6 +27556,15 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV }, }; + + +#endif /* ZSTD_CLEVELS_H */ +/**** ended inlining clevels.h ****/ + +int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } +int ZSTD_minCLevel(void) { return (int)-ZSTD_TARGETLENGTH_MAX; } +int ZSTD_defaultCLevel(void) { return ZSTD_CLEVEL_DEFAULT; } + static ZSTD_compressionParameters ZSTD_dedicatedDictSearch_getCParams(int const compressionLevel, size_t const dictSize) { ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, 0, dictSize, ZSTD_cpm_createCDict); @@ -19637,7 +27589,10 @@ static ZSTD_compressionParameters ZSTD_dedicatedDictSearch_getCParams(int const static int ZSTD_dedicatedDictSearch_isSupported( ZSTD_compressionParameters const* cParams) { - return (cParams->strategy >= ZSTD_greedy) && (cParams->strategy <= ZSTD_lazy2); + return (cParams->strategy >= ZSTD_greedy) + && (cParams->strategy <= ZSTD_lazy2) + && (cParams->hashLog > cParams->chainLog) + && (cParams->chainLog <= 24); } /** @@ -19655,6 +27610,9 @@ static void ZSTD_dedicatedDictSearch_revertCParams( case ZSTD_lazy: case ZSTD_lazy2: cParams->hashLog -= ZSTD_LAZY_DDSS_BUCKET_LOG; + if (cParams->hashLog < ZSTD_HASHLOG_MIN) { + cParams->hashLog = ZSTD_HASHLOG_MIN; + } break; case ZSTD_btlazy2: case ZSTD_btopt: @@ -19664,7 +27622,7 @@ static void ZSTD_dedicatedDictSearch_revertCParams( } } -static U64 ZSTD_getCParamRowSize(U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) +static U64 ZSTD_getCParamRowSize(U64 srcSizeHint, size_t dictSize, ZSTD_CParamMode_e mode) { switch (mode) { case ZSTD_cpm_unknown: @@ -19688,8 +27646,8 @@ static U64 ZSTD_getCParamRowSize(U64 srcSizeHint, size_t dictSize, ZSTD_cParamMo * @return ZSTD_compressionParameters structure for a selected compression level, srcSize and dictSize. * Note: srcSizeHint 0 means 0, use ZSTD_CONTENTSIZE_UNKNOWN for unknown. * Use dictSize == 0 for unknown or unused. - * Note: `mode` controls how we treat the `dictSize`. See docs for `ZSTD_cParamMode_e`. */ -static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) + * Note: `mode` controls how we treat the `dictSize`. See docs for `ZSTD_CParamMode_e`. */ +static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_CParamMode_e mode) { U64 const rSize = ZSTD_getCParamRowSize(srcSizeHint, dictSize, mode); U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); @@ -19703,13 +27661,14 @@ static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, else row = compressionLevel; { ZSTD_compressionParameters cp = ZSTD_defaultCParameters[tableID][row]; + DEBUGLOG(5, "ZSTD_getCParams_internal selected tableID: %u row: %u strat: %u", tableID, row, (U32)cp.strategy); /* acceleration factor */ if (compressionLevel < 0) { int const clampedCompressionLevel = MAX(ZSTD_minCLevel(), compressionLevel); cp.targetLength = (unsigned)(-clampedCompressionLevel); } /* refine parameters based on srcSize & dictSize */ - return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize, mode); + return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize, mode, ZSTD_ps_auto); } } @@ -19726,7 +27685,9 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long l * same idea as ZSTD_getCParams() * @return a `ZSTD_parameters` structure (instead of `ZSTD_compressionParameters`). * Fields of `ZSTD_frameParameters` are set to default values */ -static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) { +static ZSTD_parameters +ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_CParamMode_e mode) +{ ZSTD_parameters params; ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, srcSizeHint, dictSize, mode); DEBUGLOG(5, "ZSTD_getParams (cLevel=%i)", compressionLevel); @@ -19740,14 +27701,41 @@ static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned lo * same idea as ZSTD_getCParams() * @return a `ZSTD_parameters` structure (instead of `ZSTD_compressionParameters`). * Fields of `ZSTD_frameParameters` are set to default values */ -ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) { +ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) +{ if (srcSizeHint == 0) srcSizeHint = ZSTD_CONTENTSIZE_UNKNOWN; return ZSTD_getParams_internal(compressionLevel, srcSizeHint, dictSize, ZSTD_cpm_unknown); } + +void ZSTD_registerSequenceProducer( + ZSTD_CCtx* zc, + void* extSeqProdState, + ZSTD_sequenceProducer_F extSeqProdFunc) +{ + assert(zc != NULL); + ZSTD_CCtxParams_registerSequenceProducer( + &zc->requestedParams, extSeqProdState, extSeqProdFunc + ); +} + +void ZSTD_CCtxParams_registerSequenceProducer( + ZSTD_CCtx_params* params, + void* extSeqProdState, + ZSTD_sequenceProducer_F extSeqProdFunc) +{ + assert(params != NULL); + if (extSeqProdFunc != NULL) { + params->extSeqProdFunc = extSeqProdFunc; + params->extSeqProdState = extSeqProdState; + } else { + params->extSeqProdFunc = NULL; + params->extSeqProdState = NULL; + } +} /**** ended inlining compress/zstd_compress.c ****/ /**** start inlining compress/zstd_double_fast.c ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -19759,8 +27747,49 @@ ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeH /**** skipping file: zstd_compress_internal.h ****/ /**** skipping file: zstd_double_fast.h ****/ +#ifndef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR + +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +void ZSTD_fillDoubleHashTableForCDict(ZSTD_MatchState_t* ms, + void const* end, ZSTD_dictTableLoadMethod_e dtlm) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashLarge = ms->hashTable; + U32 const hBitsL = cParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS; + U32 const mls = cParams->minMatch; + U32* const hashSmall = ms->chainTable; + U32 const hBitsS = cParams->chainLog + ZSTD_SHORT_CACHE_TAG_BITS; + const BYTE* const base = ms->window.base; + const BYTE* ip = base + ms->nextToUpdate; + const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; + const U32 fastHashFillStep = 3; + + /* Always insert every fastHashFillStep position into the hash tables. + * Insert the other positions into the large hash table if their entry + * is empty. + */ + for (; ip + fastHashFillStep - 1 <= iend; ip += fastHashFillStep) { + U32 const curr = (U32)(ip - base); + U32 i; + for (i = 0; i < fastHashFillStep; ++i) { + size_t const smHashAndTag = ZSTD_hashPtr(ip + i, hBitsS, mls); + size_t const lgHashAndTag = ZSTD_hashPtr(ip + i, hBitsL, 8); + if (i == 0) { + ZSTD_writeTaggedIndex(hashSmall, smHashAndTag, curr + i); + } + if (i == 0 || hashLarge[lgHashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS] == 0) { + ZSTD_writeTaggedIndex(hashLarge, lgHashAndTag, curr + i); + } + /* Only load extra positions for ZSTD_dtlm_full */ + if (dtlm == ZSTD_dtlm_fast) + break; + } } +} -void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms, +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +void ZSTD_fillDoubleHashTableForCCtx(ZSTD_MatchState_t* ms, void const* end, ZSTD_dictTableLoadMethod_e dtlm) { const ZSTD_compressionParameters* const cParams = &ms->cParams; @@ -19791,15 +27820,251 @@ void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms, /* Only load extra positions for ZSTD_dtlm_full */ if (dtlm == ZSTD_dtlm_fast) break; - } } + } } +} + +void ZSTD_fillDoubleHashTable(ZSTD_MatchState_t* ms, + const void* const end, + ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp) +{ + if (tfp == ZSTD_tfp_forCDict) { + ZSTD_fillDoubleHashTableForCDict(ms, end, dtlm); + } else { + ZSTD_fillDoubleHashTableForCCtx(ms, end, dtlm); + } +} + + +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_compressBlock_doubleFast_noDict_generic( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, U32 const mls /* template */) +{ + ZSTD_compressionParameters const* cParams = &ms->cParams; + U32* const hashLong = ms->hashTable; + const U32 hBitsL = cParams->hashLog; + U32* const hashSmall = ms->chainTable; + const U32 hBitsS = cParams->chainLog; + const BYTE* const base = ms->window.base; + const BYTE* const istart = (const BYTE*)src; + const BYTE* anchor = istart; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + /* presumes that, if there is a dictionary, it must be using Attach mode */ + const U32 prefixLowestIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog); + const BYTE* const prefixLowest = base + prefixLowestIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - HASH_READ_SIZE; + U32 offset_1=rep[0], offset_2=rep[1]; + U32 offsetSaved1 = 0, offsetSaved2 = 0; + + size_t mLength; + U32 offset; + U32 curr; + + /* how many positions to search before increasing step size */ + const size_t kStepIncr = 1 << kSearchStrength; + /* the position at which to increment the step size if no match is found */ + const BYTE* nextStep; + size_t step; /* the current step size */ + + size_t hl0; /* the long hash at ip */ + size_t hl1; /* the long hash at ip1 */ + + U32 idxl0; /* the long match index for ip */ + U32 idxl1; /* the long match index for ip1 */ + + const BYTE* matchl0; /* the long match for ip */ + const BYTE* matchs0; /* the short match for ip */ + const BYTE* matchl1; /* the long match for ip1 */ + const BYTE* matchs0_safe; /* matchs0 or safe address */ + + const BYTE* ip = istart; /* the current position */ + const BYTE* ip1; /* the next position */ + /* Array of ~random data, should have low probability of matching data + * we load from here instead of from tables, if matchl0/matchl1 are + * invalid indices. Used to avoid unpredictable branches. */ + const BYTE dummy[] = {0x12,0x34,0x56,0x78,0x9a,0xbc,0xde,0xf0,0xe2,0xb4}; + + DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_noDict_generic"); + + /* init */ + ip += ((ip - prefixLowest) == 0); + { + U32 const current = (U32)(ip - base); + U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, current, cParams->windowLog); + U32 const maxRep = current - windowLow; + if (offset_2 > maxRep) offsetSaved2 = offset_2, offset_2 = 0; + if (offset_1 > maxRep) offsetSaved1 = offset_1, offset_1 = 0; + } + + /* Outer Loop: one iteration per match found and stored */ + while (1) { + step = 1; + nextStep = ip + kStepIncr; + ip1 = ip + step; + + if (ip1 > ilimit) { + goto _cleanup; + } + + hl0 = ZSTD_hashPtr(ip, hBitsL, 8); + idxl0 = hashLong[hl0]; + matchl0 = base + idxl0; + + /* Inner Loop: one iteration per search / position */ + do { + const size_t hs0 = ZSTD_hashPtr(ip, hBitsS, mls); + const U32 idxs0 = hashSmall[hs0]; + curr = (U32)(ip-base); + matchs0 = base + idxs0; + + hashLong[hl0] = hashSmall[hs0] = curr; /* update hash tables */ + + /* check noDict repcode */ + if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) { + mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; + ip++; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); + goto _match_stored; + } + + hl1 = ZSTD_hashPtr(ip1, hBitsL, 8); + + /* idxl0 > prefixLowestIndex is a (somewhat) unpredictable branch. + * However expression below complies into conditional move. Since + * match is unlikely and we only *branch* on idxl0 > prefixLowestIndex + * if there is a match, all branches become predictable. */ + { const BYTE* const matchl0_safe = ZSTD_selectAddr(idxl0, prefixLowestIndex, matchl0, &dummy[0]); + + /* check prefix long match */ + if (MEM_read64(matchl0_safe) == MEM_read64(ip) && matchl0_safe == matchl0) { + mLength = ZSTD_count(ip+8, matchl0+8, iend) + 8; + offset = (U32)(ip-matchl0); + while (((ip>anchor) & (matchl0>prefixLowest)) && (ip[-1] == matchl0[-1])) { ip--; matchl0--; mLength++; } /* catch up */ + goto _match_found; + } } + + idxl1 = hashLong[hl1]; + matchl1 = base + idxl1; + + /* Same optimization as matchl0 above */ + matchs0_safe = ZSTD_selectAddr(idxs0, prefixLowestIndex, matchs0, &dummy[0]); + + /* check prefix short match */ + if(MEM_read32(matchs0_safe) == MEM_read32(ip) && matchs0_safe == matchs0) { + goto _search_next_long; + } + + if (ip1 >= nextStep) { + PREFETCH_L1(ip1 + 64); + PREFETCH_L1(ip1 + 128); + step++; + nextStep += kStepIncr; + } + ip = ip1; + ip1 += step; + + hl0 = hl1; + idxl0 = idxl1; + matchl0 = matchl1; + #if defined(__aarch64__) + PREFETCH_L1(ip+256); + #endif + } while (ip1 <= ilimit); + +_cleanup: + /* If offset_1 started invalid (offsetSaved1 != 0) and became valid (offset_1 != 0), + * rotate saved offsets. See comment in ZSTD_compressBlock_fast_noDict for more context. */ + offsetSaved2 = ((offsetSaved1 != 0) && (offset_1 != 0)) ? offsetSaved1 : offsetSaved2; + + /* save reps for next block */ + rep[0] = offset_1 ? offset_1 : offsetSaved1; + rep[1] = offset_2 ? offset_2 : offsetSaved2; + + /* Return the last literals size */ + return (size_t)(iend - anchor); + +_search_next_long: + + /* short match found: let's check for a longer one */ + mLength = ZSTD_count(ip+4, matchs0+4, iend) + 4; + offset = (U32)(ip - matchs0); + + /* check long match at +1 position */ + if ((idxl1 > prefixLowestIndex) && (MEM_read64(matchl1) == MEM_read64(ip1))) { + size_t const l1len = ZSTD_count(ip1+8, matchl1+8, iend) + 8; + if (l1len > mLength) { + /* use the long match instead */ + ip = ip1; + mLength = l1len; + offset = (U32)(ip-matchl1); + matchs0 = matchl1; + } + } + + while (((ip>anchor) & (matchs0>prefixLowest)) && (ip[-1] == matchs0[-1])) { ip--; matchs0--; mLength++; } /* complete backward */ + + /* fall-through */ + +_match_found: /* requires ip, offset, mLength */ + offset_2 = offset_1; + offset_1 = offset; + + if (step < 4) { + /* It is unsafe to write this value back to the hashtable when ip1 is + * greater than or equal to the new ip we will have after we're done + * processing this match. Rather than perform that test directly + * (ip1 >= ip + mLength), which costs speed in practice, we do a simpler + * more predictable test. The minmatch even if we take a short match is + * 4 bytes, so as long as step, the distance between ip and ip1 + * (initially) is less than 4, we know ip1 < new ip. */ + hashLong[hl1] = (U32)(ip1 - base); + } + + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); + +_match_stored: + /* match found */ + ip += mLength; + anchor = ip; + + if (ip <= ilimit) { + /* Complementary insertion */ + /* done after iLimit test, as candidates could be > iend-8 */ + { U32 const indexToInsert = curr+2; + hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert; + hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base); + hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert; + hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base); + } + + /* check immediate repcode */ + while ( (ip <= ilimit) + && ( (offset_2>0) + & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) { + /* store sequence */ + size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; + U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */ + hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip-base); + hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip-base); + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, rLength); + ip += rLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } + } + } } FORCE_INLINE_TEMPLATE -size_t ZSTD_compressBlock_doubleFast_generic( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize, - U32 const mls /* template */, ZSTD_dictMode_e const dictMode) + U32 const mls /* template */) { ZSTD_compressionParameters const* cParams = &ms->cParams; U32* const hashLong = ms->hashTable; @@ -19817,57 +28082,39 @@ size_t ZSTD_compressBlock_doubleFast_generic( const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - HASH_READ_SIZE; U32 offset_1=rep[0], offset_2=rep[1]; - U32 offsetSaved = 0; - - const ZSTD_matchState_t* const dms = ms->dictMatchState; - const ZSTD_compressionParameters* const dictCParams = - dictMode == ZSTD_dictMatchState ? - &dms->cParams : NULL; - const U32* const dictHashLong = dictMode == ZSTD_dictMatchState ? - dms->hashTable : NULL; - const U32* const dictHashSmall = dictMode == ZSTD_dictMatchState ? - dms->chainTable : NULL; - const U32 dictStartIndex = dictMode == ZSTD_dictMatchState ? - dms->window.dictLimit : 0; - const BYTE* const dictBase = dictMode == ZSTD_dictMatchState ? - dms->window.base : NULL; - const BYTE* const dictStart = dictMode == ZSTD_dictMatchState ? - dictBase + dictStartIndex : NULL; - const BYTE* const dictEnd = dictMode == ZSTD_dictMatchState ? - dms->window.nextSrc : NULL; - const U32 dictIndexDelta = dictMode == ZSTD_dictMatchState ? - prefixLowestIndex - (U32)(dictEnd - dictBase) : - 0; - const U32 dictHBitsL = dictMode == ZSTD_dictMatchState ? - dictCParams->hashLog : hBitsL; - const U32 dictHBitsS = dictMode == ZSTD_dictMatchState ? - dictCParams->chainLog : hBitsS; - const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictStart)); - DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_generic"); + const ZSTD_MatchState_t* const dms = ms->dictMatchState; + const ZSTD_compressionParameters* const dictCParams = &dms->cParams; + const U32* const dictHashLong = dms->hashTable; + const U32* const dictHashSmall = dms->chainTable; + const U32 dictStartIndex = dms->window.dictLimit; + const BYTE* const dictBase = dms->window.base; + const BYTE* const dictStart = dictBase + dictStartIndex; + const BYTE* const dictEnd = dms->window.nextSrc; + const U32 dictIndexDelta = prefixLowestIndex - (U32)(dictEnd - dictBase); + const U32 dictHBitsL = dictCParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS; + const U32 dictHBitsS = dictCParams->chainLog + ZSTD_SHORT_CACHE_TAG_BITS; + const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictStart)); - assert(dictMode == ZSTD_noDict || dictMode == ZSTD_dictMatchState); + DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_dictMatchState_generic"); /* if a dictionary is attached, it must be within window range */ - if (dictMode == ZSTD_dictMatchState) { - assert(ms->window.dictLimit + (1U << cParams->windowLog) >= endIndex); + assert(ms->window.dictLimit + (1U << cParams->windowLog) >= endIndex); + + if (ms->prefetchCDictTables) { + size_t const hashTableBytes = (((size_t)1) << dictCParams->hashLog) * sizeof(U32); + size_t const chainTableBytes = (((size_t)1) << dictCParams->chainLog) * sizeof(U32); + PREFETCH_AREA(dictHashLong, hashTableBytes); + PREFETCH_AREA(dictHashSmall, chainTableBytes); } /* init */ ip += (dictAndPrefixLength == 0); - if (dictMode == ZSTD_noDict) { - U32 const curr = (U32)(ip - base); - U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, curr, cParams->windowLog); - U32 const maxRep = curr - windowLow; - if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0; - if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0; - } - if (dictMode == ZSTD_dictMatchState) { - /* dictMatchState repCode checks don't currently handle repCode == 0 - * disabling. */ - assert(offset_1 <= dictAndPrefixLength); - assert(offset_2 <= dictAndPrefixLength); - } + + /* dictMatchState repCode checks don't currently handle repCode == 0 + * disabling. */ + assert(offset_1 <= dictAndPrefixLength); + assert(offset_2 <= dictAndPrefixLength); /* Main Search Loop */ while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */ @@ -19875,51 +28122,42 @@ size_t ZSTD_compressBlock_doubleFast_generic( U32 offset; size_t const h2 = ZSTD_hashPtr(ip, hBitsL, 8); size_t const h = ZSTD_hashPtr(ip, hBitsS, mls); - size_t const dictHL = ZSTD_hashPtr(ip, dictHBitsL, 8); - size_t const dictHS = ZSTD_hashPtr(ip, dictHBitsS, mls); + size_t const dictHashAndTagL = ZSTD_hashPtr(ip, dictHBitsL, 8); + size_t const dictHashAndTagS = ZSTD_hashPtr(ip, dictHBitsS, mls); + U32 const dictMatchIndexAndTagL = dictHashLong[dictHashAndTagL >> ZSTD_SHORT_CACHE_TAG_BITS]; + U32 const dictMatchIndexAndTagS = dictHashSmall[dictHashAndTagS >> ZSTD_SHORT_CACHE_TAG_BITS]; + int const dictTagsMatchL = ZSTD_comparePackedTags(dictMatchIndexAndTagL, dictHashAndTagL); + int const dictTagsMatchS = ZSTD_comparePackedTags(dictMatchIndexAndTagS, dictHashAndTagS); U32 const curr = (U32)(ip-base); U32 const matchIndexL = hashLong[h2]; U32 matchIndexS = hashSmall[h]; const BYTE* matchLong = base + matchIndexL; const BYTE* match = base + matchIndexS; const U32 repIndex = curr + 1 - offset_1; - const BYTE* repMatch = (dictMode == ZSTD_dictMatchState - && repIndex < prefixLowestIndex) ? + const BYTE* repMatch = (repIndex < prefixLowestIndex) ? dictBase + (repIndex - dictIndexDelta) : base + repIndex; hashLong[h2] = hashSmall[h] = curr; /* update hash tables */ - /* check dictMatchState repcode */ - if (dictMode == ZSTD_dictMatchState - && ((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */) + /* check repcode */ + if ((ZSTD_index_overlap_check(prefixLowestIndex, repIndex)) && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; ip++; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, mLength-MINMATCH); - goto _match_stored; - } - - /* check noDict repcode */ - if ( dictMode == ZSTD_noDict - && ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)))) { - mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; - ip++; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); goto _match_stored; } - if (matchIndexL > prefixLowestIndex) { + if ((matchIndexL >= prefixLowestIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) { /* check prefix long match */ - if (MEM_read64(matchLong) == MEM_read64(ip)) { - mLength = ZSTD_count(ip+8, matchLong+8, iend) + 8; - offset = (U32)(ip-matchLong); - while (((ip>anchor) & (matchLong>prefixLowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ - goto _match_found; - } - } else if (dictMode == ZSTD_dictMatchState) { + mLength = ZSTD_count(ip+8, matchLong+8, iend) + 8; + offset = (U32)(ip-matchLong); + while (((ip>anchor) & (matchLong>prefixLowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ + goto _match_found; + } else if (dictTagsMatchL) { /* check dictMatchState long match */ - U32 const dictMatchIndexL = dictHashLong[dictHL]; + U32 const dictMatchIndexL = dictMatchIndexAndTagL >> ZSTD_SHORT_CACHE_TAG_BITS; const BYTE* dictMatchL = dictBase + dictMatchIndexL; assert(dictMatchL < dictEnd); @@ -19931,13 +28169,13 @@ size_t ZSTD_compressBlock_doubleFast_generic( } } if (matchIndexS > prefixLowestIndex) { - /* check prefix short match */ + /* short match candidate */ if (MEM_read32(match) == MEM_read32(ip)) { goto _search_next_long; } - } else if (dictMode == ZSTD_dictMatchState) { + } else if (dictTagsMatchS) { /* check dictMatchState short match */ - U32 const dictMatchIndexS = dictHashSmall[dictHS]; + U32 const dictMatchIndexS = dictMatchIndexAndTagS >> ZSTD_SHORT_CACHE_TAG_BITS; match = dictBase + dictMatchIndexS; matchIndexS = dictMatchIndexS + dictIndexDelta; @@ -19952,25 +28190,24 @@ size_t ZSTD_compressBlock_doubleFast_generic( continue; _search_next_long: - { size_t const hl3 = ZSTD_hashPtr(ip+1, hBitsL, 8); - size_t const dictHLNext = ZSTD_hashPtr(ip+1, dictHBitsL, 8); + size_t const dictHashAndTagL3 = ZSTD_hashPtr(ip+1, dictHBitsL, 8); U32 const matchIndexL3 = hashLong[hl3]; + U32 const dictMatchIndexAndTagL3 = dictHashLong[dictHashAndTagL3 >> ZSTD_SHORT_CACHE_TAG_BITS]; + int const dictTagsMatchL3 = ZSTD_comparePackedTags(dictMatchIndexAndTagL3, dictHashAndTagL3); const BYTE* matchL3 = base + matchIndexL3; hashLong[hl3] = curr + 1; /* check prefix long +1 match */ - if (matchIndexL3 > prefixLowestIndex) { - if (MEM_read64(matchL3) == MEM_read64(ip+1)) { - mLength = ZSTD_count(ip+9, matchL3+8, iend) + 8; - ip++; - offset = (U32)(ip-matchL3); - while (((ip>anchor) & (matchL3>prefixLowest)) && (ip[-1] == matchL3[-1])) { ip--; matchL3--; mLength++; } /* catch up */ - goto _match_found; - } - } else if (dictMode == ZSTD_dictMatchState) { + if ((matchIndexL3 >= prefixLowestIndex) && (MEM_read64(matchL3) == MEM_read64(ip+1))) { + mLength = ZSTD_count(ip+9, matchL3+8, iend) + 8; + ip++; + offset = (U32)(ip-matchL3); + while (((ip>anchor) & (matchL3>prefixLowest)) && (ip[-1] == matchL3[-1])) { ip--; matchL3--; mLength++; } /* catch up */ + goto _match_found; + } else if (dictTagsMatchL3) { /* check dict long +1 match */ - U32 const dictMatchIndexL3 = dictHashLong[dictHLNext]; + U32 const dictMatchIndexL3 = dictMatchIndexAndTagL3 >> ZSTD_SHORT_CACHE_TAG_BITS; const BYTE* dictMatchL3 = dictBase + dictMatchIndexL3; assert(dictMatchL3 < dictEnd); if (dictMatchL3 > dictStart && MEM_read64(dictMatchL3) == MEM_read64(ip+1)) { @@ -19982,7 +28219,7 @@ size_t ZSTD_compressBlock_doubleFast_generic( } } } /* if no long +1 match, explore the short match we found */ - if (dictMode == ZSTD_dictMatchState && matchIndexS < prefixLowestIndex) { + if (matchIndexS < prefixLowestIndex) { mLength = ZSTD_count_2segments(ip+4, match+4, iend, dictEnd, prefixLowest) + 4; offset = (U32)(curr - matchIndexS); while (((ip>anchor) & (match>dictStart)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ @@ -19992,13 +28229,11 @@ size_t ZSTD_compressBlock_doubleFast_generic( while (((ip>anchor) & (match>prefixLowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ } - /* fall-through */ - _match_found: offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); _match_stored: /* match found */ @@ -20016,56 +28251,58 @@ size_t ZSTD_compressBlock_doubleFast_generic( } /* check immediate repcode */ - if (dictMode == ZSTD_dictMatchState) { - while (ip <= ilimit) { - U32 const current2 = (U32)(ip-base); - U32 const repIndex2 = current2 - offset_2; - const BYTE* repMatch2 = dictMode == ZSTD_dictMatchState - && repIndex2 < prefixLowestIndex ? - dictBase + repIndex2 - dictIndexDelta : - base + repIndex2; - if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */) - && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { - const BYTE* const repEnd2 = repIndex2 < prefixLowestIndex ? dictEnd : iend; - size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixLowest) + 4; - U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, repLength2-MINMATCH); - hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; - hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2; - ip += repLength2; - anchor = ip; - continue; - } - break; - } } - - if (dictMode == ZSTD_noDict) { - while ( (ip <= ilimit) - && ( (offset_2>0) - & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) { - /* store sequence */ - size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; - U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */ - hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip-base); - hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip-base); - ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, rLength-MINMATCH); - ip += rLength; + while (ip <= ilimit) { + U32 const current2 = (U32)(ip-base); + U32 const repIndex2 = current2 - offset_2; + const BYTE* repMatch2 = repIndex2 < prefixLowestIndex ? + dictBase + repIndex2 - dictIndexDelta : + base + repIndex2; + if ( (ZSTD_index_overlap_check(prefixLowestIndex, repIndex2)) + && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { + const BYTE* const repEnd2 = repIndex2 < prefixLowestIndex ? dictEnd : iend; + size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixLowest) + 4; + U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2); + hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; + hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2; + ip += repLength2; anchor = ip; - continue; /* faster when present ... (?) */ - } } } + continue; + } + break; + } + } } /* while (ip < ilimit) */ /* save reps for next block */ - rep[0] = offset_1 ? offset_1 : offsetSaved; - rep[1] = offset_2 ? offset_2 : offsetSaved; + rep[0] = offset_1; + rep[1] = offset_2; /* Return the last literals size */ return (size_t)(iend - anchor); } +#define ZSTD_GEN_DFAST_FN(dictMode, mls) \ + static size_t ZSTD_compressBlock_doubleFast_##dictMode##_##mls( \ + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ + void const* src, size_t srcSize) \ + { \ + return ZSTD_compressBlock_doubleFast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mls); \ + } + +ZSTD_GEN_DFAST_FN(noDict, 4) +ZSTD_GEN_DFAST_FN(noDict, 5) +ZSTD_GEN_DFAST_FN(noDict, 6) +ZSTD_GEN_DFAST_FN(noDict, 7) + +ZSTD_GEN_DFAST_FN(dictMatchState, 4) +ZSTD_GEN_DFAST_FN(dictMatchState, 5) +ZSTD_GEN_DFAST_FN(dictMatchState, 6) +ZSTD_GEN_DFAST_FN(dictMatchState, 7) + size_t ZSTD_compressBlock_doubleFast( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { const U32 mls = ms->cParams.minMatch; @@ -20073,19 +28310,19 @@ size_t ZSTD_compressBlock_doubleFast( { default: /* includes case 3 */ case 4 : - return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_noDict); + return ZSTD_compressBlock_doubleFast_noDict_4(ms, seqStore, rep, src, srcSize); case 5 : - return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_noDict); + return ZSTD_compressBlock_doubleFast_noDict_5(ms, seqStore, rep, src, srcSize); case 6 : - return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_noDict); + return ZSTD_compressBlock_doubleFast_noDict_6(ms, seqStore, rep, src, srcSize); case 7 : - return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_noDict); + return ZSTD_compressBlock_doubleFast_noDict_7(ms, seqStore, rep, src, srcSize); } } size_t ZSTD_compressBlock_doubleFast_dictMatchState( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { const U32 mls = ms->cParams.minMatch; @@ -20093,19 +28330,21 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState( { default: /* includes case 3 */ case 4 : - return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_dictMatchState); + return ZSTD_compressBlock_doubleFast_dictMatchState_4(ms, seqStore, rep, src, srcSize); case 5 : - return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_dictMatchState); + return ZSTD_compressBlock_doubleFast_dictMatchState_5(ms, seqStore, rep, src, srcSize); case 6 : - return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_dictMatchState); + return ZSTD_compressBlock_doubleFast_dictMatchState_6(ms, seqStore, rep, src, srcSize); case 7 : - return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_dictMatchState); + return ZSTD_compressBlock_doubleFast_dictMatchState_7(ms, seqStore, rep, src, srcSize); } } -static size_t ZSTD_compressBlock_doubleFast_extDict_generic( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_compressBlock_doubleFast_extDict_generic( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize, U32 const mls /* template */) { @@ -20135,7 +28374,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( /* if extDict is invalidated due to maxDistance, switch to "regular" variant */ if (prefixStartIndex == dictStartIndex) - return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, mls, ZSTD_noDict); + return ZSTD_compressBlock_doubleFast(ms, seqStore, rep, src, srcSize); /* Search Loop */ while (ip < ilimit) { /* < instead of <=, because (ip+1) */ @@ -20156,13 +28395,13 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( size_t mLength; hashSmall[hSmall] = hashLong[hLong] = curr; /* update hash table */ - if ((((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow : ensure repIndex doesn't overlap dict + prefix */ - & (repIndex > dictStartIndex)) + if (((ZSTD_index_overlap_check(prefixStartIndex, repIndex)) + & (offset_1 <= curr+1 - dictStartIndex)) /* note: we are searching at curr+1 */ && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; ip++; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); } else { if ((matchLongIndex > dictStartIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) { const BYTE* const matchEnd = matchLongIndex < prefixStartIndex ? dictEnd : iend; @@ -20173,7 +28412,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); } else if ((matchIndex > dictStartIndex) && (MEM_read32(match) == MEM_read32(ip))) { size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8); @@ -20198,7 +28437,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( } offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); } else { ip += ((ip-anchor) >> kSearchStrength) + 1; @@ -20224,13 +28463,13 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( U32 const current2 = (U32)(ip-base); U32 const repIndex2 = current2 - offset_2; const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2; - if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) /* intentional overflow : ensure repIndex2 doesn't overlap dict + prefix */ - & (repIndex2 > dictStartIndex)) + if ( ((ZSTD_index_overlap_check(prefixStartIndex, repIndex2)) + & (offset_2 <= current2 - dictStartIndex)) && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, repLength2-MINMATCH); + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2); hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2; ip += repLength2; @@ -20248,9 +28487,13 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( return (size_t)(iend - anchor); } +ZSTD_GEN_DFAST_FN(extDict, 4) +ZSTD_GEN_DFAST_FN(extDict, 5) +ZSTD_GEN_DFAST_FN(extDict, 6) +ZSTD_GEN_DFAST_FN(extDict, 7) size_t ZSTD_compressBlock_doubleFast_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { U32 const mls = ms->cParams.minMatch; @@ -20258,19 +28501,21 @@ size_t ZSTD_compressBlock_doubleFast_extDict( { default: /* includes case 3 */ case 4 : - return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 4); + return ZSTD_compressBlock_doubleFast_extDict_4(ms, seqStore, rep, src, srcSize); case 5 : - return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 5); + return ZSTD_compressBlock_doubleFast_extDict_5(ms, seqStore, rep, src, srcSize); case 6 : - return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 6); + return ZSTD_compressBlock_doubleFast_extDict_6(ms, seqStore, rep, src, srcSize); case 7 : - return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 7); + return ZSTD_compressBlock_doubleFast_extDict_7(ms, seqStore, rep, src, srcSize); } } + +#endif /* ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR */ /**** ended inlining compress/zstd_double_fast.c ****/ /**** start inlining compress/zstd_fast.c ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -20282,8 +28527,46 @@ size_t ZSTD_compressBlock_doubleFast_extDict( /**** skipping file: zstd_compress_internal.h ****/ /**** skipping file: zstd_fast.h ****/ +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +void ZSTD_fillHashTableForCDict(ZSTD_MatchState_t* ms, + const void* const end, + ZSTD_dictTableLoadMethod_e dtlm) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hBits = cParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS; + U32 const mls = cParams->minMatch; + const BYTE* const base = ms->window.base; + const BYTE* ip = base + ms->nextToUpdate; + const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; + const U32 fastHashFillStep = 3; + + /* Currently, we always use ZSTD_dtlm_full for filling CDict tables. + * Feel free to remove this assert if there's a good reason! */ + assert(dtlm == ZSTD_dtlm_full); + + /* Always insert every fastHashFillStep position into the hash table. + * Insert the other positions if their hash entry is empty. + */ + for ( ; ip + fastHashFillStep < iend + 2; ip += fastHashFillStep) { + U32 const curr = (U32)(ip - base); + { size_t const hashAndTag = ZSTD_hashPtr(ip, hBits, mls); + ZSTD_writeTaggedIndex(hashTable, hashAndTag, curr); } + + if (dtlm == ZSTD_dtlm_fast) continue; + /* Only load extra positions for ZSTD_dtlm_full */ + { U32 p; + for (p = 1; p < fastHashFillStep; ++p) { + size_t const hashAndTag = ZSTD_hashPtr(ip + p, hBits, mls); + if (hashTable[hashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS] == 0) { /* not yet filled */ + ZSTD_writeTaggedIndex(hashTable, hashAndTag, curr + p); + } } } } +} -void ZSTD_fillHashTable(ZSTD_matchState_t* ms, +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +void ZSTD_fillHashTableForCCtx(ZSTD_MatchState_t* ms, const void* const end, ZSTD_dictTableLoadMethod_e dtlm) { @@ -20296,6 +28579,10 @@ void ZSTD_fillHashTable(ZSTD_matchState_t* ms, const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; const U32 fastHashFillStep = 3; + /* Currently, we always use ZSTD_dtlm_fast for filling CCtx tables. + * Feel free to remove this assert if there's a good reason! */ + assert(dtlm == ZSTD_dtlm_fast); + /* Always insert every fastHashFillStep position into the hash table. * Insert the other positions if their hash entry is empty. */ @@ -20313,171 +28600,405 @@ void ZSTD_fillHashTable(ZSTD_matchState_t* ms, } } } } } +void ZSTD_fillHashTable(ZSTD_MatchState_t* ms, + const void* const end, + ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp) +{ + if (tfp == ZSTD_tfp_forCDict) { + ZSTD_fillHashTableForCDict(ms, end, dtlm); + } else { + ZSTD_fillHashTableForCCtx(ms, end, dtlm); + } +} + -FORCE_INLINE_TEMPLATE size_t -ZSTD_compressBlock_fast_generic( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], +typedef int (*ZSTD_match4Found) (const BYTE* currentPtr, const BYTE* matchAddress, U32 matchIdx, U32 idxLowLimit); + +static int +ZSTD_match4Found_cmov(const BYTE* currentPtr, const BYTE* matchAddress, U32 matchIdx, U32 idxLowLimit) +{ + /* Array of ~random data, should have low probability of matching data. + * Load from here if the index is invalid. + * Used to avoid unpredictable branches. */ + static const BYTE dummy[] = {0x12,0x34,0x56,0x78}; + + /* currentIdx >= lowLimit is a (somewhat) unpredictable branch. + * However expression below compiles into conditional move. + */ + const BYTE* mvalAddr = ZSTD_selectAddr(matchIdx, idxLowLimit, matchAddress, dummy); + /* Note: this used to be written as : return test1 && test2; + * Unfortunately, once inlined, these tests become branches, + * in which case it becomes critical that they are executed in the right order (test1 then test2). + * So we have to write these tests in a specific manner to ensure their ordering. + */ + if (MEM_read32(currentPtr) != MEM_read32(mvalAddr)) return 0; + /* force ordering of these tests, which matters once the function is inlined, as they become branches */ +#if defined(__GNUC__) + __asm__(""); +#endif + return matchIdx >= idxLowLimit; +} + +static int +ZSTD_match4Found_branch(const BYTE* currentPtr, const BYTE* matchAddress, U32 matchIdx, U32 idxLowLimit) +{ + /* using a branch instead of a cmov, + * because it's faster in scenarios where matchIdx >= idxLowLimit is generally true, + * aka almost all candidates are within range */ + U32 mval; + if (matchIdx >= idxLowLimit) { + mval = MEM_read32(matchAddress); + } else { + mval = MEM_read32(currentPtr) ^ 1; /* guaranteed to not match. */ + } + + return (MEM_read32(currentPtr) == mval); +} + + +/** + * If you squint hard enough (and ignore repcodes), the search operation at any + * given position is broken into 4 stages: + * + * 1. Hash (map position to hash value via input read) + * 2. Lookup (map hash val to index via hashtable read) + * 3. Load (map index to value at that position via input read) + * 4. Compare + * + * Each of these steps involves a memory read at an address which is computed + * from the previous step. This means these steps must be sequenced and their + * latencies are cumulative. + * + * Rather than do 1->2->3->4 sequentially for a single position before moving + * onto the next, this implementation interleaves these operations across the + * next few positions: + * + * R = Repcode Read & Compare + * H = Hash + * T = Table Lookup + * M = Match Read & Compare + * + * Pos | Time --> + * ----+------------------- + * N | ... M + * N+1 | ... TM + * N+2 | R H T M + * N+3 | H TM + * N+4 | R H T M + * N+5 | H ... + * N+6 | R ... + * + * This is very much analogous to the pipelining of execution in a CPU. And just + * like a CPU, we have to dump the pipeline when we find a match (i.e., take a + * branch). + * + * When this happens, we throw away our current state, and do the following prep + * to re-enter the loop: + * + * Pos | Time --> + * ----+------------------- + * N | H T + * N+1 | H + * + * This is also the work we do at the beginning to enter the loop initially. + */ +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_compressBlock_fast_noDict_generic( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize, - U32 const mls) + U32 const mls, int useCmov) { const ZSTD_compressionParameters* const cParams = &ms->cParams; U32* const hashTable = ms->hashTable; U32 const hlog = cParams->hashLog; - /* support stepSize of 0 */ - size_t const stepSize = cParams->targetLength + !(cParams->targetLength) + 1; + size_t const stepSize = cParams->targetLength + !(cParams->targetLength) + 1; /* min 2 */ const BYTE* const base = ms->window.base; const BYTE* const istart = (const BYTE*)src; - /* We check ip0 (ip + 0) and ip1 (ip + 1) each loop */ - const BYTE* ip0 = istart; - const BYTE* ip1; - const BYTE* anchor = istart; const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); const U32 prefixStartIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog); const BYTE* const prefixStart = base + prefixStartIndex; const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - HASH_READ_SIZE; - U32 offset_1=rep[0], offset_2=rep[1]; - U32 offsetSaved = 0; - /* init */ + const BYTE* anchor = istart; + const BYTE* ip0 = istart; + const BYTE* ip1; + const BYTE* ip2; + const BYTE* ip3; + U32 current0; + + U32 rep_offset1 = rep[0]; + U32 rep_offset2 = rep[1]; + U32 offsetSaved1 = 0, offsetSaved2 = 0; + + size_t hash0; /* hash for ip0 */ + size_t hash1; /* hash for ip1 */ + U32 matchIdx; /* match idx for ip0 */ + + U32 offcode; + const BYTE* match0; + size_t mLength; + + /* ip0 and ip1 are always adjacent. The targetLength skipping and + * uncompressibility acceleration is applied to every other position, + * matching the behavior of #1562. step therefore represents the gap + * between pairs of positions, from ip0 to ip2 or ip1 to ip3. */ + size_t step; + const BYTE* nextStep; + const size_t kStepIncr = (1 << (kSearchStrength - 1)); + const ZSTD_match4Found matchFound = useCmov ? ZSTD_match4Found_cmov : ZSTD_match4Found_branch; + DEBUGLOG(5, "ZSTD_compressBlock_fast_generic"); ip0 += (ip0 == prefixStart); - ip1 = ip0 + 1; { U32 const curr = (U32)(ip0 - base); U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, curr, cParams->windowLog); U32 const maxRep = curr - windowLow; - if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0; - if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0; + if (rep_offset2 > maxRep) offsetSaved2 = rep_offset2, rep_offset2 = 0; + if (rep_offset1 > maxRep) offsetSaved1 = rep_offset1, rep_offset1 = 0; } - /* Main Search Loop */ -#ifdef __INTEL_COMPILER - /* From intel 'The vector pragma indicates that the loop should be - * vectorized if it is legal to do so'. Can be used together with - * #pragma ivdep (but have opted to exclude that because intel - * warns against using it).*/ - #pragma vector always -#endif - while (ip1 < ilimit) { /* < instead of <=, because check at ip0+2 */ - size_t mLength; - BYTE const* ip2 = ip0 + 2; - size_t const h0 = ZSTD_hashPtr(ip0, hlog, mls); - U32 const val0 = MEM_read32(ip0); - size_t const h1 = ZSTD_hashPtr(ip1, hlog, mls); - U32 const val1 = MEM_read32(ip1); - U32 const current0 = (U32)(ip0-base); - U32 const current1 = (U32)(ip1-base); - U32 const matchIndex0 = hashTable[h0]; - U32 const matchIndex1 = hashTable[h1]; - BYTE const* repMatch = ip2 - offset_1; - const BYTE* match0 = base + matchIndex0; - const BYTE* match1 = base + matchIndex1; - U32 offcode; + /* start each op */ +_start: /* Requires: ip0 */ -#if defined(__aarch64__) - PREFETCH_L1(ip0+256); -#endif + step = stepSize; + nextStep = ip0 + kStepIncr; - hashTable[h0] = current0; /* update hash table */ - hashTable[h1] = current1; /* update hash table */ + /* calculate positions, ip0 - anchor == 0, so we skip step calc */ + ip1 = ip0 + 1; + ip2 = ip0 + step; + ip3 = ip2 + 1; + + if (ip3 >= ilimit) { + goto _cleanup; + } - assert(ip0 + 1 == ip1); + hash0 = ZSTD_hashPtr(ip0, hlog, mls); + hash1 = ZSTD_hashPtr(ip1, hlog, mls); - if ((offset_1 > 0) & (MEM_read32(repMatch) == MEM_read32(ip2))) { - mLength = (ip2[-1] == repMatch[-1]) ? 1 : 0; - ip0 = ip2 - mLength; - match0 = repMatch - mLength; + matchIdx = hashTable[hash0]; + + do { + /* load repcode match for ip[2]*/ + const U32 rval = MEM_read32(ip2 - rep_offset1); + + /* write back hash table entry */ + current0 = (U32)(ip0 - base); + hashTable[hash0] = current0; + + /* check repcode at ip[2] */ + if ((MEM_read32(ip2) == rval) & (rep_offset1 > 0)) { + ip0 = ip2; + match0 = ip0 - rep_offset1; + mLength = ip0[-1] == match0[-1]; + ip0 -= mLength; + match0 -= mLength; + offcode = REPCODE1_TO_OFFBASE; mLength += 4; - offcode = 0; + + /* Write next hash table entry: it's already calculated. + * This write is known to be safe because ip1 is before the + * repcode (ip2). */ + hashTable[hash1] = (U32)(ip1 - base); + goto _match; } - if ((matchIndex0 > prefixStartIndex) && MEM_read32(match0) == val0) { - /* found a regular match */ + + if (matchFound(ip0, base + matchIdx, matchIdx, prefixStartIndex)) { + /* Write next hash table entry (it's already calculated). + * This write is known to be safe because the ip1 == ip0 + 1, + * so searching will resume after ip1 */ + hashTable[hash1] = (U32)(ip1 - base); + goto _offset; } - if ((matchIndex1 > prefixStartIndex) && MEM_read32(match1) == val1) { - /* found a regular match after one literal */ - ip0 = ip1; - match0 = match1; + + /* lookup ip[1] */ + matchIdx = hashTable[hash1]; + + /* hash ip[2] */ + hash0 = hash1; + hash1 = ZSTD_hashPtr(ip2, hlog, mls); + + /* advance to next positions */ + ip0 = ip1; + ip1 = ip2; + ip2 = ip3; + + /* write back hash table entry */ + current0 = (U32)(ip0 - base); + hashTable[hash0] = current0; + + if (matchFound(ip0, base + matchIdx, matchIdx, prefixStartIndex)) { + /* Write next hash table entry, since it's already calculated */ + if (step <= 4) { + /* Avoid writing an index if it's >= position where search will resume. + * The minimum possible match has length 4, so search can resume at ip0 + 4. + */ + hashTable[hash1] = (U32)(ip1 - base); + } goto _offset; } - { size_t const step = ((size_t)(ip0-anchor) >> (kSearchStrength - 1)) + stepSize; - assert(step >= 2); - ip0 += step; - ip1 += step; - continue; - } -_offset: /* Requires: ip0, match0 */ - /* Compute the offset code */ - offset_2 = offset_1; - offset_1 = (U32)(ip0-match0); - offcode = offset_1 + ZSTD_REP_MOVE; - mLength = 4; - /* Count the backwards match length */ - while (((ip0>anchor) & (match0>prefixStart)) - && (ip0[-1] == match0[-1])) { ip0--; match0--; mLength++; } /* catch up */ -_match: /* Requires: ip0, match0, offcode */ - /* Count the forward length */ - mLength += ZSTD_count(ip0+mLength, match0+mLength, iend); - ZSTD_storeSeq(seqStore, (size_t)(ip0-anchor), anchor, iend, offcode, mLength-MINMATCH); - /* match found */ - ip0 += mLength; - anchor = ip0; + /* lookup ip[1] */ + matchIdx = hashTable[hash1]; - if (ip0 <= ilimit) { - /* Fill Table */ - assert(base+current0+2 > istart); /* check base overflow */ - hashTable[ZSTD_hashPtr(base+current0+2, hlog, mls)] = current0+2; /* here because current+2 could be > iend-8 */ - hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base); + /* hash ip[2] */ + hash0 = hash1; + hash1 = ZSTD_hashPtr(ip2, hlog, mls); - if (offset_2 > 0) { /* offset_2==0 means offset_2 is invalidated */ - while ( (ip0 <= ilimit) && (MEM_read32(ip0) == MEM_read32(ip0 - offset_2)) ) { - /* store sequence */ - size_t const rLength = ZSTD_count(ip0+4, ip0+4-offset_2, iend) + 4; - { U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; } /* swap offset_2 <=> offset_1 */ - hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base); - ip0 += rLength; - ZSTD_storeSeq(seqStore, 0 /*litLen*/, anchor, iend, 0 /*offCode*/, rLength-MINMATCH); - anchor = ip0; - continue; /* faster when present (confirmed on gcc-8) ... (?) */ - } } } - ip1 = ip0 + 1; - } + /* advance to next positions */ + ip0 = ip1; + ip1 = ip2; + ip2 = ip0 + step; + ip3 = ip1 + step; + + /* calculate step */ + if (ip2 >= nextStep) { + step++; + PREFETCH_L1(ip1 + 64); + PREFETCH_L1(ip1 + 128); + nextStep += kStepIncr; + } + } while (ip3 < ilimit); + +_cleanup: + /* Note that there are probably still a couple positions one could search. + * However, it seems to be a meaningful performance hit to try to search + * them. So let's not. */ + + /* When the repcodes are outside of the prefix, we set them to zero before the loop. + * When the offsets are still zero, we need to restore them after the block to have a correct + * repcode history. If only one offset was invalid, it is easy. The tricky case is when both + * offsets were invalid. We need to figure out which offset to refill with. + * - If both offsets are zero they are in the same order. + * - If both offsets are non-zero, we won't restore the offsets from `offsetSaved[12]`. + * - If only one is zero, we need to decide which offset to restore. + * - If rep_offset1 is non-zero, then rep_offset2 must be offsetSaved1. + * - It is impossible for rep_offset2 to be non-zero. + * + * So if rep_offset1 started invalid (offsetSaved1 != 0) and became valid (rep_offset1 != 0), then + * set rep[0] = rep_offset1 and rep[1] = offsetSaved1. + */ + offsetSaved2 = ((offsetSaved1 != 0) && (rep_offset1 != 0)) ? offsetSaved1 : offsetSaved2; /* save reps for next block */ - rep[0] = offset_1 ? offset_1 : offsetSaved; - rep[1] = offset_2 ? offset_2 : offsetSaved; + rep[0] = rep_offset1 ? rep_offset1 : offsetSaved1; + rep[1] = rep_offset2 ? rep_offset2 : offsetSaved2; /* Return the last literals size */ return (size_t)(iend - anchor); + +_offset: /* Requires: ip0, idx */ + + /* Compute the offset code. */ + match0 = base + matchIdx; + rep_offset2 = rep_offset1; + rep_offset1 = (U32)(ip0-match0); + offcode = OFFSET_TO_OFFBASE(rep_offset1); + mLength = 4; + + /* Count the backwards match length. */ + while (((ip0>anchor) & (match0>prefixStart)) && (ip0[-1] == match0[-1])) { + ip0--; + match0--; + mLength++; + } + +_match: /* Requires: ip0, match0, offcode */ + + /* Count the forward length. */ + mLength += ZSTD_count(ip0 + mLength, match0 + mLength, iend); + + ZSTD_storeSeq(seqStore, (size_t)(ip0 - anchor), anchor, iend, offcode, mLength); + + ip0 += mLength; + anchor = ip0; + + /* Fill table and check for immediate repcode. */ + if (ip0 <= ilimit) { + /* Fill Table */ + assert(base+current0+2 > istart); /* check base overflow */ + hashTable[ZSTD_hashPtr(base+current0+2, hlog, mls)] = current0+2; /* here because current+2 could be > iend-8 */ + hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base); + + if (rep_offset2 > 0) { /* rep_offset2==0 means rep_offset2 is invalidated */ + while ( (ip0 <= ilimit) && (MEM_read32(ip0) == MEM_read32(ip0 - rep_offset2)) ) { + /* store sequence */ + size_t const rLength = ZSTD_count(ip0+4, ip0+4-rep_offset2, iend) + 4; + { U32 const tmpOff = rep_offset2; rep_offset2 = rep_offset1; rep_offset1 = tmpOff; } /* swap rep_offset2 <=> rep_offset1 */ + hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base); + ip0 += rLength; + ZSTD_storeSeq(seqStore, 0 /*litLen*/, anchor, iend, REPCODE1_TO_OFFBASE, rLength); + anchor = ip0; + continue; /* faster when present (confirmed on gcc-8) ... (?) */ + } } } + + goto _start; } +#define ZSTD_GEN_FAST_FN(dictMode, mml, cmov) \ + static size_t ZSTD_compressBlock_fast_##dictMode##_##mml##_##cmov( \ + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ + void const* src, size_t srcSize) \ + { \ + return ZSTD_compressBlock_fast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mml, cmov); \ + } + +ZSTD_GEN_FAST_FN(noDict, 4, 1) +ZSTD_GEN_FAST_FN(noDict, 5, 1) +ZSTD_GEN_FAST_FN(noDict, 6, 1) +ZSTD_GEN_FAST_FN(noDict, 7, 1) + +ZSTD_GEN_FAST_FN(noDict, 4, 0) +ZSTD_GEN_FAST_FN(noDict, 5, 0) +ZSTD_GEN_FAST_FN(noDict, 6, 0) +ZSTD_GEN_FAST_FN(noDict, 7, 0) size_t ZSTD_compressBlock_fast( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { - U32 const mls = ms->cParams.minMatch; + U32 const mml = ms->cParams.minMatch; + /* use cmov when "candidate in range" branch is likely unpredictable */ + int const useCmov = ms->cParams.windowLog < 19; assert(ms->dictMatchState == NULL); - switch(mls) - { - default: /* includes case 3 */ - case 4 : - return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 4); - case 5 : - return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 5); - case 6 : - return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 6); - case 7 : - return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 7); + if (useCmov) { + switch(mml) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_fast_noDict_4_1(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_fast_noDict_5_1(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_fast_noDict_6_1(ms, seqStore, rep, src, srcSize); + case 7 : + return ZSTD_compressBlock_fast_noDict_7_1(ms, seqStore, rep, src, srcSize); + } + } else { + /* use a branch instead */ + switch(mml) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_fast_noDict_4_0(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_fast_noDict_5_0(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_fast_noDict_6_0(ms, seqStore, rep, src, srcSize); + case 7 : + return ZSTD_compressBlock_fast_noDict_7_0(ms, seqStore, rep, src, srcSize); + } } } FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR size_t ZSTD_compressBlock_fast_dictMatchState_generic( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize, U32 const mls) + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, U32 const mls, U32 const hasStep) { const ZSTD_compressionParameters* const cParams = &ms->cParams; U32* const hashTable = ms->hashTable; @@ -20486,16 +29007,16 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( U32 const stepSize = cParams->targetLength + !(cParams->targetLength); const BYTE* const base = ms->window.base; const BYTE* const istart = (const BYTE*)src; - const BYTE* ip = istart; + const BYTE* ip0 = istart; + const BYTE* ip1 = ip0 + stepSize; /* we assert below that stepSize >= 1 */ const BYTE* anchor = istart; const U32 prefixStartIndex = ms->window.dictLimit; const BYTE* const prefixStart = base + prefixStartIndex; const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - HASH_READ_SIZE; U32 offset_1=rep[0], offset_2=rep[1]; - U32 offsetSaved = 0; - const ZSTD_matchState_t* const dms = ms->dictMatchState; + const ZSTD_MatchState_t* const dms = ms->dictMatchState; const ZSTD_compressionParameters* const dictCParams = &dms->cParams ; const U32* const dictHashTable = dms->hashTable; const U32 dictStartIndex = dms->window.dictLimit; @@ -20503,127 +29024,183 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( const BYTE* const dictStart = dictBase + dictStartIndex; const BYTE* const dictEnd = dms->window.nextSrc; const U32 dictIndexDelta = prefixStartIndex - (U32)(dictEnd - dictBase); - const U32 dictAndPrefixLength = (U32)(ip - prefixStart + dictEnd - dictStart); - const U32 dictHLog = dictCParams->hashLog; + const U32 dictAndPrefixLength = (U32)(istart - prefixStart + dictEnd - dictStart); + const U32 dictHBits = dictCParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS; /* if a dictionary is still attached, it necessarily means that * it is within window size. So we just check it. */ const U32 maxDistance = 1U << cParams->windowLog; - const U32 endIndex = (U32)((size_t)(ip - base) + srcSize); + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); assert(endIndex - prefixStartIndex <= maxDistance); (void)maxDistance; (void)endIndex; /* these variables are not used when assert() is disabled */ + (void)hasStep; /* not currently specialized on whether it's accelerated */ + /* ensure there will be no underflow * when translating a dict index into a local index */ assert(prefixStartIndex >= (U32)(dictEnd - dictBase)); + if (ms->prefetchCDictTables) { + size_t const hashTableBytes = (((size_t)1) << dictCParams->hashLog) * sizeof(U32); + PREFETCH_AREA(dictHashTable, hashTableBytes); + } + /* init */ DEBUGLOG(5, "ZSTD_compressBlock_fast_dictMatchState_generic"); - ip += (dictAndPrefixLength == 0); + ip0 += (dictAndPrefixLength == 0); /* dictMatchState repCode checks don't currently handle repCode == 0 * disabling. */ assert(offset_1 <= dictAndPrefixLength); assert(offset_2 <= dictAndPrefixLength); - /* Main Search Loop */ - while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */ + /* Outer search loop */ + assert(stepSize >= 1); + while (ip1 <= ilimit) { /* repcode check at (ip0 + 1) is safe because ip0 < ip1 */ size_t mLength; - size_t const h = ZSTD_hashPtr(ip, hlog, mls); - U32 const curr = (U32)(ip-base); - U32 const matchIndex = hashTable[h]; - const BYTE* match = base + matchIndex; - const U32 repIndex = curr + 1 - offset_1; - const BYTE* repMatch = (repIndex < prefixStartIndex) ? - dictBase + (repIndex - dictIndexDelta) : - base + repIndex; - hashTable[h] = curr; /* update hash table */ + size_t hash0 = ZSTD_hashPtr(ip0, hlog, mls); - if ( ((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow : ensure repIndex isn't overlapping dict + prefix */ - && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { - const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; - mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; - ip++; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, mLength-MINMATCH); - } else if ( (matchIndex <= prefixStartIndex) ) { - size_t const dictHash = ZSTD_hashPtr(ip, dictHLog, mls); - U32 const dictMatchIndex = dictHashTable[dictHash]; - const BYTE* dictMatch = dictBase + dictMatchIndex; - if (dictMatchIndex <= dictStartIndex || - MEM_read32(dictMatch) != MEM_read32(ip)) { - assert(stepSize >= 1); - ip += ((ip-anchor) >> kSearchStrength) + stepSize; - continue; - } else { - /* found a dict match */ - U32 const offset = (U32)(curr-dictMatchIndex-dictIndexDelta); - mLength = ZSTD_count_2segments(ip+4, dictMatch+4, iend, dictEnd, prefixStart) + 4; - while (((ip>anchor) & (dictMatch>dictStart)) - && (ip[-1] == dictMatch[-1])) { - ip--; dictMatch--; mLength++; + size_t const dictHashAndTag0 = ZSTD_hashPtr(ip0, dictHBits, mls); + U32 dictMatchIndexAndTag = dictHashTable[dictHashAndTag0 >> ZSTD_SHORT_CACHE_TAG_BITS]; + int dictTagsMatch = ZSTD_comparePackedTags(dictMatchIndexAndTag, dictHashAndTag0); + + U32 matchIndex = hashTable[hash0]; + U32 curr = (U32)(ip0 - base); + size_t step = stepSize; + const size_t kStepIncr = 1 << kSearchStrength; + const BYTE* nextStep = ip0 + kStepIncr; + + /* Inner search loop */ + while (1) { + const BYTE* match = base + matchIndex; + const U32 repIndex = curr + 1 - offset_1; + const BYTE* repMatch = (repIndex < prefixStartIndex) ? + dictBase + (repIndex - dictIndexDelta) : + base + repIndex; + const size_t hash1 = ZSTD_hashPtr(ip1, hlog, mls); + size_t const dictHashAndTag1 = ZSTD_hashPtr(ip1, dictHBits, mls); + hashTable[hash0] = curr; /* update hash table */ + + if ((ZSTD_index_overlap_check(prefixStartIndex, repIndex)) + && (MEM_read32(repMatch) == MEM_read32(ip0 + 1))) { + const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; + mLength = ZSTD_count_2segments(ip0 + 1 + 4, repMatch + 4, iend, repMatchEnd, prefixStart) + 4; + ip0++; + ZSTD_storeSeq(seqStore, (size_t) (ip0 - anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); + break; + } + + if (dictTagsMatch) { + /* Found a possible dict match */ + const U32 dictMatchIndex = dictMatchIndexAndTag >> ZSTD_SHORT_CACHE_TAG_BITS; + const BYTE* dictMatch = dictBase + dictMatchIndex; + if (dictMatchIndex > dictStartIndex && + MEM_read32(dictMatch) == MEM_read32(ip0)) { + /* To replicate extDict parse behavior, we only use dict matches when the normal matchIndex is invalid */ + if (matchIndex <= prefixStartIndex) { + U32 const offset = (U32) (curr - dictMatchIndex - dictIndexDelta); + mLength = ZSTD_count_2segments(ip0 + 4, dictMatch + 4, iend, dictEnd, prefixStart) + 4; + while (((ip0 > anchor) & (dictMatch > dictStart)) + && (ip0[-1] == dictMatch[-1])) { + ip0--; + dictMatch--; + mLength++; + } /* catch up */ + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStore, (size_t) (ip0 - anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); + break; + } + } + } + + if (ZSTD_match4Found_cmov(ip0, match, matchIndex, prefixStartIndex)) { + /* found a regular match of size >= 4 */ + U32 const offset = (U32) (ip0 - match); + mLength = ZSTD_count(ip0 + 4, match + 4, iend) + 4; + while (((ip0 > anchor) & (match > prefixStart)) + && (ip0[-1] == match[-1])) { + ip0--; + match--; + mLength++; } /* catch up */ offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t) (ip0 - anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); + break; } - } else if (MEM_read32(match) != MEM_read32(ip)) { - /* it's not a match, and we're not going to check the dictionary */ - assert(stepSize >= 1); - ip += ((ip-anchor) >> kSearchStrength) + stepSize; - continue; - } else { - /* found a regular match */ - U32 const offset = (U32)(ip-match); - mLength = ZSTD_count(ip+4, match+4, iend) + 4; - while (((ip>anchor) & (match>prefixStart)) - && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ - offset_2 = offset_1; - offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH); - } + + /* Prepare for next iteration */ + dictMatchIndexAndTag = dictHashTable[dictHashAndTag1 >> ZSTD_SHORT_CACHE_TAG_BITS]; + dictTagsMatch = ZSTD_comparePackedTags(dictMatchIndexAndTag, dictHashAndTag1); + matchIndex = hashTable[hash1]; + + if (ip1 >= nextStep) { + step++; + nextStep += kStepIncr; + } + ip0 = ip1; + ip1 = ip1 + step; + if (ip1 > ilimit) goto _cleanup; + + curr = (U32)(ip0 - base); + hash0 = hash1; + } /* end inner search loop */ /* match found */ - ip += mLength; - anchor = ip; + assert(mLength); + ip0 += mLength; + anchor = ip0; - if (ip <= ilimit) { + if (ip0 <= ilimit) { /* Fill Table */ assert(base+curr+2 > istart); /* check base overflow */ hashTable[ZSTD_hashPtr(base+curr+2, hlog, mls)] = curr+2; /* here because curr+2 could be > iend-8 */ - hashTable[ZSTD_hashPtr(ip-2, hlog, mls)] = (U32)(ip-2-base); + hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base); /* check immediate repcode */ - while (ip <= ilimit) { - U32 const current2 = (U32)(ip-base); + while (ip0 <= ilimit) { + U32 const current2 = (U32)(ip0-base); U32 const repIndex2 = current2 - offset_2; const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? dictBase - dictIndexDelta + repIndex2 : base + repIndex2; - if ( ((U32)((prefixStartIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */) - && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { + if ( (ZSTD_index_overlap_check(prefixStartIndex, repIndex2)) + && (MEM_read32(repMatch2) == MEM_read32(ip0))) { const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; - size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; + size_t const repLength2 = ZSTD_count_2segments(ip0+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, repLength2-MINMATCH); - hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2; - ip += repLength2; - anchor = ip; + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2); + hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = current2; + ip0 += repLength2; + anchor = ip0; continue; } break; } } + + /* Prepare for next iteration */ + assert(ip0 == anchor); + ip1 = ip0 + stepSize; } +_cleanup: /* save reps for next block */ - rep[0] = offset_1 ? offset_1 : offsetSaved; - rep[1] = offset_2 ? offset_2 : offsetSaved; + rep[0] = offset_1; + rep[1] = offset_2; /* Return the last literals size */ return (size_t)(iend - anchor); } + +ZSTD_GEN_FAST_FN(dictMatchState, 4, 0) +ZSTD_GEN_FAST_FN(dictMatchState, 5, 0) +ZSTD_GEN_FAST_FN(dictMatchState, 6, 0) +ZSTD_GEN_FAST_FN(dictMatchState, 7, 0) + size_t ZSTD_compressBlock_fast_dictMatchState( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { U32 const mls = ms->cParams.minMatch; @@ -20632,30 +29209,31 @@ size_t ZSTD_compressBlock_fast_dictMatchState( { default: /* includes case 3 */ case 4 : - return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 4); + return ZSTD_compressBlock_fast_dictMatchState_4_0(ms, seqStore, rep, src, srcSize); case 5 : - return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 5); + return ZSTD_compressBlock_fast_dictMatchState_5_0(ms, seqStore, rep, src, srcSize); case 6 : - return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 6); + return ZSTD_compressBlock_fast_dictMatchState_6_0(ms, seqStore, rep, src, srcSize); case 7 : - return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 7); + return ZSTD_compressBlock_fast_dictMatchState_7_0(ms, seqStore, rep, src, srcSize); } } -static size_t ZSTD_compressBlock_fast_extDict_generic( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize, U32 const mls) +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_compressBlock_fast_extDict_generic( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, U32 const mls, U32 const hasStep) { const ZSTD_compressionParameters* const cParams = &ms->cParams; U32* const hashTable = ms->hashTable; U32 const hlog = cParams->hashLog; /* support stepSize of 0 */ - U32 const stepSize = cParams->targetLength + !(cParams->targetLength); + size_t const stepSize = cParams->targetLength + !(cParams->targetLength) + 1; const BYTE* const base = ms->window.base; const BYTE* const dictBase = ms->window.dictBase; const BYTE* const istart = (const BYTE*)src; - const BYTE* ip = istart; const BYTE* anchor = istart; const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); const U32 lowLimit = ZSTD_getLowestMatchIndex(ms, endIndex, cParams->windowLog); @@ -20668,7920 +29246,8774 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - 8; U32 offset_1=rep[0], offset_2=rep[1]; + U32 offsetSaved1 = 0, offsetSaved2 = 0; - DEBUGLOG(5, "ZSTD_compressBlock_fast_extDict_generic (offset_1=%u)", offset_1); - - /* switch to "regular" variant if extDict is invalidated due to maxDistance */ - if (prefixStartIndex == dictStartIndex) - return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, mls); - - /* Search Loop */ - while (ip < ilimit) { /* < instead of <=, because (ip+1) */ - const size_t h = ZSTD_hashPtr(ip, hlog, mls); - const U32 matchIndex = hashTable[h]; - const BYTE* const matchBase = matchIndex < prefixStartIndex ? dictBase : base; - const BYTE* match = matchBase + matchIndex; - const U32 curr = (U32)(ip-base); - const U32 repIndex = curr + 1 - offset_1; - const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base; - const BYTE* const repMatch = repBase + repIndex; - hashTable[h] = curr; /* update hash table */ - DEBUGLOG(7, "offset_1 = %u , curr = %u", offset_1, curr); - assert(offset_1 <= curr +1); /* check repIndex */ - - if ( (((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > dictStartIndex)) - && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { - const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; - size_t const rLength = ZSTD_count_2segments(ip+1 +4, repMatch +4, iend, repMatchEnd, prefixStart) + 4; - ip++; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, rLength-MINMATCH); - ip += rLength; - anchor = ip; - } else { - if ( (matchIndex < dictStartIndex) || - (MEM_read32(match) != MEM_read32(ip)) ) { - assert(stepSize >= 1); - ip += ((ip-anchor) >> kSearchStrength) + stepSize; - continue; - } - { const BYTE* const matchEnd = matchIndex < prefixStartIndex ? dictEnd : iend; - const BYTE* const lowMatchPtr = matchIndex < prefixStartIndex ? dictStart : prefixStart; - U32 const offset = curr - matchIndex; - size_t mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, prefixStart) + 4; - while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ - offset_2 = offset_1; offset_1 = offset; /* update offset history */ - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH); - ip += mLength; - anchor = ip; - } } - - if (ip <= ilimit) { - /* Fill Table */ - hashTable[ZSTD_hashPtr(base+curr+2, hlog, mls)] = curr+2; - hashTable[ZSTD_hashPtr(ip-2, hlog, mls)] = (U32)(ip-2-base); - /* check immediate repcode */ - while (ip <= ilimit) { - U32 const current2 = (U32)(ip-base); - U32 const repIndex2 = current2 - offset_2; - const BYTE* const repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2; - if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) & (repIndex2 > dictStartIndex)) /* intentional overflow */ - && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { - const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; - size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; - { U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; } /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStore, 0 /*litlen*/, anchor, iend, 0 /*offcode*/, repLength2-MINMATCH); - hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2; - ip += repLength2; - anchor = ip; - continue; - } - break; - } } } - - /* save reps for next block */ - rep[0] = offset_1; - rep[1] = offset_2; - - /* Return the last literals size */ - return (size_t)(iend - anchor); -} - - -size_t ZSTD_compressBlock_fast_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) -{ - U32 const mls = ms->cParams.minMatch; - switch(mls) - { - default: /* includes case 3 */ - case 4 : - return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 4); - case 5 : - return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 5); - case 6 : - return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 6); - case 7 : - return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 7); - } -} -/**** ended inlining compress/zstd_fast.c ****/ -/**** start inlining compress/zstd_lazy.c ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -/**** skipping file: zstd_compress_internal.h ****/ -/**** skipping file: zstd_lazy.h ****/ - - -/*-************************************* -* Binary Tree search -***************************************/ - -static void -ZSTD_updateDUBT(ZSTD_matchState_t* ms, - const BYTE* ip, const BYTE* iend, - U32 mls) -{ - const ZSTD_compressionParameters* const cParams = &ms->cParams; - U32* const hashTable = ms->hashTable; - U32 const hashLog = cParams->hashLog; - - U32* const bt = ms->chainTable; - U32 const btLog = cParams->chainLog - 1; - U32 const btMask = (1 << btLog) - 1; - - const BYTE* const base = ms->window.base; - U32 const target = (U32)(ip - base); - U32 idx = ms->nextToUpdate; - - if (idx != target) - DEBUGLOG(7, "ZSTD_updateDUBT, from %u to %u (dictLimit:%u)", - idx, target, ms->window.dictLimit); - assert(ip + 8 <= iend); /* condition for ZSTD_hashPtr */ - (void)iend; - - assert(idx >= ms->window.dictLimit); /* condition for valid base+idx */ - for ( ; idx < target ; idx++) { - size_t const h = ZSTD_hashPtr(base + idx, hashLog, mls); /* assumption : ip + 8 <= iend */ - U32 const matchIndex = hashTable[h]; - - U32* const nextCandidatePtr = bt + 2*(idx&btMask); - U32* const sortMarkPtr = nextCandidatePtr + 1; - - DEBUGLOG(8, "ZSTD_updateDUBT: insert %u", idx); - hashTable[h] = idx; /* Update Hash Table */ - *nextCandidatePtr = matchIndex; /* update BT like a chain */ - *sortMarkPtr = ZSTD_DUBT_UNSORTED_MARK; - } - ms->nextToUpdate = target; -} - - -/** ZSTD_insertDUBT1() : - * sort one already inserted but unsorted position - * assumption : curr >= btlow == (curr - btmask) - * doesn't fail */ -static void -ZSTD_insertDUBT1(ZSTD_matchState_t* ms, - U32 curr, const BYTE* inputEnd, - U32 nbCompares, U32 btLow, - const ZSTD_dictMode_e dictMode) -{ - const ZSTD_compressionParameters* const cParams = &ms->cParams; - U32* const bt = ms->chainTable; - U32 const btLog = cParams->chainLog - 1; - U32 const btMask = (1 << btLog) - 1; - size_t commonLengthSmaller=0, commonLengthLarger=0; - const BYTE* const base = ms->window.base; - const BYTE* const dictBase = ms->window.dictBase; - const U32 dictLimit = ms->window.dictLimit; - const BYTE* const ip = (curr>=dictLimit) ? base + curr : dictBase + curr; - const BYTE* const iend = (curr>=dictLimit) ? inputEnd : dictBase + dictLimit; - const BYTE* const dictEnd = dictBase + dictLimit; - const BYTE* const prefixStart = base + dictLimit; - const BYTE* match; - U32* smallerPtr = bt + 2*(curr&btMask); - U32* largerPtr = smallerPtr + 1; - U32 matchIndex = *smallerPtr; /* this candidate is unsorted : next sorted candidate is reached through *smallerPtr, while *largerPtr contains previous unsorted candidate (which is already saved and can be overwritten) */ - U32 dummy32; /* to be nullified at the end */ - U32 const windowValid = ms->window.lowLimit; - U32 const maxDistance = 1U << cParams->windowLog; - U32 const windowLow = (curr - windowValid > maxDistance) ? curr - maxDistance : windowValid; - - - DEBUGLOG(8, "ZSTD_insertDUBT1(%u) (dictLimit=%u, lowLimit=%u)", - curr, dictLimit, windowLow); - assert(curr >= btLow); - assert(ip < iend); /* condition for ZSTD_count */ - - while (nbCompares-- && (matchIndex > windowLow)) { - U32* const nextPtr = bt + 2*(matchIndex & btMask); - size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ - assert(matchIndex < curr); - /* note : all candidates are now supposed sorted, - * but it's still possible to have nextPtr[1] == ZSTD_DUBT_UNSORTED_MARK - * when a real index has the same value as ZSTD_DUBT_UNSORTED_MARK */ - - if ( (dictMode != ZSTD_extDict) - || (matchIndex+matchLength >= dictLimit) /* both in current segment*/ - || (curr < dictLimit) /* both in extDict */) { - const BYTE* const mBase = ( (dictMode != ZSTD_extDict) - || (matchIndex+matchLength >= dictLimit)) ? - base : dictBase; - assert( (matchIndex+matchLength >= dictLimit) /* might be wrong if extDict is incorrectly set to 0 */ - || (curr < dictLimit) ); - match = mBase + matchIndex; - matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend); - } else { - match = dictBase + matchIndex; - matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); - if (matchIndex+matchLength >= dictLimit) - match = base + matchIndex; /* preparation for next read of match[matchLength] */ - } - - DEBUGLOG(8, "ZSTD_insertDUBT1: comparing %u with %u : found %u common bytes ", - curr, matchIndex, (U32)matchLength); - - if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */ - break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt tree */ - } - - if (match[matchLength] < ip[matchLength]) { /* necessarily within buffer */ - /* match is smaller than current */ - *smallerPtr = matchIndex; /* update smaller idx */ - commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ - if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop searching */ - DEBUGLOG(8, "ZSTD_insertDUBT1: %u (>btLow=%u) is smaller : next => %u", - matchIndex, btLow, nextPtr[1]); - smallerPtr = nextPtr+1; /* new "candidate" => larger than match, which was smaller than target */ - matchIndex = nextPtr[1]; /* new matchIndex, larger than previous and closer to current */ - } else { - /* match is larger than current */ - *largerPtr = matchIndex; - commonLengthLarger = matchLength; - if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop searching */ - DEBUGLOG(8, "ZSTD_insertDUBT1: %u (>btLow=%u) is larger => %u", - matchIndex, btLow, nextPtr[0]); - largerPtr = nextPtr; - matchIndex = nextPtr[0]; - } } - - *smallerPtr = *largerPtr = 0; -} - - -static size_t -ZSTD_DUBT_findBetterDictMatch ( - ZSTD_matchState_t* ms, - const BYTE* const ip, const BYTE* const iend, - size_t* offsetPtr, - size_t bestLength, - U32 nbCompares, - U32 const mls, - const ZSTD_dictMode_e dictMode) -{ - const ZSTD_matchState_t * const dms = ms->dictMatchState; - const ZSTD_compressionParameters* const dmsCParams = &dms->cParams; - const U32 * const dictHashTable = dms->hashTable; - U32 const hashLog = dmsCParams->hashLog; - size_t const h = ZSTD_hashPtr(ip, hashLog, mls); - U32 dictMatchIndex = dictHashTable[h]; + const BYTE* ip0 = istart; + const BYTE* ip1; + const BYTE* ip2; + const BYTE* ip3; + U32 current0; - const BYTE* const base = ms->window.base; - const BYTE* const prefixStart = base + ms->window.dictLimit; - U32 const curr = (U32)(ip-base); - const BYTE* const dictBase = dms->window.base; - const BYTE* const dictEnd = dms->window.nextSrc; - U32 const dictHighLimit = (U32)(dms->window.nextSrc - dms->window.base); - U32 const dictLowLimit = dms->window.lowLimit; - U32 const dictIndexDelta = ms->window.lowLimit - dictHighLimit; - U32* const dictBt = dms->chainTable; - U32 const btLog = dmsCParams->chainLog - 1; - U32 const btMask = (1 << btLog) - 1; - U32 const btLow = (btMask >= dictHighLimit - dictLowLimit) ? dictLowLimit : dictHighLimit - btMask; + size_t hash0; /* hash for ip0 */ + size_t hash1; /* hash for ip1 */ + U32 idx; /* match idx for ip0 */ + const BYTE* idxBase; /* base pointer for idx */ - size_t commonLengthSmaller=0, commonLengthLarger=0; + U32 offcode; + const BYTE* match0; + size_t mLength; + const BYTE* matchEnd = 0; /* initialize to avoid warning, assert != 0 later */ - (void)dictMode; - assert(dictMode == ZSTD_dictMatchState); + size_t step; + const BYTE* nextStep; + const size_t kStepIncr = (1 << (kSearchStrength - 1)); - while (nbCompares-- && (dictMatchIndex > dictLowLimit)) { - U32* const nextPtr = dictBt + 2*(dictMatchIndex & btMask); - size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ - const BYTE* match = dictBase + dictMatchIndex; - matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); - if (dictMatchIndex+matchLength >= dictHighLimit) - match = base + dictMatchIndex + dictIndexDelta; /* to prepare for next usage of match[matchLength] */ + (void)hasStep; /* not currently specialized on whether it's accelerated */ - if (matchLength > bestLength) { - U32 matchIndex = dictMatchIndex + dictIndexDelta; - if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) { - DEBUGLOG(9, "ZSTD_DUBT_findBetterDictMatch(%u) : found better match length %u -> %u and offsetCode %u -> %u (dictMatchIndex %u, matchIndex %u)", - curr, (U32)bestLength, (U32)matchLength, (U32)*offsetPtr, ZSTD_REP_MOVE + curr - matchIndex, dictMatchIndex, matchIndex); - bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + curr - matchIndex; - } - if (ip+matchLength == iend) { /* reached end of input : ip[matchLength] is not valid, no way to know if it's larger or smaller than match */ - break; /* drop, to guarantee consistency (miss a little bit of compression) */ - } - } + DEBUGLOG(5, "ZSTD_compressBlock_fast_extDict_generic (offset_1=%u)", offset_1); - if (match[matchLength] < ip[matchLength]) { - if (dictMatchIndex <= btLow) { break; } /* beyond tree size, stop the search */ - commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ - dictMatchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ - } else { - /* match is larger than current */ - if (dictMatchIndex <= btLow) { break; } /* beyond tree size, stop the search */ - commonLengthLarger = matchLength; - dictMatchIndex = nextPtr[0]; - } - } + /* switch to "regular" variant if extDict is invalidated due to maxDistance */ + if (prefixStartIndex == dictStartIndex) + return ZSTD_compressBlock_fast(ms, seqStore, rep, src, srcSize); - if (bestLength >= MINMATCH) { - U32 const mIndex = curr - ((U32)*offsetPtr - ZSTD_REP_MOVE); (void)mIndex; - DEBUGLOG(8, "ZSTD_DUBT_findBetterDictMatch(%u) : found match of length %u and offsetCode %u (pos %u)", - curr, (U32)bestLength, (U32)*offsetPtr, mIndex); + { U32 const curr = (U32)(ip0 - base); + U32 const maxRep = curr - dictStartIndex; + if (offset_2 >= maxRep) offsetSaved2 = offset_2, offset_2 = 0; + if (offset_1 >= maxRep) offsetSaved1 = offset_1, offset_1 = 0; } - return bestLength; -} + /* start each op */ +_start: /* Requires: ip0 */ + step = stepSize; + nextStep = ip0 + kStepIncr; -static size_t -ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms, - const BYTE* const ip, const BYTE* const iend, - size_t* offsetPtr, - U32 const mls, - const ZSTD_dictMode_e dictMode) -{ - const ZSTD_compressionParameters* const cParams = &ms->cParams; - U32* const hashTable = ms->hashTable; - U32 const hashLog = cParams->hashLog; - size_t const h = ZSTD_hashPtr(ip, hashLog, mls); - U32 matchIndex = hashTable[h]; + /* calculate positions, ip0 - anchor == 0, so we skip step calc */ + ip1 = ip0 + 1; + ip2 = ip0 + step; + ip3 = ip2 + 1; - const BYTE* const base = ms->window.base; - U32 const curr = (U32)(ip-base); - U32 const windowLow = ZSTD_getLowestMatchIndex(ms, curr, cParams->windowLog); + if (ip3 >= ilimit) { + goto _cleanup; + } - U32* const bt = ms->chainTable; - U32 const btLog = cParams->chainLog - 1; - U32 const btMask = (1 << btLog) - 1; - U32 const btLow = (btMask >= curr) ? 0 : curr - btMask; - U32 const unsortLimit = MAX(btLow, windowLow); + hash0 = ZSTD_hashPtr(ip0, hlog, mls); + hash1 = ZSTD_hashPtr(ip1, hlog, mls); - U32* nextCandidate = bt + 2*(matchIndex&btMask); - U32* unsortedMark = bt + 2*(matchIndex&btMask) + 1; - U32 nbCompares = 1U << cParams->searchLog; - U32 nbCandidates = nbCompares; - U32 previousCandidate = 0; + idx = hashTable[hash0]; + idxBase = idx < prefixStartIndex ? dictBase : base; - DEBUGLOG(7, "ZSTD_DUBT_findBestMatch (%u) ", curr); - assert(ip <= iend-8); /* required for h calculation */ - assert(dictMode != ZSTD_dedicatedDictSearch); + do { + { /* load repcode match for ip[2] */ + U32 const current2 = (U32)(ip2 - base); + U32 const repIndex = current2 - offset_1; + const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base; + U32 rval; + if ( ((U32)(prefixStartIndex - repIndex) >= 4) /* intentional underflow */ + & (offset_1 > 0) ) { + rval = MEM_read32(repBase + repIndex); + } else { + rval = MEM_read32(ip2) ^ 1; /* guaranteed to not match. */ + } - /* reach end of unsorted candidates list */ - while ( (matchIndex > unsortLimit) - && (*unsortedMark == ZSTD_DUBT_UNSORTED_MARK) - && (nbCandidates > 1) ) { - DEBUGLOG(8, "ZSTD_DUBT_findBestMatch: candidate %u is unsorted", - matchIndex); - *unsortedMark = previousCandidate; /* the unsortedMark becomes a reversed chain, to move up back to original position */ - previousCandidate = matchIndex; - matchIndex = *nextCandidate; - nextCandidate = bt + 2*(matchIndex&btMask); - unsortedMark = bt + 2*(matchIndex&btMask) + 1; - nbCandidates --; - } + /* write back hash table entry */ + current0 = (U32)(ip0 - base); + hashTable[hash0] = current0; + + /* check repcode at ip[2] */ + if (MEM_read32(ip2) == rval) { + ip0 = ip2; + match0 = repBase + repIndex; + matchEnd = repIndex < prefixStartIndex ? dictEnd : iend; + assert((match0 != prefixStart) & (match0 != dictStart)); + mLength = ip0[-1] == match0[-1]; + ip0 -= mLength; + match0 -= mLength; + offcode = REPCODE1_TO_OFFBASE; + mLength += 4; + goto _match; + } } - /* nullify last candidate if it's still unsorted - * simplification, detrimental to compression ratio, beneficial for speed */ - if ( (matchIndex > unsortLimit) - && (*unsortedMark==ZSTD_DUBT_UNSORTED_MARK) ) { - DEBUGLOG(7, "ZSTD_DUBT_findBestMatch: nullify last unsorted candidate %u", - matchIndex); - *nextCandidate = *unsortedMark = 0; - } + { /* load match for ip[0] */ + U32 const mval = idx >= dictStartIndex ? + MEM_read32(idxBase + idx) : + MEM_read32(ip0) ^ 1; /* guaranteed not to match */ - /* batch sort stacked candidates */ - matchIndex = previousCandidate; - while (matchIndex) { /* will end on matchIndex == 0 */ - U32* const nextCandidateIdxPtr = bt + 2*(matchIndex&btMask) + 1; - U32 const nextCandidateIdx = *nextCandidateIdxPtr; - ZSTD_insertDUBT1(ms, matchIndex, iend, - nbCandidates, unsortLimit, dictMode); - matchIndex = nextCandidateIdx; - nbCandidates++; - } + /* check match at ip[0] */ + if (MEM_read32(ip0) == mval) { + /* found a match! */ + goto _offset; + } } - /* find longest match */ - { size_t commonLengthSmaller = 0, commonLengthLarger = 0; - const BYTE* const dictBase = ms->window.dictBase; - const U32 dictLimit = ms->window.dictLimit; - const BYTE* const dictEnd = dictBase + dictLimit; - const BYTE* const prefixStart = base + dictLimit; - U32* smallerPtr = bt + 2*(curr&btMask); - U32* largerPtr = bt + 2*(curr&btMask) + 1; - U32 matchEndIdx = curr + 8 + 1; - U32 dummy32; /* to be nullified at the end */ - size_t bestLength = 0; + /* lookup ip[1] */ + idx = hashTable[hash1]; + idxBase = idx < prefixStartIndex ? dictBase : base; - matchIndex = hashTable[h]; - hashTable[h] = curr; /* Update Hash Table */ + /* hash ip[2] */ + hash0 = hash1; + hash1 = ZSTD_hashPtr(ip2, hlog, mls); - while (nbCompares-- && (matchIndex > windowLow)) { - U32* const nextPtr = bt + 2*(matchIndex & btMask); - size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ - const BYTE* match; + /* advance to next positions */ + ip0 = ip1; + ip1 = ip2; + ip2 = ip3; - if ((dictMode != ZSTD_extDict) || (matchIndex+matchLength >= dictLimit)) { - match = base + matchIndex; - matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend); - } else { - match = dictBase + matchIndex; - matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); - if (matchIndex+matchLength >= dictLimit) - match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ - } + /* write back hash table entry */ + current0 = (U32)(ip0 - base); + hashTable[hash0] = current0; - if (matchLength > bestLength) { - if (matchLength > matchEndIdx - matchIndex) - matchEndIdx = matchIndex + (U32)matchLength; - if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) - bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + curr - matchIndex; - if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */ - if (dictMode == ZSTD_dictMatchState) { - nbCompares = 0; /* in addition to avoiding checking any - * further in this loop, make sure we - * skip checking in the dictionary. */ - } - break; /* drop, to guarantee consistency (miss a little bit of compression) */ - } - } + { /* load match for ip[0] */ + U32 const mval = idx >= dictStartIndex ? + MEM_read32(idxBase + idx) : + MEM_read32(ip0) ^ 1; /* guaranteed not to match */ - if (match[matchLength] < ip[matchLength]) { - /* match is smaller than current */ - *smallerPtr = matchIndex; /* update smaller idx */ - commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ - if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ - smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ - matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ - } else { - /* match is larger than current */ - *largerPtr = matchIndex; - commonLengthLarger = matchLength; - if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ - largerPtr = nextPtr; - matchIndex = nextPtr[0]; + /* check match at ip[0] */ + if (MEM_read32(ip0) == mval) { + /* found a match! */ + goto _offset; } } - *smallerPtr = *largerPtr = 0; + /* lookup ip[1] */ + idx = hashTable[hash1]; + idxBase = idx < prefixStartIndex ? dictBase : base; - if (dictMode == ZSTD_dictMatchState && nbCompares) { - bestLength = ZSTD_DUBT_findBetterDictMatch( - ms, ip, iend, - offsetPtr, bestLength, nbCompares, - mls, dictMode); - } + /* hash ip[2] */ + hash0 = hash1; + hash1 = ZSTD_hashPtr(ip2, hlog, mls); - assert(matchEndIdx > curr+8); /* ensure nextToUpdate is increased */ - ms->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */ - if (bestLength >= MINMATCH) { - U32 const mIndex = curr - ((U32)*offsetPtr - ZSTD_REP_MOVE); (void)mIndex; - DEBUGLOG(8, "ZSTD_DUBT_findBestMatch(%u) : found match of length %u and offsetCode %u (pos %u)", - curr, (U32)bestLength, (U32)*offsetPtr, mIndex); + /* advance to next positions */ + ip0 = ip1; + ip1 = ip2; + ip2 = ip0 + step; + ip3 = ip1 + step; + + /* calculate step */ + if (ip2 >= nextStep) { + step++; + PREFETCH_L1(ip1 + 64); + PREFETCH_L1(ip1 + 128); + nextStep += kStepIncr; } - return bestLength; - } -} + } while (ip3 < ilimit); +_cleanup: + /* Note that there are probably still a couple positions we could search. + * However, it seems to be a meaningful performance hit to try to search + * them. So let's not. */ -/** ZSTD_BtFindBestMatch() : Tree updater, providing best match */ -FORCE_INLINE_TEMPLATE size_t -ZSTD_BtFindBestMatch( ZSTD_matchState_t* ms, - const BYTE* const ip, const BYTE* const iLimit, - size_t* offsetPtr, - const U32 mls /* template */, - const ZSTD_dictMode_e dictMode) -{ - DEBUGLOG(7, "ZSTD_BtFindBestMatch"); - if (ip < ms->window.base + ms->nextToUpdate) return 0; /* skipped area */ - ZSTD_updateDUBT(ms, ip, iLimit, mls); - return ZSTD_DUBT_findBestMatch(ms, ip, iLimit, offsetPtr, mls, dictMode); -} + /* If offset_1 started invalid (offsetSaved1 != 0) and became valid (offset_1 != 0), + * rotate saved offsets. See comment in ZSTD_compressBlock_fast_noDict for more context. */ + offsetSaved2 = ((offsetSaved1 != 0) && (offset_1 != 0)) ? offsetSaved1 : offsetSaved2; + /* save reps for next block */ + rep[0] = offset_1 ? offset_1 : offsetSaved1; + rep[1] = offset_2 ? offset_2 : offsetSaved2; -static size_t -ZSTD_BtFindBestMatch_selectMLS ( ZSTD_matchState_t* ms, - const BYTE* ip, const BYTE* const iLimit, - size_t* offsetPtr) -{ - switch(ms->cParams.minMatch) - { - default : /* includes case 3 */ - case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_noDict); - case 5 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_noDict); - case 7 : - case 6 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_noDict); - } -} + /* Return the last literals size */ + return (size_t)(iend - anchor); +_offset: /* Requires: ip0, idx, idxBase */ -static size_t ZSTD_BtFindBestMatch_dictMatchState_selectMLS ( - ZSTD_matchState_t* ms, - const BYTE* ip, const BYTE* const iLimit, - size_t* offsetPtr) -{ - switch(ms->cParams.minMatch) - { - default : /* includes case 3 */ - case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState); - case 5 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_dictMatchState); - case 7 : - case 6 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_dictMatchState); - } + /* Compute the offset code. */ + { U32 const offset = current0 - idx; + const BYTE* const lowMatchPtr = idx < prefixStartIndex ? dictStart : prefixStart; + matchEnd = idx < prefixStartIndex ? dictEnd : iend; + match0 = idxBase + idx; + offset_2 = offset_1; + offset_1 = offset; + offcode = OFFSET_TO_OFFBASE(offset); + mLength = 4; + + /* Count the backwards match length. */ + while (((ip0>anchor) & (match0>lowMatchPtr)) && (ip0[-1] == match0[-1])) { + ip0--; + match0--; + mLength++; + } } + +_match: /* Requires: ip0, match0, offcode, matchEnd */ + + /* Count the forward length. */ + assert(matchEnd != 0); + mLength += ZSTD_count_2segments(ip0 + mLength, match0 + mLength, iend, matchEnd, prefixStart); + + ZSTD_storeSeq(seqStore, (size_t)(ip0 - anchor), anchor, iend, offcode, mLength); + + ip0 += mLength; + anchor = ip0; + + /* write next hash table entry */ + if (ip1 < ip0) { + hashTable[hash1] = (U32)(ip1 - base); + } + + /* Fill table and check for immediate repcode. */ + if (ip0 <= ilimit) { + /* Fill Table */ + assert(base+current0+2 > istart); /* check base overflow */ + hashTable[ZSTD_hashPtr(base+current0+2, hlog, mls)] = current0+2; /* here because current+2 could be > iend-8 */ + hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base); + + while (ip0 <= ilimit) { + U32 const repIndex2 = (U32)(ip0-base) - offset_2; + const BYTE* const repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2; + if ( ((ZSTD_index_overlap_check(prefixStartIndex, repIndex2)) & (offset_2 > 0)) + && (MEM_read32(repMatch2) == MEM_read32(ip0)) ) { + const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; + size_t const repLength2 = ZSTD_count_2segments(ip0+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; + { U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; } /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0 /*litlen*/, anchor, iend, REPCODE1_TO_OFFBASE, repLength2); + hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base); + ip0 += repLength2; + anchor = ip0; + continue; + } + break; + } } + + goto _start; } +ZSTD_GEN_FAST_FN(extDict, 4, 0) +ZSTD_GEN_FAST_FN(extDict, 5, 0) +ZSTD_GEN_FAST_FN(extDict, 6, 0) +ZSTD_GEN_FAST_FN(extDict, 7, 0) -static size_t ZSTD_BtFindBestMatch_extDict_selectMLS ( - ZSTD_matchState_t* ms, - const BYTE* ip, const BYTE* const iLimit, - size_t* offsetPtr) +size_t ZSTD_compressBlock_fast_extDict( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) { - switch(ms->cParams.minMatch) + U32 const mls = ms->cParams.minMatch; + assert(ms->dictMatchState == NULL); + switch(mls) { - default : /* includes case 3 */ - case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_extDict); - case 5 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_extDict); + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_fast_extDict_4_0(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_fast_extDict_5_0(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_fast_extDict_6_0(ms, seqStore, rep, src, srcSize); case 7 : - case 6 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_extDict); + return ZSTD_compressBlock_fast_extDict_7_0(ms, seqStore, rep, src, srcSize); } } +/**** ended inlining compress/zstd_fast.c ****/ +/**** start inlining compress/zstd_lazy.c ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ +/**** skipping file: zstd_compress_internal.h ****/ +/**** skipping file: zstd_lazy.h ****/ +/**** skipping file: ../common/bits.h ****/ +#if !defined(ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR) -/* ********************************* -* Hash Chain -***********************************/ -#define NEXT_IN_CHAIN(d, mask) chainTable[(d) & (mask)] - -/* Update chains up to ip (excluded) - Assumption : always within prefix (i.e. not within extDict) */ -FORCE_INLINE_TEMPLATE U32 ZSTD_insertAndFindFirstIndex_internal( - ZSTD_matchState_t* ms, - const ZSTD_compressionParameters* const cParams, - const BYTE* ip, U32 const mls) -{ - U32* const hashTable = ms->hashTable; - const U32 hashLog = cParams->hashLog; - U32* const chainTable = ms->chainTable; - const U32 chainMask = (1 << cParams->chainLog) - 1; - const BYTE* const base = ms->window.base; - const U32 target = (U32)(ip - base); - U32 idx = ms->nextToUpdate; - - while(idx < target) { /* catch up */ - size_t const h = ZSTD_hashPtr(base+idx, hashLog, mls); - NEXT_IN_CHAIN(idx, chainMask) = hashTable[h]; - hashTable[h] = idx; - idx++; - } - - ms->nextToUpdate = target; - return hashTable[ZSTD_hashPtr(ip, hashLog, mls)]; -} +#define kLazySkippingStep 8 -U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip) { - const ZSTD_compressionParameters* const cParams = &ms->cParams; - return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, ms->cParams.minMatch); -} -void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const BYTE* const ip) +/*-************************************* +* Binary Tree search +***************************************/ + +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +void ZSTD_updateDUBT(ZSTD_MatchState_t* ms, + const BYTE* ip, const BYTE* iend, + U32 mls) { - const BYTE* const base = ms->window.base; - U32 const target = (U32)(ip - base); + const ZSTD_compressionParameters* const cParams = &ms->cParams; U32* const hashTable = ms->hashTable; - U32* const chainTable = ms->chainTable; - U32 const chainSize = 1 << ms->cParams.chainLog; - U32 idx = ms->nextToUpdate; - U32 const minChain = chainSize < target ? target - chainSize : idx; - U32 const bucketSize = 1 << ZSTD_LAZY_DDSS_BUCKET_LOG; - U32 const cacheSize = bucketSize - 1; - U32 const chainAttempts = (1 << ms->cParams.searchLog) - cacheSize; - U32 const chainLimit = chainAttempts > 255 ? 255 : chainAttempts; - - /* We know the hashtable is oversized by a factor of `bucketSize`. - * We are going to temporarily pretend `bucketSize == 1`, keeping only a - * single entry. We will use the rest of the space to construct a temporary - * chaintable. - */ - U32 const hashLog = ms->cParams.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG; - U32* const tmpHashTable = hashTable; - U32* const tmpChainTable = hashTable + ((size_t)1 << hashLog); - U32 const tmpChainSize = ((1 << ZSTD_LAZY_DDSS_BUCKET_LOG) - 1) << hashLog; - U32 const tmpMinChain = tmpChainSize < target ? target - tmpChainSize : idx; + U32 const hashLog = cParams->hashLog; - U32 hashIdx; + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; + U32 const btMask = (1 << btLog) - 1; - assert(ms->cParams.chainLog <= 24); - assert(ms->cParams.hashLog >= ms->cParams.chainLog); - assert(idx != 0); - assert(tmpMinChain <= minChain); + const BYTE* const base = ms->window.base; + U32 const target = (U32)(ip - base); + U32 idx = ms->nextToUpdate; - /* fill conventional hash table and conventional chain table */ - for ( ; idx < target; idx++) { - U32 const h = (U32)ZSTD_hashPtr(base + idx, hashLog, ms->cParams.minMatch); - if (idx >= tmpMinChain) { - tmpChainTable[idx - tmpMinChain] = hashTable[h]; - } - tmpHashTable[h] = idx; - } + if (idx != target) + DEBUGLOG(7, "ZSTD_updateDUBT, from %u to %u (dictLimit:%u)", + idx, target, ms->window.dictLimit); + assert(ip + 8 <= iend); /* condition for ZSTD_hashPtr */ + (void)iend; - /* sort chains into ddss chain table */ - { - U32 chainPos = 0; - for (hashIdx = 0; hashIdx < (1U << hashLog); hashIdx++) { - U32 count; - U32 countBeyondMinChain = 0; - U32 i = tmpHashTable[hashIdx]; - for (count = 0; i >= tmpMinChain && count < cacheSize; count++) { - /* skip through the chain to the first position that won't be - * in the hash cache bucket */ - if (i < minChain) { - countBeyondMinChain++; - } - i = tmpChainTable[i - tmpMinChain]; - } - if (count == cacheSize) { - for (count = 0; count < chainLimit;) { - if (i < minChain) { - if (!i || countBeyondMinChain++ > cacheSize) { - /* only allow pulling `cacheSize` number of entries - * into the cache or chainTable beyond `minChain`, - * to replace the entries pulled out of the - * chainTable into the cache. This lets us reach - * back further without increasing the total number - * of entries in the chainTable, guaranteeing the - * DDSS chain table will fit into the space - * allocated for the regular one. */ - break; - } - } - chainTable[chainPos++] = i; - count++; - if (i < tmpMinChain) { - break; - } - i = tmpChainTable[i - tmpMinChain]; - } - } else { - count = 0; - } - if (count) { - tmpHashTable[hashIdx] = ((chainPos - count) << 8) + count; - } else { - tmpHashTable[hashIdx] = 0; - } - } - assert(chainPos <= chainSize); /* I believe this is guaranteed... */ - } + assert(idx >= ms->window.dictLimit); /* condition for valid base+idx */ + for ( ; idx < target ; idx++) { + size_t const h = ZSTD_hashPtr(base + idx, hashLog, mls); /* assumption : ip + 8 <= iend */ + U32 const matchIndex = hashTable[h]; - /* move chain pointers into the last entry of each hash bucket */ - for (hashIdx = (1 << hashLog); hashIdx; ) { - U32 const bucketIdx = --hashIdx << ZSTD_LAZY_DDSS_BUCKET_LOG; - U32 const chainPackedPointer = tmpHashTable[hashIdx]; - U32 i; - for (i = 0; i < cacheSize; i++) { - hashTable[bucketIdx + i] = 0; - } - hashTable[bucketIdx + bucketSize - 1] = chainPackedPointer; - } + U32* const nextCandidatePtr = bt + 2*(idx&btMask); + U32* const sortMarkPtr = nextCandidatePtr + 1; - /* fill the buckets of the hash table */ - for (idx = ms->nextToUpdate; idx < target; idx++) { - U32 const h = (U32)ZSTD_hashPtr(base + idx, hashLog, ms->cParams.minMatch) - << ZSTD_LAZY_DDSS_BUCKET_LOG; - U32 i; - /* Shift hash cache down 1. */ - for (i = cacheSize - 1; i; i--) - hashTable[h + i] = hashTable[h + i - 1]; - hashTable[h] = idx; + DEBUGLOG(8, "ZSTD_updateDUBT: insert %u", idx); + hashTable[h] = idx; /* Update Hash Table */ + *nextCandidatePtr = matchIndex; /* update BT like a chain */ + *sortMarkPtr = ZSTD_DUBT_UNSORTED_MARK; } - ms->nextToUpdate = target; } -/* inlining is important to hardwire a hot branch (template emulation) */ -FORCE_INLINE_TEMPLATE -size_t ZSTD_HcFindBestMatch_generic ( - ZSTD_matchState_t* ms, - const BYTE* const ip, const BYTE* const iLimit, - size_t* offsetPtr, - const U32 mls, const ZSTD_dictMode_e dictMode) +/** ZSTD_insertDUBT1() : + * sort one already inserted but unsorted position + * assumption : curr >= btlow == (curr - btmask) + * doesn't fail */ +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +void ZSTD_insertDUBT1(const ZSTD_MatchState_t* ms, + U32 curr, const BYTE* inputEnd, + U32 nbCompares, U32 btLow, + const ZSTD_dictMode_e dictMode) { const ZSTD_compressionParameters* const cParams = &ms->cParams; - U32* const chainTable = ms->chainTable; - const U32 chainSize = (1 << cParams->chainLog); - const U32 chainMask = chainSize-1; + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + size_t commonLengthSmaller=0, commonLengthLarger=0; const BYTE* const base = ms->window.base; const BYTE* const dictBase = ms->window.dictBase; const U32 dictLimit = ms->window.dictLimit; - const BYTE* const prefixStart = base + dictLimit; + const BYTE* const ip = (curr>=dictLimit) ? base + curr : dictBase + curr; + const BYTE* const iend = (curr>=dictLimit) ? inputEnd : dictBase + dictLimit; const BYTE* const dictEnd = dictBase + dictLimit; - const U32 curr = (U32)(ip-base); - const U32 maxDistance = 1U << cParams->windowLog; - const U32 lowestValid = ms->window.lowLimit; - const U32 withinMaxDistance = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid; - const U32 isDictionary = (ms->loadedDictEnd != 0); - const U32 lowLimit = isDictionary ? lowestValid : withinMaxDistance; - const U32 minChain = curr > chainSize ? curr - chainSize : 0; - U32 nbAttempts = 1U << cParams->searchLog; - size_t ml=4-1; - - const ZSTD_matchState_t* const dms = ms->dictMatchState; - const U32 ddsHashLog = dictMode == ZSTD_dedicatedDictSearch - ? dms->cParams.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG : 0; - const size_t ddsIdx = dictMode == ZSTD_dedicatedDictSearch - ? ZSTD_hashPtr(ip, ddsHashLog, mls) << ZSTD_LAZY_DDSS_BUCKET_LOG : 0; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* match; + U32* smallerPtr = bt + 2*(curr&btMask); + U32* largerPtr = smallerPtr + 1; + U32 matchIndex = *smallerPtr; /* this candidate is unsorted : next sorted candidate is reached through *smallerPtr, while *largerPtr contains previous unsorted candidate (which is already saved and can be overwritten) */ + U32 dummy32; /* to be nullified at the end */ + U32 const windowValid = ms->window.lowLimit; + U32 const maxDistance = 1U << cParams->windowLog; + U32 const windowLow = (curr - windowValid > maxDistance) ? curr - maxDistance : windowValid; - U32 matchIndex; - if (dictMode == ZSTD_dedicatedDictSearch) { - const U32* entry = &dms->hashTable[ddsIdx]; - PREFETCH_L1(entry); - } + DEBUGLOG(8, "ZSTD_insertDUBT1(%u) (dictLimit=%u, lowLimit=%u)", + curr, dictLimit, windowLow); + assert(curr >= btLow); + assert(ip < iend); /* condition for ZSTD_count */ - /* HC4 match finder */ - matchIndex = ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, mls); + for (; nbCompares && (matchIndex > windowLow); --nbCompares) { + U32* const nextPtr = bt + 2*(matchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + assert(matchIndex < curr); + /* note : all candidates are now supposed sorted, + * but it's still possible to have nextPtr[1] == ZSTD_DUBT_UNSORTED_MARK + * when a real index has the same value as ZSTD_DUBT_UNSORTED_MARK */ - for ( ; (matchIndex>=lowLimit) & (nbAttempts>0) ; nbAttempts--) { - size_t currentMl=0; - if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) { - const BYTE* const match = base + matchIndex; - assert(matchIndex >= dictLimit); /* ensures this is true if dictMode != ZSTD_extDict */ - if (match[ml] == ip[ml]) /* potentially better */ - currentMl = ZSTD_count(ip, match, iLimit); + if ( (dictMode != ZSTD_extDict) + || (matchIndex+matchLength >= dictLimit) /* both in current segment*/ + || (curr < dictLimit) /* both in extDict */) { + const BYTE* const mBase = ( (dictMode != ZSTD_extDict) + || (matchIndex+matchLength >= dictLimit)) ? + base : dictBase; + assert( (matchIndex+matchLength >= dictLimit) /* might be wrong if extDict is incorrectly set to 0 */ + || (curr < dictLimit) ); + match = mBase + matchIndex; + matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend); } else { - const BYTE* const match = dictBase + matchIndex; - assert(match+4 <= dictEnd); - if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */ - currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dictEnd, prefixStart) + 4; - } - - /* save best solution */ - if (currentMl > ml) { - ml = currentMl; - *offsetPtr = curr - matchIndex + ZSTD_REP_MOVE; - if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ - } - - if (matchIndex <= minChain) break; - matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask); - } - - if (dictMode == ZSTD_dedicatedDictSearch) { - const U32 ddsLowestIndex = dms->window.dictLimit; - const BYTE* const ddsBase = dms->window.base; - const BYTE* const ddsEnd = dms->window.nextSrc; - const U32 ddsSize = (U32)(ddsEnd - ddsBase); - const U32 ddsIndexDelta = dictLimit - ddsSize; - const U32 bucketSize = (1 << ZSTD_LAZY_DDSS_BUCKET_LOG); - const U32 bucketLimit = nbAttempts < bucketSize - 1 ? nbAttempts : bucketSize - 1; - U32 ddsAttempt; - - for (ddsAttempt = 0; ddsAttempt < bucketSize - 1; ddsAttempt++) { - PREFETCH_L1(ddsBase + dms->hashTable[ddsIdx + ddsAttempt]); + match = dictBase + matchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); + if (matchIndex+matchLength >= dictLimit) + match = base + matchIndex; /* preparation for next read of match[matchLength] */ } - { - U32 const chainPackedPointer = dms->hashTable[ddsIdx + bucketSize - 1]; - U32 const chainIndex = chainPackedPointer >> 8; + DEBUGLOG(8, "ZSTD_insertDUBT1: comparing %u with %u : found %u common bytes ", + curr, matchIndex, (U32)matchLength); - PREFETCH_L1(&dms->chainTable[chainIndex]); + if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */ + break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt tree */ } - for (ddsAttempt = 0; ddsAttempt < bucketLimit; ddsAttempt++) { - size_t currentMl=0; - const BYTE* match; - matchIndex = dms->hashTable[ddsIdx + ddsAttempt]; - match = ddsBase + matchIndex; + if (match[matchLength] < ip[matchLength]) { /* necessarily within buffer */ + /* match is smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop searching */ + DEBUGLOG(8, "ZSTD_insertDUBT1: %u (>btLow=%u) is smaller : next => %u", + matchIndex, btLow, nextPtr[1]); + smallerPtr = nextPtr+1; /* new "candidate" => larger than match, which was smaller than target */ + matchIndex = nextPtr[1]; /* new matchIndex, larger than previous and closer to current */ + } else { + /* match is larger than current */ + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop searching */ + DEBUGLOG(8, "ZSTD_insertDUBT1: %u (>btLow=%u) is larger => %u", + matchIndex, btLow, nextPtr[0]); + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } } - if (!matchIndex) { - return ml; - } + *smallerPtr = *largerPtr = 0; +} - /* guaranteed by table construction */ - (void)ddsLowestIndex; - assert(matchIndex >= ddsLowestIndex); - assert(match+4 <= ddsEnd); - if (MEM_read32(match) == MEM_read32(ip)) { - /* assumption : matchIndex <= dictLimit-4 (by table construction) */ - currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, ddsEnd, prefixStart) + 4; - } - /* save best solution */ - if (currentMl > ml) { - ml = currentMl; - *offsetPtr = curr - (matchIndex + ddsIndexDelta) + ZSTD_REP_MOVE; - if (ip+currentMl == iLimit) { - /* best possible, avoids read overflow on next attempt */ - return ml; - } - } - } +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_DUBT_findBetterDictMatch ( + const ZSTD_MatchState_t* ms, + const BYTE* const ip, const BYTE* const iend, + size_t* offsetPtr, + size_t bestLength, + U32 nbCompares, + U32 const mls, + const ZSTD_dictMode_e dictMode) +{ + const ZSTD_MatchState_t * const dms = ms->dictMatchState; + const ZSTD_compressionParameters* const dmsCParams = &dms->cParams; + const U32 * const dictHashTable = dms->hashTable; + U32 const hashLog = dmsCParams->hashLog; + size_t const h = ZSTD_hashPtr(ip, hashLog, mls); + U32 dictMatchIndex = dictHashTable[h]; - { - U32 const chainPackedPointer = dms->hashTable[ddsIdx + bucketSize - 1]; - U32 chainIndex = chainPackedPointer >> 8; - U32 const chainLength = chainPackedPointer & 0xFF; - U32 const chainAttempts = nbAttempts - ddsAttempt; - U32 const chainLimit = chainAttempts > chainLength ? chainLength : chainAttempts; - U32 chainAttempt; - - for (chainAttempt = 0 ; chainAttempt < chainLimit; chainAttempt++) { - PREFETCH_L1(ddsBase + dms->chainTable[chainIndex + chainAttempt]); - } + const BYTE* const base = ms->window.base; + const BYTE* const prefixStart = base + ms->window.dictLimit; + U32 const curr = (U32)(ip-base); + const BYTE* const dictBase = dms->window.base; + const BYTE* const dictEnd = dms->window.nextSrc; + U32 const dictHighLimit = (U32)(dms->window.nextSrc - dms->window.base); + U32 const dictLowLimit = dms->window.lowLimit; + U32 const dictIndexDelta = ms->window.lowLimit - dictHighLimit; - for (chainAttempt = 0 ; chainAttempt < chainLimit; chainAttempt++, chainIndex++) { - size_t currentMl=0; - const BYTE* match; - matchIndex = dms->chainTable[chainIndex]; - match = ddsBase + matchIndex; - - /* guaranteed by table construction */ - assert(matchIndex >= ddsLowestIndex); - assert(match+4 <= ddsEnd); - if (MEM_read32(match) == MEM_read32(ip)) { - /* assumption : matchIndex <= dictLimit-4 (by table construction) */ - currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, ddsEnd, prefixStart) + 4; - } + U32* const dictBt = dms->chainTable; + U32 const btLog = dmsCParams->chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + U32 const btLow = (btMask >= dictHighLimit - dictLowLimit) ? dictLowLimit : dictHighLimit - btMask; - /* save best solution */ - if (currentMl > ml) { - ml = currentMl; - *offsetPtr = curr - (matchIndex + ddsIndexDelta) + ZSTD_REP_MOVE; - if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ - } - } - } - } else if (dictMode == ZSTD_dictMatchState) { - const U32* const dmsChainTable = dms->chainTable; - const U32 dmsChainSize = (1 << dms->cParams.chainLog); - const U32 dmsChainMask = dmsChainSize - 1; - const U32 dmsLowestIndex = dms->window.dictLimit; - const BYTE* const dmsBase = dms->window.base; - const BYTE* const dmsEnd = dms->window.nextSrc; - const U32 dmsSize = (U32)(dmsEnd - dmsBase); - const U32 dmsIndexDelta = dictLimit - dmsSize; - const U32 dmsMinChain = dmsSize > dmsChainSize ? dmsSize - dmsChainSize : 0; + size_t commonLengthSmaller=0, commonLengthLarger=0; - matchIndex = dms->hashTable[ZSTD_hashPtr(ip, dms->cParams.hashLog, mls)]; + (void)dictMode; + assert(dictMode == ZSTD_dictMatchState); - for ( ; (matchIndex>=dmsLowestIndex) & (nbAttempts>0) ; nbAttempts--) { - size_t currentMl=0; - const BYTE* const match = dmsBase + matchIndex; - assert(match+4 <= dmsEnd); - if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */ - currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dmsEnd, prefixStart) + 4; + for (; nbCompares && (dictMatchIndex > dictLowLimit); --nbCompares) { + U32* const nextPtr = dictBt + 2*(dictMatchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + const BYTE* match = dictBase + dictMatchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); + if (dictMatchIndex+matchLength >= dictHighLimit) + match = base + dictMatchIndex + dictIndexDelta; /* to prepare for next usage of match[matchLength] */ - /* save best solution */ - if (currentMl > ml) { - ml = currentMl; - *offsetPtr = curr - (matchIndex + dmsIndexDelta) + ZSTD_REP_MOVE; - if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ + if (matchLength > bestLength) { + U32 matchIndex = dictMatchIndex + dictIndexDelta; + if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) { + DEBUGLOG(9, "ZSTD_DUBT_findBetterDictMatch(%u) : found better match length %u -> %u and offsetCode %u -> %u (dictMatchIndex %u, matchIndex %u)", + curr, (U32)bestLength, (U32)matchLength, (U32)*offsetPtr, OFFSET_TO_OFFBASE(curr - matchIndex), dictMatchIndex, matchIndex); + bestLength = matchLength, *offsetPtr = OFFSET_TO_OFFBASE(curr - matchIndex); + } + if (ip+matchLength == iend) { /* reached end of input : ip[matchLength] is not valid, no way to know if it's larger or smaller than match */ + break; /* drop, to guarantee consistency (miss a little bit of compression) */ } - - if (matchIndex <= dmsMinChain) break; - - matchIndex = dmsChainTable[matchIndex & dmsChainMask]; } - } - - return ml; -} - -FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_selectMLS ( - ZSTD_matchState_t* ms, - const BYTE* ip, const BYTE* const iLimit, - size_t* offsetPtr) -{ - switch(ms->cParams.minMatch) - { - default : /* includes case 3 */ - case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_noDict); - case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_noDict); - case 7 : - case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_noDict); + if (match[matchLength] < ip[matchLength]) { + if (dictMatchIndex <= btLow) { break; } /* beyond tree size, stop the search */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + dictMatchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + } else { + /* match is larger than current */ + if (dictMatchIndex <= btLow) { break; } /* beyond tree size, stop the search */ + commonLengthLarger = matchLength; + dictMatchIndex = nextPtr[0]; + } } -} - -static size_t ZSTD_HcFindBestMatch_dictMatchState_selectMLS ( - ZSTD_matchState_t* ms, - const BYTE* ip, const BYTE* const iLimit, - size_t* offsetPtr) -{ - switch(ms->cParams.minMatch) - { - default : /* includes case 3 */ - case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState); - case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_dictMatchState); - case 7 : - case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_dictMatchState); + if (bestLength >= MINMATCH) { + U32 const mIndex = curr - (U32)OFFBASE_TO_OFFSET(*offsetPtr); (void)mIndex; + DEBUGLOG(8, "ZSTD_DUBT_findBetterDictMatch(%u) : found match of length %u and offsetCode %u (pos %u)", + curr, (U32)bestLength, (U32)*offsetPtr, mIndex); } -} - + return bestLength; -static size_t ZSTD_HcFindBestMatch_dedicatedDictSearch_selectMLS ( - ZSTD_matchState_t* ms, - const BYTE* ip, const BYTE* const iLimit, - size_t* offsetPtr) -{ - switch(ms->cParams.minMatch) - { - default : /* includes case 3 */ - case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_dedicatedDictSearch); - case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_dedicatedDictSearch); - case 7 : - case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_dedicatedDictSearch); - } } -FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_extDict_selectMLS ( - ZSTD_matchState_t* ms, - const BYTE* ip, const BYTE* const iLimit, - size_t* offsetPtr) +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_DUBT_findBestMatch(ZSTD_MatchState_t* ms, + const BYTE* const ip, const BYTE* const iend, + size_t* offBasePtr, + U32 const mls, + const ZSTD_dictMode_e dictMode) { - switch(ms->cParams.minMatch) - { - default : /* includes case 3 */ - case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_extDict); - case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_extDict); - case 7 : - case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_extDict); - } -} - - -/* ******************************* -* Common parser - lazy strategy -*********************************/ -typedef enum { search_hashChain, search_binaryTree } searchMethod_e; + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hashLog = cParams->hashLog; + size_t const h = ZSTD_hashPtr(ip, hashLog, mls); + U32 matchIndex = hashTable[h]; -FORCE_INLINE_TEMPLATE size_t -ZSTD_compressBlock_lazy_generic( - ZSTD_matchState_t* ms, seqStore_t* seqStore, - U32 rep[ZSTD_REP_NUM], - const void* src, size_t srcSize, - const searchMethod_e searchMethod, const U32 depth, - ZSTD_dictMode_e const dictMode) -{ - const BYTE* const istart = (const BYTE*)src; - const BYTE* ip = istart; - const BYTE* anchor = istart; - const BYTE* const iend = istart + srcSize; - const BYTE* const ilimit = iend - 8; const BYTE* const base = ms->window.base; - const U32 prefixLowestIndex = ms->window.dictLimit; - const BYTE* const prefixLowest = base + prefixLowestIndex; - - typedef size_t (*searchMax_f)( - ZSTD_matchState_t* ms, - const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); - - /** - * This table is indexed first by the four ZSTD_dictMode_e values, and then - * by the two searchMethod_e values. NULLs are placed for configurations - * that should never occur (extDict modes go to the other implementation - * below and there is no DDSS for binary tree search yet). - */ - const searchMax_f searchFuncs[4][2] = { - { - ZSTD_HcFindBestMatch_selectMLS, - ZSTD_BtFindBestMatch_selectMLS - }, - { - NULL, - NULL - }, - { - ZSTD_HcFindBestMatch_dictMatchState_selectMLS, - ZSTD_BtFindBestMatch_dictMatchState_selectMLS - }, - { - ZSTD_HcFindBestMatch_dedicatedDictSearch_selectMLS, - NULL - } - }; - - searchMax_f const searchMax = searchFuncs[dictMode][searchMethod == search_binaryTree]; - U32 offset_1 = rep[0], offset_2 = rep[1], savedOffset=0; + U32 const curr = (U32)(ip-base); + U32 const windowLow = ZSTD_getLowestMatchIndex(ms, curr, cParams->windowLog); - const int isDMS = dictMode == ZSTD_dictMatchState; - const int isDDS = dictMode == ZSTD_dedicatedDictSearch; - const int isDxS = isDMS || isDDS; - const ZSTD_matchState_t* const dms = ms->dictMatchState; - const U32 dictLowestIndex = isDxS ? dms->window.dictLimit : 0; - const BYTE* const dictBase = isDxS ? dms->window.base : NULL; - const BYTE* const dictLowest = isDxS ? dictBase + dictLowestIndex : NULL; - const BYTE* const dictEnd = isDxS ? dms->window.nextSrc : NULL; - const U32 dictIndexDelta = isDxS ? - prefixLowestIndex - (U32)(dictEnd - dictBase) : - 0; - const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictLowest)); + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + U32 const btLow = (btMask >= curr) ? 0 : curr - btMask; + U32 const unsortLimit = MAX(btLow, windowLow); - assert(searchMax != NULL); + U32* nextCandidate = bt + 2*(matchIndex&btMask); + U32* unsortedMark = bt + 2*(matchIndex&btMask) + 1; + U32 nbCompares = 1U << cParams->searchLog; + U32 nbCandidates = nbCompares; + U32 previousCandidate = 0; - DEBUGLOG(5, "ZSTD_compressBlock_lazy_generic (dictMode=%u)", (U32)dictMode); + DEBUGLOG(7, "ZSTD_DUBT_findBestMatch (%u) ", curr); + assert(ip <= iend-8); /* required for h calculation */ + assert(dictMode != ZSTD_dedicatedDictSearch); - /* init */ - ip += (dictAndPrefixLength == 0); - if (dictMode == ZSTD_noDict) { - U32 const curr = (U32)(ip - base); - U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, curr, ms->cParams.windowLog); - U32 const maxRep = curr - windowLow; - if (offset_2 > maxRep) savedOffset = offset_2, offset_2 = 0; - if (offset_1 > maxRep) savedOffset = offset_1, offset_1 = 0; - } - if (isDxS) { - /* dictMatchState repCode checks don't currently handle repCode == 0 - * disabling. */ - assert(offset_1 <= dictAndPrefixLength); - assert(offset_2 <= dictAndPrefixLength); + /* reach end of unsorted candidates list */ + while ( (matchIndex > unsortLimit) + && (*unsortedMark == ZSTD_DUBT_UNSORTED_MARK) + && (nbCandidates > 1) ) { + DEBUGLOG(8, "ZSTD_DUBT_findBestMatch: candidate %u is unsorted", + matchIndex); + *unsortedMark = previousCandidate; /* the unsortedMark becomes a reversed chain, to move up back to original position */ + previousCandidate = matchIndex; + matchIndex = *nextCandidate; + nextCandidate = bt + 2*(matchIndex&btMask); + unsortedMark = bt + 2*(matchIndex&btMask) + 1; + nbCandidates --; } - /* Match Loop */ -#if defined(__GNUC__) && defined(__x86_64__) - /* I've measured random a 5% speed loss on levels 5 & 6 (greedy) when the - * code alignment is perturbed. To fix the instability align the loop on 32-bytes. - */ - __asm__(".p2align 5"); -#endif - while (ip < ilimit) { - size_t matchLength=0; - size_t offset=0; - const BYTE* start=ip+1; - - /* check repCode */ - if (isDxS) { - const U32 repIndex = (U32)(ip - base) + 1 - offset_1; - const BYTE* repMatch = ((dictMode == ZSTD_dictMatchState || dictMode == ZSTD_dedicatedDictSearch) - && repIndex < prefixLowestIndex) ? - dictBase + (repIndex - dictIndexDelta) : - base + repIndex; - if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */) - && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { - const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; - matchLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; - if (depth==0) goto _storeSequence; - } - } - if ( dictMode == ZSTD_noDict - && ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)))) { - matchLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; - if (depth==0) goto _storeSequence; - } + /* nullify last candidate if it's still unsorted + * simplification, detrimental to compression ratio, beneficial for speed */ + if ( (matchIndex > unsortLimit) + && (*unsortedMark==ZSTD_DUBT_UNSORTED_MARK) ) { + DEBUGLOG(7, "ZSTD_DUBT_findBestMatch: nullify last unsorted candidate %u", + matchIndex); + *nextCandidate = *unsortedMark = 0; + } - /* first search (depth 0) */ - { size_t offsetFound = 999999999; - size_t const ml2 = searchMax(ms, ip, iend, &offsetFound); - if (ml2 > matchLength) - matchLength = ml2, start = ip, offset=offsetFound; - } + /* batch sort stacked candidates */ + matchIndex = previousCandidate; + while (matchIndex) { /* will end on matchIndex == 0 */ + U32* const nextCandidateIdxPtr = bt + 2*(matchIndex&btMask) + 1; + U32 const nextCandidateIdx = *nextCandidateIdxPtr; + ZSTD_insertDUBT1(ms, matchIndex, iend, + nbCandidates, unsortLimit, dictMode); + matchIndex = nextCandidateIdx; + nbCandidates++; + } - if (matchLength < 4) { - ip += ((ip-anchor) >> kSearchStrength) + 1; /* jump faster over incompressible sections */ - continue; - } + /* find longest match */ + { size_t commonLengthSmaller = 0, commonLengthLarger = 0; + const BYTE* const dictBase = ms->window.dictBase; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + U32* smallerPtr = bt + 2*(curr&btMask); + U32* largerPtr = bt + 2*(curr&btMask) + 1; + U32 matchEndIdx = curr + 8 + 1; + U32 dummy32; /* to be nullified at the end */ + size_t bestLength = 0; - /* let's try to find a better solution */ - if (depth>=1) - while (ip0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) { - size_t const mlRep = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4; - int const gain2 = (int)(mlRep * 3); - int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offset+1) + 1); - if ((mlRep >= 4) && (gain2 > gain1)) - matchLength = mlRep, offset = 0, start = ip; - } - if (isDxS) { - const U32 repIndex = (U32)(ip - base) - offset_1; - const BYTE* repMatch = repIndex < prefixLowestIndex ? - dictBase + (repIndex - dictIndexDelta) : - base + repIndex; - if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */) - && (MEM_read32(repMatch) == MEM_read32(ip)) ) { - const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; - size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; - int const gain2 = (int)(mlRep * 3); - int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offset+1) + 1); - if ((mlRep >= 4) && (gain2 > gain1)) - matchLength = mlRep, offset = 0, start = ip; - } - } - { size_t offset2=999999999; - size_t const ml2 = searchMax(ms, ip, iend, &offset2); - int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ - int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4); - if ((ml2 >= 4) && (gain2 > gain1)) { - matchLength = ml2, offset = offset2, start = ip; - continue; /* search a better one */ - } } + matchIndex = hashTable[h]; + hashTable[h] = curr; /* Update Hash Table */ - /* let's find an even better one */ - if ((depth==2) && (ip0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) { - size_t const mlRep = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4; - int const gain2 = (int)(mlRep * 4); - int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1); - if ((mlRep >= 4) && (gain2 > gain1)) - matchLength = mlRep, offset = 0, start = ip; - } - if (isDxS) { - const U32 repIndex = (U32)(ip - base) - offset_1; - const BYTE* repMatch = repIndex < prefixLowestIndex ? - dictBase + (repIndex - dictIndexDelta) : - base + repIndex; - if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */) - && (MEM_read32(repMatch) == MEM_read32(ip)) ) { - const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; - size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; - int const gain2 = (int)(mlRep * 4); - int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1); - if ((mlRep >= 4) && (gain2 > gain1)) - matchLength = mlRep, offset = 0, start = ip; - } - } - { size_t offset2=999999999; - size_t const ml2 = searchMax(ms, ip, iend, &offset2); - int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ - int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7); - if ((ml2 >= 4) && (gain2 > gain1)) { - matchLength = ml2, offset = offset2, start = ip; - continue; - } } } - break; /* nothing found : store previous solution */ - } + for (; nbCompares && (matchIndex > windowLow); --nbCompares) { + U32* const nextPtr = bt + 2*(matchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + const BYTE* match; - /* NOTE: - * start[-offset+ZSTD_REP_MOVE-1] is undefined behavior. - * (-offset+ZSTD_REP_MOVE-1) is unsigned, and is added to start, which - * overflows the pointer, which is undefined behavior. - */ - /* catch up */ - if (offset) { - if (dictMode == ZSTD_noDict) { - while ( ((start > anchor) & (start - (offset-ZSTD_REP_MOVE) > prefixLowest)) - && (start[-1] == (start-(offset-ZSTD_REP_MOVE))[-1]) ) /* only search for offset within prefix */ - { start--; matchLength++; } - } - if (isDxS) { - U32 const matchIndex = (U32)((start-base) - (offset - ZSTD_REP_MOVE)); - const BYTE* match = (matchIndex < prefixLowestIndex) ? dictBase + matchIndex - dictIndexDelta : base + matchIndex; - const BYTE* const mStart = (matchIndex < prefixLowestIndex) ? dictLowest : prefixLowest; - while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */ + if ((dictMode != ZSTD_extDict) || (matchIndex+matchLength >= dictLimit)) { + match = base + matchIndex; + matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend); + } else { + match = dictBase + matchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); + if (matchIndex+matchLength >= dictLimit) + match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ } - offset_2 = offset_1; offset_1 = (U32)(offset - ZSTD_REP_MOVE); - } - /* store sequence */ -_storeSequence: - { size_t const litLength = start - anchor; - ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offset, matchLength-MINMATCH); - anchor = ip = start + matchLength; - } - /* check immediate repcode */ - if (isDxS) { - while (ip <= ilimit) { - U32 const current2 = (U32)(ip-base); - U32 const repIndex = current2 - offset_2; - const BYTE* repMatch = repIndex < prefixLowestIndex ? - dictBase - dictIndexDelta + repIndex : - base + repIndex; - if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex) >= 3 /* intentional overflow */) - && (MEM_read32(repMatch) == MEM_read32(ip)) ) { - const BYTE* const repEnd2 = repIndex < prefixLowestIndex ? dictEnd : iend; - matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd2, prefixLowest) + 4; - offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, matchLength-MINMATCH); - ip += matchLength; - anchor = ip; - continue; + if (matchLength > bestLength) { + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + (U32)matchLength; + if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr - matchIndex + 1) - ZSTD_highbit32((U32)*offBasePtr)) ) + bestLength = matchLength, *offBasePtr = OFFSET_TO_OFFBASE(curr - matchIndex); + if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */ + if (dictMode == ZSTD_dictMatchState) { + nbCompares = 0; /* in addition to avoiding checking any + * further in this loop, make sure we + * skip checking in the dictionary. */ + } + break; /* drop, to guarantee consistency (miss a little bit of compression) */ } - break; } - } - - if (dictMode == ZSTD_noDict) { - while ( ((ip <= ilimit) & (offset_2>0)) - && (MEM_read32(ip) == MEM_read32(ip - offset_2)) ) { - /* store sequence */ - matchLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; - offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap repcodes */ - ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, matchLength-MINMATCH); - ip += matchLength; - anchor = ip; - continue; /* faster when present ... (?) */ - } } } - /* Save reps for next block */ - rep[0] = offset_1 ? offset_1 : savedOffset; - rep[1] = offset_2 ? offset_2 : savedOffset; + if (match[matchLength] < ip[matchLength]) { + /* match is smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ + matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + } else { + /* match is larger than current */ + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } } - /* Return the last literals size */ - return (size_t)(iend - anchor); -} + *smallerPtr = *largerPtr = 0; + assert(nbCompares <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */ + if (dictMode == ZSTD_dictMatchState && nbCompares) { + bestLength = ZSTD_DUBT_findBetterDictMatch( + ms, ip, iend, + offBasePtr, bestLength, nbCompares, + mls, dictMode); + } -size_t ZSTD_compressBlock_btlazy2( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) -{ - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2, ZSTD_noDict); + assert(matchEndIdx > curr+8); /* ensure nextToUpdate is increased */ + ms->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */ + if (bestLength >= MINMATCH) { + U32 const mIndex = curr - (U32)OFFBASE_TO_OFFSET(*offBasePtr); (void)mIndex; + DEBUGLOG(8, "ZSTD_DUBT_findBestMatch(%u) : found match of length %u and offsetCode %u (pos %u)", + curr, (U32)bestLength, (U32)*offBasePtr, mIndex); + } + return bestLength; + } } -size_t ZSTD_compressBlock_lazy2( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) -{ - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_noDict); -} -size_t ZSTD_compressBlock_lazy( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) +/** ZSTD_BtFindBestMatch() : Tree updater, providing best match */ +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_BtFindBestMatch( ZSTD_MatchState_t* ms, + const BYTE* const ip, const BYTE* const iLimit, + size_t* offBasePtr, + const U32 mls /* template */, + const ZSTD_dictMode_e dictMode) { - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_noDict); + DEBUGLOG(7, "ZSTD_BtFindBestMatch"); + if (ip < ms->window.base + ms->nextToUpdate) return 0; /* skipped area */ + ZSTD_updateDUBT(ms, ip, iLimit, mls); + return ZSTD_DUBT_findBestMatch(ms, ip, iLimit, offBasePtr, mls, dictMode); } -size_t ZSTD_compressBlock_greedy( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) -{ - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_noDict); -} +/*********************************** +* Dedicated dict search +***********************************/ -size_t ZSTD_compressBlock_btlazy2_dictMatchState( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) +void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_MatchState_t* ms, const BYTE* const ip) { - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2, ZSTD_dictMatchState); -} + const BYTE* const base = ms->window.base; + U32 const target = (U32)(ip - base); + U32* const hashTable = ms->hashTable; + U32* const chainTable = ms->chainTable; + U32 const chainSize = 1 << ms->cParams.chainLog; + U32 idx = ms->nextToUpdate; + U32 const minChain = chainSize < target - idx ? target - chainSize : idx; + U32 const bucketSize = 1 << ZSTD_LAZY_DDSS_BUCKET_LOG; + U32 const cacheSize = bucketSize - 1; + U32 const chainAttempts = (1 << ms->cParams.searchLog) - cacheSize; + U32 const chainLimit = chainAttempts > 255 ? 255 : chainAttempts; -size_t ZSTD_compressBlock_lazy2_dictMatchState( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) -{ - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_dictMatchState); -} + /* We know the hashtable is oversized by a factor of `bucketSize`. + * We are going to temporarily pretend `bucketSize == 1`, keeping only a + * single entry. We will use the rest of the space to construct a temporary + * chaintable. + */ + U32 const hashLog = ms->cParams.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG; + U32* const tmpHashTable = hashTable; + U32* const tmpChainTable = hashTable + ((size_t)1 << hashLog); + U32 const tmpChainSize = (U32)((1 << ZSTD_LAZY_DDSS_BUCKET_LOG) - 1) << hashLog; + U32 const tmpMinChain = tmpChainSize < target ? target - tmpChainSize : idx; + U32 hashIdx; -size_t ZSTD_compressBlock_lazy_dictMatchState( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) -{ - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_dictMatchState); -} + assert(ms->cParams.chainLog <= 24); + assert(ms->cParams.hashLog > ms->cParams.chainLog); + assert(idx != 0); + assert(tmpMinChain <= minChain); -size_t ZSTD_compressBlock_greedy_dictMatchState( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) -{ - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_dictMatchState); -} + /* fill conventional hash table and conventional chain table */ + for ( ; idx < target; idx++) { + U32 const h = (U32)ZSTD_hashPtr(base + idx, hashLog, ms->cParams.minMatch); + if (idx >= tmpMinChain) { + tmpChainTable[idx - tmpMinChain] = hashTable[h]; + } + tmpHashTable[h] = idx; + } + /* sort chains into ddss chain table */ + { + U32 chainPos = 0; + for (hashIdx = 0; hashIdx < (1U << hashLog); hashIdx++) { + U32 count; + U32 countBeyondMinChain = 0; + U32 i = tmpHashTable[hashIdx]; + for (count = 0; i >= tmpMinChain && count < cacheSize; count++) { + /* skip through the chain to the first position that won't be + * in the hash cache bucket */ + if (i < minChain) { + countBeyondMinChain++; + } + i = tmpChainTable[i - tmpMinChain]; + } + if (count == cacheSize) { + for (count = 0; count < chainLimit;) { + if (i < minChain) { + if (!i || ++countBeyondMinChain > cacheSize) { + /* only allow pulling `cacheSize` number of entries + * into the cache or chainTable beyond `minChain`, + * to replace the entries pulled out of the + * chainTable into the cache. This lets us reach + * back further without increasing the total number + * of entries in the chainTable, guaranteeing the + * DDSS chain table will fit into the space + * allocated for the regular one. */ + break; + } + } + chainTable[chainPos++] = i; + count++; + if (i < tmpMinChain) { + break; + } + i = tmpChainTable[i - tmpMinChain]; + } + } else { + count = 0; + } + if (count) { + tmpHashTable[hashIdx] = ((chainPos - count) << 8) + count; + } else { + tmpHashTable[hashIdx] = 0; + } + } + assert(chainPos <= chainSize); /* I believe this is guaranteed... */ + } -size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) -{ - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_dedicatedDictSearch); -} + /* move chain pointers into the last entry of each hash bucket */ + for (hashIdx = (1 << hashLog); hashIdx; ) { + U32 const bucketIdx = --hashIdx << ZSTD_LAZY_DDSS_BUCKET_LOG; + U32 const chainPackedPointer = tmpHashTable[hashIdx]; + U32 i; + for (i = 0; i < cacheSize; i++) { + hashTable[bucketIdx + i] = 0; + } + hashTable[bucketIdx + bucketSize - 1] = chainPackedPointer; + } -size_t ZSTD_compressBlock_lazy_dedicatedDictSearch( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) -{ - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_dedicatedDictSearch); -} + /* fill the buckets of the hash table */ + for (idx = ms->nextToUpdate; idx < target; idx++) { + U32 const h = (U32)ZSTD_hashPtr(base + idx, hashLog, ms->cParams.minMatch) + << ZSTD_LAZY_DDSS_BUCKET_LOG; + U32 i; + /* Shift hash cache down 1. */ + for (i = cacheSize - 1; i; i--) + hashTable[h + i] = hashTable[h + i - 1]; + hashTable[h] = idx; + } -size_t ZSTD_compressBlock_greedy_dedicatedDictSearch( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) -{ - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_dedicatedDictSearch); + ms->nextToUpdate = target; } - +/* Returns the longest match length found in the dedicated dict search structure. + * If none are longer than the argument ml, then ml will be returned. + */ FORCE_INLINE_TEMPLATE -size_t ZSTD_compressBlock_lazy_extDict_generic( - ZSTD_matchState_t* ms, seqStore_t* seqStore, - U32 rep[ZSTD_REP_NUM], - const void* src, size_t srcSize, - const searchMethod_e searchMethod, const U32 depth) -{ - const BYTE* const istart = (const BYTE*)src; - const BYTE* ip = istart; - const BYTE* anchor = istart; - const BYTE* const iend = istart + srcSize; - const BYTE* const ilimit = iend - 8; - const BYTE* const base = ms->window.base; - const U32 dictLimit = ms->window.dictLimit; - const BYTE* const prefixStart = base + dictLimit; - const BYTE* const dictBase = ms->window.dictBase; - const BYTE* const dictEnd = dictBase + dictLimit; - const BYTE* const dictStart = dictBase + ms->window.lowLimit; - const U32 windowLog = ms->cParams.windowLog; - - typedef size_t (*searchMax_f)( - ZSTD_matchState_t* ms, - const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); - searchMax_f searchMax = searchMethod==search_binaryTree ? ZSTD_BtFindBestMatch_extDict_selectMLS : ZSTD_HcFindBestMatch_extDict_selectMLS; - - U32 offset_1 = rep[0], offset_2 = rep[1]; +size_t ZSTD_dedicatedDictSearch_lazy_search(size_t* offsetPtr, size_t ml, U32 nbAttempts, + const ZSTD_MatchState_t* const dms, + const BYTE* const ip, const BYTE* const iLimit, + const BYTE* const prefixStart, const U32 curr, + const U32 dictLimit, const size_t ddsIdx) { + const U32 ddsLowestIndex = dms->window.dictLimit; + const BYTE* const ddsBase = dms->window.base; + const BYTE* const ddsEnd = dms->window.nextSrc; + const U32 ddsSize = (U32)(ddsEnd - ddsBase); + const U32 ddsIndexDelta = dictLimit - ddsSize; + const U32 bucketSize = (1 << ZSTD_LAZY_DDSS_BUCKET_LOG); + const U32 bucketLimit = nbAttempts < bucketSize - 1 ? nbAttempts : bucketSize - 1; + U32 ddsAttempt; + U32 matchIndex; - DEBUGLOG(5, "ZSTD_compressBlock_lazy_extDict_generic"); + for (ddsAttempt = 0; ddsAttempt < bucketSize - 1; ddsAttempt++) { + PREFETCH_L1(ddsBase + dms->hashTable[ddsIdx + ddsAttempt]); + } - /* init */ - ip += (ip == prefixStart); + { + U32 const chainPackedPointer = dms->hashTable[ddsIdx + bucketSize - 1]; + U32 const chainIndex = chainPackedPointer >> 8; - /* Match Loop */ -#if defined(__GNUC__) && defined(__x86_64__) - /* I've measured random a 5% speed loss on levels 5 & 6 (greedy) when the - * code alignment is perturbed. To fix the instability align the loop on 32-bytes. - */ - __asm__(".p2align 5"); -#endif - while (ip < ilimit) { - size_t matchLength=0; - size_t offset=0; - const BYTE* start=ip+1; - U32 curr = (U32)(ip-base); + PREFETCH_L1(&dms->chainTable[chainIndex]); + } - /* check repCode */ - { const U32 windowLow = ZSTD_getLowestMatchIndex(ms, curr+1, windowLog); - const U32 repIndex = (U32)(curr+1 - offset_1); - const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; - const BYTE* const repMatch = repBase + repIndex; - if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > windowLow)) /* intentional overflow */ - if (MEM_read32(ip+1) == MEM_read32(repMatch)) { - /* repcode detected we should take it */ - const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; - matchLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repEnd, prefixStart) + 4; - if (depth==0) goto _storeSequence; - } } + for (ddsAttempt = 0; ddsAttempt < bucketLimit; ddsAttempt++) { + size_t currentMl=0; + const BYTE* match; + matchIndex = dms->hashTable[ddsIdx + ddsAttempt]; + match = ddsBase + matchIndex; - /* first search (depth 0) */ - { size_t offsetFound = 999999999; - size_t const ml2 = searchMax(ms, ip, iend, &offsetFound); - if (ml2 > matchLength) - matchLength = ml2, start = ip, offset=offsetFound; + if (!matchIndex) { + return ml; } - if (matchLength < 4) { - ip += ((ip-anchor) >> kSearchStrength) + 1; /* jump faster over incompressible sections */ - continue; + /* guaranteed by table construction */ + (void)ddsLowestIndex; + assert(matchIndex >= ddsLowestIndex); + assert(match+4 <= ddsEnd); + if (MEM_read32(match) == MEM_read32(ip)) { + /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, ddsEnd, prefixStart) + 4; } - /* let's try to find a better solution */ - if (depth>=1) - while (ip= 3) & (repIndex > windowLow)) /* intentional overflow */ - if (MEM_read32(ip) == MEM_read32(repMatch)) { - /* repcode detected */ - const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; - size_t const repLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; - int const gain2 = (int)(repLength * 3); - int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offset+1) + 1); - if ((repLength >= 4) && (gain2 > gain1)) - matchLength = repLength, offset = 0, start = ip; - } } - - /* search match, depth 1 */ - { size_t offset2=999999999; - size_t const ml2 = searchMax(ms, ip, iend, &offset2); - int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ - int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4); - if ((ml2 >= 4) && (gain2 > gain1)) { - matchLength = ml2, offset = offset2, start = ip; - continue; /* search a better one */ - } } - - /* let's find an even better one */ - if ((depth==2) && (ip= 3) & (repIndex > windowLow)) /* intentional overflow */ - if (MEM_read32(ip) == MEM_read32(repMatch)) { - /* repcode detected */ - const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; - size_t const repLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; - int const gain2 = (int)(repLength * 4); - int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1); - if ((repLength >= 4) && (gain2 > gain1)) - matchLength = repLength, offset = 0, start = ip; - } } - - /* search match, depth 2 */ - { size_t offset2=999999999; - size_t const ml2 = searchMax(ms, ip, iend, &offset2); - int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ - int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7); - if ((ml2 >= 4) && (gain2 > gain1)) { - matchLength = ml2, offset = offset2, start = ip; - continue; - } } } - break; /* nothing found : store previous solution */ + /* save best solution */ + if (currentMl > ml) { + ml = currentMl; + *offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + ddsIndexDelta)); + if (ip+currentMl == iLimit) { + /* best possible, avoids read overflow on next attempt */ + return ml; + } } + } - /* catch up */ - if (offset) { - U32 const matchIndex = (U32)((start-base) - (offset - ZSTD_REP_MOVE)); - const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex; - const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart; - while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */ - offset_2 = offset_1; offset_1 = (U32)(offset - ZSTD_REP_MOVE); - } + { + U32 const chainPackedPointer = dms->hashTable[ddsIdx + bucketSize - 1]; + U32 chainIndex = chainPackedPointer >> 8; + U32 const chainLength = chainPackedPointer & 0xFF; + U32 const chainAttempts = nbAttempts - ddsAttempt; + U32 const chainLimit = chainAttempts > chainLength ? chainLength : chainAttempts; + U32 chainAttempt; - /* store sequence */ -_storeSequence: - { size_t const litLength = start - anchor; - ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offset, matchLength-MINMATCH); - anchor = ip = start + matchLength; + for (chainAttempt = 0 ; chainAttempt < chainLimit; chainAttempt++) { + PREFETCH_L1(ddsBase + dms->chainTable[chainIndex + chainAttempt]); } - /* check immediate repcode */ - while (ip <= ilimit) { - const U32 repCurrent = (U32)(ip-base); - const U32 windowLow = ZSTD_getLowestMatchIndex(ms, repCurrent, windowLog); - const U32 repIndex = repCurrent - offset_2; - const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; - const BYTE* const repMatch = repBase + repIndex; - if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > windowLow)) /* intentional overflow */ - if (MEM_read32(ip) == MEM_read32(repMatch)) { - /* repcode detected we should take it */ - const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; - matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; - offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap offset history */ - ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, matchLength-MINMATCH); - ip += matchLength; - anchor = ip; - continue; /* faster when present ... (?) */ - } - break; - } } + for (chainAttempt = 0 ; chainAttempt < chainLimit; chainAttempt++, chainIndex++) { + size_t currentMl=0; + const BYTE* match; + matchIndex = dms->chainTable[chainIndex]; + match = ddsBase + matchIndex; - /* Save reps for next block */ - rep[0] = offset_1; - rep[1] = offset_2; + /* guaranteed by table construction */ + assert(matchIndex >= ddsLowestIndex); + assert(match+4 <= ddsEnd); + if (MEM_read32(match) == MEM_read32(ip)) { + /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, ddsEnd, prefixStart) + 4; + } - /* Return the last literals size */ - return (size_t)(iend - anchor); + /* save best solution */ + if (currentMl > ml) { + ml = currentMl; + *offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + ddsIndexDelta)); + if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ + } + } + } + return ml; } -size_t ZSTD_compressBlock_greedy_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) -{ - return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0); -} - -size_t ZSTD_compressBlock_lazy_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) +/* ********************************* +* Hash Chain +***********************************/ +#define NEXT_IN_CHAIN(d, mask) chainTable[(d) & (mask)] +/* Update chains up to ip (excluded) + Assumption : always within prefix (i.e. not within extDict) */ +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +U32 ZSTD_insertAndFindFirstIndex_internal( + ZSTD_MatchState_t* ms, + const ZSTD_compressionParameters* const cParams, + const BYTE* ip, U32 const mls, U32 const lazySkipping) { - return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1); -} + U32* const hashTable = ms->hashTable; + const U32 hashLog = cParams->hashLog; + U32* const chainTable = ms->chainTable; + const U32 chainMask = (1 << cParams->chainLog) - 1; + const BYTE* const base = ms->window.base; + const U32 target = (U32)(ip - base); + U32 idx = ms->nextToUpdate; -size_t ZSTD_compressBlock_lazy2_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) + while(idx < target) { /* catch up */ + size_t const h = ZSTD_hashPtr(base+idx, hashLog, mls); + NEXT_IN_CHAIN(idx, chainMask) = hashTable[h]; + hashTable[h] = idx; + idx++; + /* Stop inserting every position when in the lazy skipping mode. */ + if (lazySkipping) + break; + } -{ - return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2); + ms->nextToUpdate = target; + return hashTable[ZSTD_hashPtr(ip, hashLog, mls)]; } -size_t ZSTD_compressBlock_btlazy2_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) - -{ - return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2); +U32 ZSTD_insertAndFindFirstIndex(ZSTD_MatchState_t* ms, const BYTE* ip) { + const ZSTD_compressionParameters* const cParams = &ms->cParams; + return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, ms->cParams.minMatch, /* lazySkipping*/ 0); } -/**** ended inlining compress/zstd_lazy.c ****/ -/**** start inlining compress/zstd_ldm.c ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ -/**** skipping file: zstd_ldm.h ****/ +/* inlining is important to hardwire a hot branch (template emulation) */ +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_HcFindBestMatch( + ZSTD_MatchState_t* ms, + const BYTE* const ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 mls, const ZSTD_dictMode_e dictMode) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const chainTable = ms->chainTable; + const U32 chainSize = (1 << cParams->chainLog); + const U32 chainMask = chainSize-1; + const BYTE* const base = ms->window.base; + const BYTE* const dictBase = ms->window.dictBase; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const U32 curr = (U32)(ip-base); + const U32 maxDistance = 1U << cParams->windowLog; + const U32 lowestValid = ms->window.lowLimit; + const U32 withinMaxDistance = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid; + const U32 isDictionary = (ms->loadedDictEnd != 0); + const U32 lowLimit = isDictionary ? lowestValid : withinMaxDistance; + const U32 minChain = curr > chainSize ? curr - chainSize : 0; + U32 nbAttempts = 1U << cParams->searchLog; + size_t ml=4-1; -/**** skipping file: ../common/debug.h ****/ -/**** skipping file: ../common/xxhash.h ****/ -/**** skipping file: zstd_fast.h ****/ -/**** skipping file: zstd_double_fast.h ****/ -/**** start inlining zstd_ldm_geartab.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ + const ZSTD_MatchState_t* const dms = ms->dictMatchState; + const U32 ddsHashLog = dictMode == ZSTD_dedicatedDictSearch + ? dms->cParams.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG : 0; + const size_t ddsIdx = dictMode == ZSTD_dedicatedDictSearch + ? ZSTD_hashPtr(ip, ddsHashLog, mls) << ZSTD_LAZY_DDSS_BUCKET_LOG : 0; -#ifndef ZSTD_LDM_GEARTAB_H -#define ZSTD_LDM_GEARTAB_H + U32 matchIndex; -static U64 ZSTD_ldm_gearTab[256] = { - 0xf5b8f72c5f77775c, 0x84935f266b7ac412, 0xb647ada9ca730ccc, - 0xb065bb4b114fb1de, 0x34584e7e8c3a9fd0, 0x4e97e17c6ae26b05, - 0x3a03d743bc99a604, 0xcecd042422c4044f, 0x76de76c58524259e, - 0x9c8528f65badeaca, 0x86563706e2097529, 0x2902475fa375d889, - 0xafb32a9739a5ebe6, 0xce2714da3883e639, 0x21eaf821722e69e, - 0x37b628620b628, 0x49a8d455d88caf5, 0x8556d711e6958140, - 0x4f7ae74fc605c1f, 0x829f0c3468bd3a20, 0x4ffdc885c625179e, - 0x8473de048a3daf1b, 0x51008822b05646b2, 0x69d75d12b2d1cc5f, - 0x8c9d4a19159154bc, 0xc3cc10f4abbd4003, 0xd06ddc1cecb97391, - 0xbe48e6e7ed80302e, 0x3481db31cee03547, 0xacc3f67cdaa1d210, - 0x65cb771d8c7f96cc, 0x8eb27177055723dd, 0xc789950d44cd94be, - 0x934feadc3700b12b, 0x5e485f11edbdf182, 0x1e2e2a46fd64767a, - 0x2969ca71d82efa7c, 0x9d46e9935ebbba2e, 0xe056b67e05e6822b, - 0x94d73f55739d03a0, 0xcd7010bdb69b5a03, 0x455ef9fcd79b82f4, - 0x869cb54a8749c161, 0x38d1a4fa6185d225, 0xb475166f94bbe9bb, - 0xa4143548720959f1, 0x7aed4780ba6b26ba, 0xd0ce264439e02312, - 0x84366d746078d508, 0xa8ce973c72ed17be, 0x21c323a29a430b01, - 0x9962d617e3af80ee, 0xab0ce91d9c8cf75b, 0x530e8ee6d19a4dbc, - 0x2ef68c0cf53f5d72, 0xc03a681640a85506, 0x496e4e9f9c310967, - 0x78580472b59b14a0, 0x273824c23b388577, 0x66bf923ad45cb553, - 0x47ae1a5a2492ba86, 0x35e304569e229659, 0x4765182a46870b6f, - 0x6cbab625e9099412, 0xddac9a2e598522c1, 0x7172086e666624f2, - 0xdf5003ca503b7837, 0x88c0c1db78563d09, 0x58d51865acfc289d, - 0x177671aec65224f1, 0xfb79d8a241e967d7, 0x2be1e101cad9a49a, - 0x6625682f6e29186b, 0x399553457ac06e50, 0x35dffb4c23abb74, - 0x429db2591f54aade, 0xc52802a8037d1009, 0x6acb27381f0b25f3, - 0xf45e2551ee4f823b, 0x8b0ea2d99580c2f7, 0x3bed519cbcb4e1e1, - 0xff452823dbb010a, 0x9d42ed614f3dd267, 0x5b9313c06257c57b, - 0xa114b8008b5e1442, 0xc1fe311c11c13d4b, 0x66e8763ea34c5568, - 0x8b982af1c262f05d, 0xee8876faaa75fbb7, 0x8a62a4d0d172bb2a, - 0xc13d94a3b7449a97, 0x6dbbba9dc15d037c, 0xc786101f1d92e0f1, - 0xd78681a907a0b79b, 0xf61aaf2962c9abb9, 0x2cfd16fcd3cb7ad9, - 0x868c5b6744624d21, 0x25e650899c74ddd7, 0xba042af4a7c37463, - 0x4eb1a539465a3eca, 0xbe09dbf03b05d5ca, 0x774e5a362b5472ba, - 0x47a1221229d183cd, 0x504b0ca18ef5a2df, 0xdffbdfbde2456eb9, - 0x46cd2b2fbee34634, 0xf2aef8fe819d98c3, 0x357f5276d4599d61, - 0x24a5483879c453e3, 0x88026889192b4b9, 0x28da96671782dbec, - 0x4ef37c40588e9aaa, 0x8837b90651bc9fb3, 0xc164f741d3f0e5d6, - 0xbc135a0a704b70ba, 0x69cd868f7622ada, 0xbc37ba89e0b9c0ab, - 0x47c14a01323552f6, 0x4f00794bacee98bb, 0x7107de7d637a69d5, - 0x88af793bb6f2255e, 0xf3c6466b8799b598, 0xc288c616aa7f3b59, - 0x81ca63cf42fca3fd, 0x88d85ace36a2674b, 0xd056bd3792389e7, - 0xe55c396c4e9dd32d, 0xbefb504571e6c0a6, 0x96ab32115e91e8cc, - 0xbf8acb18de8f38d1, 0x66dae58801672606, 0x833b6017872317fb, - 0xb87c16f2d1c92864, 0xdb766a74e58b669c, 0x89659f85c61417be, - 0xc8daad856011ea0c, 0x76a4b565b6fe7eae, 0xa469d085f6237312, - 0xaaf0365683a3e96c, 0x4dbb746f8424f7b8, 0x638755af4e4acc1, - 0x3d7807f5bde64486, 0x17be6d8f5bbb7639, 0x903f0cd44dc35dc, - 0x67b672eafdf1196c, 0xa676ff93ed4c82f1, 0x521d1004c5053d9d, - 0x37ba9ad09ccc9202, 0x84e54d297aacfb51, 0xa0b4b776a143445, - 0x820d471e20b348e, 0x1874383cb83d46dc, 0x97edeec7a1efe11c, - 0xb330e50b1bdc42aa, 0x1dd91955ce70e032, 0xa514cdb88f2939d5, - 0x2791233fd90db9d3, 0x7b670a4cc50f7a9b, 0x77c07d2a05c6dfa5, - 0xe3778b6646d0a6fa, 0xb39c8eda47b56749, 0x933ed448addbef28, - 0xaf846af6ab7d0bf4, 0xe5af208eb666e49, 0x5e6622f73534cd6a, - 0x297daeca42ef5b6e, 0x862daef3d35539a6, 0xe68722498f8e1ea9, - 0x981c53093dc0d572, 0xfa09b0bfbf86fbf5, 0x30b1e96166219f15, - 0x70e7d466bdc4fb83, 0x5a66736e35f2a8e9, 0xcddb59d2b7c1baef, - 0xd6c7d247d26d8996, 0xea4e39eac8de1ba3, 0x539c8bb19fa3aff2, - 0x9f90e4c5fd508d8, 0xa34e5956fbaf3385, 0x2e2f8e151d3ef375, - 0x173691e9b83faec1, 0xb85a8d56bf016379, 0x8382381267408ae3, - 0xb90f901bbdc0096d, 0x7c6ad32933bcec65, 0x76bb5e2f2c8ad595, - 0x390f851a6cf46d28, 0xc3e6064da1c2da72, 0xc52a0c101cfa5389, - 0xd78eaf84a3fbc530, 0x3781b9e2288b997e, 0x73c2f6dea83d05c4, - 0x4228e364c5b5ed7, 0x9d7a3edf0da43911, 0x8edcfeda24686756, - 0x5e7667a7b7a9b3a1, 0x4c4f389fa143791d, 0xb08bc1023da7cddc, - 0x7ab4be3ae529b1cc, 0x754e6132dbe74ff9, 0x71635442a839df45, - 0x2f6fb1643fbe52de, 0x961e0a42cf7a8177, 0xf3b45d83d89ef2ea, - 0xee3de4cf4a6e3e9b, 0xcd6848542c3295e7, 0xe4cee1664c78662f, - 0x9947548b474c68c4, 0x25d73777a5ed8b0b, 0xc915b1d636b7fc, - 0x21c2ba75d9b0d2da, 0x5f6b5dcf608a64a1, 0xdcf333255ff9570c, - 0x633b922418ced4ee, 0xc136dde0b004b34a, 0x58cc83b05d4b2f5a, - 0x5eb424dda28e42d2, 0x62df47369739cd98, 0xb4e0b42485e4ce17, - 0x16e1f0c1f9a8d1e7, 0x8ec3916707560ebf, 0x62ba6e2df2cc9db3, - 0xcbf9f4ff77d83a16, 0x78d9d7d07d2bbcc4, 0xef554ce1e02c41f4, - 0x8d7581127eccf94d, 0xa9b53336cb3c8a05, 0x38c42c0bf45c4f91, - 0x640893cdf4488863, 0x80ec34bc575ea568, 0x39f324f5b48eaa40, - 0xe9d9ed1f8eff527f, 0x9224fc058cc5a214, 0xbaba00b04cfe7741, - 0x309a9f120fcf52af, 0xa558f3ec65626212, 0x424bec8b7adabe2f, - 0x41622513a6aea433, 0xb88da2d5324ca798, 0xd287733b245528a4, - 0x9a44697e6d68aec3, 0x7b1093be2f49bb28, 0x50bbec632e3d8aad, - 0x6cd90723e1ea8283, 0x897b9e7431b02bf3, 0x219efdcb338a7047, - 0x3b0311f0a27c0656, 0xdb17bf91c0db96e7, 0x8cd4fd6b4e85a5b2, - 0xfab071054ba6409d, 0x40d6fe831fa9dfd9, 0xaf358debad7d791e, - 0xeb8d0e25a65e3e58, 0xbbcbd3df14e08580, 0xcf751f27ecdab2b, - 0x2b4da14f2613d8f4 -}; + if (dictMode == ZSTD_dedicatedDictSearch) { + const U32* entry = &dms->hashTable[ddsIdx]; + PREFETCH_L1(entry); + } -#endif /* ZSTD_LDM_GEARTAB_H */ -/**** ended inlining zstd_ldm_geartab.h ****/ + /* HC4 match finder */ + matchIndex = ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, mls, ms->lazySkipping); -#define LDM_BUCKET_SIZE_LOG 3 -#define LDM_MIN_MATCH_LENGTH 64 -#define LDM_HASH_RLOG 7 + for ( ; (matchIndex>=lowLimit) & (nbAttempts>0) ; nbAttempts--) { + size_t currentMl=0; + if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) { + const BYTE* const match = base + matchIndex; + assert(matchIndex >= dictLimit); /* ensures this is true if dictMode != ZSTD_extDict */ + /* read 4B starting from (match + ml + 1 - sizeof(U32)) */ + if (MEM_read32(match + ml - 3) == MEM_read32(ip + ml - 3)) /* potentially better */ + currentMl = ZSTD_count(ip, match, iLimit); + } else { + const BYTE* const match = dictBase + matchIndex; + assert(match+4 <= dictEnd); + if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dictEnd, prefixStart) + 4; + } -typedef struct { - U64 rolling; - U64 stopMask; -} ldmRollingHashState_t; + /* save best solution */ + if (currentMl > ml) { + ml = currentMl; + *offsetPtr = OFFSET_TO_OFFBASE(curr - matchIndex); + if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ + } -/** ZSTD_ldm_gear_init(): - * - * Initializes the rolling hash state such that it will honor the - * settings in params. */ -static void ZSTD_ldm_gear_init(ldmRollingHashState_t* state, ldmParams_t const* params) -{ - unsigned maxBitsInMask = MIN(params->minMatchLength, 64); - unsigned hashRateLog = params->hashRateLog; + if (matchIndex <= minChain) break; + matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask); + } - state->rolling = ~(U32)0; + assert(nbAttempts <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */ + if (dictMode == ZSTD_dedicatedDictSearch) { + ml = ZSTD_dedicatedDictSearch_lazy_search(offsetPtr, ml, nbAttempts, dms, + ip, iLimit, prefixStart, curr, dictLimit, ddsIdx); + } else if (dictMode == ZSTD_dictMatchState) { + const U32* const dmsChainTable = dms->chainTable; + const U32 dmsChainSize = (1 << dms->cParams.chainLog); + const U32 dmsChainMask = dmsChainSize - 1; + const U32 dmsLowestIndex = dms->window.dictLimit; + const BYTE* const dmsBase = dms->window.base; + const BYTE* const dmsEnd = dms->window.nextSrc; + const U32 dmsSize = (U32)(dmsEnd - dmsBase); + const U32 dmsIndexDelta = dictLimit - dmsSize; + const U32 dmsMinChain = dmsSize > dmsChainSize ? dmsSize - dmsChainSize : 0; - /* The choice of the splitting criterion is subject to two conditions: - * 1. it has to trigger on average every 2^(hashRateLog) bytes; - * 2. ideally, it has to depend on a window of minMatchLength bytes. - * - * In the gear hash algorithm, bit n depends on the last n bytes; - * so in order to obtain a good quality splitting criterion it is - * preferable to use bits with high weight. - * - * To match condition 1 we use a mask with hashRateLog bits set - * and, because of the previous remark, we make sure these bits - * have the highest possible weight while still respecting - * condition 2. - */ - if (hashRateLog > 0 && hashRateLog <= maxBitsInMask) { - state->stopMask = (((U64)1 << hashRateLog) - 1) << (maxBitsInMask - hashRateLog); - } else { - /* In this degenerate case we simply honor the hash rate. */ - state->stopMask = ((U64)1 << hashRateLog) - 1; - } -} + matchIndex = dms->hashTable[ZSTD_hashPtr(ip, dms->cParams.hashLog, mls)]; -/** ZSTD_ldm_gear_feed(): - * - * Registers in the splits array all the split points found in the first - * size bytes following the data pointer. This function terminates when - * either all the data has been processed or LDM_BATCH_SIZE splits are - * present in the splits array. - * - * Precondition: The splits array must not be full. - * Returns: The number of bytes processed. */ -static size_t ZSTD_ldm_gear_feed(ldmRollingHashState_t* state, - BYTE const* data, size_t size, - size_t* splits, unsigned* numSplits) -{ - size_t n; - U64 hash, mask; + for ( ; (matchIndex>=dmsLowestIndex) & (nbAttempts>0) ; nbAttempts--) { + size_t currentMl=0; + const BYTE* const match = dmsBase + matchIndex; + assert(match+4 <= dmsEnd); + if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dmsEnd, prefixStart) + 4; - hash = state->rolling; - mask = state->stopMask; - n = 0; + /* save best solution */ + if (currentMl > ml) { + ml = currentMl; + assert(curr > matchIndex + dmsIndexDelta); + *offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + dmsIndexDelta)); + if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ + } -#define GEAR_ITER_ONCE() do { \ - hash = (hash << 1) + ZSTD_ldm_gearTab[data[n] & 0xff]; \ - n += 1; \ - if (UNLIKELY((hash & mask) == 0)) { \ - splits[*numSplits] = n; \ - *numSplits += 1; \ - if (*numSplits == LDM_BATCH_SIZE) \ - goto done; \ - } \ - } while (0) + if (matchIndex <= dmsMinChain) break; - while (n + 3 < size) { - GEAR_ITER_ONCE(); - GEAR_ITER_ONCE(); - GEAR_ITER_ONCE(); - GEAR_ITER_ONCE(); - } - while (n < size) { - GEAR_ITER_ONCE(); + matchIndex = dmsChainTable[matchIndex & dmsChainMask]; + } } -#undef GEAR_ITER_ONCE - -done: - state->rolling = hash; - return n; + return ml; } -void ZSTD_ldm_adjustParameters(ldmParams_t* params, - ZSTD_compressionParameters const* cParams) -{ - params->windowLog = cParams->windowLog; - ZSTD_STATIC_ASSERT(LDM_BUCKET_SIZE_LOG <= ZSTD_LDM_BUCKETSIZELOG_MAX); - DEBUGLOG(4, "ZSTD_ldm_adjustParameters"); - if (!params->bucketSizeLog) params->bucketSizeLog = LDM_BUCKET_SIZE_LOG; - if (!params->minMatchLength) params->minMatchLength = LDM_MIN_MATCH_LENGTH; - if (params->hashLog == 0) { - params->hashLog = MAX(ZSTD_HASHLOG_MIN, params->windowLog - LDM_HASH_RLOG); - assert(params->hashLog <= ZSTD_HASHLOG_MAX); - } - if (params->hashRateLog == 0) { - params->hashRateLog = params->windowLog < params->hashLog - ? 0 - : params->windowLog - params->hashLog; - } - params->bucketSizeLog = MIN(params->bucketSizeLog, params->hashLog); +/* ********************************* +* (SIMD) Row-based matchfinder +***********************************/ +/* Constants for row-based hash */ +#define ZSTD_ROW_HASH_TAG_MASK ((1u << ZSTD_ROW_HASH_TAG_BITS) - 1) +#define ZSTD_ROW_HASH_MAX_ENTRIES 64 /* absolute maximum number of entries per row, for all configurations */ + +#define ZSTD_ROW_HASH_CACHE_MASK (ZSTD_ROW_HASH_CACHE_SIZE - 1) + +typedef U64 ZSTD_VecMask; /* Clarifies when we are interacting with a U64 representing a mask of matches */ + +/* ZSTD_VecMask_next(): + * Starting from the LSB, returns the idx of the next non-zero bit. + * Basically counting the nb of trailing zeroes. + */ +MEM_STATIC U32 ZSTD_VecMask_next(ZSTD_VecMask val) { + return ZSTD_countTrailingZeros64(val); } -size_t ZSTD_ldm_getTableSize(ldmParams_t params) -{ - size_t const ldmHSize = ((size_t)1) << params.hashLog; - size_t const ldmBucketSizeLog = MIN(params.bucketSizeLog, params.hashLog); - size_t const ldmBucketSize = ((size_t)1) << (params.hashLog - ldmBucketSizeLog); - size_t const totalSize = ZSTD_cwksp_alloc_size(ldmBucketSize) - + ZSTD_cwksp_alloc_size(ldmHSize * sizeof(ldmEntry_t)); - return params.enableLdm ? totalSize : 0; +/* ZSTD_row_nextIndex(): + * Returns the next index to insert at within a tagTable row, and updates the "head" + * value to reflect the update. Essentially cycles backwards from [1, {entries per row}) + */ +FORCE_INLINE_TEMPLATE U32 ZSTD_row_nextIndex(BYTE* const tagRow, U32 const rowMask) { + U32 next = (*tagRow-1) & rowMask; + next += (next == 0) ? rowMask : 0; /* skip first position */ + *tagRow = (BYTE)next; + return next; } -size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize) -{ - return params.enableLdm ? (maxChunkSize / params.minMatchLength) : 0; +/* ZSTD_isAligned(): + * Checks that a pointer is aligned to "align" bytes which must be a power of 2. + */ +MEM_STATIC int ZSTD_isAligned(void const* ptr, size_t align) { + assert((align & (align - 1)) == 0); + return (((size_t)ptr) & (align - 1)) == 0; } -/** ZSTD_ldm_getBucket() : - * Returns a pointer to the start of the bucket associated with hash. */ -static ldmEntry_t* ZSTD_ldm_getBucket( - ldmState_t* ldmState, size_t hash, ldmParams_t const ldmParams) -{ - return ldmState->hashTable + (hash << ldmParams.bucketSizeLog); +/* ZSTD_row_prefetch(): + * Performs prefetching for the hashTable and tagTable at a given row. + */ +FORCE_INLINE_TEMPLATE void ZSTD_row_prefetch(U32 const* hashTable, BYTE const* tagTable, U32 const relRow, U32 const rowLog) { + PREFETCH_L1(hashTable + relRow); + if (rowLog >= 5) { + PREFETCH_L1(hashTable + relRow + 16); + /* Note: prefetching more of the hash table does not appear to be beneficial for 128-entry rows */ + } + PREFETCH_L1(tagTable + relRow); + if (rowLog == 6) { + PREFETCH_L1(tagTable + relRow + 32); + } + assert(rowLog == 4 || rowLog == 5 || rowLog == 6); + assert(ZSTD_isAligned(hashTable + relRow, 64)); /* prefetched hash row always 64-byte aligned */ + assert(ZSTD_isAligned(tagTable + relRow, (size_t)1 << rowLog)); /* prefetched tagRow sits on correct multiple of bytes (32,64,128) */ } -/** ZSTD_ldm_insertEntry() : - * Insert the entry with corresponding hash into the hash table */ -static void ZSTD_ldm_insertEntry(ldmState_t* ldmState, - size_t const hash, const ldmEntry_t entry, - ldmParams_t const ldmParams) +/* ZSTD_row_fillHashCache(): + * Fill up the hash cache starting at idx, prefetching up to ZSTD_ROW_HASH_CACHE_SIZE entries, + * but not beyond iLimit. + */ +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +void ZSTD_row_fillHashCache(ZSTD_MatchState_t* ms, const BYTE* base, + U32 const rowLog, U32 const mls, + U32 idx, const BYTE* const iLimit) { - BYTE* const pOffset = ldmState->bucketOffsets + hash; - unsigned const offset = *pOffset; + U32 const* const hashTable = ms->hashTable; + BYTE const* const tagTable = ms->tagTable; + U32 const hashLog = ms->rowHashLog; + U32 const maxElemsToPrefetch = (base + idx) > iLimit ? 0 : (U32)(iLimit - (base + idx) + 1); + U32 const lim = idx + MIN(ZSTD_ROW_HASH_CACHE_SIZE, maxElemsToPrefetch); - *(ZSTD_ldm_getBucket(ldmState, hash, ldmParams) + offset) = entry; - *pOffset = (BYTE)((offset + 1) & ((1u << ldmParams.bucketSizeLog) - 1)); + for (; idx < lim; ++idx) { + U32 const hash = (U32)ZSTD_hashPtrSalted(base + idx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, ms->hashSalt); + U32 const row = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog; + ZSTD_row_prefetch(hashTable, tagTable, row, rowLog); + ms->hashCache[idx & ZSTD_ROW_HASH_CACHE_MASK] = hash; + } + DEBUGLOG(6, "ZSTD_row_fillHashCache(): [%u %u %u %u %u %u %u %u]", ms->hashCache[0], ms->hashCache[1], + ms->hashCache[2], ms->hashCache[3], ms->hashCache[4], + ms->hashCache[5], ms->hashCache[6], ms->hashCache[7]); } -/** ZSTD_ldm_countBackwardsMatch() : - * Returns the number of bytes that match backwards before pIn and pMatch. - * - * We count only bytes where pMatch >= pBase and pIn >= pAnchor. */ -static size_t ZSTD_ldm_countBackwardsMatch( - const BYTE* pIn, const BYTE* pAnchor, - const BYTE* pMatch, const BYTE* pMatchBase) +/* ZSTD_row_nextCachedHash(): + * Returns the hash of base + idx, and replaces the hash in the hash cache with the byte at + * base + idx + ZSTD_ROW_HASH_CACHE_SIZE. Also prefetches the appropriate rows from hashTable and tagTable. + */ +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +U32 ZSTD_row_nextCachedHash(U32* cache, U32 const* hashTable, + BYTE const* tagTable, BYTE const* base, + U32 idx, U32 const hashLog, + U32 const rowLog, U32 const mls, + U64 const hashSalt) { - size_t matchLength = 0; - while (pIn > pAnchor && pMatch > pMatchBase && pIn[-1] == pMatch[-1]) { - pIn--; - pMatch--; - matchLength++; + U32 const newHash = (U32)ZSTD_hashPtrSalted(base+idx+ZSTD_ROW_HASH_CACHE_SIZE, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, hashSalt); + U32 const row = (newHash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog; + ZSTD_row_prefetch(hashTable, tagTable, row, rowLog); + { U32 const hash = cache[idx & ZSTD_ROW_HASH_CACHE_MASK]; + cache[idx & ZSTD_ROW_HASH_CACHE_MASK] = newHash; + return hash; } - return matchLength; } -/** ZSTD_ldm_countBackwardsMatch_2segments() : - * Returns the number of bytes that match backwards from pMatch, - * even with the backwards match spanning 2 different segments. - * - * On reaching `pMatchBase`, start counting from mEnd */ -static size_t ZSTD_ldm_countBackwardsMatch_2segments( - const BYTE* pIn, const BYTE* pAnchor, - const BYTE* pMatch, const BYTE* pMatchBase, - const BYTE* pExtDictStart, const BYTE* pExtDictEnd) +/* ZSTD_row_update_internalImpl(): + * Updates the hash table with positions starting from updateStartIdx until updateEndIdx. + */ +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +void ZSTD_row_update_internalImpl(ZSTD_MatchState_t* ms, + U32 updateStartIdx, U32 const updateEndIdx, + U32 const mls, U32 const rowLog, + U32 const rowMask, U32 const useCache) { - size_t matchLength = ZSTD_ldm_countBackwardsMatch(pIn, pAnchor, pMatch, pMatchBase); - if (pMatch - matchLength != pMatchBase || pMatchBase == pExtDictStart) { - /* If backwards match is entirely in the extDict or prefix, immediately return */ - return matchLength; + U32* const hashTable = ms->hashTable; + BYTE* const tagTable = ms->tagTable; + U32 const hashLog = ms->rowHashLog; + const BYTE* const base = ms->window.base; + + DEBUGLOG(6, "ZSTD_row_update_internalImpl(): updateStartIdx=%u, updateEndIdx=%u", updateStartIdx, updateEndIdx); + for (; updateStartIdx < updateEndIdx; ++updateStartIdx) { + U32 const hash = useCache ? ZSTD_row_nextCachedHash(ms->hashCache, hashTable, tagTable, base, updateStartIdx, hashLog, rowLog, mls, ms->hashSalt) + : (U32)ZSTD_hashPtrSalted(base + updateStartIdx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, ms->hashSalt); + U32 const relRow = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog; + U32* const row = hashTable + relRow; + BYTE* tagRow = tagTable + relRow; + U32 const pos = ZSTD_row_nextIndex(tagRow, rowMask); + + assert(hash == ZSTD_hashPtrSalted(base + updateStartIdx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, ms->hashSalt)); + tagRow[pos] = hash & ZSTD_ROW_HASH_TAG_MASK; + row[pos] = updateStartIdx; } - DEBUGLOG(7, "ZSTD_ldm_countBackwardsMatch_2segments: found 2-parts backwards match (length in prefix==%zu)", matchLength); - matchLength += ZSTD_ldm_countBackwardsMatch(pIn - matchLength, pAnchor, pExtDictEnd, pExtDictStart); - DEBUGLOG(7, "final backwards match length = %zu", matchLength); - return matchLength; } -/** ZSTD_ldm_fillFastTables() : - * - * Fills the relevant tables for the ZSTD_fast and ZSTD_dfast strategies. - * This is similar to ZSTD_loadDictionaryContent. - * - * The tables for the other strategies are filled within their - * block compressors. */ -static size_t ZSTD_ldm_fillFastTables(ZSTD_matchState_t* ms, - void const* end) +/* ZSTD_row_update_internal(): + * Inserts the byte at ip into the appropriate position in the hash table, and updates ms->nextToUpdate. + * Skips sections of long matches as is necessary. + */ +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +void ZSTD_row_update_internal(ZSTD_MatchState_t* ms, const BYTE* ip, + U32 const mls, U32 const rowLog, + U32 const rowMask, U32 const useCache) { - const BYTE* const iend = (const BYTE*)end; + U32 idx = ms->nextToUpdate; + const BYTE* const base = ms->window.base; + const U32 target = (U32)(ip - base); + const U32 kSkipThreshold = 384; + const U32 kMaxMatchStartPositionsToUpdate = 96; + const U32 kMaxMatchEndPositionsToUpdate = 32; + + if (useCache) { + /* Only skip positions when using hash cache, i.e. + * if we are loading a dict, don't skip anything. + * If we decide to skip, then we only update a set number + * of positions at the beginning and end of the match. + */ + if (UNLIKELY(target - idx > kSkipThreshold)) { + U32 const bound = idx + kMaxMatchStartPositionsToUpdate; + ZSTD_row_update_internalImpl(ms, idx, bound, mls, rowLog, rowMask, useCache); + idx = target - kMaxMatchEndPositionsToUpdate; + ZSTD_row_fillHashCache(ms, base, rowLog, mls, idx, ip+1); + } + } + assert(target >= idx); + ZSTD_row_update_internalImpl(ms, idx, target, mls, rowLog, rowMask, useCache); + ms->nextToUpdate = target; +} - switch(ms->cParams.strategy) - { - case ZSTD_fast: - ZSTD_fillHashTable(ms, iend, ZSTD_dtlm_fast); - break; +/* ZSTD_row_update(): + * External wrapper for ZSTD_row_update_internal(). Used for filling the hashtable during dictionary + * processing. + */ +void ZSTD_row_update(ZSTD_MatchState_t* const ms, const BYTE* ip) { + const U32 rowLog = BOUNDED(4, ms->cParams.searchLog, 6); + const U32 rowMask = (1u << rowLog) - 1; + const U32 mls = MIN(ms->cParams.minMatch, 6 /* mls caps out at 6 */); - case ZSTD_dfast: - ZSTD_fillDoubleHashTable(ms, iend, ZSTD_dtlm_fast); - break; + DEBUGLOG(5, "ZSTD_row_update(), rowLog=%u", rowLog); + ZSTD_row_update_internal(ms, ip, mls, rowLog, rowMask, 0 /* don't use cache */); +} - case ZSTD_greedy: - case ZSTD_lazy: - case ZSTD_lazy2: - case ZSTD_btlazy2: - case ZSTD_btopt: - case ZSTD_btultra: - case ZSTD_btultra2: - break; - default: - assert(0); /* not possible : not a valid strategy id */ +/* Returns the mask width of bits group of which will be set to 1. Given not all + * architectures have easy movemask instruction, this helps to iterate over + * groups of bits easier and faster. + */ +FORCE_INLINE_TEMPLATE U32 +ZSTD_row_matchMaskGroupWidth(const U32 rowEntries) +{ + assert((rowEntries == 16) || (rowEntries == 32) || rowEntries == 64); + assert(rowEntries <= ZSTD_ROW_HASH_MAX_ENTRIES); + (void)rowEntries; +#if defined(ZSTD_ARCH_ARM_NEON) + /* NEON path only works for little endian */ + if (!MEM_isLittleEndian()) { + return 1; } - - return 0; + if (rowEntries == 16) { + return 4; + } + if (rowEntries == 32) { + return 2; + } + if (rowEntries == 64) { + return 1; + } +#endif + return 1; } -void ZSTD_ldm_fillHashTable( - ldmState_t* ldmState, const BYTE* ip, - const BYTE* iend, ldmParams_t const* params) +#if defined(ZSTD_ARCH_X86_SSE2) +FORCE_INLINE_TEMPLATE ZSTD_VecMask +ZSTD_row_getSSEMask(int nbChunks, const BYTE* const src, const BYTE tag, const U32 head) { - U32 const minMatchLength = params->minMatchLength; - U32 const hBits = params->hashLog - params->bucketSizeLog; - BYTE const* const base = ldmState->window.base; - BYTE const* const istart = ip; - ldmRollingHashState_t hashState; - size_t* const splits = ldmState->splitIndices; - unsigned numSplits; - - DEBUGLOG(5, "ZSTD_ldm_fillHashTable"); - - ZSTD_ldm_gear_init(&hashState, params); - while (ip < iend) { - size_t hashed; - unsigned n; - - numSplits = 0; - hashed = ZSTD_ldm_gear_feed(&hashState, ip, iend - ip, splits, &numSplits); - - for (n = 0; n < numSplits; n++) { - if (ip + splits[n] >= istart + minMatchLength) { - BYTE const* const split = ip + splits[n] - minMatchLength; - U64 const xxhash = XXH64(split, minMatchLength, 0); - U32 const hash = (U32)(xxhash & (((U32)1 << hBits) - 1)); - ldmEntry_t entry; - - entry.offset = (U32)(split - base); - entry.checksum = (U32)(xxhash >> 32); - ZSTD_ldm_insertEntry(ldmState, hash, entry, *params); - } - } - - ip += hashed; + const __m128i comparisonMask = _mm_set1_epi8((char)tag); + int matches[4] = {0}; + int i; + assert(nbChunks == 1 || nbChunks == 2 || nbChunks == 4); + for (i=0; inextToUpdate to a position corresponding closer to anchor - * if it is far way - * (after a long match, only update tables a limited amount). */ -static void ZSTD_ldm_limitTableUpdate(ZSTD_matchState_t* ms, const BYTE* anchor) +#if defined(ZSTD_ARCH_ARM_NEON) +FORCE_INLINE_TEMPLATE ZSTD_VecMask +ZSTD_row_getNEONMask(const U32 rowEntries, const BYTE* const src, const BYTE tag, const U32 headGrouped) { - U32 const curr = (U32)(anchor - ms->window.base); - if (curr > ms->nextToUpdate + 1024) { - ms->nextToUpdate = - curr - MIN(512, curr - ms->nextToUpdate - 1024); + assert((rowEntries == 16) || (rowEntries == 32) || rowEntries == 64); + if (rowEntries == 16) { + /* vshrn_n_u16 shifts by 4 every u16 and narrows to 8 lower bits. + * After that groups of 4 bits represent the equalMask. We lower + * all bits except the highest in these groups by doing AND with + * 0x88 = 0b10001000. + */ + const uint8x16_t chunk = vld1q_u8(src); + const uint16x8_t equalMask = vreinterpretq_u16_u8(vceqq_u8(chunk, vdupq_n_u8(tag))); + const uint8x8_t res = vshrn_n_u16(equalMask, 4); + const U64 matches = vget_lane_u64(vreinterpret_u64_u8(res), 0); + return ZSTD_rotateRight_U64(matches, headGrouped) & 0x8888888888888888ull; + } else if (rowEntries == 32) { + /* Same idea as with rowEntries == 16 but doing AND with + * 0x55 = 0b01010101. + */ + const uint16x8x2_t chunk = vld2q_u16((const uint16_t*)(const void*)src); + const uint8x16_t chunk0 = vreinterpretq_u8_u16(chunk.val[0]); + const uint8x16_t chunk1 = vreinterpretq_u8_u16(chunk.val[1]); + const uint8x16_t dup = vdupq_n_u8(tag); + const uint8x8_t t0 = vshrn_n_u16(vreinterpretq_u16_u8(vceqq_u8(chunk0, dup)), 6); + const uint8x8_t t1 = vshrn_n_u16(vreinterpretq_u16_u8(vceqq_u8(chunk1, dup)), 6); + const uint8x8_t res = vsli_n_u8(t0, t1, 4); + const U64 matches = vget_lane_u64(vreinterpret_u64_u8(res), 0) ; + return ZSTD_rotateRight_U64(matches, headGrouped) & 0x5555555555555555ull; + } else { /* rowEntries == 64 */ + const uint8x16x4_t chunk = vld4q_u8(src); + const uint8x16_t dup = vdupq_n_u8(tag); + const uint8x16_t cmp0 = vceqq_u8(chunk.val[0], dup); + const uint8x16_t cmp1 = vceqq_u8(chunk.val[1], dup); + const uint8x16_t cmp2 = vceqq_u8(chunk.val[2], dup); + const uint8x16_t cmp3 = vceqq_u8(chunk.val[3], dup); + + const uint8x16_t t0 = vsriq_n_u8(cmp1, cmp0, 1); + const uint8x16_t t1 = vsriq_n_u8(cmp3, cmp2, 1); + const uint8x16_t t2 = vsriq_n_u8(t1, t0, 2); + const uint8x16_t t3 = vsriq_n_u8(t2, t2, 4); + const uint8x8_t t4 = vshrn_n_u16(vreinterpretq_u16_u8(t3), 4); + const U64 matches = vget_lane_u64(vreinterpret_u64_u8(t4), 0); + return ZSTD_rotateRight_U64(matches, headGrouped); + } +} +#endif + +/* Returns a ZSTD_VecMask (U64) that has the nth group (determined by + * ZSTD_row_matchMaskGroupWidth) of bits set to 1 if the newly-computed "tag" + * matches the hash at the nth position in a row of the tagTable. + * Each row is a circular buffer beginning at the value of "headGrouped". So we + * must rotate the "matches" bitfield to match up with the actual layout of the + * entries within the hashTable */ +FORCE_INLINE_TEMPLATE ZSTD_VecMask +ZSTD_row_getMatchMask(const BYTE* const tagRow, const BYTE tag, const U32 headGrouped, const U32 rowEntries) +{ + const BYTE* const src = tagRow; + assert((rowEntries == 16) || (rowEntries == 32) || rowEntries == 64); + assert(rowEntries <= ZSTD_ROW_HASH_MAX_ENTRIES); + assert(ZSTD_row_matchMaskGroupWidth(rowEntries) * rowEntries <= sizeof(ZSTD_VecMask) * 8); + +#if defined(ZSTD_ARCH_X86_SSE2) + + return ZSTD_row_getSSEMask(rowEntries / 16, src, tag, headGrouped); + +#else /* SW or NEON-LE */ + +# if defined(ZSTD_ARCH_ARM_NEON) + /* This NEON path only works for little endian - otherwise use SWAR below */ + if (MEM_isLittleEndian()) { + return ZSTD_row_getNEONMask(rowEntries, src, tag, headGrouped); + } +# endif /* ZSTD_ARCH_ARM_NEON */ + /* SWAR */ + { const int chunkSize = sizeof(size_t); + const size_t shiftAmount = ((chunkSize * 8) - chunkSize); + const size_t xFF = ~((size_t)0); + const size_t x01 = xFF / 0xFF; + const size_t x80 = x01 << 7; + const size_t splatChar = tag * x01; + ZSTD_VecMask matches = 0; + int i = rowEntries - chunkSize; + assert((sizeof(size_t) == 4) || (sizeof(size_t) == 8)); + if (MEM_isLittleEndian()) { /* runtime check so have two loops */ + const size_t extractMagic = (xFF / 0x7F) >> chunkSize; + do { + size_t chunk = MEM_readST(&src[i]); + chunk ^= splatChar; + chunk = (((chunk | x80) - x01) | chunk) & x80; + matches <<= chunkSize; + matches |= (chunk * extractMagic) >> shiftAmount; + i -= chunkSize; + } while (i >= 0); + } else { /* big endian: reverse bits during extraction */ + const size_t msb = xFF ^ (xFF >> 1); + const size_t extractMagic = (msb / 0x1FF) | msb; + do { + size_t chunk = MEM_readST(&src[i]); + chunk ^= splatChar; + chunk = (((chunk | x80) - x01) | chunk) & x80; + matches <<= chunkSize; + matches |= ((chunk >> 7) * extractMagic) >> shiftAmount; + i -= chunkSize; + } while (i >= 0); + } + matches = ~matches; + if (rowEntries == 16) { + return ZSTD_rotateRight_U16((U16)matches, headGrouped); + } else if (rowEntries == 32) { + return ZSTD_rotateRight_U32((U32)matches, headGrouped); + } else { + return ZSTD_rotateRight_U64((U64)matches, headGrouped); + } } +#endif } -static size_t ZSTD_ldm_generateSequences_internal( - ldmState_t* ldmState, rawSeqStore_t* rawSeqStore, - ldmParams_t const* params, void const* src, size_t srcSize) +/* The high-level approach of the SIMD row based match finder is as follows: + * - Figure out where to insert the new entry: + * - Generate a hash for current input position and split it into a one byte of tag and `rowHashLog` bits of index. + * - The hash is salted by a value that changes on every context reset, so when the same table is used + * we will avoid collisions that would otherwise slow us down by introducing phantom matches. + * - The hashTable is effectively split into groups or "rows" of 15 or 31 entries of U32, and the index determines + * which row to insert into. + * - Determine the correct position within the row to insert the entry into. Each row of 15 or 31 can + * be considered as a circular buffer with a "head" index that resides in the tagTable (overall 16 or 32 bytes + * per row). + * - Use SIMD to efficiently compare the tags in the tagTable to the 1-byte tag calculated for the position and + * generate a bitfield that we can cycle through to check the collisions in the hash table. + * - Pick the longest match. + * - Insert the tag into the equivalent row and position in the tagTable. + */ +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_RowFindBestMatch( + ZSTD_MatchState_t* ms, + const BYTE* const ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 mls, const ZSTD_dictMode_e dictMode, + const U32 rowLog) { - /* LDM parameters */ - int const extDict = ZSTD_window_hasExtDict(ldmState->window); - U32 const minMatchLength = params->minMatchLength; - U32 const entsPerBucket = 1U << params->bucketSizeLog; - U32 const hBits = params->hashLog - params->bucketSizeLog; - /* Prefix and extDict parameters */ - U32 const dictLimit = ldmState->window.dictLimit; - U32 const lowestIndex = extDict ? ldmState->window.lowLimit : dictLimit; - BYTE const* const base = ldmState->window.base; - BYTE const* const dictBase = extDict ? ldmState->window.dictBase : NULL; - BYTE const* const dictStart = extDict ? dictBase + lowestIndex : NULL; - BYTE const* const dictEnd = extDict ? dictBase + dictLimit : NULL; - BYTE const* const lowPrefixPtr = base + dictLimit; - /* Input bounds */ - BYTE const* const istart = (BYTE const*)src; - BYTE const* const iend = istart + srcSize; - BYTE const* const ilimit = iend - HASH_READ_SIZE; - /* Input positions */ - BYTE const* anchor = istart; - BYTE const* ip = istart; - /* Rolling hash state */ - ldmRollingHashState_t hashState; - /* Arrays for staged-processing */ - size_t* const splits = ldmState->splitIndices; - ldmMatchCandidate_t* const candidates = ldmState->matchCandidates; - unsigned numSplits; + U32* const hashTable = ms->hashTable; + BYTE* const tagTable = ms->tagTable; + U32* const hashCache = ms->hashCache; + const U32 hashLog = ms->rowHashLog; + const ZSTD_compressionParameters* const cParams = &ms->cParams; + const BYTE* const base = ms->window.base; + const BYTE* const dictBase = ms->window.dictBase; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const U32 curr = (U32)(ip-base); + const U32 maxDistance = 1U << cParams->windowLog; + const U32 lowestValid = ms->window.lowLimit; + const U32 withinMaxDistance = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid; + const U32 isDictionary = (ms->loadedDictEnd != 0); + const U32 lowLimit = isDictionary ? lowestValid : withinMaxDistance; + const U32 rowEntries = (1U << rowLog); + const U32 rowMask = rowEntries - 1; + const U32 cappedSearchLog = MIN(cParams->searchLog, rowLog); /* nb of searches is capped at nb entries per row */ + const U32 groupWidth = ZSTD_row_matchMaskGroupWidth(rowEntries); + const U64 hashSalt = ms->hashSalt; + U32 nbAttempts = 1U << cappedSearchLog; + size_t ml=4-1; + U32 hash; - if (srcSize < minMatchLength) - return iend - anchor; + /* DMS/DDS variables that may be referenced laster */ + const ZSTD_MatchState_t* const dms = ms->dictMatchState; - /* Initialize the rolling hash state with the first minMatchLength bytes */ - ZSTD_ldm_gear_init(&hashState, params); - { - size_t n = 0; + /* Initialize the following variables to satisfy static analyzer */ + size_t ddsIdx = 0; + U32 ddsExtraAttempts = 0; /* cctx hash tables are limited in searches, but allow extra searches into DDS */ + U32 dmsTag = 0; + U32* dmsRow = NULL; + BYTE* dmsTagRow = NULL; - while (n < minMatchLength) { - numSplits = 0; - n += ZSTD_ldm_gear_feed(&hashState, ip + n, minMatchLength - n, - splits, &numSplits); + if (dictMode == ZSTD_dedicatedDictSearch) { + const U32 ddsHashLog = dms->cParams.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG; + { /* Prefetch DDS hashtable entry */ + ddsIdx = ZSTD_hashPtr(ip, ddsHashLog, mls) << ZSTD_LAZY_DDSS_BUCKET_LOG; + PREFETCH_L1(&dms->hashTable[ddsIdx]); } - ip += minMatchLength; + ddsExtraAttempts = cParams->searchLog > rowLog ? 1U << (cParams->searchLog - rowLog) : 0; } - while (ip < ilimit) { - size_t hashed; - unsigned n; + if (dictMode == ZSTD_dictMatchState) { + /* Prefetch DMS rows */ + U32* const dmsHashTable = dms->hashTable; + BYTE* const dmsTagTable = dms->tagTable; + U32 const dmsHash = (U32)ZSTD_hashPtr(ip, dms->rowHashLog + ZSTD_ROW_HASH_TAG_BITS, mls); + U32 const dmsRelRow = (dmsHash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog; + dmsTag = dmsHash & ZSTD_ROW_HASH_TAG_MASK; + dmsTagRow = (BYTE*)(dmsTagTable + dmsRelRow); + dmsRow = dmsHashTable + dmsRelRow; + ZSTD_row_prefetch(dmsHashTable, dmsTagTable, dmsRelRow, rowLog); + } + + /* Update the hashTable and tagTable up to (but not including) ip */ + if (!ms->lazySkipping) { + ZSTD_row_update_internal(ms, ip, mls, rowLog, rowMask, 1 /* useCache */); + hash = ZSTD_row_nextCachedHash(hashCache, hashTable, tagTable, base, curr, hashLog, rowLog, mls, hashSalt); + } else { + /* Stop inserting every position when in the lazy skipping mode. + * The hash cache is also not kept up to date in this mode. + */ + hash = (U32)ZSTD_hashPtrSalted(ip, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, hashSalt); + ms->nextToUpdate = curr; + } + ms->hashSaltEntropy += hash; /* collect salt entropy */ + + { /* Get the hash for ip, compute the appropriate row */ + U32 const relRow = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog; + U32 const tag = hash & ZSTD_ROW_HASH_TAG_MASK; + U32* const row = hashTable + relRow; + BYTE* tagRow = (BYTE*)(tagTable + relRow); + U32 const headGrouped = (*tagRow & rowMask) * groupWidth; + U32 matchBuffer[ZSTD_ROW_HASH_MAX_ENTRIES]; + size_t numMatches = 0; + size_t currMatch = 0; + ZSTD_VecMask matches = ZSTD_row_getMatchMask(tagRow, (BYTE)tag, headGrouped, rowEntries); + + /* Cycle through the matches and prefetch */ + for (; (matches > 0) && (nbAttempts > 0); matches &= (matches - 1)) { + U32 const matchPos = ((headGrouped + ZSTD_VecMask_next(matches)) / groupWidth) & rowMask; + U32 const matchIndex = row[matchPos]; + if(matchPos == 0) continue; + assert(numMatches < rowEntries); + if (matchIndex < lowLimit) + break; + if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) { + PREFETCH_L1(base + matchIndex); + } else { + PREFETCH_L1(dictBase + matchIndex); + } + matchBuffer[numMatches++] = matchIndex; + --nbAttempts; + } - numSplits = 0; - hashed = ZSTD_ldm_gear_feed(&hashState, ip, ilimit - ip, - splits, &numSplits); + /* Speed opt: insert current byte into hashtable too. This allows us to avoid one iteration of the loop + in ZSTD_row_update_internal() at the next search. */ + { + U32 const pos = ZSTD_row_nextIndex(tagRow, rowMask); + tagRow[pos] = (BYTE)tag; + row[pos] = ms->nextToUpdate++; + } - for (n = 0; n < numSplits; n++) { - BYTE const* const split = ip + splits[n] - minMatchLength; - U64 const xxhash = XXH64(split, minMatchLength, 0); - U32 const hash = (U32)(xxhash & (((U32)1 << hBits) - 1)); + /* Return the longest match */ + for (; currMatch < numMatches; ++currMatch) { + U32 const matchIndex = matchBuffer[currMatch]; + size_t currentMl=0; + assert(matchIndex < curr); + assert(matchIndex >= lowLimit); + + if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) { + const BYTE* const match = base + matchIndex; + assert(matchIndex >= dictLimit); /* ensures this is true if dictMode != ZSTD_extDict */ + /* read 4B starting from (match + ml + 1 - sizeof(U32)) */ + if (MEM_read32(match + ml - 3) == MEM_read32(ip + ml - 3)) /* potentially better */ + currentMl = ZSTD_count(ip, match, iLimit); + } else { + const BYTE* const match = dictBase + matchIndex; + assert(match+4 <= dictEnd); + if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dictEnd, prefixStart) + 4; + } - candidates[n].split = split; - candidates[n].hash = hash; - candidates[n].checksum = (U32)(xxhash >> 32); - candidates[n].bucket = ZSTD_ldm_getBucket(ldmState, hash, *params); - PREFETCH_L1(candidates[n].bucket); + /* Save best solution */ + if (currentMl > ml) { + ml = currentMl; + *offsetPtr = OFFSET_TO_OFFBASE(curr - matchIndex); + if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ + } } + } - for (n = 0; n < numSplits; n++) { - size_t forwardMatchLength = 0, backwardMatchLength = 0, - bestMatchLength = 0, mLength; - BYTE const* const split = candidates[n].split; - U32 const checksum = candidates[n].checksum; - U32 const hash = candidates[n].hash; - ldmEntry_t* const bucket = candidates[n].bucket; - ldmEntry_t const* cur; - ldmEntry_t const* bestEntry = NULL; - ldmEntry_t newEntry; - - newEntry.offset = (U32)(split - base); - newEntry.checksum = checksum; + assert(nbAttempts <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */ + if (dictMode == ZSTD_dedicatedDictSearch) { + ml = ZSTD_dedicatedDictSearch_lazy_search(offsetPtr, ml, nbAttempts + ddsExtraAttempts, dms, + ip, iLimit, prefixStart, curr, dictLimit, ddsIdx); + } else if (dictMode == ZSTD_dictMatchState) { + /* TODO: Measure and potentially add prefetching to DMS */ + const U32 dmsLowestIndex = dms->window.dictLimit; + const BYTE* const dmsBase = dms->window.base; + const BYTE* const dmsEnd = dms->window.nextSrc; + const U32 dmsSize = (U32)(dmsEnd - dmsBase); + const U32 dmsIndexDelta = dictLimit - dmsSize; - /* If a split point would generate a sequence overlapping with - * the previous one, we merely register it in the hash table and - * move on */ - if (split < anchor) { - ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params); - continue; + { U32 const headGrouped = (*dmsTagRow & rowMask) * groupWidth; + U32 matchBuffer[ZSTD_ROW_HASH_MAX_ENTRIES]; + size_t numMatches = 0; + size_t currMatch = 0; + ZSTD_VecMask matches = ZSTD_row_getMatchMask(dmsTagRow, (BYTE)dmsTag, headGrouped, rowEntries); + + for (; (matches > 0) && (nbAttempts > 0); matches &= (matches - 1)) { + U32 const matchPos = ((headGrouped + ZSTD_VecMask_next(matches)) / groupWidth) & rowMask; + U32 const matchIndex = dmsRow[matchPos]; + if(matchPos == 0) continue; + if (matchIndex < dmsLowestIndex) + break; + PREFETCH_L1(dmsBase + matchIndex); + matchBuffer[numMatches++] = matchIndex; + --nbAttempts; } - for (cur = bucket; cur < bucket + entsPerBucket; cur++) { - size_t curForwardMatchLength, curBackwardMatchLength, - curTotalMatchLength; - if (cur->checksum != checksum || cur->offset <= lowestIndex) { - continue; - } - if (extDict) { - BYTE const* const curMatchBase = - cur->offset < dictLimit ? dictBase : base; - BYTE const* const pMatch = curMatchBase + cur->offset; - BYTE const* const matchEnd = - cur->offset < dictLimit ? dictEnd : iend; - BYTE const* const lowMatchPtr = - cur->offset < dictLimit ? dictStart : lowPrefixPtr; - curForwardMatchLength = - ZSTD_count_2segments(split, pMatch, iend, matchEnd, lowPrefixPtr); - if (curForwardMatchLength < minMatchLength) { - continue; - } - curBackwardMatchLength = ZSTD_ldm_countBackwardsMatch_2segments( - split, anchor, pMatch, lowMatchPtr, dictStart, dictEnd); - } else { /* !extDict */ - BYTE const* const pMatch = base + cur->offset; - curForwardMatchLength = ZSTD_count(split, pMatch, iend); - if (curForwardMatchLength < minMatchLength) { - continue; - } - curBackwardMatchLength = - ZSTD_ldm_countBackwardsMatch(split, anchor, pMatch, lowPrefixPtr); - } - curTotalMatchLength = curForwardMatchLength + curBackwardMatchLength; + /* Return the longest match */ + for (; currMatch < numMatches; ++currMatch) { + U32 const matchIndex = matchBuffer[currMatch]; + size_t currentMl=0; + assert(matchIndex >= dmsLowestIndex); + assert(matchIndex < curr); - if (curTotalMatchLength > bestMatchLength) { - bestMatchLength = curTotalMatchLength; - forwardMatchLength = curForwardMatchLength; - backwardMatchLength = curBackwardMatchLength; - bestEntry = cur; + { const BYTE* const match = dmsBase + matchIndex; + assert(match+4 <= dmsEnd); + if (MEM_read32(match) == MEM_read32(ip)) + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dmsEnd, prefixStart) + 4; } - } - /* No match found -- insert an entry into the hash table - * and process the next candidate match */ - if (bestEntry == NULL) { - ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params); - continue; + if (currentMl > ml) { + ml = currentMl; + assert(curr > matchIndex + dmsIndexDelta); + *offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + dmsIndexDelta)); + if (ip+currentMl == iLimit) break; + } } + } + } + return ml; +} - /* Match found */ - mLength = forwardMatchLength + backwardMatchLength; - { - U32 const offset = (U32)(split - base) - bestEntry->offset; - rawSeq* const seq = rawSeqStore->seq + rawSeqStore->size; - - /* Out of sequence storage */ - if (rawSeqStore->size == rawSeqStore->capacity) - return ERROR(dstSize_tooSmall); - seq->litLength = (U32)(split - backwardMatchLength - anchor); - seq->matchLength = (U32)mLength; - seq->offset = offset; - rawSeqStore->size++; - } - /* Insert the current entry into the hash table --- it must be - * done after the previous block to avoid clobbering bestEntry */ - ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params); +/** + * Generate search functions templated on (dictMode, mls, rowLog). + * These functions are outlined for code size & compilation time. + * ZSTD_searchMax() dispatches to the correct implementation function. + * + * TODO: The start of the search function involves loading and calculating a + * bunch of constants from the ZSTD_MatchState_t. These computations could be + * done in an initialization function, and saved somewhere in the match state. + * Then we could pass a pointer to the saved state instead of the match state, + * and avoid duplicate computations. + * + * TODO: Move the match re-winding into searchMax. This improves compression + * ratio, and unlocks further simplifications with the next TODO. + * + * TODO: Try moving the repcode search into searchMax. After the re-winding + * and repcode search are in searchMax, there is no more logic in the match + * finder loop that requires knowledge about the dictMode. So we should be + * able to avoid force inlining it, and we can join the extDict loop with + * the single segment loop. It should go in searchMax instead of its own + * function to avoid having multiple virtual function calls per search. + */ + +#define ZSTD_BT_SEARCH_FN(dictMode, mls) ZSTD_BtFindBestMatch_##dictMode##_##mls +#define ZSTD_HC_SEARCH_FN(dictMode, mls) ZSTD_HcFindBestMatch_##dictMode##_##mls +#define ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog) ZSTD_RowFindBestMatch_##dictMode##_##mls##_##rowLog + +#define ZSTD_SEARCH_FN_ATTRS FORCE_NOINLINE + +#define GEN_ZSTD_BT_SEARCH_FN(dictMode, mls) \ + ZSTD_SEARCH_FN_ATTRS size_t ZSTD_BT_SEARCH_FN(dictMode, mls)( \ + ZSTD_MatchState_t* ms, \ + const BYTE* ip, const BYTE* const iLimit, \ + size_t* offBasePtr) \ + { \ + assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \ + return ZSTD_BtFindBestMatch(ms, ip, iLimit, offBasePtr, mls, ZSTD_##dictMode); \ + } \ + +#define GEN_ZSTD_HC_SEARCH_FN(dictMode, mls) \ + ZSTD_SEARCH_FN_ATTRS size_t ZSTD_HC_SEARCH_FN(dictMode, mls)( \ + ZSTD_MatchState_t* ms, \ + const BYTE* ip, const BYTE* const iLimit, \ + size_t* offsetPtr) \ + { \ + assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \ + return ZSTD_HcFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode); \ + } \ + +#define GEN_ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog) \ + ZSTD_SEARCH_FN_ATTRS size_t ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog)( \ + ZSTD_MatchState_t* ms, \ + const BYTE* ip, const BYTE* const iLimit, \ + size_t* offsetPtr) \ + { \ + assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \ + assert(MAX(4, MIN(6, ms->cParams.searchLog)) == rowLog); \ + return ZSTD_RowFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode, rowLog); \ + } \ + +#define ZSTD_FOR_EACH_ROWLOG(X, dictMode, mls) \ + X(dictMode, mls, 4) \ + X(dictMode, mls, 5) \ + X(dictMode, mls, 6) + +#define ZSTD_FOR_EACH_MLS_ROWLOG(X, dictMode) \ + ZSTD_FOR_EACH_ROWLOG(X, dictMode, 4) \ + ZSTD_FOR_EACH_ROWLOG(X, dictMode, 5) \ + ZSTD_FOR_EACH_ROWLOG(X, dictMode, 6) + +#define ZSTD_FOR_EACH_MLS(X, dictMode) \ + X(dictMode, 4) \ + X(dictMode, 5) \ + X(dictMode, 6) + +#define ZSTD_FOR_EACH_DICT_MODE(X, ...) \ + X(__VA_ARGS__, noDict) \ + X(__VA_ARGS__, extDict) \ + X(__VA_ARGS__, dictMatchState) \ + X(__VA_ARGS__, dedicatedDictSearch) + +/* Generate row search fns for each combination of (dictMode, mls, rowLog) */ +ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS_ROWLOG, GEN_ZSTD_ROW_SEARCH_FN) +/* Generate binary Tree search fns for each combination of (dictMode, mls) */ +ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_BT_SEARCH_FN) +/* Generate hash chain search fns for each combination of (dictMode, mls) */ +ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_HC_SEARCH_FN) + +typedef enum { search_hashChain=0, search_binaryTree=1, search_rowHash=2 } searchMethod_e; + +#define GEN_ZSTD_CALL_BT_SEARCH_FN(dictMode, mls) \ + case mls: \ + return ZSTD_BT_SEARCH_FN(dictMode, mls)(ms, ip, iend, offsetPtr); +#define GEN_ZSTD_CALL_HC_SEARCH_FN(dictMode, mls) \ + case mls: \ + return ZSTD_HC_SEARCH_FN(dictMode, mls)(ms, ip, iend, offsetPtr); +#define GEN_ZSTD_CALL_ROW_SEARCH_FN(dictMode, mls, rowLog) \ + case rowLog: \ + return ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog)(ms, ip, iend, offsetPtr); + +#define ZSTD_SWITCH_MLS(X, dictMode) \ + switch (mls) { \ + ZSTD_FOR_EACH_MLS(X, dictMode) \ + } + +#define ZSTD_SWITCH_ROWLOG(dictMode, mls) \ + case mls: \ + switch (rowLog) { \ + ZSTD_FOR_EACH_ROWLOG(GEN_ZSTD_CALL_ROW_SEARCH_FN, dictMode, mls) \ + } \ + ZSTD_UNREACHABLE; \ + break; - anchor = split + forwardMatchLength; - } +#define ZSTD_SWITCH_SEARCH_METHOD(dictMode) \ + switch (searchMethod) { \ + case search_hashChain: \ + ZSTD_SWITCH_MLS(GEN_ZSTD_CALL_HC_SEARCH_FN, dictMode) \ + break; \ + case search_binaryTree: \ + ZSTD_SWITCH_MLS(GEN_ZSTD_CALL_BT_SEARCH_FN, dictMode) \ + break; \ + case search_rowHash: \ + ZSTD_SWITCH_MLS(ZSTD_SWITCH_ROWLOG, dictMode) \ + break; \ + } \ + ZSTD_UNREACHABLE; - ip += hashed; +/** + * Searches for the longest match at @p ip. + * Dispatches to the correct implementation function based on the + * (searchMethod, dictMode, mls, rowLog). We use switch statements + * here instead of using an indirect function call through a function + * pointer because after Spectre and Meltdown mitigations, indirect + * function calls can be very costly, especially in the kernel. + * + * NOTE: dictMode and searchMethod should be templated, so those switch + * statements should be optimized out. Only the mls & rowLog switches + * should be left. + * + * @param ms The match state. + * @param ip The position to search at. + * @param iend The end of the input data. + * @param[out] offsetPtr Stores the match offset into this pointer. + * @param mls The minimum search length, in the range [4, 6]. + * @param rowLog The row log (if applicable), in the range [4, 6]. + * @param searchMethod The search method to use (templated). + * @param dictMode The dictMode (templated). + * + * @returns The length of the longest match found, or < mls if no match is found. + * If a match is found its offset is stored in @p offsetPtr. + */ +FORCE_INLINE_TEMPLATE size_t ZSTD_searchMax( + ZSTD_MatchState_t* ms, + const BYTE* ip, + const BYTE* iend, + size_t* offsetPtr, + U32 const mls, + U32 const rowLog, + searchMethod_e const searchMethod, + ZSTD_dictMode_e const dictMode) +{ + if (dictMode == ZSTD_noDict) { + ZSTD_SWITCH_SEARCH_METHOD(noDict) + } else if (dictMode == ZSTD_extDict) { + ZSTD_SWITCH_SEARCH_METHOD(extDict) + } else if (dictMode == ZSTD_dictMatchState) { + ZSTD_SWITCH_SEARCH_METHOD(dictMatchState) + } else if (dictMode == ZSTD_dedicatedDictSearch) { + ZSTD_SWITCH_SEARCH_METHOD(dedicatedDictSearch) } - - return iend - anchor; + ZSTD_UNREACHABLE; + return 0; } -/*! ZSTD_ldm_reduceTable() : - * reduce table indexes by `reducerValue` */ -static void ZSTD_ldm_reduceTable(ldmEntry_t* const table, U32 const size, - U32 const reducerValue) +/* ******************************* +* Common parser - lazy strategy +*********************************/ + +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_compressBlock_lazy_generic( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, + U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize, + const searchMethod_e searchMethod, const U32 depth, + ZSTD_dictMode_e const dictMode) { - U32 u; - for (u = 0; u < size; u++) { - if (table[u].offset < reducerValue) table[u].offset = 0; - else table[u].offset -= reducerValue; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = (searchMethod == search_rowHash) ? iend - 8 - ZSTD_ROW_HASH_CACHE_SIZE : iend - 8; + const BYTE* const base = ms->window.base; + const U32 prefixLowestIndex = ms->window.dictLimit; + const BYTE* const prefixLowest = base + prefixLowestIndex; + const U32 mls = BOUNDED(4, ms->cParams.minMatch, 6); + const U32 rowLog = BOUNDED(4, ms->cParams.searchLog, 6); + + U32 offset_1 = rep[0], offset_2 = rep[1]; + U32 offsetSaved1 = 0, offsetSaved2 = 0; + + const int isDMS = dictMode == ZSTD_dictMatchState; + const int isDDS = dictMode == ZSTD_dedicatedDictSearch; + const int isDxS = isDMS || isDDS; + const ZSTD_MatchState_t* const dms = ms->dictMatchState; + const U32 dictLowestIndex = isDxS ? dms->window.dictLimit : 0; + const BYTE* const dictBase = isDxS ? dms->window.base : NULL; + const BYTE* const dictLowest = isDxS ? dictBase + dictLowestIndex : NULL; + const BYTE* const dictEnd = isDxS ? dms->window.nextSrc : NULL; + const U32 dictIndexDelta = isDxS ? + prefixLowestIndex - (U32)(dictEnd - dictBase) : + 0; + const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictLowest)); + + DEBUGLOG(5, "ZSTD_compressBlock_lazy_generic (dictMode=%u) (searchFunc=%u)", (U32)dictMode, (U32)searchMethod); + ip += (dictAndPrefixLength == 0); + if (dictMode == ZSTD_noDict) { + U32 const curr = (U32)(ip - base); + U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, curr, ms->cParams.windowLog); + U32 const maxRep = curr - windowLow; + if (offset_2 > maxRep) offsetSaved2 = offset_2, offset_2 = 0; + if (offset_1 > maxRep) offsetSaved1 = offset_1, offset_1 = 0; + } + if (isDxS) { + /* dictMatchState repCode checks don't currently handle repCode == 0 + * disabling. */ + assert(offset_1 <= dictAndPrefixLength); + assert(offset_2 <= dictAndPrefixLength); } -} -size_t ZSTD_ldm_generateSequences( - ldmState_t* ldmState, rawSeqStore_t* sequences, - ldmParams_t const* params, void const* src, size_t srcSize) -{ - U32 const maxDist = 1U << params->windowLog; - BYTE const* const istart = (BYTE const*)src; - BYTE const* const iend = istart + srcSize; - size_t const kMaxChunkSize = 1 << 20; - size_t const nbChunks = (srcSize / kMaxChunkSize) + ((srcSize % kMaxChunkSize) != 0); - size_t chunk; - size_t leftoverSize = 0; + /* Reset the lazy skipping state */ + ms->lazySkipping = 0; - assert(ZSTD_CHUNKSIZE_MAX >= kMaxChunkSize); - /* Check that ZSTD_window_update() has been called for this chunk prior - * to passing it to this function. - */ - assert(ldmState->window.nextSrc >= (BYTE const*)src + srcSize); - /* The input could be very large (in zstdmt), so it must be broken up into - * chunks to enforce the maximum distance and handle overflow correction. + if (searchMethod == search_rowHash) { + ZSTD_row_fillHashCache(ms, base, rowLog, mls, ms->nextToUpdate, ilimit); + } + + /* Match Loop */ +#if defined(__GNUC__) && defined(__x86_64__) + /* I've measured random a 5% speed loss on levels 5 & 6 (greedy) when the + * code alignment is perturbed. To fix the instability align the loop on 32-bytes. */ - assert(sequences->pos <= sequences->size); - assert(sequences->size <= sequences->capacity); - for (chunk = 0; chunk < nbChunks && sequences->size < sequences->capacity; ++chunk) { - BYTE const* const chunkStart = istart + chunk * kMaxChunkSize; - size_t const remaining = (size_t)(iend - chunkStart); - BYTE const *const chunkEnd = - (remaining < kMaxChunkSize) ? iend : chunkStart + kMaxChunkSize; - size_t const chunkSize = chunkEnd - chunkStart; - size_t newLeftoverSize; - size_t const prevSize = sequences->size; + __asm__(".p2align 5"); +#endif + while (ip < ilimit) { + size_t matchLength=0; + size_t offBase = REPCODE1_TO_OFFBASE; + const BYTE* start=ip+1; + DEBUGLOG(7, "search baseline (depth 0)"); - assert(chunkStart < iend); - /* 1. Perform overflow correction if necessary. */ - if (ZSTD_window_needOverflowCorrection(ldmState->window, chunkEnd)) { - U32 const ldmHSize = 1U << params->hashLog; - U32 const correction = ZSTD_window_correctOverflow( - &ldmState->window, /* cycleLog */ 0, maxDist, chunkStart); - ZSTD_ldm_reduceTable(ldmState->hashTable, ldmHSize, correction); - /* invalidate dictionaries on overflow correction */ - ldmState->loadedDictEnd = 0; + /* check repCode */ + if (isDxS) { + const U32 repIndex = (U32)(ip - base) + 1 - offset_1; + const BYTE* repMatch = ((dictMode == ZSTD_dictMatchState || dictMode == ZSTD_dedicatedDictSearch) + && repIndex < prefixLowestIndex) ? + dictBase + (repIndex - dictIndexDelta) : + base + repIndex; + if ((ZSTD_index_overlap_check(prefixLowestIndex, repIndex)) + && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { + const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; + if (depth==0) goto _storeSequence; + } } - /* 2. We enforce the maximum offset allowed. - * - * kMaxChunkSize should be small enough that we don't lose too much of - * the window through early invalidation. - * TODO: * Test the chunk size. - * * Try invalidation after the sequence generation and test the - * the offset against maxDist directly. - * - * NOTE: Because of dictionaries + sequence splitting we MUST make sure - * that any offset used is valid at the END of the sequence, since it may - * be split into two sequences. This condition holds when using - * ZSTD_window_enforceMaxDist(), but if we move to checking offsets - * against maxDist directly, we'll have to carefully handle that case. - */ - ZSTD_window_enforceMaxDist(&ldmState->window, chunkEnd, maxDist, &ldmState->loadedDictEnd, NULL); - /* 3. Generate the sequences for the chunk, and get newLeftoverSize. */ - newLeftoverSize = ZSTD_ldm_generateSequences_internal( - ldmState, sequences, params, chunkStart, chunkSize); - if (ZSTD_isError(newLeftoverSize)) - return newLeftoverSize; - /* 4. We add the leftover literals from previous iterations to the first - * newly generated sequence, or add the `newLeftoverSize` if none are - * generated. - */ - /* Prepend the leftover literals from the last call */ - if (prevSize < sequences->size) { - sequences->seq[prevSize].litLength += (U32)leftoverSize; - leftoverSize = newLeftoverSize; - } else { - assert(newLeftoverSize == chunkSize); - leftoverSize += chunkSize; + if ( dictMode == ZSTD_noDict + && ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)))) { + matchLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; + if (depth==0) goto _storeSequence; } - } - return 0; -} -void ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 const minMatch) { - while (srcSize > 0 && rawSeqStore->pos < rawSeqStore->size) { - rawSeq* seq = rawSeqStore->seq + rawSeqStore->pos; - if (srcSize <= seq->litLength) { - /* Skip past srcSize literals */ - seq->litLength -= (U32)srcSize; - return; + /* first search (depth 0) */ + { size_t offbaseFound = 999999999; + size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &offbaseFound, mls, rowLog, searchMethod, dictMode); + if (ml2 > matchLength) + matchLength = ml2, start = ip, offBase = offbaseFound; } - srcSize -= seq->litLength; - seq->litLength = 0; - if (srcSize < seq->matchLength) { - /* Skip past the first srcSize of the match */ - seq->matchLength -= (U32)srcSize; - if (seq->matchLength < minMatch) { - /* The match is too short, omit it */ - if (rawSeqStore->pos + 1 < rawSeqStore->size) { - seq[1].litLength += seq[0].matchLength; + + if (matchLength < 4) { + size_t const step = ((size_t)(ip-anchor) >> kSearchStrength) + 1; /* jump faster over incompressible sections */; + ip += step; + /* Enter the lazy skipping mode once we are skipping more than 8 bytes at a time. + * In this mode we stop inserting every position into our tables, and only insert + * positions that we search, which is one in step positions. + * The exact cutoff is flexible, I've just chosen a number that is reasonably high, + * so we minimize the compression ratio loss in "normal" scenarios. This mode gets + * triggered once we've gone 2KB without finding any matches. + */ + ms->lazySkipping = step > kLazySkippingStep; + continue; + } + + /* let's try to find a better solution */ + if (depth>=1) + while (ip0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) { + size_t const mlRep = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4; + int const gain2 = (int)(mlRep * 3); + int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offBase) + 1); + if ((mlRep >= 4) && (gain2 > gain1)) + matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip; + } + if (isDxS) { + const U32 repIndex = (U32)(ip - base) - offset_1; + const BYTE* repMatch = repIndex < prefixLowestIndex ? + dictBase + (repIndex - dictIndexDelta) : + base + repIndex; + if ((ZSTD_index_overlap_check(prefixLowestIndex, repIndex)) + && (MEM_read32(repMatch) == MEM_read32(ip)) ) { + const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; + size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; + int const gain2 = (int)(mlRep * 3); + int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offBase) + 1); + if ((mlRep >= 4) && (gain2 > gain1)) + matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip; } - rawSeqStore->pos++; } - return; - } - srcSize -= seq->matchLength; - seq->matchLength = 0; - rawSeqStore->pos++; - } -} + { size_t ofbCandidate=999999999; + size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, dictMode); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 4); + if ((ml2 >= 4) && (gain2 > gain1)) { + matchLength = ml2, offBase = ofbCandidate, start = ip; + continue; /* search a better one */ + } } -/** - * If the sequence length is longer than remaining then the sequence is split - * between this block and the next. - * - * Returns the current sequence to handle, or if the rest of the block should - * be literals, it returns a sequence with offset == 0. - */ -static rawSeq maybeSplitSequence(rawSeqStore_t* rawSeqStore, - U32 const remaining, U32 const minMatch) -{ - rawSeq sequence = rawSeqStore->seq[rawSeqStore->pos]; - assert(sequence.offset > 0); - /* Likely: No partial sequence */ - if (remaining >= sequence.litLength + sequence.matchLength) { - rawSeqStore->pos++; - return sequence; - } - /* Cut the sequence short (offset == 0 ==> rest is literals). */ - if (remaining <= sequence.litLength) { - sequence.offset = 0; - } else if (remaining < sequence.litLength + sequence.matchLength) { - sequence.matchLength = remaining - sequence.litLength; - if (sequence.matchLength < minMatch) { - sequence.offset = 0; + /* let's find an even better one */ + if ((depth==2) && (ip0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) { + size_t const mlRep = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4; + int const gain2 = (int)(mlRep * 4); + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 1); + if ((mlRep >= 4) && (gain2 > gain1)) + matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip; + } + if (isDxS) { + const U32 repIndex = (U32)(ip - base) - offset_1; + const BYTE* repMatch = repIndex < prefixLowestIndex ? + dictBase + (repIndex - dictIndexDelta) : + base + repIndex; + if ((ZSTD_index_overlap_check(prefixLowestIndex, repIndex)) + && (MEM_read32(repMatch) == MEM_read32(ip)) ) { + const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; + size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; + int const gain2 = (int)(mlRep * 4); + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 1); + if ((mlRep >= 4) && (gain2 > gain1)) + matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip; + } + } + { size_t ofbCandidate=999999999; + size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, dictMode); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 7); + if ((ml2 >= 4) && (gain2 > gain1)) { + matchLength = ml2, offBase = ofbCandidate, start = ip; + continue; + } } } + break; /* nothing found : store previous solution */ } - } - /* Skip past `remaining` bytes for the future sequences. */ - ZSTD_ldm_skipSequences(rawSeqStore, remaining, minMatch); - return sequence; -} -void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) { - U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes); - while (currPos && rawSeqStore->pos < rawSeqStore->size) { - rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos]; - if (currPos >= currSeq.litLength + currSeq.matchLength) { - currPos -= currSeq.litLength + currSeq.matchLength; - rawSeqStore->pos++; - } else { - rawSeqStore->posInSequence = currPos; - break; + /* NOTE: + * Pay attention that `start[-value]` can lead to strange undefined behavior + * notably if `value` is unsigned, resulting in a large positive `-value`. + */ + /* catch up */ + if (OFFBASE_IS_OFFSET(offBase)) { + if (dictMode == ZSTD_noDict) { + while ( ((start > anchor) & (start - OFFBASE_TO_OFFSET(offBase) > prefixLowest)) + && (start[-1] == (start-OFFBASE_TO_OFFSET(offBase))[-1]) ) /* only search for offset within prefix */ + { start--; matchLength++; } + } + if (isDxS) { + U32 const matchIndex = (U32)((size_t)(start-base) - OFFBASE_TO_OFFSET(offBase)); + const BYTE* match = (matchIndex < prefixLowestIndex) ? dictBase + matchIndex - dictIndexDelta : base + matchIndex; + const BYTE* const mStart = (matchIndex < prefixLowestIndex) ? dictLowest : prefixLowest; + while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */ + } + offset_2 = offset_1; offset_1 = (U32)OFFBASE_TO_OFFSET(offBase); + } + /* store sequence */ +_storeSequence: + { size_t const litLength = (size_t)(start - anchor); + ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offBase, matchLength); + anchor = ip = start + matchLength; + } + if (ms->lazySkipping) { + /* We've found a match, disable lazy skipping mode, and refill the hash cache. */ + if (searchMethod == search_rowHash) { + ZSTD_row_fillHashCache(ms, base, rowLog, mls, ms->nextToUpdate, ilimit); + } + ms->lazySkipping = 0; } - } - if (currPos == 0 || rawSeqStore->pos == rawSeqStore->size) { - rawSeqStore->posInSequence = 0; - } -} - -size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) -{ - const ZSTD_compressionParameters* const cParams = &ms->cParams; - unsigned const minMatch = cParams->minMatch; - ZSTD_blockCompressor const blockCompressor = - ZSTD_selectBlockCompressor(cParams->strategy, ZSTD_matchState_dictMode(ms)); - /* Input bounds */ - BYTE const* const istart = (BYTE const*)src; - BYTE const* const iend = istart + srcSize; - /* Input positions */ - BYTE const* ip = istart; - DEBUGLOG(5, "ZSTD_ldm_blockCompress: srcSize=%zu", srcSize); - /* If using opt parser, use LDMs only as candidates rather than always accepting them */ - if (cParams->strategy >= ZSTD_btopt) { - size_t lastLLSize; - ms->ldmSeqStore = rawSeqStore; - lastLLSize = blockCompressor(ms, seqStore, rep, src, srcSize); - ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore, srcSize); - return lastLLSize; - } + /* check immediate repcode */ + if (isDxS) { + while (ip <= ilimit) { + U32 const current2 = (U32)(ip-base); + U32 const repIndex = current2 - offset_2; + const BYTE* repMatch = repIndex < prefixLowestIndex ? + dictBase - dictIndexDelta + repIndex : + base + repIndex; + if ( (ZSTD_index_overlap_check(prefixLowestIndex, repIndex)) + && (MEM_read32(repMatch) == MEM_read32(ip)) ) { + const BYTE* const repEnd2 = repIndex < prefixLowestIndex ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd2, prefixLowest) + 4; + offBase = offset_2; offset_2 = offset_1; offset_1 = (U32)offBase; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, matchLength); + ip += matchLength; + anchor = ip; + continue; + } + break; + } + } - assert(rawSeqStore->pos <= rawSeqStore->size); - assert(rawSeqStore->size <= rawSeqStore->capacity); - /* Loop through each sequence and apply the block compressor to the literals */ - while (rawSeqStore->pos < rawSeqStore->size && ip < iend) { - /* maybeSplitSequence updates rawSeqStore->pos */ - rawSeq const sequence = maybeSplitSequence(rawSeqStore, - (U32)(iend - ip), minMatch); - int i; - /* End signal */ - if (sequence.offset == 0) - break; + if (dictMode == ZSTD_noDict) { + while ( ((ip <= ilimit) & (offset_2>0)) + && (MEM_read32(ip) == MEM_read32(ip - offset_2)) ) { + /* store sequence */ + matchLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; + offBase = offset_2; offset_2 = offset_1; offset_1 = (U32)offBase; /* swap repcodes */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, matchLength); + ip += matchLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } } } - assert(ip + sequence.litLength + sequence.matchLength <= iend); + /* If offset_1 started invalid (offsetSaved1 != 0) and became valid (offset_1 != 0), + * rotate saved offsets. See comment in ZSTD_compressBlock_fast_noDict for more context. */ + offsetSaved2 = ((offsetSaved1 != 0) && (offset_1 != 0)) ? offsetSaved1 : offsetSaved2; - /* Fill tables for block compressor */ - ZSTD_ldm_limitTableUpdate(ms, ip); - ZSTD_ldm_fillFastTables(ms, ip); - /* Run the block compressor */ - DEBUGLOG(5, "pos %u : calling block compressor on segment of size %u", (unsigned)(ip-istart), sequence.litLength); - { - size_t const newLitLength = - blockCompressor(ms, seqStore, rep, ip, sequence.litLength); - ip += sequence.litLength; - /* Update the repcodes */ - for (i = ZSTD_REP_NUM - 1; i > 0; i--) - rep[i] = rep[i-1]; - rep[0] = sequence.offset; - /* Store the sequence */ - ZSTD_storeSeq(seqStore, newLitLength, ip - newLitLength, iend, - sequence.offset + ZSTD_REP_MOVE, - sequence.matchLength - MINMATCH); - ip += sequence.matchLength; - } - } - /* Fill the tables for the block compressor */ - ZSTD_ldm_limitTableUpdate(ms, ip); - ZSTD_ldm_fillFastTables(ms, ip); - /* Compress the last literals */ - return blockCompressor(ms, seqStore, rep, ip, iend - ip); + /* save reps for next block */ + rep[0] = offset_1 ? offset_1 : offsetSaved1; + rep[1] = offset_2 ? offset_2 : offsetSaved2; + + /* Return the last literals size */ + return (size_t)(iend - anchor); } -/**** ended inlining compress/zstd_ldm.c ****/ -/**** start inlining compress/zstd_opt.c ****/ -/* - * Copyright (c) 2016-2021, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ +#endif /* build exclusions */ -/**** skipping file: zstd_compress_internal.h ****/ -/**** skipping file: hist.h ****/ -/**** skipping file: zstd_opt.h ****/ +#ifndef ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_greedy( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_noDict); +} -#define ZSTD_LITFREQ_ADD 2 /* scaling factor for litFreq, so that frequencies adapt faster to new stats */ -#define ZSTD_FREQ_DIV 4 /* log factor when using previous stats to init next stats */ -#define ZSTD_MAX_PRICE (1<<30) +size_t ZSTD_compressBlock_greedy_dictMatchState( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_dictMatchState); +} -#define ZSTD_PREDEF_THRESHOLD 1024 /* if srcSize < ZSTD_PREDEF_THRESHOLD, symbols' cost is assumed static, directly determined by pre-defined distributions */ +size_t ZSTD_compressBlock_greedy_dedicatedDictSearch( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_dedicatedDictSearch); +} +size_t ZSTD_compressBlock_greedy_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0, ZSTD_noDict); +} -/*-************************************* -* Price functions for optimal parser -***************************************/ +size_t ZSTD_compressBlock_greedy_dictMatchState_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0, ZSTD_dictMatchState); +} -#if 0 /* approximation at bit level */ -# define BITCOST_ACCURACY 0 -# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) -# define WEIGHT(stat) ((void)opt, ZSTD_bitWeight(stat)) -#elif 0 /* fractional bit accuracy */ -# define BITCOST_ACCURACY 8 -# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) -# define WEIGHT(stat,opt) ((void)opt, ZSTD_fracWeight(stat)) -#else /* opt==approx, ultra==accurate */ -# define BITCOST_ACCURACY 8 -# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) -# define WEIGHT(stat,opt) (opt ? ZSTD_fracWeight(stat) : ZSTD_bitWeight(stat)) +size_t ZSTD_compressBlock_greedy_dedicatedDictSearch_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0, ZSTD_dedicatedDictSearch); +} #endif -MEM_STATIC U32 ZSTD_bitWeight(U32 stat) +#ifndef ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_lazy( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) { - return (ZSTD_highbit32(stat+1) * BITCOST_MULTIPLIER); + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_noDict); } -MEM_STATIC U32 ZSTD_fracWeight(U32 rawStat) +size_t ZSTD_compressBlock_lazy_dictMatchState( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) { - U32 const stat = rawStat + 1; - U32 const hb = ZSTD_highbit32(stat); - U32 const BWeight = hb * BITCOST_MULTIPLIER; - U32 const FWeight = (stat << BITCOST_ACCURACY) >> hb; - U32 const weight = BWeight + FWeight; - assert(hb + BITCOST_ACCURACY < 31); - return weight; + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_dictMatchState); } -#if (DEBUGLEVEL>=2) -/* debugging function, - * @return price in bytes as fractional value - * for debug messages only */ -MEM_STATIC double ZSTD_fCost(U32 price) +size_t ZSTD_compressBlock_lazy_dedicatedDictSearch( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) { - return (double)price / (BITCOST_MULTIPLIER*8); + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_dedicatedDictSearch); +} + +size_t ZSTD_compressBlock_lazy_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_lazy_dictMatchState_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_lazy_dedicatedDictSearch_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1, ZSTD_dedicatedDictSearch); } #endif -static int ZSTD_compressedLiterals(optState_t const* const optPtr) +#ifndef ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_lazy2( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) { - return optPtr->literalCompressionMode != ZSTD_lcm_uncompressed; + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_noDict); } -static void ZSTD_setBasePrices(optState_t* optPtr, int optLevel) +size_t ZSTD_compressBlock_lazy2_dictMatchState( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) { - if (ZSTD_compressedLiterals(optPtr)) - optPtr->litSumBasePrice = WEIGHT(optPtr->litSum, optLevel); - optPtr->litLengthSumBasePrice = WEIGHT(optPtr->litLengthSum, optLevel); - optPtr->matchLengthSumBasePrice = WEIGHT(optPtr->matchLengthSum, optLevel); - optPtr->offCodeSumBasePrice = WEIGHT(optPtr->offCodeSum, optLevel); + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_dictMatchState); } +size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_dedicatedDictSearch); +} -/* ZSTD_downscaleStat() : - * reduce all elements in table by a factor 2^(ZSTD_FREQ_DIV+malus) - * return the resulting sum of elements */ -static U32 ZSTD_downscaleStat(unsigned* table, U32 lastEltIndex, int malus) +size_t ZSTD_compressBlock_lazy2_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) { - U32 s, sum=0; - DEBUGLOG(5, "ZSTD_downscaleStat (nbElts=%u)", (unsigned)lastEltIndex+1); - assert(ZSTD_FREQ_DIV+malus > 0 && ZSTD_FREQ_DIV+malus < 31); - for (s=0; s> (ZSTD_FREQ_DIV+malus)); - sum += table[s]; - } - return sum; + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2, ZSTD_noDict); } -/* ZSTD_rescaleFreqs() : - * if first block (detected by optPtr->litLengthSum == 0) : init statistics - * take hints from dictionary if there is one - * or init from zero, using src for literals stats, or flat 1 for match symbols - * otherwise downscale existing stats, to be used as seed for next block. - */ -static void -ZSTD_rescaleFreqs(optState_t* const optPtr, - const BYTE* const src, size_t const srcSize, - int const optLevel) +size_t ZSTD_compressBlock_lazy2_dictMatchState_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) { - int const compressedLiterals = ZSTD_compressedLiterals(optPtr); - DEBUGLOG(5, "ZSTD_rescaleFreqs (srcSize=%u)", (unsigned)srcSize); - optPtr->priceType = zop_dynamic; + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2, ZSTD_dictMatchState); +} - if (optPtr->litLengthSum == 0) { /* first block : init */ - if (srcSize <= ZSTD_PREDEF_THRESHOLD) { /* heuristic */ - DEBUGLOG(5, "(srcSize <= ZSTD_PREDEF_THRESHOLD) => zop_predef"); - optPtr->priceType = zop_predef; - } +size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2, ZSTD_dedicatedDictSearch); +} +#endif - assert(optPtr->symbolCosts != NULL); - if (optPtr->symbolCosts->huf.repeatMode == HUF_repeat_valid) { - /* huffman table presumed generated by dictionary */ - optPtr->priceType = zop_dynamic; +#ifndef ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_btlazy2( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2, ZSTD_noDict); +} - if (compressedLiterals) { - unsigned lit; - assert(optPtr->litFreq != NULL); - optPtr->litSum = 0; - for (lit=0; lit<=MaxLit; lit++) { - U32 const scaleLog = 11; /* scale to 2K */ - U32 const bitCost = HUF_getNbBits(optPtr->symbolCosts->huf.CTable, lit); - assert(bitCost <= scaleLog); - optPtr->litFreq[lit] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/; - optPtr->litSum += optPtr->litFreq[lit]; - } } +size_t ZSTD_compressBlock_btlazy2_dictMatchState( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2, ZSTD_dictMatchState); +} +#endif - { unsigned ll; - FSE_CState_t llstate; - FSE_initCState(&llstate, optPtr->symbolCosts->fse.litlengthCTable); - optPtr->litLengthSum = 0; - for (ll=0; ll<=MaxLL; ll++) { - U32 const scaleLog = 10; /* scale to 1K */ - U32 const bitCost = FSE_getMaxNbBits(llstate.symbolTT, ll); - assert(bitCost < scaleLog); - optPtr->litLengthFreq[ll] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/; - optPtr->litLengthSum += optPtr->litLengthFreq[ll]; - } } +#if !defined(ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR) +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_compressBlock_lazy_extDict_generic( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, + U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize, + const searchMethod_e searchMethod, const U32 depth) +{ + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = searchMethod == search_rowHash ? iend - 8 - ZSTD_ROW_HASH_CACHE_SIZE : iend - 8; + const BYTE* const base = ms->window.base; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* const dictBase = ms->window.dictBase; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const dictStart = dictBase + ms->window.lowLimit; + const U32 windowLog = ms->cParams.windowLog; + const U32 mls = BOUNDED(4, ms->cParams.minMatch, 6); + const U32 rowLog = BOUNDED(4, ms->cParams.searchLog, 6); - { unsigned ml; - FSE_CState_t mlstate; - FSE_initCState(&mlstate, optPtr->symbolCosts->fse.matchlengthCTable); - optPtr->matchLengthSum = 0; - for (ml=0; ml<=MaxML; ml++) { - U32 const scaleLog = 10; - U32 const bitCost = FSE_getMaxNbBits(mlstate.symbolTT, ml); - assert(bitCost < scaleLog); - optPtr->matchLengthFreq[ml] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/; - optPtr->matchLengthSum += optPtr->matchLengthFreq[ml]; - } } + U32 offset_1 = rep[0], offset_2 = rep[1]; - { unsigned of; - FSE_CState_t ofstate; - FSE_initCState(&ofstate, optPtr->symbolCosts->fse.offcodeCTable); - optPtr->offCodeSum = 0; - for (of=0; of<=MaxOff; of++) { - U32 const scaleLog = 10; - U32 const bitCost = FSE_getMaxNbBits(ofstate.symbolTT, of); - assert(bitCost < scaleLog); - optPtr->offCodeFreq[of] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/; - optPtr->offCodeSum += optPtr->offCodeFreq[of]; + DEBUGLOG(5, "ZSTD_compressBlock_lazy_extDict_generic (searchFunc=%u)", (U32)searchMethod); + + /* Reset the lazy skipping state */ + ms->lazySkipping = 0; + + /* init */ + ip += (ip == prefixStart); + if (searchMethod == search_rowHash) { + ZSTD_row_fillHashCache(ms, base, rowLog, mls, ms->nextToUpdate, ilimit); + } + + /* Match Loop */ +#if defined(__GNUC__) && defined(__x86_64__) + /* I've measured random a 5% speed loss on levels 5 & 6 (greedy) when the + * code alignment is perturbed. To fix the instability align the loop on 32-bytes. + */ + __asm__(".p2align 5"); +#endif + while (ip < ilimit) { + size_t matchLength=0; + size_t offBase = REPCODE1_TO_OFFBASE; + const BYTE* start=ip+1; + U32 curr = (U32)(ip-base); + + /* check repCode */ + { const U32 windowLow = ZSTD_getLowestMatchIndex(ms, curr+1, windowLog); + const U32 repIndex = (U32)(curr+1 - offset_1); + const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + if ( (ZSTD_index_overlap_check(dictLimit, repIndex)) + & (offset_1 <= curr+1 - windowLow) ) /* note: we are searching at curr+1 */ + if (MEM_read32(ip+1) == MEM_read32(repMatch)) { + /* repcode detected we should take it */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repEnd, prefixStart) + 4; + if (depth==0) goto _storeSequence; + } } + + /* first search (depth 0) */ + { size_t ofbCandidate = 999999999; + size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, ZSTD_extDict); + if (ml2 > matchLength) + matchLength = ml2, start = ip, offBase = ofbCandidate; + } + + if (matchLength < 4) { + size_t const step = ((size_t)(ip-anchor) >> kSearchStrength); + ip += step + 1; /* jump faster over incompressible sections */ + /* Enter the lazy skipping mode once we are skipping more than 8 bytes at a time. + * In this mode we stop inserting every position into our tables, and only insert + * positions that we search, which is one in step positions. + * The exact cutoff is flexible, I've just chosen a number that is reasonably high, + * so we minimize the compression ratio loss in "normal" scenarios. This mode gets + * triggered once we've gone 2KB without finding any matches. + */ + ms->lazySkipping = step > kLazySkippingStep; + continue; + } + + /* let's try to find a better solution */ + if (depth>=1) + while (ip repIndex >= windowLow` */ + if (MEM_read32(ip) == MEM_read32(repMatch)) { + /* repcode detected */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + size_t const repLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; + int const gain2 = (int)(repLength * 3); + int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offBase) + 1); + if ((repLength >= 4) && (gain2 > gain1)) + matchLength = repLength, offBase = REPCODE1_TO_OFFBASE, start = ip; } } - } else { /* not a dictionary */ + /* search match, depth 1 */ + { size_t ofbCandidate = 999999999; + size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, ZSTD_extDict); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 4); + if ((ml2 >= 4) && (gain2 > gain1)) { + matchLength = ml2, offBase = ofbCandidate, start = ip; + continue; /* search a better one */ + } } - assert(optPtr->litFreq != NULL); - if (compressedLiterals) { - unsigned lit = MaxLit; - HIST_count_simple(optPtr->litFreq, &lit, src, srcSize); /* use raw first block to init statistics */ - optPtr->litSum = ZSTD_downscaleStat(optPtr->litFreq, MaxLit, 1); - } + /* let's find an even better one */ + if ((depth==2) && (ip repIndex >= windowLow` */ + if (MEM_read32(ip) == MEM_read32(repMatch)) { + /* repcode detected */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + size_t const repLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; + int const gain2 = (int)(repLength * 4); + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 1); + if ((repLength >= 4) && (gain2 > gain1)) + matchLength = repLength, offBase = REPCODE1_TO_OFFBASE, start = ip; + } } - { unsigned ll; - for (ll=0; ll<=MaxLL; ll++) - optPtr->litLengthFreq[ll] = 1; - } - optPtr->litLengthSum = MaxLL+1; + /* search match, depth 2 */ + { size_t ofbCandidate = 999999999; + size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, ZSTD_extDict); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 7); + if ((ml2 >= 4) && (gain2 > gain1)) { + matchLength = ml2, offBase = ofbCandidate, start = ip; + continue; + } } } + break; /* nothing found : store previous solution */ + } - { unsigned ml; - for (ml=0; ml<=MaxML; ml++) - optPtr->matchLengthFreq[ml] = 1; - } - optPtr->matchLengthSum = MaxML+1; + /* catch up */ + if (OFFBASE_IS_OFFSET(offBase)) { + U32 const matchIndex = (U32)((size_t)(start-base) - OFFBASE_TO_OFFSET(offBase)); + const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex; + const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart; + while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */ + offset_2 = offset_1; offset_1 = (U32)OFFBASE_TO_OFFSET(offBase); + } - { unsigned of; - for (of=0; of<=MaxOff; of++) - optPtr->offCodeFreq[of] = 1; + /* store sequence */ +_storeSequence: + { size_t const litLength = (size_t)(start - anchor); + ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offBase, matchLength); + anchor = ip = start + matchLength; + } + if (ms->lazySkipping) { + /* We've found a match, disable lazy skipping mode, and refill the hash cache. */ + if (searchMethod == search_rowHash) { + ZSTD_row_fillHashCache(ms, base, rowLog, mls, ms->nextToUpdate, ilimit); } - optPtr->offCodeSum = MaxOff+1; - + ms->lazySkipping = 0; } - } else { /* new block : re-use previous statistics, scaled down */ + /* check immediate repcode */ + while (ip <= ilimit) { + const U32 repCurrent = (U32)(ip-base); + const U32 windowLow = ZSTD_getLowestMatchIndex(ms, repCurrent, windowLog); + const U32 repIndex = repCurrent - offset_2; + const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + if ( (ZSTD_index_overlap_check(dictLimit, repIndex)) + & (offset_2 <= repCurrent - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */ + if (MEM_read32(ip) == MEM_read32(repMatch)) { + /* repcode detected we should take it */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; + offBase = offset_2; offset_2 = offset_1; offset_1 = (U32)offBase; /* swap offset history */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, matchLength); + ip += matchLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } + break; + } } - if (compressedLiterals) - optPtr->litSum = ZSTD_downscaleStat(optPtr->litFreq, MaxLit, 1); - optPtr->litLengthSum = ZSTD_downscaleStat(optPtr->litLengthFreq, MaxLL, 0); - optPtr->matchLengthSum = ZSTD_downscaleStat(optPtr->matchLengthFreq, MaxML, 0); - optPtr->offCodeSum = ZSTD_downscaleStat(optPtr->offCodeFreq, MaxOff, 0); - } + /* Save reps for next block */ + rep[0] = offset_1; + rep[1] = offset_2; - ZSTD_setBasePrices(optPtr, optLevel); + /* Return the last literals size */ + return (size_t)(iend - anchor); } +#endif /* build exclusions */ -/* ZSTD_rawLiteralsCost() : - * price of literals (only) in specified segment (which length can be 0). - * does not include price of literalLength symbol */ -static U32 ZSTD_rawLiteralsCost(const BYTE* const literals, U32 const litLength, - const optState_t* const optPtr, - int optLevel) +#ifndef ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_greedy_extDict( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) { - if (litLength == 0) return 0; + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0); +} - if (!ZSTD_compressedLiterals(optPtr)) - return (litLength << 3) * BITCOST_MULTIPLIER; /* Uncompressed - 8 bytes per literal. */ +size_t ZSTD_compressBlock_greedy_extDict_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0); +} +#endif - if (optPtr->priceType == zop_predef) - return (litLength*6) * BITCOST_MULTIPLIER; /* 6 bit per literal - no statistic used */ +#ifndef ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_lazy_extDict( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) - /* dynamic statistics */ - { U32 price = litLength * optPtr->litSumBasePrice; - U32 u; - for (u=0; u < litLength; u++) { - assert(WEIGHT(optPtr->litFreq[literals[u]], optLevel) <= optPtr->litSumBasePrice); /* literal cost should never be negative */ - price -= WEIGHT(optPtr->litFreq[literals[u]], optLevel); - } - return price; - } +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1); } -/* ZSTD_litLengthPrice() : - * cost of literalLength symbol */ -static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optPtr, int optLevel) +size_t ZSTD_compressBlock_lazy_extDict_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) + { - if (optPtr->priceType == zop_predef) return WEIGHT(litLength, optLevel); + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1); +} +#endif - /* dynamic statistics */ - { U32 const llCode = ZSTD_LLcode(litLength); - return (LL_bits[llCode] * BITCOST_MULTIPLIER) - + optPtr->litLengthSumBasePrice - - WEIGHT(optPtr->litLengthFreq[llCode], optLevel); - } +#ifndef ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_lazy2_extDict( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) + +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2); } -/* ZSTD_getMatchPrice() : - * Provides the cost of the match part (offset + matchLength) of a sequence - * Must be combined with ZSTD_fullLiteralsCost() to get the full cost of a sequence. - * optLevel: when <2, favors small offset for decompression speed (improved cache efficiency) */ -FORCE_INLINE_TEMPLATE U32 -ZSTD_getMatchPrice(U32 const offset, - U32 const matchLength, - const optState_t* const optPtr, - int const optLevel) +size_t ZSTD_compressBlock_lazy2_extDict_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) { - U32 price; - U32 const offCode = ZSTD_highbit32(offset+1); - U32 const mlBase = matchLength - MINMATCH; - assert(matchLength >= MINMATCH); + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2); +} +#endif - if (optPtr->priceType == zop_predef) /* fixed scheme, do not use statistics */ - return WEIGHT(mlBase, optLevel) + ((16 + offCode) * BITCOST_MULTIPLIER); +#ifndef ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_btlazy2_extDict( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) - /* dynamic statistics */ - price = (offCode * BITCOST_MULTIPLIER) + (optPtr->offCodeSumBasePrice - WEIGHT(optPtr->offCodeFreq[offCode], optLevel)); - if ((optLevel<2) /*static*/ && offCode >= 20) - price += (offCode-19)*2 * BITCOST_MULTIPLIER; /* handicap for long distance offsets, favor decompression speed */ +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2); +} +#endif +/**** ended inlining compress/zstd_lazy.c ****/ +/**** start inlining compress/zstd_ldm.c ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/**** skipping file: zstd_ldm.h ****/ + +/**** skipping file: ../common/debug.h ****/ +/**** skipping file: ../common/xxhash.h ****/ +/**** skipping file: zstd_fast.h ****/ +/**** skipping file: zstd_double_fast.h ****/ +/**** start inlining zstd_ldm_geartab.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_LDM_GEARTAB_H +#define ZSTD_LDM_GEARTAB_H + +/**** skipping file: ../common/compiler.h ****/ +/**** skipping file: ../common/mem.h ****/ + +static UNUSED_ATTR const U64 ZSTD_ldm_gearTab[256] = { + 0xf5b8f72c5f77775c, 0x84935f266b7ac412, 0xb647ada9ca730ccc, + 0xb065bb4b114fb1de, 0x34584e7e8c3a9fd0, 0x4e97e17c6ae26b05, + 0x3a03d743bc99a604, 0xcecd042422c4044f, 0x76de76c58524259e, + 0x9c8528f65badeaca, 0x86563706e2097529, 0x2902475fa375d889, + 0xafb32a9739a5ebe6, 0xce2714da3883e639, 0x21eaf821722e69e, + 0x37b628620b628, 0x49a8d455d88caf5, 0x8556d711e6958140, + 0x4f7ae74fc605c1f, 0x829f0c3468bd3a20, 0x4ffdc885c625179e, + 0x8473de048a3daf1b, 0x51008822b05646b2, 0x69d75d12b2d1cc5f, + 0x8c9d4a19159154bc, 0xc3cc10f4abbd4003, 0xd06ddc1cecb97391, + 0xbe48e6e7ed80302e, 0x3481db31cee03547, 0xacc3f67cdaa1d210, + 0x65cb771d8c7f96cc, 0x8eb27177055723dd, 0xc789950d44cd94be, + 0x934feadc3700b12b, 0x5e485f11edbdf182, 0x1e2e2a46fd64767a, + 0x2969ca71d82efa7c, 0x9d46e9935ebbba2e, 0xe056b67e05e6822b, + 0x94d73f55739d03a0, 0xcd7010bdb69b5a03, 0x455ef9fcd79b82f4, + 0x869cb54a8749c161, 0x38d1a4fa6185d225, 0xb475166f94bbe9bb, + 0xa4143548720959f1, 0x7aed4780ba6b26ba, 0xd0ce264439e02312, + 0x84366d746078d508, 0xa8ce973c72ed17be, 0x21c323a29a430b01, + 0x9962d617e3af80ee, 0xab0ce91d9c8cf75b, 0x530e8ee6d19a4dbc, + 0x2ef68c0cf53f5d72, 0xc03a681640a85506, 0x496e4e9f9c310967, + 0x78580472b59b14a0, 0x273824c23b388577, 0x66bf923ad45cb553, + 0x47ae1a5a2492ba86, 0x35e304569e229659, 0x4765182a46870b6f, + 0x6cbab625e9099412, 0xddac9a2e598522c1, 0x7172086e666624f2, + 0xdf5003ca503b7837, 0x88c0c1db78563d09, 0x58d51865acfc289d, + 0x177671aec65224f1, 0xfb79d8a241e967d7, 0x2be1e101cad9a49a, + 0x6625682f6e29186b, 0x399553457ac06e50, 0x35dffb4c23abb74, + 0x429db2591f54aade, 0xc52802a8037d1009, 0x6acb27381f0b25f3, + 0xf45e2551ee4f823b, 0x8b0ea2d99580c2f7, 0x3bed519cbcb4e1e1, + 0xff452823dbb010a, 0x9d42ed614f3dd267, 0x5b9313c06257c57b, + 0xa114b8008b5e1442, 0xc1fe311c11c13d4b, 0x66e8763ea34c5568, + 0x8b982af1c262f05d, 0xee8876faaa75fbb7, 0x8a62a4d0d172bb2a, + 0xc13d94a3b7449a97, 0x6dbbba9dc15d037c, 0xc786101f1d92e0f1, + 0xd78681a907a0b79b, 0xf61aaf2962c9abb9, 0x2cfd16fcd3cb7ad9, + 0x868c5b6744624d21, 0x25e650899c74ddd7, 0xba042af4a7c37463, + 0x4eb1a539465a3eca, 0xbe09dbf03b05d5ca, 0x774e5a362b5472ba, + 0x47a1221229d183cd, 0x504b0ca18ef5a2df, 0xdffbdfbde2456eb9, + 0x46cd2b2fbee34634, 0xf2aef8fe819d98c3, 0x357f5276d4599d61, + 0x24a5483879c453e3, 0x88026889192b4b9, 0x28da96671782dbec, + 0x4ef37c40588e9aaa, 0x8837b90651bc9fb3, 0xc164f741d3f0e5d6, + 0xbc135a0a704b70ba, 0x69cd868f7622ada, 0xbc37ba89e0b9c0ab, + 0x47c14a01323552f6, 0x4f00794bacee98bb, 0x7107de7d637a69d5, + 0x88af793bb6f2255e, 0xf3c6466b8799b598, 0xc288c616aa7f3b59, + 0x81ca63cf42fca3fd, 0x88d85ace36a2674b, 0xd056bd3792389e7, + 0xe55c396c4e9dd32d, 0xbefb504571e6c0a6, 0x96ab32115e91e8cc, + 0xbf8acb18de8f38d1, 0x66dae58801672606, 0x833b6017872317fb, + 0xb87c16f2d1c92864, 0xdb766a74e58b669c, 0x89659f85c61417be, + 0xc8daad856011ea0c, 0x76a4b565b6fe7eae, 0xa469d085f6237312, + 0xaaf0365683a3e96c, 0x4dbb746f8424f7b8, 0x638755af4e4acc1, + 0x3d7807f5bde64486, 0x17be6d8f5bbb7639, 0x903f0cd44dc35dc, + 0x67b672eafdf1196c, 0xa676ff93ed4c82f1, 0x521d1004c5053d9d, + 0x37ba9ad09ccc9202, 0x84e54d297aacfb51, 0xa0b4b776a143445, + 0x820d471e20b348e, 0x1874383cb83d46dc, 0x97edeec7a1efe11c, + 0xb330e50b1bdc42aa, 0x1dd91955ce70e032, 0xa514cdb88f2939d5, + 0x2791233fd90db9d3, 0x7b670a4cc50f7a9b, 0x77c07d2a05c6dfa5, + 0xe3778b6646d0a6fa, 0xb39c8eda47b56749, 0x933ed448addbef28, + 0xaf846af6ab7d0bf4, 0xe5af208eb666e49, 0x5e6622f73534cd6a, + 0x297daeca42ef5b6e, 0x862daef3d35539a6, 0xe68722498f8e1ea9, + 0x981c53093dc0d572, 0xfa09b0bfbf86fbf5, 0x30b1e96166219f15, + 0x70e7d466bdc4fb83, 0x5a66736e35f2a8e9, 0xcddb59d2b7c1baef, + 0xd6c7d247d26d8996, 0xea4e39eac8de1ba3, 0x539c8bb19fa3aff2, + 0x9f90e4c5fd508d8, 0xa34e5956fbaf3385, 0x2e2f8e151d3ef375, + 0x173691e9b83faec1, 0xb85a8d56bf016379, 0x8382381267408ae3, + 0xb90f901bbdc0096d, 0x7c6ad32933bcec65, 0x76bb5e2f2c8ad595, + 0x390f851a6cf46d28, 0xc3e6064da1c2da72, 0xc52a0c101cfa5389, + 0xd78eaf84a3fbc530, 0x3781b9e2288b997e, 0x73c2f6dea83d05c4, + 0x4228e364c5b5ed7, 0x9d7a3edf0da43911, 0x8edcfeda24686756, + 0x5e7667a7b7a9b3a1, 0x4c4f389fa143791d, 0xb08bc1023da7cddc, + 0x7ab4be3ae529b1cc, 0x754e6132dbe74ff9, 0x71635442a839df45, + 0x2f6fb1643fbe52de, 0x961e0a42cf7a8177, 0xf3b45d83d89ef2ea, + 0xee3de4cf4a6e3e9b, 0xcd6848542c3295e7, 0xe4cee1664c78662f, + 0x9947548b474c68c4, 0x25d73777a5ed8b0b, 0xc915b1d636b7fc, + 0x21c2ba75d9b0d2da, 0x5f6b5dcf608a64a1, 0xdcf333255ff9570c, + 0x633b922418ced4ee, 0xc136dde0b004b34a, 0x58cc83b05d4b2f5a, + 0x5eb424dda28e42d2, 0x62df47369739cd98, 0xb4e0b42485e4ce17, + 0x16e1f0c1f9a8d1e7, 0x8ec3916707560ebf, 0x62ba6e2df2cc9db3, + 0xcbf9f4ff77d83a16, 0x78d9d7d07d2bbcc4, 0xef554ce1e02c41f4, + 0x8d7581127eccf94d, 0xa9b53336cb3c8a05, 0x38c42c0bf45c4f91, + 0x640893cdf4488863, 0x80ec34bc575ea568, 0x39f324f5b48eaa40, + 0xe9d9ed1f8eff527f, 0x9224fc058cc5a214, 0xbaba00b04cfe7741, + 0x309a9f120fcf52af, 0xa558f3ec65626212, 0x424bec8b7adabe2f, + 0x41622513a6aea433, 0xb88da2d5324ca798, 0xd287733b245528a4, + 0x9a44697e6d68aec3, 0x7b1093be2f49bb28, 0x50bbec632e3d8aad, + 0x6cd90723e1ea8283, 0x897b9e7431b02bf3, 0x219efdcb338a7047, + 0x3b0311f0a27c0656, 0xdb17bf91c0db96e7, 0x8cd4fd6b4e85a5b2, + 0xfab071054ba6409d, 0x40d6fe831fa9dfd9, 0xaf358debad7d791e, + 0xeb8d0e25a65e3e58, 0xbbcbd3df14e08580, 0xcf751f27ecdab2b, + 0x2b4da14f2613d8f4 +}; - /* match Length */ - { U32 const mlCode = ZSTD_MLcode(mlBase); - price += (ML_bits[mlCode] * BITCOST_MULTIPLIER) + (optPtr->matchLengthSumBasePrice - WEIGHT(optPtr->matchLengthFreq[mlCode], optLevel)); - } +#endif /* ZSTD_LDM_GEARTAB_H */ +/**** ended inlining zstd_ldm_geartab.h ****/ - price += BITCOST_MULTIPLIER / 5; /* heuristic : make matches a bit more costly to favor less sequences -> faster decompression speed */ +#define LDM_BUCKET_SIZE_LOG 4 +#define LDM_MIN_MATCH_LENGTH 64 +#define LDM_HASH_RLOG 7 - DEBUGLOG(8, "ZSTD_getMatchPrice(ml:%u) = %u", matchLength, price); - return price; -} +typedef struct { + U64 rolling; + U64 stopMask; +} ldmRollingHashState_t; -/* ZSTD_updateStats() : - * assumption : literals + litLengtn <= iend */ -static void ZSTD_updateStats(optState_t* const optPtr, - U32 litLength, const BYTE* literals, - U32 offsetCode, U32 matchLength) +/** ZSTD_ldm_gear_init(): + * + * Initializes the rolling hash state such that it will honor the + * settings in params. */ +static void ZSTD_ldm_gear_init(ldmRollingHashState_t* state, ldmParams_t const* params) { - /* literals */ - if (ZSTD_compressedLiterals(optPtr)) { - U32 u; - for (u=0; u < litLength; u++) - optPtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD; - optPtr->litSum += litLength*ZSTD_LITFREQ_ADD; - } - - /* literal Length */ - { U32 const llCode = ZSTD_LLcode(litLength); - optPtr->litLengthFreq[llCode]++; - optPtr->litLengthSum++; - } - - /* match offset code (0-2=>repCode; 3+=>offset+2) */ - { U32 const offCode = ZSTD_highbit32(offsetCode+1); - assert(offCode <= MaxOff); - optPtr->offCodeFreq[offCode]++; - optPtr->offCodeSum++; - } - - /* match Length */ - { U32 const mlBase = matchLength - MINMATCH; - U32 const mlCode = ZSTD_MLcode(mlBase); - optPtr->matchLengthFreq[mlCode]++; - optPtr->matchLengthSum++; - } -} + unsigned maxBitsInMask = MIN(params->minMatchLength, 64); + unsigned hashRateLog = params->hashRateLog; + state->rolling = ~(U32)0; -/* ZSTD_readMINMATCH() : - * function safe only for comparisons - * assumption : memPtr must be at least 4 bytes before end of buffer */ -MEM_STATIC U32 ZSTD_readMINMATCH(const void* memPtr, U32 length) -{ - switch (length) - { - default : - case 4 : return MEM_read32(memPtr); - case 3 : if (MEM_isLittleEndian()) - return MEM_read32(memPtr)<<8; - else - return MEM_read32(memPtr)>>8; + /* The choice of the splitting criterion is subject to two conditions: + * 1. it has to trigger on average every 2^(hashRateLog) bytes; + * 2. ideally, it has to depend on a window of minMatchLength bytes. + * + * In the gear hash algorithm, bit n depends on the last n bytes; + * so in order to obtain a good quality splitting criterion it is + * preferable to use bits with high weight. + * + * To match condition 1 we use a mask with hashRateLog bits set + * and, because of the previous remark, we make sure these bits + * have the highest possible weight while still respecting + * condition 2. + */ + if (hashRateLog > 0 && hashRateLog <= maxBitsInMask) { + state->stopMask = (((U64)1 << hashRateLog) - 1) << (maxBitsInMask - hashRateLog); + } else { + /* In this degenerate case we simply honor the hash rate. */ + state->stopMask = ((U64)1 << hashRateLog) - 1; } } - -/* Update hashTable3 up to ip (excluded) - Assumption : always within prefix (i.e. not within extDict) */ -static U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_matchState_t* ms, - U32* nextToUpdate3, - const BYTE* const ip) +/** ZSTD_ldm_gear_reset() + * Feeds [data, data + minMatchLength) into the hash without registering any + * splits. This effectively resets the hash state. This is used when skipping + * over data, either at the beginning of a block, or skipping sections. + */ +static void ZSTD_ldm_gear_reset(ldmRollingHashState_t* state, + BYTE const* data, size_t minMatchLength) { - U32* const hashTable3 = ms->hashTable3; - U32 const hashLog3 = ms->hashLog3; - const BYTE* const base = ms->window.base; - U32 idx = *nextToUpdate3; - U32 const target = (U32)(ip - base); - size_t const hash3 = ZSTD_hash3Ptr(ip, hashLog3); - assert(hashLog3 > 0); + U64 hash = state->rolling; + size_t n = 0; - while(idx < target) { - hashTable3[ZSTD_hash3Ptr(base+idx, hashLog3)] = idx; - idx++; +#define GEAR_ITER_ONCE() do { \ + hash = (hash << 1) + ZSTD_ldm_gearTab[data[n] & 0xff]; \ + n += 1; \ + } while (0) + while (n + 3 < minMatchLength) { + GEAR_ITER_ONCE(); + GEAR_ITER_ONCE(); + GEAR_ITER_ONCE(); + GEAR_ITER_ONCE(); } - - *nextToUpdate3 = target; - return hashTable3[hash3]; + while (n < minMatchLength) { + GEAR_ITER_ONCE(); + } +#undef GEAR_ITER_ONCE } - -/*-************************************* -* Binary Tree search -***************************************/ -/** ZSTD_insertBt1() : add one or multiple positions to tree. - * ip : assumed <= iend-8 . - * @return : nb of positions added */ -static U32 ZSTD_insertBt1( - ZSTD_matchState_t* ms, - const BYTE* const ip, const BYTE* const iend, - U32 const mls, const int extDict) +/** ZSTD_ldm_gear_feed(): + * + * Registers in the splits array all the split points found in the first + * size bytes following the data pointer. This function terminates when + * either all the data has been processed or LDM_BATCH_SIZE splits are + * present in the splits array. + * + * Precondition: The splits array must not be full. + * Returns: The number of bytes processed. */ +static size_t ZSTD_ldm_gear_feed(ldmRollingHashState_t* state, + BYTE const* data, size_t size, + size_t* splits, unsigned* numSplits) { - const ZSTD_compressionParameters* const cParams = &ms->cParams; - U32* const hashTable = ms->hashTable; - U32 const hashLog = cParams->hashLog; - size_t const h = ZSTD_hashPtr(ip, hashLog, mls); - U32* const bt = ms->chainTable; - U32 const btLog = cParams->chainLog - 1; - U32 const btMask = (1 << btLog) - 1; - U32 matchIndex = hashTable[h]; - size_t commonLengthSmaller=0, commonLengthLarger=0; - const BYTE* const base = ms->window.base; - const BYTE* const dictBase = ms->window.dictBase; - const U32 dictLimit = ms->window.dictLimit; - const BYTE* const dictEnd = dictBase + dictLimit; - const BYTE* const prefixStart = base + dictLimit; - const BYTE* match; - const U32 curr = (U32)(ip-base); - const U32 btLow = btMask >= curr ? 0 : curr - btMask; - U32* smallerPtr = bt + 2*(curr&btMask); - U32* largerPtr = smallerPtr + 1; - U32 dummy32; /* to be nullified at the end */ - U32 const windowLow = ms->window.lowLimit; - U32 matchEndIdx = curr+8+1; - size_t bestLength = 8; - U32 nbCompares = 1U << cParams->searchLog; -#ifdef ZSTD_C_PREDICT - U32 predictedSmall = *(bt + 2*((curr-1)&btMask) + 0); - U32 predictedLarge = *(bt + 2*((curr-1)&btMask) + 1); - predictedSmall += (predictedSmall>0); - predictedLarge += (predictedLarge>0); -#endif /* ZSTD_C_PREDICT */ - - DEBUGLOG(8, "ZSTD_insertBt1 (%u)", curr); - - assert(ip <= iend-8); /* required for h calculation */ - hashTable[h] = curr; /* Update Hash Table */ - - assert(windowLow > 0); - while (nbCompares-- && (matchIndex >= windowLow)) { - U32* const nextPtr = bt + 2*(matchIndex & btMask); - size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ - assert(matchIndex < curr); - -#ifdef ZSTD_C_PREDICT /* note : can create issues when hlog small <= 11 */ - const U32* predictPtr = bt + 2*((matchIndex-1) & btMask); /* written this way, as bt is a roll buffer */ - if (matchIndex == predictedSmall) { - /* no need to check length, result known */ - *smallerPtr = matchIndex; - if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ - smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ - matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ - predictedSmall = predictPtr[1] + (predictPtr[1]>0); - continue; - } - if (matchIndex == predictedLarge) { - *largerPtr = matchIndex; - if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ - largerPtr = nextPtr; - matchIndex = nextPtr[0]; - predictedLarge = predictPtr[0] + (predictPtr[0]>0); - continue; - } -#endif - - if (!extDict || (matchIndex+matchLength >= dictLimit)) { - assert(matchIndex+matchLength >= dictLimit); /* might be wrong if actually extDict */ - match = base + matchIndex; - matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend); - } else { - match = dictBase + matchIndex; - matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); - if (matchIndex+matchLength >= dictLimit) - match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ - } - - if (matchLength > bestLength) { - bestLength = matchLength; - if (matchLength > matchEndIdx - matchIndex) - matchEndIdx = matchIndex + (U32)matchLength; - } - - if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */ - break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt tree */ - } + size_t n; + U64 hash, mask; - if (match[matchLength] < ip[matchLength]) { /* necessarily within buffer */ - /* match is smaller than current */ - *smallerPtr = matchIndex; /* update smaller idx */ - commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ - if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop searching */ - smallerPtr = nextPtr+1; /* new "candidate" => larger than match, which was smaller than target */ - matchIndex = nextPtr[1]; /* new matchIndex, larger than previous and closer to current */ - } else { - /* match is larger than current */ - *largerPtr = matchIndex; - commonLengthLarger = matchLength; - if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop searching */ - largerPtr = nextPtr; - matchIndex = nextPtr[0]; - } } + hash = state->rolling; + mask = state->stopMask; + n = 0; - *smallerPtr = *largerPtr = 0; - { U32 positions = 0; - if (bestLength > 384) positions = MIN(192, (U32)(bestLength - 384)); /* speed optimization */ - assert(matchEndIdx > curr + 8); - return MAX(positions, matchEndIdx - (curr + 8)); +#define GEAR_ITER_ONCE() do { \ + hash = (hash << 1) + ZSTD_ldm_gearTab[data[n] & 0xff]; \ + n += 1; \ + if (UNLIKELY((hash & mask) == 0)) { \ + splits[*numSplits] = n; \ + *numSplits += 1; \ + if (*numSplits == LDM_BATCH_SIZE) \ + goto done; \ + } \ + } while (0) + + while (n + 3 < size) { + GEAR_ITER_ONCE(); + GEAR_ITER_ONCE(); + GEAR_ITER_ONCE(); + GEAR_ITER_ONCE(); } + while (n < size) { + GEAR_ITER_ONCE(); + } + +#undef GEAR_ITER_ONCE + +done: + state->rolling = hash; + return n; } -FORCE_INLINE_TEMPLATE -void ZSTD_updateTree_internal( - ZSTD_matchState_t* ms, - const BYTE* const ip, const BYTE* const iend, - const U32 mls, const ZSTD_dictMode_e dictMode) +void ZSTD_ldm_adjustParameters(ldmParams_t* params, + const ZSTD_compressionParameters* cParams) { - const BYTE* const base = ms->window.base; - U32 const target = (U32)(ip - base); - U32 idx = ms->nextToUpdate; - DEBUGLOG(6, "ZSTD_updateTree_internal, from %u to %u (dictMode:%u)", - idx, target, dictMode); - - while(idx < target) { - U32 const forward = ZSTD_insertBt1(ms, base+idx, iend, mls, dictMode == ZSTD_extDict); - assert(idx < (U32)(idx + forward)); - idx += forward; + params->windowLog = cParams->windowLog; + ZSTD_STATIC_ASSERT(LDM_BUCKET_SIZE_LOG <= ZSTD_LDM_BUCKETSIZELOG_MAX); + DEBUGLOG(4, "ZSTD_ldm_adjustParameters"); + if (params->hashRateLog == 0) { + if (params->hashLog > 0) { + /* if params->hashLog is set, derive hashRateLog from it */ + assert(params->hashLog <= ZSTD_HASHLOG_MAX); + if (params->windowLog > params->hashLog) { + params->hashRateLog = params->windowLog - params->hashLog; + } + } else { + assert(1 <= (int)cParams->strategy && (int)cParams->strategy <= 9); + /* mapping from [fast, rate7] to [btultra2, rate4] */ + params->hashRateLog = 7 - (cParams->strategy/3); + } } - assert((size_t)(ip - base) <= (size_t)(U32)(-1)); - assert((size_t)(iend - base) <= (size_t)(U32)(-1)); - ms->nextToUpdate = target; + if (params->hashLog == 0) { + params->hashLog = BOUNDED(ZSTD_HASHLOG_MIN, params->windowLog - params->hashRateLog, ZSTD_HASHLOG_MAX); + } + if (params->minMatchLength == 0) { + params->minMatchLength = LDM_MIN_MATCH_LENGTH; + if (cParams->strategy >= ZSTD_btultra) + params->minMatchLength /= 2; + } + if (params->bucketSizeLog==0) { + assert(1 <= (int)cParams->strategy && (int)cParams->strategy <= 9); + params->bucketSizeLog = BOUNDED(LDM_BUCKET_SIZE_LOG, (U32)cParams->strategy, ZSTD_LDM_BUCKETSIZELOG_MAX); + } + params->bucketSizeLog = MIN(params->bucketSizeLog, params->hashLog); } -void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend) { - ZSTD_updateTree_internal(ms, ip, iend, ms->cParams.minMatch, ZSTD_noDict); +size_t ZSTD_ldm_getTableSize(ldmParams_t params) +{ + size_t const ldmHSize = ((size_t)1) << params.hashLog; + size_t const ldmBucketSizeLog = MIN(params.bucketSizeLog, params.hashLog); + size_t const ldmBucketSize = ((size_t)1) << (params.hashLog - ldmBucketSizeLog); + size_t const totalSize = ZSTD_cwksp_alloc_size(ldmBucketSize) + + ZSTD_cwksp_alloc_size(ldmHSize * sizeof(ldmEntry_t)); + return params.enableLdm == ZSTD_ps_enable ? totalSize : 0; } -FORCE_INLINE_TEMPLATE -U32 ZSTD_insertBtAndGetAllMatches ( - ZSTD_match_t* matches, /* store result (found matches) in this table (presumed large enough) */ - ZSTD_matchState_t* ms, - U32* nextToUpdate3, - const BYTE* const ip, const BYTE* const iLimit, const ZSTD_dictMode_e dictMode, - const U32 rep[ZSTD_REP_NUM], - U32 const ll0, /* tells if associated literal length is 0 or not. This value must be 0 or 1 */ - const U32 lengthToBeat, - U32 const mls /* template */) +size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize) { - const ZSTD_compressionParameters* const cParams = &ms->cParams; - U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1); - const BYTE* const base = ms->window.base; - U32 const curr = (U32)(ip-base); - U32 const hashLog = cParams->hashLog; - U32 const minMatch = (mls==3) ? 3 : 4; - U32* const hashTable = ms->hashTable; - size_t const h = ZSTD_hashPtr(ip, hashLog, mls); - U32 matchIndex = hashTable[h]; - U32* const bt = ms->chainTable; - U32 const btLog = cParams->chainLog - 1; - U32 const btMask= (1U << btLog) - 1; - size_t commonLengthSmaller=0, commonLengthLarger=0; - const BYTE* const dictBase = ms->window.dictBase; - U32 const dictLimit = ms->window.dictLimit; - const BYTE* const dictEnd = dictBase + dictLimit; - const BYTE* const prefixStart = base + dictLimit; - U32 const btLow = (btMask >= curr) ? 0 : curr - btMask; - U32 const windowLow = ZSTD_getLowestMatchIndex(ms, curr, cParams->windowLog); - U32 const matchLow = windowLow ? windowLow : 1; - U32* smallerPtr = bt + 2*(curr&btMask); - U32* largerPtr = bt + 2*(curr&btMask) + 1; - U32 matchEndIdx = curr+8+1; /* farthest referenced position of any match => detects repetitive patterns */ - U32 dummy32; /* to be nullified at the end */ - U32 mnum = 0; - U32 nbCompares = 1U << cParams->searchLog; + return params.enableLdm == ZSTD_ps_enable ? (maxChunkSize / params.minMatchLength) : 0; +} - const ZSTD_matchState_t* dms = dictMode == ZSTD_dictMatchState ? ms->dictMatchState : NULL; - const ZSTD_compressionParameters* const dmsCParams = - dictMode == ZSTD_dictMatchState ? &dms->cParams : NULL; - const BYTE* const dmsBase = dictMode == ZSTD_dictMatchState ? dms->window.base : NULL; - const BYTE* const dmsEnd = dictMode == ZSTD_dictMatchState ? dms->window.nextSrc : NULL; - U32 const dmsHighLimit = dictMode == ZSTD_dictMatchState ? (U32)(dmsEnd - dmsBase) : 0; - U32 const dmsLowLimit = dictMode == ZSTD_dictMatchState ? dms->window.lowLimit : 0; - U32 const dmsIndexDelta = dictMode == ZSTD_dictMatchState ? windowLow - dmsHighLimit : 0; - U32 const dmsHashLog = dictMode == ZSTD_dictMatchState ? dmsCParams->hashLog : hashLog; - U32 const dmsBtLog = dictMode == ZSTD_dictMatchState ? dmsCParams->chainLog - 1 : btLog; - U32 const dmsBtMask = dictMode == ZSTD_dictMatchState ? (1U << dmsBtLog) - 1 : 0; - U32 const dmsBtLow = dictMode == ZSTD_dictMatchState && dmsBtMask < dmsHighLimit - dmsLowLimit ? dmsHighLimit - dmsBtMask : dmsLowLimit; +/** ZSTD_ldm_getBucket() : + * Returns a pointer to the start of the bucket associated with hash. */ +static ldmEntry_t* ZSTD_ldm_getBucket( + const ldmState_t* ldmState, size_t hash, U32 const bucketSizeLog) +{ + return ldmState->hashTable + (hash << bucketSizeLog); +} - size_t bestLength = lengthToBeat-1; - DEBUGLOG(8, "ZSTD_insertBtAndGetAllMatches: current=%u", curr); +/** ZSTD_ldm_insertEntry() : + * Insert the entry with corresponding hash into the hash table */ +static void ZSTD_ldm_insertEntry(ldmState_t* ldmState, + size_t const hash, const ldmEntry_t entry, + U32 const bucketSizeLog) +{ + BYTE* const pOffset = ldmState->bucketOffsets + hash; + unsigned const offset = *pOffset; - /* check repCode */ - assert(ll0 <= 1); /* necessarily 1 or 0 */ - { U32 const lastR = ZSTD_REP_NUM + ll0; - U32 repCode; - for (repCode = ll0; repCode < lastR; repCode++) { - U32 const repOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode]; - U32 const repIndex = curr - repOffset; - U32 repLen = 0; - assert(curr >= dictLimit); - if (repOffset-1 /* intentional overflow, discards 0 and -1 */ < curr-dictLimit) { /* equivalent to `curr > repIndex >= dictLimit` */ - /* We must validate the repcode offset because when we're using a dictionary the - * valid offset range shrinks when the dictionary goes out of bounds. - */ - if ((repIndex >= windowLow) & (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(ip - repOffset, minMatch))) { - repLen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-repOffset, iLimit) + minMatch; - } - } else { /* repIndex < dictLimit || repIndex >= curr */ - const BYTE* const repMatch = dictMode == ZSTD_dictMatchState ? - dmsBase + repIndex - dmsIndexDelta : - dictBase + repIndex; - assert(curr >= windowLow); - if ( dictMode == ZSTD_extDict - && ( ((repOffset-1) /*intentional overflow*/ < curr - windowLow) /* equivalent to `curr > repIndex >= windowLow` */ - & (((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */) - && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) { - repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dictEnd, prefixStart) + minMatch; - } - if (dictMode == ZSTD_dictMatchState - && ( ((repOffset-1) /*intentional overflow*/ < curr - (dmsLowLimit + dmsIndexDelta)) /* equivalent to `curr > repIndex >= dmsLowLimit` */ - & ((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */ - && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) { - repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dmsEnd, prefixStart) + minMatch; - } } - /* save longer solution */ - if (repLen > bestLength) { - DEBUGLOG(8, "found repCode %u (ll0:%u, offset:%u) of length %u", - repCode, ll0, repOffset, repLen); - bestLength = repLen; - matches[mnum].off = repCode - ll0; - matches[mnum].len = (U32)repLen; - mnum++; - if ( (repLen > sufficient_len) - | (ip+repLen == iLimit) ) { /* best possible */ - return mnum; - } } } } + *(ZSTD_ldm_getBucket(ldmState, hash, bucketSizeLog) + offset) = entry; + *pOffset = (BYTE)((offset + 1) & ((1u << bucketSizeLog) - 1)); - /* HC3 match finder */ - if ((mls == 3) /*static*/ && (bestLength < mls)) { - U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3(ms, nextToUpdate3, ip); - if ((matchIndex3 >= matchLow) - & (curr - matchIndex3 < (1<<18)) /*heuristic : longer distance likely too expensive*/ ) { - size_t mlen; - if ((dictMode == ZSTD_noDict) /*static*/ || (dictMode == ZSTD_dictMatchState) /*static*/ || (matchIndex3 >= dictLimit)) { - const BYTE* const match = base + matchIndex3; - mlen = ZSTD_count(ip, match, iLimit); - } else { - const BYTE* const match = dictBase + matchIndex3; - mlen = ZSTD_count_2segments(ip, match, iLimit, dictEnd, prefixStart); - } +} - /* save best solution */ - if (mlen >= mls /* == 3 > bestLength */) { - DEBUGLOG(8, "found small match with hlog3, of length %u", - (U32)mlen); - bestLength = mlen; - assert(curr > matchIndex3); - assert(mnum==0); /* no prior solution */ - matches[0].off = (curr - matchIndex3) + ZSTD_REP_MOVE; - matches[0].len = (U32)mlen; - mnum = 1; - if ( (mlen > sufficient_len) | - (ip+mlen == iLimit) ) { /* best possible length */ - ms->nextToUpdate = curr+1; /* skip insertion */ - return 1; - } } } - /* no dictMatchState lookup: dicts don't have a populated HC3 table */ +/** ZSTD_ldm_countBackwardsMatch() : + * Returns the number of bytes that match backwards before pIn and pMatch. + * + * We count only bytes where pMatch >= pBase and pIn >= pAnchor. */ +static size_t ZSTD_ldm_countBackwardsMatch( + const BYTE* pIn, const BYTE* pAnchor, + const BYTE* pMatch, const BYTE* pMatchBase) +{ + size_t matchLength = 0; + while (pIn > pAnchor && pMatch > pMatchBase && pIn[-1] == pMatch[-1]) { + pIn--; + pMatch--; + matchLength++; + } + return matchLength; +} + +/** ZSTD_ldm_countBackwardsMatch_2segments() : + * Returns the number of bytes that match backwards from pMatch, + * even with the backwards match spanning 2 different segments. + * + * On reaching `pMatchBase`, start counting from mEnd */ +static size_t ZSTD_ldm_countBackwardsMatch_2segments( + const BYTE* pIn, const BYTE* pAnchor, + const BYTE* pMatch, const BYTE* pMatchBase, + const BYTE* pExtDictStart, const BYTE* pExtDictEnd) +{ + size_t matchLength = ZSTD_ldm_countBackwardsMatch(pIn, pAnchor, pMatch, pMatchBase); + if (pMatch - matchLength != pMatchBase || pMatchBase == pExtDictStart) { + /* If backwards match is entirely in the extDict or prefix, immediately return */ + return matchLength; } + DEBUGLOG(7, "ZSTD_ldm_countBackwardsMatch_2segments: found 2-parts backwards match (length in prefix==%zu)", matchLength); + matchLength += ZSTD_ldm_countBackwardsMatch(pIn - matchLength, pAnchor, pExtDictEnd, pExtDictStart); + DEBUGLOG(7, "final backwards match length = %zu", matchLength); + return matchLength; +} - hashTable[h] = curr; /* Update Hash Table */ +/** ZSTD_ldm_fillFastTables() : + * + * Fills the relevant tables for the ZSTD_fast and ZSTD_dfast strategies. + * This is similar to ZSTD_loadDictionaryContent. + * + * The tables for the other strategies are filled within their + * block compressors. */ +static size_t ZSTD_ldm_fillFastTables(ZSTD_MatchState_t* ms, + void const* end) +{ + const BYTE* const iend = (const BYTE*)end; - while (nbCompares-- && (matchIndex >= matchLow)) { - U32* const nextPtr = bt + 2*(matchIndex & btMask); - const BYTE* match; - size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ - assert(curr > matchIndex); + switch(ms->cParams.strategy) + { + case ZSTD_fast: + ZSTD_fillHashTable(ms, iend, ZSTD_dtlm_fast, ZSTD_tfp_forCCtx); + break; - if ((dictMode == ZSTD_noDict) || (dictMode == ZSTD_dictMatchState) || (matchIndex+matchLength >= dictLimit)) { - assert(matchIndex+matchLength >= dictLimit); /* ensure the condition is correct when !extDict */ - match = base + matchIndex; - if (matchIndex >= dictLimit) assert(memcmp(match, ip, matchLength) == 0); /* ensure early section of match is equal as expected */ - matchLength += ZSTD_count(ip+matchLength, match+matchLength, iLimit); - } else { - match = dictBase + matchIndex; - assert(memcmp(match, ip, matchLength) == 0); /* ensure early section of match is equal as expected */ - matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dictEnd, prefixStart); - if (matchIndex+matchLength >= dictLimit) - match = base + matchIndex; /* prepare for match[matchLength] read */ - } + case ZSTD_dfast: +#ifndef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR + ZSTD_fillDoubleHashTable(ms, iend, ZSTD_dtlm_fast, ZSTD_tfp_forCCtx); +#else + assert(0); /* shouldn't be called: cparams should've been adjusted. */ +#endif + break; + + case ZSTD_greedy: + case ZSTD_lazy: + case ZSTD_lazy2: + case ZSTD_btlazy2: + case ZSTD_btopt: + case ZSTD_btultra: + case ZSTD_btultra2: + break; + default: + assert(0); /* not possible : not a valid strategy id */ + } - if (matchLength > bestLength) { - DEBUGLOG(8, "found match of length %u at distance %u (offCode=%u)", - (U32)matchLength, curr - matchIndex, curr - matchIndex + ZSTD_REP_MOVE); - assert(matchEndIdx > matchIndex); - if (matchLength > matchEndIdx - matchIndex) - matchEndIdx = matchIndex + (U32)matchLength; - bestLength = matchLength; - matches[mnum].off = (curr - matchIndex) + ZSTD_REP_MOVE; - matches[mnum].len = (U32)matchLength; - mnum++; - if ( (matchLength > ZSTD_OPT_NUM) - | (ip+matchLength == iLimit) /* equal : no way to know if inf or sup */) { - if (dictMode == ZSTD_dictMatchState) nbCompares = 0; /* break should also skip searching dms */ - break; /* drop, to preserve bt consistency (miss a little bit of compression) */ - } - } + return 0; +} - if (match[matchLength] < ip[matchLength]) { - /* match smaller than current */ - *smallerPtr = matchIndex; /* update smaller idx */ - commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ - if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ - smallerPtr = nextPtr+1; /* new candidate => larger than match, which was smaller than current */ - matchIndex = nextPtr[1]; /* new matchIndex, larger than previous, closer to current */ - } else { - *largerPtr = matchIndex; - commonLengthLarger = matchLength; - if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ - largerPtr = nextPtr; - matchIndex = nextPtr[0]; - } } +void ZSTD_ldm_fillHashTable( + ldmState_t* ldmState, const BYTE* ip, + const BYTE* iend, ldmParams_t const* params) +{ + U32 const minMatchLength = params->minMatchLength; + U32 const bucketSizeLog = params->bucketSizeLog; + U32 const hBits = params->hashLog - bucketSizeLog; + BYTE const* const base = ldmState->window.base; + BYTE const* const istart = ip; + ldmRollingHashState_t hashState; + size_t* const splits = ldmState->splitIndices; + unsigned numSplits; - *smallerPtr = *largerPtr = 0; + DEBUGLOG(5, "ZSTD_ldm_fillHashTable"); - if (dictMode == ZSTD_dictMatchState && nbCompares) { - size_t const dmsH = ZSTD_hashPtr(ip, dmsHashLog, mls); - U32 dictMatchIndex = dms->hashTable[dmsH]; - const U32* const dmsBt = dms->chainTable; - commonLengthSmaller = commonLengthLarger = 0; - while (nbCompares-- && (dictMatchIndex > dmsLowLimit)) { - const U32* const nextPtr = dmsBt + 2*(dictMatchIndex & dmsBtMask); - size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ - const BYTE* match = dmsBase + dictMatchIndex; - matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dmsEnd, prefixStart); - if (dictMatchIndex+matchLength >= dmsHighLimit) - match = base + dictMatchIndex + dmsIndexDelta; /* to prepare for next usage of match[matchLength] */ + ZSTD_ldm_gear_init(&hashState, params); + while (ip < iend) { + size_t hashed; + unsigned n; - if (matchLength > bestLength) { - matchIndex = dictMatchIndex + dmsIndexDelta; - DEBUGLOG(8, "found dms match of length %u at distance %u (offCode=%u)", - (U32)matchLength, curr - matchIndex, curr - matchIndex + ZSTD_REP_MOVE); - if (matchLength > matchEndIdx - matchIndex) - matchEndIdx = matchIndex + (U32)matchLength; - bestLength = matchLength; - matches[mnum].off = (curr - matchIndex) + ZSTD_REP_MOVE; - matches[mnum].len = (U32)matchLength; - mnum++; - if ( (matchLength > ZSTD_OPT_NUM) - | (ip+matchLength == iLimit) /* equal : no way to know if inf or sup */) { - break; /* drop, to guarantee consistency (miss a little bit of compression) */ - } - } + numSplits = 0; + hashed = ZSTD_ldm_gear_feed(&hashState, ip, (size_t)(iend - ip), splits, &numSplits); - if (dictMatchIndex <= dmsBtLow) { break; } /* beyond tree size, stop the search */ - if (match[matchLength] < ip[matchLength]) { - commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ - dictMatchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ - } else { - /* match is larger than current */ - commonLengthLarger = matchLength; - dictMatchIndex = nextPtr[0]; + for (n = 0; n < numSplits; n++) { + if (ip + splits[n] >= istart + minMatchLength) { + BYTE const* const split = ip + splits[n] - minMatchLength; + U64 const xxhash = XXH64(split, minMatchLength, 0); + U32 const hash = (U32)(xxhash & (((U32)1 << hBits) - 1)); + ldmEntry_t entry; + + entry.offset = (U32)(split - base); + entry.checksum = (U32)(xxhash >> 32); + ZSTD_ldm_insertEntry(ldmState, hash, entry, params->bucketSizeLog); } } - } - assert(matchEndIdx > curr+8); - ms->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */ - return mnum; + ip += hashed; + } } -FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches ( - ZSTD_match_t* matches, /* store result (match found, increasing size) in this table */ - ZSTD_matchState_t* ms, - U32* nextToUpdate3, - const BYTE* ip, const BYTE* const iHighLimit, const ZSTD_dictMode_e dictMode, - const U32 rep[ZSTD_REP_NUM], - U32 const ll0, - U32 const lengthToBeat) +/** ZSTD_ldm_limitTableUpdate() : + * + * Sets cctx->nextToUpdate to a position corresponding closer to anchor + * if it is far way + * (after a long match, only update tables a limited amount). */ +static void ZSTD_ldm_limitTableUpdate(ZSTD_MatchState_t* ms, const BYTE* anchor) { - const ZSTD_compressionParameters* const cParams = &ms->cParams; - U32 const matchLengthSearch = cParams->minMatch; - DEBUGLOG(8, "ZSTD_BtGetAllMatches"); - if (ip < ms->window.base + ms->nextToUpdate) return 0; /* skipped area */ - ZSTD_updateTree_internal(ms, ip, iHighLimit, matchLengthSearch, dictMode); - switch(matchLengthSearch) - { - case 3 : return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, 3); - default : - case 4 : return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, 4); - case 5 : return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, 5); - case 7 : - case 6 : return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, 6); + U32 const curr = (U32)(anchor - ms->window.base); + if (curr > ms->nextToUpdate + 1024) { + ms->nextToUpdate = + curr - MIN(512, curr - ms->nextToUpdate - 1024); } } -/************************* -* LDM helper functions * -*************************/ +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_ldm_generateSequences_internal( + ldmState_t* ldmState, RawSeqStore_t* rawSeqStore, + ldmParams_t const* params, void const* src, size_t srcSize) +{ + /* LDM parameters */ + int const extDict = ZSTD_window_hasExtDict(ldmState->window); + U32 const minMatchLength = params->minMatchLength; + U32 const entsPerBucket = 1U << params->bucketSizeLog; + U32 const hBits = params->hashLog - params->bucketSizeLog; + /* Prefix and extDict parameters */ + U32 const dictLimit = ldmState->window.dictLimit; + U32 const lowestIndex = extDict ? ldmState->window.lowLimit : dictLimit; + BYTE const* const base = ldmState->window.base; + BYTE const* const dictBase = extDict ? ldmState->window.dictBase : NULL; + BYTE const* const dictStart = extDict ? dictBase + lowestIndex : NULL; + BYTE const* const dictEnd = extDict ? dictBase + dictLimit : NULL; + BYTE const* const lowPrefixPtr = base + dictLimit; + /* Input bounds */ + BYTE const* const istart = (BYTE const*)src; + BYTE const* const iend = istart + srcSize; + BYTE const* const ilimit = iend - HASH_READ_SIZE; + /* Input positions */ + BYTE const* anchor = istart; + BYTE const* ip = istart; + /* Rolling hash state */ + ldmRollingHashState_t hashState; + /* Arrays for staged-processing */ + size_t* const splits = ldmState->splitIndices; + ldmMatchCandidate_t* const candidates = ldmState->matchCandidates; + unsigned numSplits; -/* Struct containing info needed to make decision about ldm inclusion */ -typedef struct { - rawSeqStore_t seqStore; /* External match candidates store for this block */ - U32 startPosInBlock; /* Start position of the current match candidate */ - U32 endPosInBlock; /* End position of the current match candidate */ - U32 offset; /* Offset of the match candidate */ -} ZSTD_optLdm_t; + if (srcSize < minMatchLength) + return iend - anchor; -/* ZSTD_optLdm_skipRawSeqStoreBytes(): - * Moves forward in rawSeqStore by nbBytes, which will update the fields 'pos' and 'posInSequence'. - */ -static void ZSTD_optLdm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) { - U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes); - while (currPos && rawSeqStore->pos < rawSeqStore->size) { - rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos]; - if (currPos >= currSeq.litLength + currSeq.matchLength) { - currPos -= currSeq.litLength + currSeq.matchLength; - rawSeqStore->pos++; - } else { - rawSeqStore->posInSequence = currPos; - break; + /* Initialize the rolling hash state with the first minMatchLength bytes */ + ZSTD_ldm_gear_init(&hashState, params); + ZSTD_ldm_gear_reset(&hashState, ip, minMatchLength); + ip += minMatchLength; + + while (ip < ilimit) { + size_t hashed; + unsigned n; + + numSplits = 0; + hashed = ZSTD_ldm_gear_feed(&hashState, ip, ilimit - ip, + splits, &numSplits); + + for (n = 0; n < numSplits; n++) { + BYTE const* const split = ip + splits[n] - minMatchLength; + U64 const xxhash = XXH64(split, minMatchLength, 0); + U32 const hash = (U32)(xxhash & (((U32)1 << hBits) - 1)); + + candidates[n].split = split; + candidates[n].hash = hash; + candidates[n].checksum = (U32)(xxhash >> 32); + candidates[n].bucket = ZSTD_ldm_getBucket(ldmState, hash, params->bucketSizeLog); + PREFETCH_L1(candidates[n].bucket); } - } - if (currPos == 0 || rawSeqStore->pos == rawSeqStore->size) { - rawSeqStore->posInSequence = 0; - } -} -/* ZSTD_opt_getNextMatchAndUpdateSeqStore(): - * Calculates the beginning and end of the next match in the current block. - * Updates 'pos' and 'posInSequence' of the ldmSeqStore. - */ -static void ZSTD_opt_getNextMatchAndUpdateSeqStore(ZSTD_optLdm_t* optLdm, U32 currPosInBlock, - U32 blockBytesRemaining) { - rawSeq currSeq; - U32 currBlockEndPos; - U32 literalsBytesRemaining; - U32 matchBytesRemaining; + for (n = 0; n < numSplits; n++) { + size_t forwardMatchLength = 0, backwardMatchLength = 0, + bestMatchLength = 0, mLength; + U32 offset; + BYTE const* const split = candidates[n].split; + U32 const checksum = candidates[n].checksum; + U32 const hash = candidates[n].hash; + ldmEntry_t* const bucket = candidates[n].bucket; + ldmEntry_t const* cur; + ldmEntry_t const* bestEntry = NULL; + ldmEntry_t newEntry; - /* Setting match end position to MAX to ensure we never use an LDM during this block */ - if (optLdm->seqStore.size == 0 || optLdm->seqStore.pos >= optLdm->seqStore.size) { - optLdm->startPosInBlock = UINT_MAX; - optLdm->endPosInBlock = UINT_MAX; - return; - } - /* Calculate appropriate bytes left in matchLength and litLength after adjusting - based on ldmSeqStore->posInSequence */ - currSeq = optLdm->seqStore.seq[optLdm->seqStore.pos]; - assert(optLdm->seqStore.posInSequence <= currSeq.litLength + currSeq.matchLength); - currBlockEndPos = currPosInBlock + blockBytesRemaining; - literalsBytesRemaining = (optLdm->seqStore.posInSequence < currSeq.litLength) ? - currSeq.litLength - (U32)optLdm->seqStore.posInSequence : - 0; - matchBytesRemaining = (literalsBytesRemaining == 0) ? - currSeq.matchLength - ((U32)optLdm->seqStore.posInSequence - currSeq.litLength) : - currSeq.matchLength; + newEntry.offset = (U32)(split - base); + newEntry.checksum = checksum; - /* If there are more literal bytes than bytes remaining in block, no ldm is possible */ - if (literalsBytesRemaining >= blockBytesRemaining) { - optLdm->startPosInBlock = UINT_MAX; - optLdm->endPosInBlock = UINT_MAX; - ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, blockBytesRemaining); - return; - } + /* If a split point would generate a sequence overlapping with + * the previous one, we merely register it in the hash table and + * move on */ + if (split < anchor) { + ZSTD_ldm_insertEntry(ldmState, hash, newEntry, params->bucketSizeLog); + continue; + } + + for (cur = bucket; cur < bucket + entsPerBucket; cur++) { + size_t curForwardMatchLength, curBackwardMatchLength, + curTotalMatchLength; + if (cur->checksum != checksum || cur->offset <= lowestIndex) { + continue; + } + if (extDict) { + BYTE const* const curMatchBase = + cur->offset < dictLimit ? dictBase : base; + BYTE const* const pMatch = curMatchBase + cur->offset; + BYTE const* const matchEnd = + cur->offset < dictLimit ? dictEnd : iend; + BYTE const* const lowMatchPtr = + cur->offset < dictLimit ? dictStart : lowPrefixPtr; + curForwardMatchLength = + ZSTD_count_2segments(split, pMatch, iend, matchEnd, lowPrefixPtr); + if (curForwardMatchLength < minMatchLength) { + continue; + } + curBackwardMatchLength = ZSTD_ldm_countBackwardsMatch_2segments( + split, anchor, pMatch, lowMatchPtr, dictStart, dictEnd); + } else { /* !extDict */ + BYTE const* const pMatch = base + cur->offset; + curForwardMatchLength = ZSTD_count(split, pMatch, iend); + if (curForwardMatchLength < minMatchLength) { + continue; + } + curBackwardMatchLength = + ZSTD_ldm_countBackwardsMatch(split, anchor, pMatch, lowPrefixPtr); + } + curTotalMatchLength = curForwardMatchLength + curBackwardMatchLength; - /* Matches may be < MINMATCH by this process. In that case, we will reject them - when we are deciding whether or not to add the ldm */ - optLdm->startPosInBlock = currPosInBlock + literalsBytesRemaining; - optLdm->endPosInBlock = optLdm->startPosInBlock + matchBytesRemaining; - optLdm->offset = currSeq.offset; + if (curTotalMatchLength > bestMatchLength) { + bestMatchLength = curTotalMatchLength; + forwardMatchLength = curForwardMatchLength; + backwardMatchLength = curBackwardMatchLength; + bestEntry = cur; + } + } - if (optLdm->endPosInBlock > currBlockEndPos) { - /* Match ends after the block ends, we can't use the whole match */ - optLdm->endPosInBlock = currBlockEndPos; - ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, currBlockEndPos - currPosInBlock); - } else { - /* Consume nb of bytes equal to size of sequence left */ - ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, literalsBytesRemaining + matchBytesRemaining); - } -} + /* No match found -- insert an entry into the hash table + * and process the next candidate match */ + if (bestEntry == NULL) { + ZSTD_ldm_insertEntry(ldmState, hash, newEntry, params->bucketSizeLog); + continue; + } -/* ZSTD_optLdm_maybeAddMatch(): - * Adds a match if it's long enough, based on it's 'matchStartPosInBlock' - * and 'matchEndPosInBlock', into 'matches'. Maintains the correct ordering of 'matches' - */ -static void ZSTD_optLdm_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches, - ZSTD_optLdm_t* optLdm, U32 currPosInBlock) { - U32 posDiff = currPosInBlock - optLdm->startPosInBlock; - /* Note: ZSTD_match_t actually contains offCode and matchLength (before subtracting MINMATCH) */ - U32 candidateMatchLength = optLdm->endPosInBlock - optLdm->startPosInBlock - posDiff; - U32 candidateOffCode = optLdm->offset + ZSTD_REP_MOVE; + /* Match found */ + offset = (U32)(split - base) - bestEntry->offset; + mLength = forwardMatchLength + backwardMatchLength; + { + rawSeq* const seq = rawSeqStore->seq + rawSeqStore->size; - /* Ensure that current block position is not outside of the match */ - if (currPosInBlock < optLdm->startPosInBlock - || currPosInBlock >= optLdm->endPosInBlock - || candidateMatchLength < MINMATCH) { - return; - } + /* Out of sequence storage */ + if (rawSeqStore->size == rawSeqStore->capacity) + return ERROR(dstSize_tooSmall); + seq->litLength = (U32)(split - backwardMatchLength - anchor); + seq->matchLength = (U32)mLength; + seq->offset = offset; + rawSeqStore->size++; + } - if (*nbMatches == 0 || ((candidateMatchLength > matches[*nbMatches-1].len) && *nbMatches < ZSTD_OPT_NUM)) { - DEBUGLOG(6, "ZSTD_optLdm_maybeAddMatch(): Adding ldm candidate match (offCode: %u matchLength %u) at block position=%u", - candidateOffCode, candidateMatchLength, currPosInBlock); - matches[*nbMatches].len = candidateMatchLength; - matches[*nbMatches].off = candidateOffCode; - (*nbMatches)++; - } -} + /* Insert the current entry into the hash table --- it must be + * done after the previous block to avoid clobbering bestEntry */ + ZSTD_ldm_insertEntry(ldmState, hash, newEntry, params->bucketSizeLog); -/* ZSTD_optLdm_processMatchCandidate(): - * Wrapper function to update ldm seq store and call ldm functions as necessary. - */ -static void ZSTD_optLdm_processMatchCandidate(ZSTD_optLdm_t* optLdm, ZSTD_match_t* matches, U32* nbMatches, - U32 currPosInBlock, U32 remainingBytes) { - if (optLdm->seqStore.size == 0 || optLdm->seqStore.pos >= optLdm->seqStore.size) { - return; - } + anchor = split + forwardMatchLength; - if (currPosInBlock >= optLdm->endPosInBlock) { - if (currPosInBlock > optLdm->endPosInBlock) { - /* The position at which ZSTD_optLdm_processMatchCandidate() is called is not necessarily - * at the end of a match from the ldm seq store, and will often be some bytes - * over beyond matchEndPosInBlock. As such, we need to correct for these "overshoots" + /* If we find a match that ends after the data that we've hashed + * then we have a repeating, overlapping, pattern. E.g. all zeros. + * If one repetition of the pattern matches our `stopMask` then all + * repetitions will. We don't need to insert them all into out table, + * only the first one. So skip over overlapping matches. + * This is a major speed boost (20x) for compressing a single byte + * repeated, when that byte ends up in the table. */ - U32 posOvershoot = currPosInBlock - optLdm->endPosInBlock; - ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, posOvershoot); + if (anchor > ip + hashed) { + ZSTD_ldm_gear_reset(&hashState, anchor - minMatchLength, minMatchLength); + /* Continue the outer loop at anchor (ip + hashed == anchor). */ + ip = anchor - hashed; + break; + } } - ZSTD_opt_getNextMatchAndUpdateSeqStore(optLdm, currPosInBlock, remainingBytes); - } - ZSTD_optLdm_maybeAddMatch(matches, nbMatches, optLdm, currPosInBlock); -} - -/*-******************************* -* Optimal parser -*********************************/ + ip += hashed; + } -static U32 ZSTD_totalLen(ZSTD_optimal_t sol) -{ - return sol.litlen + sol.mlen; + return iend - anchor; } -#if 0 /* debug */ - -static void -listStats(const U32* table, int lastEltID) +/*! ZSTD_ldm_reduceTable() : + * reduce table indexes by `reducerValue` */ +static void ZSTD_ldm_reduceTable(ldmEntry_t* const table, U32 const size, + U32 const reducerValue) { - int const nbElts = lastEltID + 1; - int enb; - for (enb=0; enb < nbElts; enb++) { - (void)table; - /* RAWLOG(2, "%3i:%3i, ", enb, table[enb]); */ - RAWLOG(2, "%4i,", table[enb]); + U32 u; + for (u = 0; u < size; u++) { + if (table[u].offset < reducerValue) table[u].offset = 0; + else table[u].offset -= reducerValue; } - RAWLOG(2, " \n"); } -#endif - -FORCE_INLINE_TEMPLATE size_t -ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, - seqStore_t* seqStore, - U32 rep[ZSTD_REP_NUM], - const void* src, size_t srcSize, - const int optLevel, - const ZSTD_dictMode_e dictMode) +size_t ZSTD_ldm_generateSequences( + ldmState_t* ldmState, RawSeqStore_t* sequences, + ldmParams_t const* params, void const* src, size_t srcSize) { - optState_t* const optStatePtr = &ms->opt; - const BYTE* const istart = (const BYTE*)src; - const BYTE* ip = istart; - const BYTE* anchor = istart; - const BYTE* const iend = istart + srcSize; - const BYTE* const ilimit = iend - 8; - const BYTE* const base = ms->window.base; - const BYTE* const prefixStart = base + ms->window.dictLimit; - const ZSTD_compressionParameters* const cParams = &ms->cParams; - - U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1); - U32 const minMatch = (cParams->minMatch == 3) ? 3 : 4; - U32 nextToUpdate3 = ms->nextToUpdate; - - ZSTD_optimal_t* const opt = optStatePtr->priceTable; - ZSTD_match_t* const matches = optStatePtr->matchTable; - ZSTD_optimal_t lastSequence; - ZSTD_optLdm_t optLdm; - - optLdm.seqStore = ms->ldmSeqStore ? *ms->ldmSeqStore : kNullRawSeqStore; - optLdm.endPosInBlock = optLdm.startPosInBlock = optLdm.offset = 0; - ZSTD_opt_getNextMatchAndUpdateSeqStore(&optLdm, (U32)(ip-istart), (U32)(iend-ip)); - - /* init */ - DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u", - (U32)(ip - base), ms->window.dictLimit, ms->nextToUpdate); - assert(optLevel <= 2); - ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize, optLevel); - ip += (ip==prefixStart); - - /* Match Loop */ - while (ip < ilimit) { - U32 cur, last_pos = 0; - - /* find first match */ - { U32 const litlen = (U32)(ip - anchor); - U32 const ll0 = !litlen; - U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, ip, iend, dictMode, rep, ll0, minMatch); - ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches, - (U32)(ip-istart), (U32)(iend - ip)); - if (!nbMatches) { ip++; continue; } - - /* initialize opt[0] */ - { U32 i ; for (i=0; i immediate encoding */ - { U32 const maxML = matches[nbMatches-1].len; - U32 const maxOffset = matches[nbMatches-1].off; - DEBUGLOG(6, "found %u matches of maxLength=%u and maxOffCode=%u at cPos=%u => start new series", - nbMatches, maxML, maxOffset, (U32)(ip-prefixStart)); + U32 const maxDist = 1U << params->windowLog; + BYTE const* const istart = (BYTE const*)src; + BYTE const* const iend = istart + srcSize; + size_t const kMaxChunkSize = 1 << 20; + size_t const nbChunks = (srcSize / kMaxChunkSize) + ((srcSize % kMaxChunkSize) != 0); + size_t chunk; + size_t leftoverSize = 0; - if (maxML > sufficient_len) { - lastSequence.litlen = litlen; - lastSequence.mlen = maxML; - lastSequence.off = maxOffset; - DEBUGLOG(6, "large match (%u>%u), immediate encoding", - maxML, sufficient_len); - cur = 0; - last_pos = ZSTD_totalLen(lastSequence); - goto _shortestPath; - } } + assert(ZSTD_CHUNKSIZE_MAX >= kMaxChunkSize); + /* Check that ZSTD_window_update() has been called for this chunk prior + * to passing it to this function. + */ + assert(ldmState->window.nextSrc >= (BYTE const*)src + srcSize); + /* The input could be very large (in zstdmt), so it must be broken up into + * chunks to enforce the maximum distance and handle overflow correction. + */ + assert(sequences->pos <= sequences->size); + assert(sequences->size <= sequences->capacity); + for (chunk = 0; chunk < nbChunks && sequences->size < sequences->capacity; ++chunk) { + BYTE const* const chunkStart = istart + chunk * kMaxChunkSize; + size_t const remaining = (size_t)(iend - chunkStart); + BYTE const *const chunkEnd = + (remaining < kMaxChunkSize) ? iend : chunkStart + kMaxChunkSize; + size_t const chunkSize = chunkEnd - chunkStart; + size_t newLeftoverSize; + size_t const prevSize = sequences->size; - /* set prices for first matches starting position == 0 */ - { U32 const literalsPrice = opt[0].price + ZSTD_litLengthPrice(0, optStatePtr, optLevel); - U32 pos; - U32 matchNb; - for (pos = 1; pos < minMatch; pos++) { - opt[pos].price = ZSTD_MAX_PRICE; /* mlen, litlen and price will be fixed during forward scanning */ - } - for (matchNb = 0; matchNb < nbMatches; matchNb++) { - U32 const offset = matches[matchNb].off; - U32 const end = matches[matchNb].len; - for ( ; pos <= end ; pos++ ) { - U32 const matchPrice = ZSTD_getMatchPrice(offset, pos, optStatePtr, optLevel); - U32 const sequencePrice = literalsPrice + matchPrice; - DEBUGLOG(7, "rPos:%u => set initial price : %.2f", - pos, ZSTD_fCost(sequencePrice)); - opt[pos].mlen = pos; - opt[pos].off = offset; - opt[pos].litlen = litlen; - opt[pos].price = sequencePrice; - } } - last_pos = pos-1; - } + assert(chunkStart < iend); + /* 1. Perform overflow correction if necessary. */ + if (ZSTD_window_needOverflowCorrection(ldmState->window, 0, maxDist, ldmState->loadedDictEnd, chunkStart, chunkEnd)) { + U32 const ldmHSize = 1U << params->hashLog; + U32 const correction = ZSTD_window_correctOverflow( + &ldmState->window, /* cycleLog */ 0, maxDist, chunkStart); + ZSTD_ldm_reduceTable(ldmState->hashTable, ldmHSize, correction); + /* invalidate dictionaries on overflow correction */ + ldmState->loadedDictEnd = 0; } + /* 2. We enforce the maximum offset allowed. + * + * kMaxChunkSize should be small enough that we don't lose too much of + * the window through early invalidation. + * TODO: * Test the chunk size. + * * Try invalidation after the sequence generation and test the + * offset against maxDist directly. + * + * NOTE: Because of dictionaries + sequence splitting we MUST make sure + * that any offset used is valid at the END of the sequence, since it may + * be split into two sequences. This condition holds when using + * ZSTD_window_enforceMaxDist(), but if we move to checking offsets + * against maxDist directly, we'll have to carefully handle that case. + */ + ZSTD_window_enforceMaxDist(&ldmState->window, chunkEnd, maxDist, &ldmState->loadedDictEnd, NULL); + /* 3. Generate the sequences for the chunk, and get newLeftoverSize. */ + newLeftoverSize = ZSTD_ldm_generateSequences_internal( + ldmState, sequences, params, chunkStart, chunkSize); + if (ZSTD_isError(newLeftoverSize)) + return newLeftoverSize; + /* 4. We add the leftover literals from previous iterations to the first + * newly generated sequence, or add the `newLeftoverSize` if none are + * generated. + */ + /* Prepend the leftover literals from the last call */ + if (prevSize < sequences->size) { + sequences->seq[prevSize].litLength += (U32)leftoverSize; + leftoverSize = newLeftoverSize; + } else { + assert(newLeftoverSize == chunkSize); + leftoverSize += chunkSize; + } + } + return 0; +} - /* check further positions */ - for (cur = 1; cur <= last_pos; cur++) { - const BYTE* const inr = ip + cur; - assert(cur < ZSTD_OPT_NUM); - DEBUGLOG(7, "cPos:%zi==rPos:%u", inr-istart, cur) - - /* Fix current position with one literal if cheaper */ - { U32 const litlen = (opt[cur-1].mlen == 0) ? opt[cur-1].litlen + 1 : 1; - int const price = opt[cur-1].price - + ZSTD_rawLiteralsCost(ip+cur-1, 1, optStatePtr, optLevel) - + ZSTD_litLengthPrice(litlen, optStatePtr, optLevel) - - ZSTD_litLengthPrice(litlen-1, optStatePtr, optLevel); - assert(price < 1000000000); /* overflow check */ - if (price <= opt[cur].price) { - DEBUGLOG(7, "cPos:%zi==rPos:%u : better price (%.2f<=%.2f) using literal (ll==%u) (hist:%u,%u,%u)", - inr-istart, cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price), litlen, - opt[cur-1].rep[0], opt[cur-1].rep[1], opt[cur-1].rep[2]); - opt[cur].mlen = 0; - opt[cur].off = 0; - opt[cur].litlen = litlen; - opt[cur].price = price; - } else { - DEBUGLOG(7, "cPos:%zi==rPos:%u : literal would cost more (%.2f>%.2f) (hist:%u,%u,%u)", - inr-istart, cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price), - opt[cur].rep[0], opt[cur].rep[1], opt[cur].rep[2]); - } - } - - /* Set the repcodes of the current position. We must do it here - * because we rely on the repcodes of the 2nd to last sequence being - * correct to set the next chunks repcodes during the backward - * traversal. - */ - ZSTD_STATIC_ASSERT(sizeof(opt[cur].rep) == sizeof(repcodes_t)); - assert(cur >= opt[cur].mlen); - if (opt[cur].mlen != 0) { - U32 const prev = cur - opt[cur].mlen; - repcodes_t newReps = ZSTD_updateRep(opt[prev].rep, opt[cur].off, opt[cur].litlen==0); - ZSTD_memcpy(opt[cur].rep, &newReps, sizeof(repcodes_t)); - } else { - ZSTD_memcpy(opt[cur].rep, opt[cur - 1].rep, sizeof(repcodes_t)); +void +ZSTD_ldm_skipSequences(RawSeqStore_t* rawSeqStore, size_t srcSize, U32 const minMatch) +{ + while (srcSize > 0 && rawSeqStore->pos < rawSeqStore->size) { + rawSeq* seq = rawSeqStore->seq + rawSeqStore->pos; + if (srcSize <= seq->litLength) { + /* Skip past srcSize literals */ + seq->litLength -= (U32)srcSize; + return; + } + srcSize -= seq->litLength; + seq->litLength = 0; + if (srcSize < seq->matchLength) { + /* Skip past the first srcSize of the match */ + seq->matchLength -= (U32)srcSize; + if (seq->matchLength < minMatch) { + /* The match is too short, omit it */ + if (rawSeqStore->pos + 1 < rawSeqStore->size) { + seq[1].litLength += seq[0].matchLength; + } + rawSeqStore->pos++; } + return; + } + srcSize -= seq->matchLength; + seq->matchLength = 0; + rawSeqStore->pos++; + } +} - /* last match must start at a minimum distance of 8 from oend */ - if (inr > ilimit) continue; - - if (cur == last_pos) break; +/** + * If the sequence length is longer than remaining then the sequence is split + * between this block and the next. + * + * Returns the current sequence to handle, or if the rest of the block should + * be literals, it returns a sequence with offset == 0. + */ +static rawSeq maybeSplitSequence(RawSeqStore_t* rawSeqStore, + U32 const remaining, U32 const minMatch) +{ + rawSeq sequence = rawSeqStore->seq[rawSeqStore->pos]; + assert(sequence.offset > 0); + /* Likely: No partial sequence */ + if (remaining >= sequence.litLength + sequence.matchLength) { + rawSeqStore->pos++; + return sequence; + } + /* Cut the sequence short (offset == 0 ==> rest is literals). */ + if (remaining <= sequence.litLength) { + sequence.offset = 0; + } else if (remaining < sequence.litLength + sequence.matchLength) { + sequence.matchLength = remaining - sequence.litLength; + if (sequence.matchLength < minMatch) { + sequence.offset = 0; + } + } + /* Skip past `remaining` bytes for the future sequences. */ + ZSTD_ldm_skipSequences(rawSeqStore, remaining, minMatch); + return sequence; +} - if ( (optLevel==0) /*static_test*/ - && (opt[cur+1].price <= opt[cur].price + (BITCOST_MULTIPLIER/2)) ) { - DEBUGLOG(7, "move to next rPos:%u : price is <=", cur+1); - continue; /* skip unpromising positions; about ~+6% speed, -0.01 ratio */ - } +void ZSTD_ldm_skipRawSeqStoreBytes(RawSeqStore_t* rawSeqStore, size_t nbBytes) { + U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes); + while (currPos && rawSeqStore->pos < rawSeqStore->size) { + rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos]; + if (currPos >= currSeq.litLength + currSeq.matchLength) { + currPos -= currSeq.litLength + currSeq.matchLength; + rawSeqStore->pos++; + } else { + rawSeqStore->posInSequence = currPos; + break; + } + } + if (currPos == 0 || rawSeqStore->pos == rawSeqStore->size) { + rawSeqStore->posInSequence = 0; + } +} - { U32 const ll0 = (opt[cur].mlen != 0); - U32 const litlen = (opt[cur].mlen == 0) ? opt[cur].litlen : 0; - U32 const previousPrice = opt[cur].price; - U32 const basePrice = previousPrice + ZSTD_litLengthPrice(0, optStatePtr, optLevel); - U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, inr, iend, dictMode, opt[cur].rep, ll0, minMatch); - U32 matchNb; +size_t ZSTD_ldm_blockCompress(RawSeqStore_t* rawSeqStore, + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_ParamSwitch_e useRowMatchFinder, + void const* src, size_t srcSize) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + unsigned const minMatch = cParams->minMatch; + ZSTD_BlockCompressor_f const blockCompressor = + ZSTD_selectBlockCompressor(cParams->strategy, useRowMatchFinder, ZSTD_matchState_dictMode(ms)); + /* Input bounds */ + BYTE const* const istart = (BYTE const*)src; + BYTE const* const iend = istart + srcSize; + /* Input positions */ + BYTE const* ip = istart; - ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches, - (U32)(inr-istart), (U32)(iend-inr)); + DEBUGLOG(5, "ZSTD_ldm_blockCompress: srcSize=%zu", srcSize); + /* If using opt parser, use LDMs only as candidates rather than always accepting them */ + if (cParams->strategy >= ZSTD_btopt) { + size_t lastLLSize; + ms->ldmSeqStore = rawSeqStore; + lastLLSize = blockCompressor(ms, seqStore, rep, src, srcSize); + ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore, srcSize); + return lastLLSize; + } - if (!nbMatches) { - DEBUGLOG(7, "rPos:%u : no match found", cur); - continue; - } + assert(rawSeqStore->pos <= rawSeqStore->size); + assert(rawSeqStore->size <= rawSeqStore->capacity); + /* Loop through each sequence and apply the block compressor to the literals */ + while (rawSeqStore->pos < rawSeqStore->size && ip < iend) { + /* maybeSplitSequence updates rawSeqStore->pos */ + rawSeq const sequence = maybeSplitSequence(rawSeqStore, + (U32)(iend - ip), minMatch); + /* End signal */ + if (sequence.offset == 0) + break; - { U32 const maxML = matches[nbMatches-1].len; - DEBUGLOG(7, "cPos:%zi==rPos:%u, found %u matches, of maxLength=%u", - inr-istart, cur, nbMatches, maxML); - - if ( (maxML > sufficient_len) - || (cur + maxML >= ZSTD_OPT_NUM) ) { - lastSequence.mlen = maxML; - lastSequence.off = matches[nbMatches-1].off; - lastSequence.litlen = litlen; - cur -= (opt[cur].mlen==0) ? opt[cur].litlen : 0; /* last sequence is actually only literals, fix cur to last match - note : may underflow, in which case, it's first sequence, and it's okay */ - last_pos = cur + ZSTD_totalLen(lastSequence); - if (cur > ZSTD_OPT_NUM) cur = 0; /* underflow => first match */ - goto _shortestPath; - } } + assert(ip + sequence.litLength + sequence.matchLength <= iend); - /* set prices using matches found at position == cur */ - for (matchNb = 0; matchNb < nbMatches; matchNb++) { - U32 const offset = matches[matchNb].off; - U32 const lastML = matches[matchNb].len; - U32 const startML = (matchNb>0) ? matches[matchNb-1].len+1 : minMatch; - U32 mlen; + /* Fill tables for block compressor */ + ZSTD_ldm_limitTableUpdate(ms, ip); + ZSTD_ldm_fillFastTables(ms, ip); + /* Run the block compressor */ + DEBUGLOG(5, "pos %u : calling block compressor on segment of size %u", (unsigned)(ip-istart), sequence.litLength); + { + int i; + size_t const newLitLength = + blockCompressor(ms, seqStore, rep, ip, sequence.litLength); + ip += sequence.litLength; + /* Update the repcodes */ + for (i = ZSTD_REP_NUM - 1; i > 0; i--) + rep[i] = rep[i-1]; + rep[0] = sequence.offset; + /* Store the sequence */ + ZSTD_storeSeq(seqStore, newLitLength, ip - newLitLength, iend, + OFFSET_TO_OFFBASE(sequence.offset), + sequence.matchLength); + ip += sequence.matchLength; + } + } + /* Fill the tables for the block compressor */ + ZSTD_ldm_limitTableUpdate(ms, ip); + ZSTD_ldm_fillFastTables(ms, ip); + /* Compress the last literals */ + return blockCompressor(ms, seqStore, rep, ip, iend - ip); +} +/**** ended inlining compress/zstd_ldm.c ****/ +/**** start inlining compress/zstd_opt.c ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ - DEBUGLOG(7, "testing match %u => offCode=%4u, mlen=%2u, llen=%2u", - matchNb, matches[matchNb].off, lastML, litlen); +/**** skipping file: zstd_compress_internal.h ****/ +/**** skipping file: hist.h ****/ +/**** skipping file: zstd_opt.h ****/ - for (mlen = lastML; mlen >= startML; mlen--) { /* scan downward */ - U32 const pos = cur + mlen; - int const price = basePrice + ZSTD_getMatchPrice(offset, mlen, optStatePtr, optLevel); +#if !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR) - if ((pos > last_pos) || (price < opt[pos].price)) { - DEBUGLOG(7, "rPos:%u (ml=%2u) => new better price (%.2f<%.2f)", - pos, mlen, ZSTD_fCost(price), ZSTD_fCost(opt[pos].price)); - while (last_pos < pos) { opt[last_pos+1].price = ZSTD_MAX_PRICE; last_pos++; } /* fill empty positions */ - opt[pos].mlen = mlen; - opt[pos].off = offset; - opt[pos].litlen = litlen; - opt[pos].price = price; - } else { - DEBUGLOG(7, "rPos:%u (ml=%2u) => new price is worse (%.2f>=%.2f)", - pos, mlen, ZSTD_fCost(price), ZSTD_fCost(opt[pos].price)); - if (optLevel==0) break; /* early update abort; gets ~+10% speed for about -0.01 ratio loss */ - } - } } } - } /* for (cur = 1; cur <= last_pos; cur++) */ +#define ZSTD_LITFREQ_ADD 2 /* scaling factor for litFreq, so that frequencies adapt faster to new stats */ +#define ZSTD_MAX_PRICE (1<<30) - lastSequence = opt[last_pos]; - cur = last_pos > ZSTD_totalLen(lastSequence) ? last_pos - ZSTD_totalLen(lastSequence) : 0; /* single sequence, and it starts before `ip` */ - assert(cur < ZSTD_OPT_NUM); /* control overflow*/ +#define ZSTD_PREDEF_THRESHOLD 8 /* if srcSize < ZSTD_PREDEF_THRESHOLD, symbols' cost is assumed static, directly determined by pre-defined distributions */ -_shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ - assert(opt[0].mlen == 0); - /* Set the next chunk's repcodes based on the repcodes of the beginning - * of the last match, and the last sequence. This avoids us having to - * update them while traversing the sequences. - */ - if (lastSequence.mlen != 0) { - repcodes_t reps = ZSTD_updateRep(opt[cur].rep, lastSequence.off, lastSequence.litlen==0); - ZSTD_memcpy(rep, &reps, sizeof(reps)); - } else { - ZSTD_memcpy(rep, opt[cur].rep, sizeof(repcodes_t)); - } +/*-************************************* +* Price functions for optimal parser +***************************************/ - { U32 const storeEnd = cur + 1; - U32 storeStart = storeEnd; - U32 seqPos = cur; +#if 0 /* approximation at bit level (for tests) */ +# define BITCOST_ACCURACY 0 +# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) +# define WEIGHT(stat, opt) ((void)(opt), ZSTD_bitWeight(stat)) +#elif 0 /* fractional bit accuracy (for tests) */ +# define BITCOST_ACCURACY 8 +# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) +# define WEIGHT(stat,opt) ((void)(opt), ZSTD_fracWeight(stat)) +#else /* opt==approx, ultra==accurate */ +# define BITCOST_ACCURACY 8 +# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) +# define WEIGHT(stat,opt) ((opt) ? ZSTD_fracWeight(stat) : ZSTD_bitWeight(stat)) +#endif - DEBUGLOG(6, "start reverse traversal (last_pos:%u, cur:%u)", - last_pos, cur); (void)last_pos; - assert(storeEnd < ZSTD_OPT_NUM); - DEBUGLOG(6, "last sequence copied into pos=%u (llen=%u,mlen=%u,ofc=%u)", - storeEnd, lastSequence.litlen, lastSequence.mlen, lastSequence.off); - opt[storeEnd] = lastSequence; - while (seqPos > 0) { - U32 const backDist = ZSTD_totalLen(opt[seqPos]); - storeStart--; - DEBUGLOG(6, "sequence from rPos=%u copied into pos=%u (llen=%u,mlen=%u,ofc=%u)", - seqPos, storeStart, opt[seqPos].litlen, opt[seqPos].mlen, opt[seqPos].off); - opt[storeStart] = opt[seqPos]; - seqPos = (seqPos > backDist) ? seqPos - backDist : 0; - } +/* ZSTD_bitWeight() : + * provide estimated "cost" of a stat in full bits only */ +MEM_STATIC U32 ZSTD_bitWeight(U32 stat) +{ + return (ZSTD_highbit32(stat+1) * BITCOST_MULTIPLIER); +} - /* save sequences */ - DEBUGLOG(6, "sending selected sequences into seqStore") - { U32 storePos; - for (storePos=storeStart; storePos <= storeEnd; storePos++) { - U32 const llen = opt[storePos].litlen; - U32 const mlen = opt[storePos].mlen; - U32 const offCode = opt[storePos].off; - U32 const advance = llen + mlen; - DEBUGLOG(6, "considering seq starting at %zi, llen=%u, mlen=%u", - anchor - istart, (unsigned)llen, (unsigned)mlen); +/* ZSTD_fracWeight() : + * provide fractional-bit "cost" of a stat, + * using linear interpolation approximation */ +MEM_STATIC U32 ZSTD_fracWeight(U32 rawStat) +{ + U32 const stat = rawStat + 1; + U32 const hb = ZSTD_highbit32(stat); + U32 const BWeight = hb * BITCOST_MULTIPLIER; + /* Fweight was meant for "Fractional weight" + * but it's effectively a value between 1 and 2 + * using fixed point arithmetic */ + U32 const FWeight = (stat << BITCOST_ACCURACY) >> hb; + U32 const weight = BWeight + FWeight; + assert(hb + BITCOST_ACCURACY < 31); + return weight; +} - if (mlen==0) { /* only literals => must be last "sequence", actually starting a new stream of sequences */ - assert(storePos == storeEnd); /* must be last sequence */ - ip = anchor + llen; /* last "sequence" is a bunch of literals => don't progress anchor */ - continue; /* will finish */ - } +#if (DEBUGLEVEL>=2) +/* debugging function, + * @return price in bytes as fractional value + * for debug messages only */ +MEM_STATIC double ZSTD_fCost(int price) +{ + return (double)price / (BITCOST_MULTIPLIER*8); +} +#endif - assert(anchor + llen <= iend); - ZSTD_updateStats(optStatePtr, llen, anchor, offCode, mlen); - ZSTD_storeSeq(seqStore, llen, anchor, iend, offCode, mlen-MINMATCH); - anchor += advance; - ip = anchor; - } } - ZSTD_setBasePrices(optStatePtr, optLevel); - } - } /* while (ip < ilimit) */ +static int ZSTD_compressedLiterals(optState_t const* const optPtr) +{ + return optPtr->literalCompressionMode != ZSTD_ps_disable; +} - /* Return the last literals size */ - return (size_t)(iend - anchor); +static void ZSTD_setBasePrices(optState_t* optPtr, int optLevel) +{ + if (ZSTD_compressedLiterals(optPtr)) + optPtr->litSumBasePrice = WEIGHT(optPtr->litSum, optLevel); + optPtr->litLengthSumBasePrice = WEIGHT(optPtr->litLengthSum, optLevel); + optPtr->matchLengthSumBasePrice = WEIGHT(optPtr->matchLengthSum, optLevel); + optPtr->offCodeSumBasePrice = WEIGHT(optPtr->offCodeSum, optLevel); } -size_t ZSTD_compressBlock_btopt( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - const void* src, size_t srcSize) +static U32 sum_u32(const unsigned table[], size_t nbElts) { - DEBUGLOG(5, "ZSTD_compressBlock_btopt"); - return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 0 /*optLevel*/, ZSTD_noDict); + size_t n; + U32 total = 0; + for (n=0; n= 0); + DEBUGLOG(5, "ZSTD_downscaleStats (nbElts=%u, shift=%u)", + (unsigned)lastEltIndex+1, (unsigned)shift ); + assert(shift < 30); for (s=0; s0); + unsigned const newStat = base + (table[s] >> shift); + sum += newStat; + table[s] = newStat; } return sum; } -/* used in 2-pass strategy */ -MEM_STATIC void ZSTD_upscaleStats(optState_t* optPtr) +/* ZSTD_scaleStats() : + * reduce all elt frequencies in table if sum too large + * return the resulting sum of elements */ +static U32 ZSTD_scaleStats(unsigned* table, U32 lastEltIndex, U32 logTarget) { - if (ZSTD_compressedLiterals(optPtr)) - optPtr->litSum = ZSTD_upscaleStat(optPtr->litFreq, MaxLit, 0); - optPtr->litLengthSum = ZSTD_upscaleStat(optPtr->litLengthFreq, MaxLL, 0); - optPtr->matchLengthSum = ZSTD_upscaleStat(optPtr->matchLengthFreq, MaxML, 0); - optPtr->offCodeSum = ZSTD_upscaleStat(optPtr->offCodeFreq, MaxOff, 0); + U32 const prevsum = sum_u32(table, lastEltIndex+1); + U32 const factor = prevsum >> logTarget; + DEBUGLOG(5, "ZSTD_scaleStats (nbElts=%u, target=%u)", (unsigned)lastEltIndex+1, (unsigned)logTarget); + assert(logTarget < 30); + if (factor <= 1) return prevsum; + return ZSTD_downscaleStats(table, lastEltIndex, ZSTD_highbit32(factor), base_1guaranteed); } -/* ZSTD_initStats_ultra(): - * make a first compression pass, just to seed stats with more accurate starting values. - * only works on first block, with no dictionary and no ldm. - * this function cannot error, hence its contract must be respected. +/* ZSTD_rescaleFreqs() : + * if first block (detected by optPtr->litLengthSum == 0) : init statistics + * take hints from dictionary if there is one + * and init from zero if there is none, + * using src for literals stats, and baseline stats for sequence symbols + * otherwise downscale existing stats, to be used as seed for next block. */ static void -ZSTD_initStats_ultra(ZSTD_matchState_t* ms, - seqStore_t* seqStore, - U32 rep[ZSTD_REP_NUM], - const void* src, size_t srcSize) +ZSTD_rescaleFreqs(optState_t* const optPtr, + const BYTE* const src, size_t const srcSize, + int const optLevel) { - U32 tmpRep[ZSTD_REP_NUM]; /* updated rep codes will sink here */ - ZSTD_memcpy(tmpRep, rep, sizeof(tmpRep)); - - DEBUGLOG(4, "ZSTD_initStats_ultra (srcSize=%zu)", srcSize); - assert(ms->opt.litLengthSum == 0); /* first block */ - assert(seqStore->sequences == seqStore->sequencesStart); /* no ldm */ - assert(ms->window.dictLimit == ms->window.lowLimit); /* no dictionary */ - assert(ms->window.dictLimit - ms->nextToUpdate <= 1); /* no prefix (note: intentional overflow, defined as 2-complement) */ - - ZSTD_compressBlock_opt_generic(ms, seqStore, tmpRep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict); /* generate stats into ms->opt*/ + int const compressedLiterals = ZSTD_compressedLiterals(optPtr); + DEBUGLOG(5, "ZSTD_rescaleFreqs (srcSize=%u)", (unsigned)srcSize); + optPtr->priceType = zop_dynamic; - /* invalidate first scan from history */ - ZSTD_resetSeqStore(seqStore); - ms->window.base -= srcSize; - ms->window.dictLimit += (U32)srcSize; - ms->window.lowLimit = ms->window.dictLimit; - ms->nextToUpdate = ms->window.dictLimit; + if (optPtr->litLengthSum == 0) { /* no literals stats collected -> first block assumed -> init */ - /* re-inforce weight of collected statistics */ - ZSTD_upscaleStats(&ms->opt); -} + /* heuristic: use pre-defined stats for too small inputs */ + if (srcSize <= ZSTD_PREDEF_THRESHOLD) { + DEBUGLOG(5, "srcSize <= %i : use predefined stats", ZSTD_PREDEF_THRESHOLD); + optPtr->priceType = zop_predef; + } -size_t ZSTD_compressBlock_btultra( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - const void* src, size_t srcSize) -{ - DEBUGLOG(5, "ZSTD_compressBlock_btultra (srcSize=%zu)", srcSize); - return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict); -} + assert(optPtr->symbolCosts != NULL); + if (optPtr->symbolCosts->huf.repeatMode == HUF_repeat_valid) { -size_t ZSTD_compressBlock_btultra2( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - const void* src, size_t srcSize) -{ - U32 const curr = (U32)((const BYTE*)src - ms->window.base); - DEBUGLOG(5, "ZSTD_compressBlock_btultra2 (srcSize=%zu)", srcSize); + /* huffman stats covering the full value set : table presumed generated by dictionary */ + optPtr->priceType = zop_dynamic; - /* 2-pass strategy: - * this strategy makes a first pass over first block to collect statistics - * and seed next round's statistics with it. - * After 1st pass, function forgets everything, and starts a new block. - * Consequently, this can only work if no data has been previously loaded in tables, - * aka, no dictionary, no prefix, no ldm preprocessing. - * The compression ratio gain is generally small (~0.5% on first block), - * the cost is 2x cpu time on first block. */ - assert(srcSize <= ZSTD_BLOCKSIZE_MAX); - if ( (ms->opt.litLengthSum==0) /* first block */ - && (seqStore->sequences == seqStore->sequencesStart) /* no ldm */ - && (ms->window.dictLimit == ms->window.lowLimit) /* no dictionary */ - && (curr == ms->window.dictLimit) /* start of frame, nothing already loaded nor skipped */ - && (srcSize > ZSTD_PREDEF_THRESHOLD) - ) { - ZSTD_initStats_ultra(ms, seqStore, rep, src, srcSize); - } + if (compressedLiterals) { + /* generate literals statistics from huffman table */ + unsigned lit; + assert(optPtr->litFreq != NULL); + optPtr->litSum = 0; + for (lit=0; lit<=MaxLit; lit++) { + U32 const scaleLog = 11; /* scale to 2K */ + U32 const bitCost = HUF_getNbBitsFromCTable(optPtr->symbolCosts->huf.CTable, lit); + assert(bitCost <= scaleLog); + optPtr->litFreq[lit] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/; + optPtr->litSum += optPtr->litFreq[lit]; + } } - return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict); -} + { unsigned ll; + FSE_CState_t llstate; + FSE_initCState(&llstate, optPtr->symbolCosts->fse.litlengthCTable); + optPtr->litLengthSum = 0; + for (ll=0; ll<=MaxLL; ll++) { + U32 const scaleLog = 10; /* scale to 1K */ + U32 const bitCost = FSE_getMaxNbBits(llstate.symbolTT, ll); + assert(bitCost < scaleLog); + optPtr->litLengthFreq[ll] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/; + optPtr->litLengthSum += optPtr->litLengthFreq[ll]; + } } -size_t ZSTD_compressBlock_btopt_dictMatchState( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - const void* src, size_t srcSize) -{ - return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 0 /*optLevel*/, ZSTD_dictMatchState); -} + { unsigned ml; + FSE_CState_t mlstate; + FSE_initCState(&mlstate, optPtr->symbolCosts->fse.matchlengthCTable); + optPtr->matchLengthSum = 0; + for (ml=0; ml<=MaxML; ml++) { + U32 const scaleLog = 10; + U32 const bitCost = FSE_getMaxNbBits(mlstate.symbolTT, ml); + assert(bitCost < scaleLog); + optPtr->matchLengthFreq[ml] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/; + optPtr->matchLengthSum += optPtr->matchLengthFreq[ml]; + } } -size_t ZSTD_compressBlock_btultra_dictMatchState( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - const void* src, size_t srcSize) -{ - return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_dictMatchState); -} + { unsigned of; + FSE_CState_t ofstate; + FSE_initCState(&ofstate, optPtr->symbolCosts->fse.offcodeCTable); + optPtr->offCodeSum = 0; + for (of=0; of<=MaxOff; of++) { + U32 const scaleLog = 10; + U32 const bitCost = FSE_getMaxNbBits(ofstate.symbolTT, of); + assert(bitCost < scaleLog); + optPtr->offCodeFreq[of] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/; + optPtr->offCodeSum += optPtr->offCodeFreq[of]; + } } -size_t ZSTD_compressBlock_btopt_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - const void* src, size_t srcSize) -{ - return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 0 /*optLevel*/, ZSTD_extDict); -} + } else { /* first block, no dictionary */ -size_t ZSTD_compressBlock_btultra_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - const void* src, size_t srcSize) -{ - return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_extDict); -} + assert(optPtr->litFreq != NULL); + if (compressedLiterals) { + /* base initial cost of literals on direct frequency within src */ + unsigned lit = MaxLit; + HIST_count_simple(optPtr->litFreq, &lit, src, srcSize); /* use raw first block to init statistics */ + optPtr->litSum = ZSTD_downscaleStats(optPtr->litFreq, MaxLit, 8, base_0possible); + } -/* note : no btultra2 variant for extDict nor dictMatchState, - * because btultra2 is not meant to work with dictionaries - * and is only specific for the first block (no prefix) */ -/**** ended inlining compress/zstd_opt.c ****/ -#ifdef ZSTD_MULTITHREAD -/**** start inlining compress/zstdmt_compress.c ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ + { unsigned const baseLLfreqs[MaxLL+1] = { + 4, 2, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1 + }; + ZSTD_memcpy(optPtr->litLengthFreq, baseLLfreqs, sizeof(baseLLfreqs)); + optPtr->litLengthSum = sum_u32(baseLLfreqs, MaxLL+1); + } + { unsigned ml; + for (ml=0; ml<=MaxML; ml++) + optPtr->matchLengthFreq[ml] = 1; + } + optPtr->matchLengthSum = MaxML+1; -/* ====== Compiler specifics ====== */ -#if defined(_MSC_VER) -# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ -#endif + { unsigned const baseOFCfreqs[MaxOff+1] = { + 6, 2, 1, 1, 2, 3, 4, 4, + 4, 3, 2, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1 + }; + ZSTD_memcpy(optPtr->offCodeFreq, baseOFCfreqs, sizeof(baseOFCfreqs)); + optPtr->offCodeSum = sum_u32(baseOFCfreqs, MaxOff+1); + } + } -/* ====== Constants ====== */ -#define ZSTDMT_OVERLAPLOG_DEFAULT 0 + } else { /* new block : scale down accumulated statistics */ + if (compressedLiterals) + optPtr->litSum = ZSTD_scaleStats(optPtr->litFreq, MaxLit, 12); + optPtr->litLengthSum = ZSTD_scaleStats(optPtr->litLengthFreq, MaxLL, 11); + optPtr->matchLengthSum = ZSTD_scaleStats(optPtr->matchLengthFreq, MaxML, 11); + optPtr->offCodeSum = ZSTD_scaleStats(optPtr->offCodeFreq, MaxOff, 11); + } -/* ====== Dependencies ====== */ -/**** skipping file: ../common/zstd_deps.h ****/ -/**** skipping file: ../common/mem.h ****/ -/**** skipping file: ../common/pool.h ****/ -/**** skipping file: ../common/threading.h ****/ -/**** skipping file: zstd_compress_internal.h ****/ -/**** skipping file: zstd_ldm.h ****/ -/**** skipping file: zstdmt_compress.h ****/ + ZSTD_setBasePrices(optPtr, optLevel); +} -/* Guards code to support resizing the SeqPool. - * We will want to resize the SeqPool to save memory in the future. - * Until then, comment the code out since it is unused. - */ -#define ZSTD_RESIZE_SEQPOOL 0 +/* ZSTD_rawLiteralsCost() : + * price of literals (only) in specified segment (which length can be 0). + * does not include price of literalLength symbol */ +static U32 ZSTD_rawLiteralsCost(const BYTE* const literals, U32 const litLength, + const optState_t* const optPtr, + int optLevel) +{ + DEBUGLOG(8, "ZSTD_rawLiteralsCost (%u literals)", litLength); + if (litLength == 0) return 0; -/* ====== Debug ====== */ -#if defined(DEBUGLEVEL) && (DEBUGLEVEL>=2) \ - && !defined(_MSC_VER) \ - && !defined(__MINGW32__) + if (!ZSTD_compressedLiterals(optPtr)) + return (litLength << 3) * BITCOST_MULTIPLIER; /* Uncompressed - 8 bytes per literal. */ -# include -# include -# include + if (optPtr->priceType == zop_predef) + return (litLength*6) * BITCOST_MULTIPLIER; /* 6 bit per literal - no statistic used */ -# define DEBUG_PRINTHEX(l,p,n) { \ - unsigned debug_u; \ - for (debug_u=0; debug_u<(n); debug_u++) \ - RAWLOG(l, "%02X ", ((const unsigned char*)(p))[debug_u]); \ - RAWLOG(l, " \n"); \ + /* dynamic statistics */ + { U32 price = optPtr->litSumBasePrice * litLength; + U32 const litPriceMax = optPtr->litSumBasePrice - BITCOST_MULTIPLIER; + U32 u; + assert(optPtr->litSumBasePrice >= BITCOST_MULTIPLIER); + for (u=0; u < litLength; u++) { + U32 litPrice = WEIGHT(optPtr->litFreq[literals[u]], optLevel); + if (UNLIKELY(litPrice > litPriceMax)) litPrice = litPriceMax; + price -= litPrice; + } + return price; + } } -static unsigned long long GetCurrentClockTimeMicroseconds(void) +/* ZSTD_litLengthPrice() : + * cost of literalLength symbol */ +static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optPtr, int optLevel) { - static clock_t _ticksPerSecond = 0; - if (_ticksPerSecond <= 0) _ticksPerSecond = sysconf(_SC_CLK_TCK); + assert(litLength <= ZSTD_BLOCKSIZE_MAX); + if (optPtr->priceType == zop_predef) + return WEIGHT(litLength, optLevel); - { struct tms junk; clock_t newTicks = (clock_t) times(&junk); - return ((((unsigned long long)newTicks)*(1000000))/_ticksPerSecond); -} } + /* ZSTD_LLcode() can't compute litLength price for sizes >= ZSTD_BLOCKSIZE_MAX + * because it isn't representable in the zstd format. + * So instead just pretend it would cost 1 bit more than ZSTD_BLOCKSIZE_MAX - 1. + * In such a case, the block would be all literals. + */ + if (litLength == ZSTD_BLOCKSIZE_MAX) + return BITCOST_MULTIPLIER + ZSTD_litLengthPrice(ZSTD_BLOCKSIZE_MAX - 1, optPtr, optLevel); -#define MUTEX_WAIT_TIME_DLEVEL 6 -#define ZSTD_PTHREAD_MUTEX_LOCK(mutex) { \ - if (DEBUGLEVEL >= MUTEX_WAIT_TIME_DLEVEL) { \ - unsigned long long const beforeTime = GetCurrentClockTimeMicroseconds(); \ - ZSTD_pthread_mutex_lock(mutex); \ - { unsigned long long const afterTime = GetCurrentClockTimeMicroseconds(); \ - unsigned long long const elapsedTime = (afterTime-beforeTime); \ - if (elapsedTime > 1000) { /* or whatever threshold you like; I'm using 1 millisecond here */ \ - DEBUGLOG(MUTEX_WAIT_TIME_DLEVEL, "Thread took %llu microseconds to acquire mutex %s \n", \ - elapsedTime, #mutex); \ - } } \ - } else { \ - ZSTD_pthread_mutex_lock(mutex); \ - } \ + /* dynamic statistics */ + { U32 const llCode = ZSTD_LLcode(litLength); + return (LL_bits[llCode] * BITCOST_MULTIPLIER) + + optPtr->litLengthSumBasePrice + - WEIGHT(optPtr->litLengthFreq[llCode], optLevel); + } } -#else +/* ZSTD_getMatchPrice() : + * Provides the cost of the match part (offset + matchLength) of a sequence. + * Must be combined with ZSTD_fullLiteralsCost() to get the full cost of a sequence. + * @offBase : sumtype, representing an offset or a repcode, and using numeric representation of ZSTD_storeSeq() + * @optLevel: when <2, favors small offset for decompression speed (improved cache efficiency) + */ +FORCE_INLINE_TEMPLATE U32 +ZSTD_getMatchPrice(U32 const offBase, + U32 const matchLength, + const optState_t* const optPtr, + int const optLevel) +{ + U32 price; + U32 const offCode = ZSTD_highbit32(offBase); + U32 const mlBase = matchLength - MINMATCH; + assert(matchLength >= MINMATCH); -# define ZSTD_PTHREAD_MUTEX_LOCK(m) ZSTD_pthread_mutex_lock(m) -# define DEBUG_PRINTHEX(l,p,n) {} + if (optPtr->priceType == zop_predef) /* fixed scheme, does not use statistics */ + return WEIGHT(mlBase, optLevel) + + ((16 + offCode) * BITCOST_MULTIPLIER); /* emulated offset cost */ -#endif + /* dynamic statistics */ + price = (offCode * BITCOST_MULTIPLIER) + (optPtr->offCodeSumBasePrice - WEIGHT(optPtr->offCodeFreq[offCode], optLevel)); + if ((optLevel<2) /*static*/ && offCode >= 20) + price += (offCode-19)*2 * BITCOST_MULTIPLIER; /* handicap for long distance offsets, favor decompression speed */ + /* match Length */ + { U32 const mlCode = ZSTD_MLcode(mlBase); + price += (ML_bits[mlCode] * BITCOST_MULTIPLIER) + (optPtr->matchLengthSumBasePrice - WEIGHT(optPtr->matchLengthFreq[mlCode], optLevel)); + } -/* ===== Buffer Pool ===== */ -/* a single Buffer Pool can be invoked from multiple threads in parallel */ + price += BITCOST_MULTIPLIER / 5; /* heuristic : make matches a bit more costly to favor less sequences -> faster decompression speed */ -typedef struct buffer_s { - void* start; - size_t capacity; -} buffer_t; + DEBUGLOG(8, "ZSTD_getMatchPrice(ml:%u) = %u", matchLength, price); + return price; +} + +/* ZSTD_updateStats() : + * assumption : literals + litLength <= iend */ +static void ZSTD_updateStats(optState_t* const optPtr, + U32 litLength, const BYTE* literals, + U32 offBase, U32 matchLength) +{ + /* literals */ + if (ZSTD_compressedLiterals(optPtr)) { + U32 u; + for (u=0; u < litLength; u++) + optPtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD; + optPtr->litSum += litLength*ZSTD_LITFREQ_ADD; + } -static const buffer_t g_nullBuffer = { NULL, 0 }; + /* literal Length */ + { U32 const llCode = ZSTD_LLcode(litLength); + optPtr->litLengthFreq[llCode]++; + optPtr->litLengthSum++; + } -typedef struct ZSTDMT_bufferPool_s { - ZSTD_pthread_mutex_t poolMutex; - size_t bufferSize; - unsigned totalBuffers; - unsigned nbBuffers; - ZSTD_customMem cMem; - buffer_t bTable[1]; /* variable size */ -} ZSTDMT_bufferPool; + /* offset code : follows storeSeq() numeric representation */ + { U32 const offCode = ZSTD_highbit32(offBase); + assert(offCode <= MaxOff); + optPtr->offCodeFreq[offCode]++; + optPtr->offCodeSum++; + } -static ZSTDMT_bufferPool* ZSTDMT_createBufferPool(unsigned nbWorkers, ZSTD_customMem cMem) -{ - unsigned const maxNbBuffers = 2*nbWorkers + 3; - ZSTDMT_bufferPool* const bufPool = (ZSTDMT_bufferPool*)ZSTD_customCalloc( - sizeof(ZSTDMT_bufferPool) + (maxNbBuffers-1) * sizeof(buffer_t), cMem); - if (bufPool==NULL) return NULL; - if (ZSTD_pthread_mutex_init(&bufPool->poolMutex, NULL)) { - ZSTD_customFree(bufPool, cMem); - return NULL; + /* match Length */ + { U32 const mlBase = matchLength - MINMATCH; + U32 const mlCode = ZSTD_MLcode(mlBase); + optPtr->matchLengthFreq[mlCode]++; + optPtr->matchLengthSum++; } - bufPool->bufferSize = 64 KB; - bufPool->totalBuffers = maxNbBuffers; - bufPool->nbBuffers = 0; - bufPool->cMem = cMem; - return bufPool; } -static void ZSTDMT_freeBufferPool(ZSTDMT_bufferPool* bufPool) + +/* ZSTD_readMINMATCH() : + * function safe only for comparisons + * assumption : memPtr must be at least 4 bytes before end of buffer */ +MEM_STATIC U32 ZSTD_readMINMATCH(const void* memPtr, U32 length) { - unsigned u; - DEBUGLOG(3, "ZSTDMT_freeBufferPool (address:%08X)", (U32)(size_t)bufPool); - if (!bufPool) return; /* compatibility with free on NULL */ - for (u=0; utotalBuffers; u++) { - DEBUGLOG(4, "free buffer %2u (address:%08X)", u, (U32)(size_t)bufPool->bTable[u].start); - ZSTD_customFree(bufPool->bTable[u].start, bufPool->cMem); + switch (length) + { + default : + case 4 : return MEM_read32(memPtr); + case 3 : if (MEM_isLittleEndian()) + return MEM_read32(memPtr)<<8; + else + return MEM_read32(memPtr)>>8; } - ZSTD_pthread_mutex_destroy(&bufPool->poolMutex); - ZSTD_customFree(bufPool, bufPool->cMem); } -/* only works at initialization, not during compression */ -static size_t ZSTDMT_sizeof_bufferPool(ZSTDMT_bufferPool* bufPool) + +/* Update hashTable3 up to ip (excluded) + Assumption : always within prefix (i.e. not within extDict) */ +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +U32 ZSTD_insertAndFindFirstIndexHash3 (const ZSTD_MatchState_t* ms, + U32* nextToUpdate3, + const BYTE* const ip) { - size_t const poolSize = sizeof(*bufPool) - + (bufPool->totalBuffers - 1) * sizeof(buffer_t); - unsigned u; - size_t totalBufferSize = 0; - ZSTD_pthread_mutex_lock(&bufPool->poolMutex); - for (u=0; utotalBuffers; u++) - totalBufferSize += bufPool->bTable[u].capacity; - ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); + U32* const hashTable3 = ms->hashTable3; + U32 const hashLog3 = ms->hashLog3; + const BYTE* const base = ms->window.base; + U32 idx = *nextToUpdate3; + U32 const target = (U32)(ip - base); + size_t const hash3 = ZSTD_hash3Ptr(ip, hashLog3); + assert(hashLog3 > 0); - return poolSize + totalBufferSize; -} + while(idx < target) { + hashTable3[ZSTD_hash3Ptr(base+idx, hashLog3)] = idx; + idx++; + } -/* ZSTDMT_setBufferSize() : - * all future buffers provided by this buffer pool will have _at least_ this size - * note : it's better for all buffers to have same size, - * as they become freely interchangeable, reducing malloc/free usages and memory fragmentation */ -static void ZSTDMT_setBufferSize(ZSTDMT_bufferPool* const bufPool, size_t const bSize) -{ - ZSTD_pthread_mutex_lock(&bufPool->poolMutex); - DEBUGLOG(4, "ZSTDMT_setBufferSize: bSize = %u", (U32)bSize); - bufPool->bufferSize = bSize; - ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); + *nextToUpdate3 = target; + return hashTable3[hash3]; } -static ZSTDMT_bufferPool* ZSTDMT_expandBufferPool(ZSTDMT_bufferPool* srcBufPool, U32 nbWorkers) +/*-************************************* +* Binary Tree search +***************************************/ +/** ZSTD_insertBt1() : add one or multiple positions to tree. + * @param ip assumed <= iend-8 . + * @param target The target of ZSTD_updateTree_internal() - we are filling to this position + * @return : nb of positions added */ +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +U32 ZSTD_insertBt1( + const ZSTD_MatchState_t* ms, + const BYTE* const ip, const BYTE* const iend, + U32 const target, + U32 const mls, const int extDict) { - unsigned const maxNbBuffers = 2*nbWorkers + 3; - if (srcBufPool==NULL) return NULL; - if (srcBufPool->totalBuffers >= maxNbBuffers) /* good enough */ - return srcBufPool; - /* need a larger buffer pool */ - { ZSTD_customMem const cMem = srcBufPool->cMem; - size_t const bSize = srcBufPool->bufferSize; /* forward parameters */ - ZSTDMT_bufferPool* newBufPool; - ZSTDMT_freeBufferPool(srcBufPool); - newBufPool = ZSTDMT_createBufferPool(nbWorkers, cMem); - if (newBufPool==NULL) return newBufPool; - ZSTDMT_setBufferSize(newBufPool, bSize); - return newBufPool; - } -} + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hashLog = cParams->hashLog; + size_t const h = ZSTD_hashPtr(ip, hashLog, mls); + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + U32 matchIndex = hashTable[h]; + size_t commonLengthSmaller=0, commonLengthLarger=0; + const BYTE* const base = ms->window.base; + const BYTE* const dictBase = ms->window.dictBase; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* match; + const U32 curr = (U32)(ip-base); + const U32 btLow = btMask >= curr ? 0 : curr - btMask; + U32* smallerPtr = bt + 2*(curr&btMask); + U32* largerPtr = smallerPtr + 1; + U32 dummy32; /* to be nullified at the end */ + /* windowLow is based on target because + * we only need positions that will be in the window at the end of the tree update. + */ + U32 const windowLow = ZSTD_getLowestMatchIndex(ms, target, cParams->windowLog); + U32 matchEndIdx = curr+8+1; + size_t bestLength = 8; + U32 nbCompares = 1U << cParams->searchLog; +#ifdef ZSTD_C_PREDICT + U32 predictedSmall = *(bt + 2*((curr-1)&btMask) + 0); + U32 predictedLarge = *(bt + 2*((curr-1)&btMask) + 1); + predictedSmall += (predictedSmall>0); + predictedLarge += (predictedLarge>0); +#endif /* ZSTD_C_PREDICT */ -/** ZSTDMT_getBuffer() : - * assumption : bufPool must be valid - * @return : a buffer, with start pointer and size - * note: allocation may fail, in this case, start==NULL and size==0 */ -static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* bufPool) -{ - size_t const bSize = bufPool->bufferSize; - DEBUGLOG(5, "ZSTDMT_getBuffer: bSize = %u", (U32)bufPool->bufferSize); - ZSTD_pthread_mutex_lock(&bufPool->poolMutex); - if (bufPool->nbBuffers) { /* try to use an existing buffer */ - buffer_t const buf = bufPool->bTable[--(bufPool->nbBuffers)]; - size_t const availBufferSize = buf.capacity; - bufPool->bTable[bufPool->nbBuffers] = g_nullBuffer; - if ((availBufferSize >= bSize) & ((availBufferSize>>3) <= bSize)) { - /* large enough, but not too much */ - DEBUGLOG(5, "ZSTDMT_getBuffer: provide buffer %u of size %u", - bufPool->nbBuffers, (U32)buf.capacity); - ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); - return buf; + DEBUGLOG(8, "ZSTD_insertBt1 (%u)", curr); + + assert(curr <= target); + assert(ip <= iend-8); /* required for h calculation */ + hashTable[h] = curr; /* Update Hash Table */ + + assert(windowLow > 0); + for (; nbCompares && (matchIndex >= windowLow); --nbCompares) { + U32* const nextPtr = bt + 2*(matchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + assert(matchIndex < curr); + +#ifdef ZSTD_C_PREDICT /* note : can create issues when hlog small <= 11 */ + const U32* predictPtr = bt + 2*((matchIndex-1) & btMask); /* written this way, as bt is a roll buffer */ + if (matchIndex == predictedSmall) { + /* no need to check length, result known */ + *smallerPtr = matchIndex; + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ + matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + predictedSmall = predictPtr[1] + (predictPtr[1]>0); + continue; } - /* size conditions not respected : scratch this buffer, create new one */ - DEBUGLOG(5, "ZSTDMT_getBuffer: existing buffer does not meet size conditions => freeing"); - ZSTD_customFree(buf.start, bufPool->cMem); - } - ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); - /* create new buffer */ - DEBUGLOG(5, "ZSTDMT_getBuffer: create a new buffer"); - { buffer_t buffer; - void* const start = ZSTD_customMalloc(bSize, bufPool->cMem); - buffer.start = start; /* note : start can be NULL if malloc fails ! */ - buffer.capacity = (start==NULL) ? 0 : bSize; - if (start==NULL) { - DEBUGLOG(5, "ZSTDMT_getBuffer: buffer allocation failure !!"); + if (matchIndex == predictedLarge) { + *largerPtr = matchIndex; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + predictedLarge = predictPtr[0] + (predictPtr[0]>0); + continue; + } +#endif + + if (!extDict || (matchIndex+matchLength >= dictLimit)) { + assert(matchIndex+matchLength >= dictLimit); /* might be wrong if actually extDict */ + match = base + matchIndex; + matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend); } else { - DEBUGLOG(5, "ZSTDMT_getBuffer: created buffer of size %u", (U32)bSize); + match = dictBase + matchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); + if (matchIndex+matchLength >= dictLimit) + match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ } - return buffer; - } -} -#if ZSTD_RESIZE_SEQPOOL -/** ZSTDMT_resizeBuffer() : - * assumption : bufPool must be valid - * @return : a buffer that is at least the buffer pool buffer size. - * If a reallocation happens, the data in the input buffer is copied. - */ -static buffer_t ZSTDMT_resizeBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buffer) -{ - size_t const bSize = bufPool->bufferSize; - if (buffer.capacity < bSize) { - void* const start = ZSTD_customMalloc(bSize, bufPool->cMem); - buffer_t newBuffer; - newBuffer.start = start; - newBuffer.capacity = start == NULL ? 0 : bSize; - if (start != NULL) { - assert(newBuffer.capacity >= buffer.capacity); - ZSTD_memcpy(newBuffer.start, buffer.start, buffer.capacity); - DEBUGLOG(5, "ZSTDMT_resizeBuffer: created buffer of size %u", (U32)bSize); - return newBuffer; + if (matchLength > bestLength) { + bestLength = matchLength; + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + (U32)matchLength; } - DEBUGLOG(5, "ZSTDMT_resizeBuffer: buffer allocation failure !!"); + + if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */ + break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt tree */ + } + + if (match[matchLength] < ip[matchLength]) { /* necessarily within buffer */ + /* match is smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop searching */ + smallerPtr = nextPtr+1; /* new "candidate" => larger than match, which was smaller than target */ + matchIndex = nextPtr[1]; /* new matchIndex, larger than previous and closer to current */ + } else { + /* match is larger than current */ + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop searching */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } } + + *smallerPtr = *largerPtr = 0; + { U32 positions = 0; + if (bestLength > 384) positions = MIN(192, (U32)(bestLength - 384)); /* speed optimization */ + assert(matchEndIdx > curr + 8); + return MAX(positions, matchEndIdx - (curr + 8)); } - return buffer; } -#endif -/* store buffer for later re-use, up to pool capacity */ -static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buf) +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +void ZSTD_updateTree_internal( + ZSTD_MatchState_t* ms, + const BYTE* const ip, const BYTE* const iend, + const U32 mls, const ZSTD_dictMode_e dictMode) { - DEBUGLOG(5, "ZSTDMT_releaseBuffer"); - if (buf.start == NULL) return; /* compatible with release on NULL */ - ZSTD_pthread_mutex_lock(&bufPool->poolMutex); - if (bufPool->nbBuffers < bufPool->totalBuffers) { - bufPool->bTable[bufPool->nbBuffers++] = buf; /* stored for later use */ - DEBUGLOG(5, "ZSTDMT_releaseBuffer: stored buffer of size %u in slot %u", - (U32)buf.capacity, (U32)(bufPool->nbBuffers-1)); - ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); - return; + const BYTE* const base = ms->window.base; + U32 const target = (U32)(ip - base); + U32 idx = ms->nextToUpdate; + DEBUGLOG(7, "ZSTD_updateTree_internal, from %u to %u (dictMode:%u)", + idx, target, dictMode); + + while(idx < target) { + U32 const forward = ZSTD_insertBt1(ms, base+idx, iend, target, mls, dictMode == ZSTD_extDict); + assert(idx < (U32)(idx + forward)); + idx += forward; } - ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); - /* Reached bufferPool capacity (should not happen) */ - DEBUGLOG(5, "ZSTDMT_releaseBuffer: pool capacity reached => freeing "); - ZSTD_customFree(buf.start, bufPool->cMem); + assert((size_t)(ip - base) <= (size_t)(U32)(-1)); + assert((size_t)(iend - base) <= (size_t)(U32)(-1)); + ms->nextToUpdate = target; } +void ZSTD_updateTree(ZSTD_MatchState_t* ms, const BYTE* ip, const BYTE* iend) { + ZSTD_updateTree_internal(ms, ip, iend, ms->cParams.minMatch, ZSTD_noDict); +} -/* ===== Seq Pool Wrapper ====== */ +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +U32 +ZSTD_insertBtAndGetAllMatches ( + ZSTD_match_t* matches, /* store result (found matches) in this table (presumed large enough) */ + ZSTD_MatchState_t* ms, + U32* nextToUpdate3, + const BYTE* const ip, const BYTE* const iLimit, + const ZSTD_dictMode_e dictMode, + const U32 rep[ZSTD_REP_NUM], + const U32 ll0, /* tells if associated literal length is 0 or not. This value must be 0 or 1 */ + const U32 lengthToBeat, + const U32 mls /* template */) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1); + const BYTE* const base = ms->window.base; + U32 const curr = (U32)(ip-base); + U32 const hashLog = cParams->hashLog; + U32 const minMatch = (mls==3) ? 3 : 4; + U32* const hashTable = ms->hashTable; + size_t const h = ZSTD_hashPtr(ip, hashLog, mls); + U32 matchIndex = hashTable[h]; + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; + U32 const btMask= (1U << btLog) - 1; + size_t commonLengthSmaller=0, commonLengthLarger=0; + const BYTE* const dictBase = ms->window.dictBase; + U32 const dictLimit = ms->window.dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + U32 const btLow = (btMask >= curr) ? 0 : curr - btMask; + U32 const windowLow = ZSTD_getLowestMatchIndex(ms, curr, cParams->windowLog); + U32 const matchLow = windowLow ? windowLow : 1; + U32* smallerPtr = bt + 2*(curr&btMask); + U32* largerPtr = bt + 2*(curr&btMask) + 1; + U32 matchEndIdx = curr+8+1; /* farthest referenced position of any match => detects repetitive patterns */ + U32 dummy32; /* to be nullified at the end */ + U32 mnum = 0; + U32 nbCompares = 1U << cParams->searchLog; -typedef ZSTDMT_bufferPool ZSTDMT_seqPool; + const ZSTD_MatchState_t* dms = dictMode == ZSTD_dictMatchState ? ms->dictMatchState : NULL; + const ZSTD_compressionParameters* const dmsCParams = + dictMode == ZSTD_dictMatchState ? &dms->cParams : NULL; + const BYTE* const dmsBase = dictMode == ZSTD_dictMatchState ? dms->window.base : NULL; + const BYTE* const dmsEnd = dictMode == ZSTD_dictMatchState ? dms->window.nextSrc : NULL; + U32 const dmsHighLimit = dictMode == ZSTD_dictMatchState ? (U32)(dmsEnd - dmsBase) : 0; + U32 const dmsLowLimit = dictMode == ZSTD_dictMatchState ? dms->window.lowLimit : 0; + U32 const dmsIndexDelta = dictMode == ZSTD_dictMatchState ? windowLow - dmsHighLimit : 0; + U32 const dmsHashLog = dictMode == ZSTD_dictMatchState ? dmsCParams->hashLog : hashLog; + U32 const dmsBtLog = dictMode == ZSTD_dictMatchState ? dmsCParams->chainLog - 1 : btLog; + U32 const dmsBtMask = dictMode == ZSTD_dictMatchState ? (1U << dmsBtLog) - 1 : 0; + U32 const dmsBtLow = dictMode == ZSTD_dictMatchState && dmsBtMask < dmsHighLimit - dmsLowLimit ? dmsHighLimit - dmsBtMask : dmsLowLimit; -static size_t ZSTDMT_sizeof_seqPool(ZSTDMT_seqPool* seqPool) -{ - return ZSTDMT_sizeof_bufferPool(seqPool); -} + size_t bestLength = lengthToBeat-1; + DEBUGLOG(8, "ZSTD_insertBtAndGetAllMatches: current=%u", curr); -static rawSeqStore_t bufferToSeq(buffer_t buffer) -{ - rawSeqStore_t seq = kNullRawSeqStore; - seq.seq = (rawSeq*)buffer.start; - seq.capacity = buffer.capacity / sizeof(rawSeq); - return seq; -} + /* check repCode */ + assert(ll0 <= 1); /* necessarily 1 or 0 */ + { U32 const lastR = ZSTD_REP_NUM + ll0; + U32 repCode; + for (repCode = ll0; repCode < lastR; repCode++) { + U32 const repOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode]; + U32 const repIndex = curr - repOffset; + U32 repLen = 0; + assert(curr >= dictLimit); + if (repOffset-1 /* intentional overflow, discards 0 and -1 */ < curr-dictLimit) { /* equivalent to `curr > repIndex >= dictLimit` */ + /* We must validate the repcode offset because when we're using a dictionary the + * valid offset range shrinks when the dictionary goes out of bounds. + */ + if ((repIndex >= windowLow) & (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(ip - repOffset, minMatch))) { + repLen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-repOffset, iLimit) + minMatch; + } + } else { /* repIndex < dictLimit || repIndex >= curr */ + const BYTE* const repMatch = dictMode == ZSTD_dictMatchState ? + dmsBase + repIndex - dmsIndexDelta : + dictBase + repIndex; + assert(curr >= windowLow); + if ( dictMode == ZSTD_extDict + && ( ((repOffset-1) /*intentional overflow*/ < curr - windowLow) /* equivalent to `curr > repIndex >= windowLow` */ + & (ZSTD_index_overlap_check(dictLimit, repIndex)) ) + && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) { + repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dictEnd, prefixStart) + minMatch; + } + if (dictMode == ZSTD_dictMatchState + && ( ((repOffset-1) /*intentional overflow*/ < curr - (dmsLowLimit + dmsIndexDelta)) /* equivalent to `curr > repIndex >= dmsLowLimit` */ + & (ZSTD_index_overlap_check(dictLimit, repIndex)) ) + && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) { + repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dmsEnd, prefixStart) + minMatch; + } } + /* save longer solution */ + if (repLen > bestLength) { + DEBUGLOG(8, "found repCode %u (ll0:%u, offset:%u) of length %u", + repCode, ll0, repOffset, repLen); + bestLength = repLen; + matches[mnum].off = REPCODE_TO_OFFBASE(repCode - ll0 + 1); /* expect value between 1 and 3 */ + matches[mnum].len = (U32)repLen; + mnum++; + if ( (repLen > sufficient_len) + | (ip+repLen == iLimit) ) { /* best possible */ + return mnum; + } } } } -static buffer_t seqToBuffer(rawSeqStore_t seq) -{ - buffer_t buffer; - buffer.start = seq.seq; - buffer.capacity = seq.capacity * sizeof(rawSeq); - return buffer; -} + /* HC3 match finder */ + if ((mls == 3) /*static*/ && (bestLength < mls)) { + U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3(ms, nextToUpdate3, ip); + if ((matchIndex3 >= matchLow) + & (curr - matchIndex3 < (1<<18)) /*heuristic : longer distance likely too expensive*/ ) { + size_t mlen; + if ((dictMode == ZSTD_noDict) /*static*/ || (dictMode == ZSTD_dictMatchState) /*static*/ || (matchIndex3 >= dictLimit)) { + const BYTE* const match = base + matchIndex3; + mlen = ZSTD_count(ip, match, iLimit); + } else { + const BYTE* const match = dictBase + matchIndex3; + mlen = ZSTD_count_2segments(ip, match, iLimit, dictEnd, prefixStart); + } -static rawSeqStore_t ZSTDMT_getSeq(ZSTDMT_seqPool* seqPool) -{ - if (seqPool->bufferSize == 0) { - return kNullRawSeqStore; - } - return bufferToSeq(ZSTDMT_getBuffer(seqPool)); -} + /* save best solution */ + if (mlen >= mls /* == 3 > bestLength */) { + DEBUGLOG(8, "found small match with hlog3, of length %u", + (U32)mlen); + bestLength = mlen; + assert(curr > matchIndex3); + assert(mnum==0); /* no prior solution */ + matches[0].off = OFFSET_TO_OFFBASE(curr - matchIndex3); + matches[0].len = (U32)mlen; + mnum = 1; + if ( (mlen > sufficient_len) | + (ip+mlen == iLimit) ) { /* best possible length */ + ms->nextToUpdate = curr+1; /* skip insertion */ + return 1; + } } } + /* no dictMatchState lookup: dicts don't have a populated HC3 table */ + } /* if (mls == 3) */ -#if ZSTD_RESIZE_SEQPOOL -static rawSeqStore_t ZSTDMT_resizeSeq(ZSTDMT_seqPool* seqPool, rawSeqStore_t seq) -{ - return bufferToSeq(ZSTDMT_resizeBuffer(seqPool, seqToBuffer(seq))); -} -#endif + hashTable[h] = curr; /* Update Hash Table */ -static void ZSTDMT_releaseSeq(ZSTDMT_seqPool* seqPool, rawSeqStore_t seq) -{ - ZSTDMT_releaseBuffer(seqPool, seqToBuffer(seq)); -} + for (; nbCompares && (matchIndex >= matchLow); --nbCompares) { + U32* const nextPtr = bt + 2*(matchIndex & btMask); + const BYTE* match; + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + assert(curr > matchIndex); -static void ZSTDMT_setNbSeq(ZSTDMT_seqPool* const seqPool, size_t const nbSeq) -{ - ZSTDMT_setBufferSize(seqPool, nbSeq * sizeof(rawSeq)); -} + if ((dictMode == ZSTD_noDict) || (dictMode == ZSTD_dictMatchState) || (matchIndex+matchLength >= dictLimit)) { + assert(matchIndex+matchLength >= dictLimit); /* ensure the condition is correct when !extDict */ + match = base + matchIndex; + if (matchIndex >= dictLimit) assert(memcmp(match, ip, matchLength) == 0); /* ensure early section of match is equal as expected */ + matchLength += ZSTD_count(ip+matchLength, match+matchLength, iLimit); + } else { + match = dictBase + matchIndex; + assert(memcmp(match, ip, matchLength) == 0); /* ensure early section of match is equal as expected */ + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dictEnd, prefixStart); + if (matchIndex+matchLength >= dictLimit) + match = base + matchIndex; /* prepare for match[matchLength] read */ + } -static ZSTDMT_seqPool* ZSTDMT_createSeqPool(unsigned nbWorkers, ZSTD_customMem cMem) -{ - ZSTDMT_seqPool* const seqPool = ZSTDMT_createBufferPool(nbWorkers, cMem); - if (seqPool == NULL) return NULL; - ZSTDMT_setNbSeq(seqPool, 0); - return seqPool; -} + if (matchLength > bestLength) { + DEBUGLOG(8, "found match of length %u at distance %u (offBase=%u)", + (U32)matchLength, curr - matchIndex, OFFSET_TO_OFFBASE(curr - matchIndex)); + assert(matchEndIdx > matchIndex); + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + (U32)matchLength; + bestLength = matchLength; + matches[mnum].off = OFFSET_TO_OFFBASE(curr - matchIndex); + matches[mnum].len = (U32)matchLength; + mnum++; + if ( (matchLength > ZSTD_OPT_NUM) + | (ip+matchLength == iLimit) /* equal : no way to know if inf or sup */) { + if (dictMode == ZSTD_dictMatchState) nbCompares = 0; /* break should also skip searching dms */ + break; /* drop, to preserve bt consistency (miss a little bit of compression) */ + } } -static void ZSTDMT_freeSeqPool(ZSTDMT_seqPool* seqPool) -{ - ZSTDMT_freeBufferPool(seqPool); -} + if (match[matchLength] < ip[matchLength]) { + /* match smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + smallerPtr = nextPtr+1; /* new candidate => larger than match, which was smaller than current */ + matchIndex = nextPtr[1]; /* new matchIndex, larger than previous, closer to current */ + } else { + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } } -static ZSTDMT_seqPool* ZSTDMT_expandSeqPool(ZSTDMT_seqPool* pool, U32 nbWorkers) -{ - return ZSTDMT_expandBufferPool(pool, nbWorkers); -} + *smallerPtr = *largerPtr = 0; + + assert(nbCompares <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */ + if (dictMode == ZSTD_dictMatchState && nbCompares) { + size_t const dmsH = ZSTD_hashPtr(ip, dmsHashLog, mls); + U32 dictMatchIndex = dms->hashTable[dmsH]; + const U32* const dmsBt = dms->chainTable; + commonLengthSmaller = commonLengthLarger = 0; + for (; nbCompares && (dictMatchIndex > dmsLowLimit); --nbCompares) { + const U32* const nextPtr = dmsBt + 2*(dictMatchIndex & dmsBtMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + const BYTE* match = dmsBase + dictMatchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dmsEnd, prefixStart); + if (dictMatchIndex+matchLength >= dmsHighLimit) + match = base + dictMatchIndex + dmsIndexDelta; /* to prepare for next usage of match[matchLength] */ + + if (matchLength > bestLength) { + matchIndex = dictMatchIndex + dmsIndexDelta; + DEBUGLOG(8, "found dms match of length %u at distance %u (offBase=%u)", + (U32)matchLength, curr - matchIndex, OFFSET_TO_OFFBASE(curr - matchIndex)); + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + (U32)matchLength; + bestLength = matchLength; + matches[mnum].off = OFFSET_TO_OFFBASE(curr - matchIndex); + matches[mnum].len = (U32)matchLength; + mnum++; + if ( (matchLength > ZSTD_OPT_NUM) + | (ip+matchLength == iLimit) /* equal : no way to know if inf or sup */) { + break; /* drop, to guarantee consistency (miss a little bit of compression) */ + } } + if (dictMatchIndex <= dmsBtLow) { break; } /* beyond tree size, stop the search */ + if (match[matchLength] < ip[matchLength]) { + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + dictMatchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + } else { + /* match is larger than current */ + commonLengthLarger = matchLength; + dictMatchIndex = nextPtr[0]; + } } } /* if (dictMode == ZSTD_dictMatchState) */ -/* ===== CCtx Pool ===== */ -/* a single CCtx Pool can be invoked from multiple threads in parallel */ + assert(matchEndIdx > curr+8); + ms->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */ + return mnum; +} -typedef struct { - ZSTD_pthread_mutex_t poolMutex; - int totalCCtx; - int availCCtx; - ZSTD_customMem cMem; - ZSTD_CCtx* cctx[1]; /* variable size */ -} ZSTDMT_CCtxPool; +typedef U32 (*ZSTD_getAllMatchesFn)( + ZSTD_match_t*, + ZSTD_MatchState_t*, + U32*, + const BYTE*, + const BYTE*, + const U32 rep[ZSTD_REP_NUM], + U32 const ll0, + U32 const lengthToBeat); -/* note : all CCtx borrowed from the pool should be released back to the pool _before_ freeing the pool */ -static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool) -{ - int cid; - for (cid=0; cidtotalCCtx; cid++) - ZSTD_freeCCtx(pool->cctx[cid]); /* note : compatible with free on NULL */ - ZSTD_pthread_mutex_destroy(&pool->poolMutex); - ZSTD_customFree(pool, pool->cMem); +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +U32 ZSTD_btGetAllMatches_internal( + ZSTD_match_t* matches, + ZSTD_MatchState_t* ms, + U32* nextToUpdate3, + const BYTE* ip, + const BYTE* const iHighLimit, + const U32 rep[ZSTD_REP_NUM], + U32 const ll0, + U32 const lengthToBeat, + const ZSTD_dictMode_e dictMode, + const U32 mls) +{ + assert(BOUNDED(3, ms->cParams.minMatch, 6) == mls); + DEBUGLOG(8, "ZSTD_BtGetAllMatches(dictMode=%d, mls=%u)", (int)dictMode, mls); + if (ip < ms->window.base + ms->nextToUpdate) + return 0; /* skipped area */ + ZSTD_updateTree_internal(ms, ip, iHighLimit, mls, dictMode); + return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, mls); +} + +#define ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, mls) ZSTD_btGetAllMatches_##dictMode##_##mls + +#define GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, mls) \ + static U32 ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, mls)( \ + ZSTD_match_t* matches, \ + ZSTD_MatchState_t* ms, \ + U32* nextToUpdate3, \ + const BYTE* ip, \ + const BYTE* const iHighLimit, \ + const U32 rep[ZSTD_REP_NUM], \ + U32 const ll0, \ + U32 const lengthToBeat) \ + { \ + return ZSTD_btGetAllMatches_internal( \ + matches, ms, nextToUpdate3, ip, iHighLimit, \ + rep, ll0, lengthToBeat, ZSTD_##dictMode, mls); \ + } + +#define GEN_ZSTD_BT_GET_ALL_MATCHES(dictMode) \ + GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 3) \ + GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 4) \ + GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 5) \ + GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 6) + +GEN_ZSTD_BT_GET_ALL_MATCHES(noDict) +GEN_ZSTD_BT_GET_ALL_MATCHES(extDict) +GEN_ZSTD_BT_GET_ALL_MATCHES(dictMatchState) + +#define ZSTD_BT_GET_ALL_MATCHES_ARRAY(dictMode) \ + { \ + ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 3), \ + ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 4), \ + ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 5), \ + ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 6) \ + } + +static ZSTD_getAllMatchesFn +ZSTD_selectBtGetAllMatches(ZSTD_MatchState_t const* ms, ZSTD_dictMode_e const dictMode) +{ + ZSTD_getAllMatchesFn const getAllMatchesFns[3][4] = { + ZSTD_BT_GET_ALL_MATCHES_ARRAY(noDict), + ZSTD_BT_GET_ALL_MATCHES_ARRAY(extDict), + ZSTD_BT_GET_ALL_MATCHES_ARRAY(dictMatchState) + }; + U32 const mls = BOUNDED(3, ms->cParams.minMatch, 6); + assert((U32)dictMode < 3); + assert(mls - 3 < 4); + return getAllMatchesFns[(int)dictMode][mls - 3]; } -/* ZSTDMT_createCCtxPool() : - * implies nbWorkers >= 1 , checked by caller ZSTDMT_createCCtx() */ -static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(int nbWorkers, - ZSTD_customMem cMem) -{ - ZSTDMT_CCtxPool* const cctxPool = (ZSTDMT_CCtxPool*) ZSTD_customCalloc( - sizeof(ZSTDMT_CCtxPool) + (nbWorkers-1)*sizeof(ZSTD_CCtx*), cMem); - assert(nbWorkers > 0); - if (!cctxPool) return NULL; - if (ZSTD_pthread_mutex_init(&cctxPool->poolMutex, NULL)) { - ZSTD_customFree(cctxPool, cMem); - return NULL; - } - cctxPool->cMem = cMem; - cctxPool->totalCCtx = nbWorkers; - cctxPool->availCCtx = 1; /* at least one cctx for single-thread mode */ - cctxPool->cctx[0] = ZSTD_createCCtx_advanced(cMem); - if (!cctxPool->cctx[0]) { ZSTDMT_freeCCtxPool(cctxPool); return NULL; } - DEBUGLOG(3, "cctxPool created, with %u workers", nbWorkers); - return cctxPool; -} +/************************* +* LDM helper functions * +*************************/ -static ZSTDMT_CCtxPool* ZSTDMT_expandCCtxPool(ZSTDMT_CCtxPool* srcPool, - int nbWorkers) -{ - if (srcPool==NULL) return NULL; - if (nbWorkers <= srcPool->totalCCtx) return srcPool; /* good enough */ - /* need a larger cctx pool */ - { ZSTD_customMem const cMem = srcPool->cMem; - ZSTDMT_freeCCtxPool(srcPool); - return ZSTDMT_createCCtxPool(nbWorkers, cMem); - } -} +/* Struct containing info needed to make decision about ldm inclusion */ +typedef struct { + RawSeqStore_t seqStore; /* External match candidates store for this block */ + U32 startPosInBlock; /* Start position of the current match candidate */ + U32 endPosInBlock; /* End position of the current match candidate */ + U32 offset; /* Offset of the match candidate */ +} ZSTD_optLdm_t; -/* only works during initialization phase, not during compression */ -static size_t ZSTDMT_sizeof_CCtxPool(ZSTDMT_CCtxPool* cctxPool) +/* ZSTD_optLdm_skipRawSeqStoreBytes(): + * Moves forward in @rawSeqStore by @nbBytes, + * which will update the fields 'pos' and 'posInSequence'. + */ +static void ZSTD_optLdm_skipRawSeqStoreBytes(RawSeqStore_t* rawSeqStore, size_t nbBytes) { - ZSTD_pthread_mutex_lock(&cctxPool->poolMutex); - { unsigned const nbWorkers = cctxPool->totalCCtx; - size_t const poolSize = sizeof(*cctxPool) - + (nbWorkers-1) * sizeof(ZSTD_CCtx*); - unsigned u; - size_t totalCCtxSize = 0; - for (u=0; ucctx[u]); + U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes); + while (currPos && rawSeqStore->pos < rawSeqStore->size) { + rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos]; + if (currPos >= currSeq.litLength + currSeq.matchLength) { + currPos -= currSeq.litLength + currSeq.matchLength; + rawSeqStore->pos++; + } else { + rawSeqStore->posInSequence = currPos; + break; } - ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex); - assert(nbWorkers > 0); - return poolSize + totalCCtxSize; + } + if (currPos == 0 || rawSeqStore->pos == rawSeqStore->size) { + rawSeqStore->posInSequence = 0; } } -static ZSTD_CCtx* ZSTDMT_getCCtx(ZSTDMT_CCtxPool* cctxPool) +/* ZSTD_opt_getNextMatchAndUpdateSeqStore(): + * Calculates the beginning and end of the next match in the current block. + * Updates 'pos' and 'posInSequence' of the ldmSeqStore. + */ +static void +ZSTD_opt_getNextMatchAndUpdateSeqStore(ZSTD_optLdm_t* optLdm, U32 currPosInBlock, + U32 blockBytesRemaining) { - DEBUGLOG(5, "ZSTDMT_getCCtx"); - ZSTD_pthread_mutex_lock(&cctxPool->poolMutex); - if (cctxPool->availCCtx) { - cctxPool->availCCtx--; - { ZSTD_CCtx* const cctx = cctxPool->cctx[cctxPool->availCCtx]; - ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex); - return cctx; - } } - ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex); - DEBUGLOG(5, "create one more CCtx"); - return ZSTD_createCCtx_advanced(cctxPool->cMem); /* note : can be NULL, when creation fails ! */ -} + rawSeq currSeq; + U32 currBlockEndPos; + U32 literalsBytesRemaining; + U32 matchBytesRemaining; -static void ZSTDMT_releaseCCtx(ZSTDMT_CCtxPool* pool, ZSTD_CCtx* cctx) -{ - if (cctx==NULL) return; /* compatibility with release on NULL */ - ZSTD_pthread_mutex_lock(&pool->poolMutex); - if (pool->availCCtx < pool->totalCCtx) - pool->cctx[pool->availCCtx++] = cctx; - else { - /* pool overflow : should not happen, since totalCCtx==nbWorkers */ - DEBUGLOG(4, "CCtx pool overflow : free cctx"); - ZSTD_freeCCtx(cctx); + /* Setting match end position to MAX to ensure we never use an LDM during this block */ + if (optLdm->seqStore.size == 0 || optLdm->seqStore.pos >= optLdm->seqStore.size) { + optLdm->startPosInBlock = UINT_MAX; + optLdm->endPosInBlock = UINT_MAX; + return; } - ZSTD_pthread_mutex_unlock(&pool->poolMutex); -} - -/* ==== Serial State ==== */ - -typedef struct { - void const* start; - size_t size; -} range_t; - -typedef struct { - /* All variables in the struct are protected by mutex. */ - ZSTD_pthread_mutex_t mutex; - ZSTD_pthread_cond_t cond; - ZSTD_CCtx_params params; - ldmState_t ldmState; - XXH64_state_t xxhState; - unsigned nextJobID; - /* Protects ldmWindow. - * Must be acquired after the main mutex when acquiring both. - */ - ZSTD_pthread_mutex_t ldmWindowMutex; - ZSTD_pthread_cond_t ldmWindowCond; /* Signaled when ldmWindow is updated */ - ZSTD_window_t ldmWindow; /* A thread-safe copy of ldmState.window */ -} serialState_t; + /* Calculate appropriate bytes left in matchLength and litLength + * after adjusting based on ldmSeqStore->posInSequence */ + currSeq = optLdm->seqStore.seq[optLdm->seqStore.pos]; + assert(optLdm->seqStore.posInSequence <= currSeq.litLength + currSeq.matchLength); + currBlockEndPos = currPosInBlock + blockBytesRemaining; + literalsBytesRemaining = (optLdm->seqStore.posInSequence < currSeq.litLength) ? + currSeq.litLength - (U32)optLdm->seqStore.posInSequence : + 0; + matchBytesRemaining = (literalsBytesRemaining == 0) ? + currSeq.matchLength - ((U32)optLdm->seqStore.posInSequence - currSeq.litLength) : + currSeq.matchLength; -static int -ZSTDMT_serialState_reset(serialState_t* serialState, - ZSTDMT_seqPool* seqPool, - ZSTD_CCtx_params params, - size_t jobSize, - const void* dict, size_t const dictSize, - ZSTD_dictContentType_e dictContentType) -{ - /* Adjust parameters */ - if (params.ldmParams.enableLdm) { - DEBUGLOG(4, "LDM window size = %u KB", (1U << params.cParams.windowLog) >> 10); - ZSTD_ldm_adjustParameters(¶ms.ldmParams, ¶ms.cParams); - assert(params.ldmParams.hashLog >= params.ldmParams.bucketSizeLog); - assert(params.ldmParams.hashRateLog < 32); - } else { - ZSTD_memset(¶ms.ldmParams, 0, sizeof(params.ldmParams)); + /* If there are more literal bytes than bytes remaining in block, no ldm is possible */ + if (literalsBytesRemaining >= blockBytesRemaining) { + optLdm->startPosInBlock = UINT_MAX; + optLdm->endPosInBlock = UINT_MAX; + ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, blockBytesRemaining); + return; } - serialState->nextJobID = 0; - if (params.fParams.checksumFlag) - XXH64_reset(&serialState->xxhState, 0); - if (params.ldmParams.enableLdm) { - ZSTD_customMem cMem = params.customMem; - unsigned const hashLog = params.ldmParams.hashLog; - size_t const hashSize = ((size_t)1 << hashLog) * sizeof(ldmEntry_t); - unsigned const bucketLog = - params.ldmParams.hashLog - params.ldmParams.bucketSizeLog; - unsigned const prevBucketLog = - serialState->params.ldmParams.hashLog - - serialState->params.ldmParams.bucketSizeLog; - size_t const numBuckets = (size_t)1 << bucketLog; - /* Size the seq pool tables */ - ZSTDMT_setNbSeq(seqPool, ZSTD_ldm_getMaxNbSeq(params.ldmParams, jobSize)); - /* Reset the window */ - ZSTD_window_init(&serialState->ldmState.window); - /* Resize tables and output space if necessary. */ - if (serialState->ldmState.hashTable == NULL || serialState->params.ldmParams.hashLog < hashLog) { - ZSTD_customFree(serialState->ldmState.hashTable, cMem); - serialState->ldmState.hashTable = (ldmEntry_t*)ZSTD_customMalloc(hashSize, cMem); - } - if (serialState->ldmState.bucketOffsets == NULL || prevBucketLog < bucketLog) { - ZSTD_customFree(serialState->ldmState.bucketOffsets, cMem); - serialState->ldmState.bucketOffsets = (BYTE*)ZSTD_customMalloc(numBuckets, cMem); - } - if (!serialState->ldmState.hashTable || !serialState->ldmState.bucketOffsets) - return 1; - /* Zero the tables */ - ZSTD_memset(serialState->ldmState.hashTable, 0, hashSize); - ZSTD_memset(serialState->ldmState.bucketOffsets, 0, numBuckets); - /* Update window state and fill hash table with dict */ - serialState->ldmState.loadedDictEnd = 0; - if (dictSize > 0) { - if (dictContentType == ZSTD_dct_rawContent) { - BYTE const* const dictEnd = (const BYTE*)dict + dictSize; - ZSTD_window_update(&serialState->ldmState.window, dict, dictSize); - ZSTD_ldm_fillHashTable(&serialState->ldmState, (const BYTE*)dict, dictEnd, ¶ms.ldmParams); - serialState->ldmState.loadedDictEnd = params.forceWindow ? 0 : (U32)(dictEnd - serialState->ldmState.window.base); - } else { - /* don't even load anything */ - } - } + /* Matches may be < minMatch by this process. In that case, we will reject them + when we are deciding whether or not to add the ldm */ + optLdm->startPosInBlock = currPosInBlock + literalsBytesRemaining; + optLdm->endPosInBlock = optLdm->startPosInBlock + matchBytesRemaining; + optLdm->offset = currSeq.offset; - /* Initialize serialState's copy of ldmWindow. */ - serialState->ldmWindow = serialState->ldmState.window; + if (optLdm->endPosInBlock > currBlockEndPos) { + /* Match ends after the block ends, we can't use the whole match */ + optLdm->endPosInBlock = currBlockEndPos; + ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, currBlockEndPos - currPosInBlock); + } else { + /* Consume nb of bytes equal to size of sequence left */ + ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, literalsBytesRemaining + matchBytesRemaining); } - - serialState->params = params; - serialState->params.jobSize = (U32)jobSize; - return 0; } -static int ZSTDMT_serialState_init(serialState_t* serialState) +/* ZSTD_optLdm_maybeAddMatch(): + * Adds a match if it's long enough, + * based on it's 'matchStartPosInBlock' and 'matchEndPosInBlock', + * into 'matches'. Maintains the correct ordering of 'matches'. + */ +static void ZSTD_optLdm_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches, + const ZSTD_optLdm_t* optLdm, U32 currPosInBlock, + U32 minMatch) { - int initError = 0; - ZSTD_memset(serialState, 0, sizeof(*serialState)); - initError |= ZSTD_pthread_mutex_init(&serialState->mutex, NULL); - initError |= ZSTD_pthread_cond_init(&serialState->cond, NULL); - initError |= ZSTD_pthread_mutex_init(&serialState->ldmWindowMutex, NULL); - initError |= ZSTD_pthread_cond_init(&serialState->ldmWindowCond, NULL); - return initError; -} + U32 const posDiff = currPosInBlock - optLdm->startPosInBlock; + /* Note: ZSTD_match_t actually contains offBase and matchLength (before subtracting MINMATCH) */ + U32 const candidateMatchLength = optLdm->endPosInBlock - optLdm->startPosInBlock - posDiff; -static void ZSTDMT_serialState_free(serialState_t* serialState) -{ - ZSTD_customMem cMem = serialState->params.customMem; - ZSTD_pthread_mutex_destroy(&serialState->mutex); - ZSTD_pthread_cond_destroy(&serialState->cond); - ZSTD_pthread_mutex_destroy(&serialState->ldmWindowMutex); - ZSTD_pthread_cond_destroy(&serialState->ldmWindowCond); - ZSTD_customFree(serialState->ldmState.hashTable, cMem); - ZSTD_customFree(serialState->ldmState.bucketOffsets, cMem); + /* Ensure that current block position is not outside of the match */ + if (currPosInBlock < optLdm->startPosInBlock + || currPosInBlock >= optLdm->endPosInBlock + || candidateMatchLength < minMatch) { + return; + } + + if (*nbMatches == 0 || ((candidateMatchLength > matches[*nbMatches-1].len) && *nbMatches < ZSTD_OPT_NUM)) { + U32 const candidateOffBase = OFFSET_TO_OFFBASE(optLdm->offset); + DEBUGLOG(6, "ZSTD_optLdm_maybeAddMatch(): Adding ldm candidate match (offBase: %u matchLength %u) at block position=%u", + candidateOffBase, candidateMatchLength, currPosInBlock); + matches[*nbMatches].len = candidateMatchLength; + matches[*nbMatches].off = candidateOffBase; + (*nbMatches)++; + } } -static void ZSTDMT_serialState_update(serialState_t* serialState, - ZSTD_CCtx* jobCCtx, rawSeqStore_t seqStore, - range_t src, unsigned jobID) +/* ZSTD_optLdm_processMatchCandidate(): + * Wrapper function to update ldm seq store and call ldm functions as necessary. + */ +static void +ZSTD_optLdm_processMatchCandidate(ZSTD_optLdm_t* optLdm, + ZSTD_match_t* matches, U32* nbMatches, + U32 currPosInBlock, U32 remainingBytes, + U32 minMatch) { - /* Wait for our turn */ - ZSTD_PTHREAD_MUTEX_LOCK(&serialState->mutex); - while (serialState->nextJobID < jobID) { - DEBUGLOG(5, "wait for serialState->cond"); - ZSTD_pthread_cond_wait(&serialState->cond, &serialState->mutex); + if (optLdm->seqStore.size == 0 || optLdm->seqStore.pos >= optLdm->seqStore.size) { + return; } - /* A future job may error and skip our job */ - if (serialState->nextJobID == jobID) { - /* It is now our turn, do any processing necessary */ - if (serialState->params.ldmParams.enableLdm) { - size_t error; - assert(seqStore.seq != NULL && seqStore.pos == 0 && - seqStore.size == 0 && seqStore.capacity > 0); - assert(src.size <= serialState->params.jobSize); - ZSTD_window_update(&serialState->ldmState.window, src.start, src.size); - error = ZSTD_ldm_generateSequences( - &serialState->ldmState, &seqStore, - &serialState->params.ldmParams, src.start, src.size); - /* We provide a large enough buffer to never fail. */ - assert(!ZSTD_isError(error)); (void)error; - /* Update ldmWindow to match the ldmState.window and signal the main - * thread if it is waiting for a buffer. + + if (currPosInBlock >= optLdm->endPosInBlock) { + if (currPosInBlock > optLdm->endPosInBlock) { + /* The position at which ZSTD_optLdm_processMatchCandidate() is called is not necessarily + * at the end of a match from the ldm seq store, and will often be some bytes + * over beyond matchEndPosInBlock. As such, we need to correct for these "overshoots" */ - ZSTD_PTHREAD_MUTEX_LOCK(&serialState->ldmWindowMutex); - serialState->ldmWindow = serialState->ldmState.window; - ZSTD_pthread_cond_signal(&serialState->ldmWindowCond); - ZSTD_pthread_mutex_unlock(&serialState->ldmWindowMutex); + U32 const posOvershoot = currPosInBlock - optLdm->endPosInBlock; + ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, posOvershoot); } - if (serialState->params.fParams.checksumFlag && src.size > 0) - XXH64_update(&serialState->xxhState, src.start, src.size); + ZSTD_opt_getNextMatchAndUpdateSeqStore(optLdm, currPosInBlock, remainingBytes); } - /* Now it is the next jobs turn */ - serialState->nextJobID++; - ZSTD_pthread_cond_broadcast(&serialState->cond); - ZSTD_pthread_mutex_unlock(&serialState->mutex); + ZSTD_optLdm_maybeAddMatch(matches, nbMatches, optLdm, currPosInBlock, minMatch); +} - if (seqStore.size > 0) { - size_t const err = ZSTD_referenceExternalSequences( - jobCCtx, seqStore.seq, seqStore.size); - assert(serialState->params.ldmParams.enableLdm); - assert(!ZSTD_isError(err)); - (void)err; + +/*-******************************* +* Optimal parser +*********************************/ + +#if 0 /* debug */ + +static void +listStats(const U32* table, int lastEltID) +{ + int const nbElts = lastEltID + 1; + int enb; + for (enb=0; enb < nbElts; enb++) { + (void)table; + /* RAWLOG(2, "%3i:%3i, ", enb, table[enb]); */ + RAWLOG(2, "%4i,", table[enb]); } + RAWLOG(2, " \n"); } -static void ZSTDMT_serialState_ensureFinished(serialState_t* serialState, - unsigned jobID, size_t cSize) +#endif + +#define LIT_PRICE(_p) (int)ZSTD_rawLiteralsCost(_p, 1, optStatePtr, optLevel) +#define LL_PRICE(_l) (int)ZSTD_litLengthPrice(_l, optStatePtr, optLevel) +#define LL_INCPRICE(_l) (LL_PRICE(_l) - LL_PRICE(_l-1)) + +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t +ZSTD_compressBlock_opt_generic(ZSTD_MatchState_t* ms, + SeqStore_t* seqStore, + U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize, + const int optLevel, + const ZSTD_dictMode_e dictMode) { - ZSTD_PTHREAD_MUTEX_LOCK(&serialState->mutex); - if (serialState->nextJobID <= jobID) { - assert(ZSTD_isError(cSize)); (void)cSize; - DEBUGLOG(5, "Skipping past job %u because of error", jobID); - serialState->nextJobID = jobID + 1; - ZSTD_pthread_cond_broadcast(&serialState->cond); + optState_t* const optStatePtr = &ms->opt; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + const BYTE* const base = ms->window.base; + const BYTE* const prefixStart = base + ms->window.dictLimit; + const ZSTD_compressionParameters* const cParams = &ms->cParams; - ZSTD_PTHREAD_MUTEX_LOCK(&serialState->ldmWindowMutex); - ZSTD_window_clear(&serialState->ldmWindow); - ZSTD_pthread_cond_signal(&serialState->ldmWindowCond); - ZSTD_pthread_mutex_unlock(&serialState->ldmWindowMutex); - } - ZSTD_pthread_mutex_unlock(&serialState->mutex); + ZSTD_getAllMatchesFn getAllMatches = ZSTD_selectBtGetAllMatches(ms, dictMode); -} + U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1); + U32 const minMatch = (cParams->minMatch == 3) ? 3 : 4; + U32 nextToUpdate3 = ms->nextToUpdate; + ZSTD_optimal_t* const opt = optStatePtr->priceTable; + ZSTD_match_t* const matches = optStatePtr->matchTable; + ZSTD_optimal_t lastStretch; + ZSTD_optLdm_t optLdm; -/* ------------------------------------------ */ -/* ===== Worker thread ===== */ -/* ------------------------------------------ */ + ZSTD_memset(&lastStretch, 0, sizeof(ZSTD_optimal_t)); -static const range_t kNullRange = { NULL, 0 }; + optLdm.seqStore = ms->ldmSeqStore ? *ms->ldmSeqStore : kNullRawSeqStore; + optLdm.endPosInBlock = optLdm.startPosInBlock = optLdm.offset = 0; + ZSTD_opt_getNextMatchAndUpdateSeqStore(&optLdm, (U32)(ip-istart), (U32)(iend-ip)); -typedef struct { - size_t consumed; /* SHARED - set0 by mtctx, then modified by worker AND read by mtctx */ - size_t cSize; /* SHARED - set0 by mtctx, then modified by worker AND read by mtctx, then set0 by mtctx */ - ZSTD_pthread_mutex_t job_mutex; /* Thread-safe - used by mtctx and worker */ - ZSTD_pthread_cond_t job_cond; /* Thread-safe - used by mtctx and worker */ - ZSTDMT_CCtxPool* cctxPool; /* Thread-safe - used by mtctx and (all) workers */ - ZSTDMT_bufferPool* bufPool; /* Thread-safe - used by mtctx and (all) workers */ - ZSTDMT_seqPool* seqPool; /* Thread-safe - used by mtctx and (all) workers */ - serialState_t* serial; /* Thread-safe - used by mtctx and (all) workers */ - buffer_t dstBuff; /* set by worker (or mtctx), then read by worker & mtctx, then modified by mtctx => no barrier */ - range_t prefix; /* set by mtctx, then read by worker & mtctx => no barrier */ - range_t src; /* set by mtctx, then read by worker & mtctx => no barrier */ - unsigned jobID; /* set by mtctx, then read by worker => no barrier */ - unsigned firstJob; /* set by mtctx, then read by worker => no barrier */ - unsigned lastJob; /* set by mtctx, then read by worker => no barrier */ - ZSTD_CCtx_params params; /* set by mtctx, then read by worker => no barrier */ - const ZSTD_CDict* cdict; /* set by mtctx, then read by worker => no barrier */ - unsigned long long fullFrameSize; /* set by mtctx, then read by worker => no barrier */ - size_t dstFlushed; /* used only by mtctx */ - unsigned frameChecksumNeeded; /* used only by mtctx */ -} ZSTDMT_jobDescription; + /* init */ + DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u", + (U32)(ip - base), ms->window.dictLimit, ms->nextToUpdate); + assert(optLevel <= 2); + ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize, optLevel); + ip += (ip==prefixStart); -#define JOB_ERROR(e) { \ - ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex); \ - job->cSize = e; \ - ZSTD_pthread_mutex_unlock(&job->job_mutex); \ - goto _endJob; \ -} + /* Match Loop */ + while (ip < ilimit) { + U32 cur, last_pos = 0; -/* ZSTDMT_compressionJob() is a POOL_function type */ -static void ZSTDMT_compressionJob(void* jobDescription) -{ - ZSTDMT_jobDescription* const job = (ZSTDMT_jobDescription*)jobDescription; - ZSTD_CCtx_params jobParams = job->params; /* do not modify job->params ! copy it, modify the copy */ - ZSTD_CCtx* const cctx = ZSTDMT_getCCtx(job->cctxPool); - rawSeqStore_t rawSeqStore = ZSTDMT_getSeq(job->seqPool); - buffer_t dstBuff = job->dstBuff; - size_t lastCBlockSize = 0; + /* find first match */ + { U32 const litlen = (U32)(ip - anchor); + U32 const ll0 = !litlen; + U32 nbMatches = getAllMatches(matches, ms, &nextToUpdate3, ip, iend, rep, ll0, minMatch); + ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches, + (U32)(ip-istart), (U32)(iend-ip), + minMatch); + if (!nbMatches) { + DEBUGLOG(8, "no match found at cPos %u", (unsigned)(ip-istart)); + ip++; + continue; + } - /* resources */ - if (cctx==NULL) JOB_ERROR(ERROR(memory_allocation)); - if (dstBuff.start == NULL) { /* streaming job : doesn't provide a dstBuffer */ - dstBuff = ZSTDMT_getBuffer(job->bufPool); - if (dstBuff.start==NULL) JOB_ERROR(ERROR(memory_allocation)); - job->dstBuff = dstBuff; /* this value can be read in ZSTDMT_flush, when it copies the whole job */ - } - if (jobParams.ldmParams.enableLdm && rawSeqStore.seq == NULL) - JOB_ERROR(ERROR(memory_allocation)); + /* Match found: let's store this solution, and eventually find more candidates. + * During this forward pass, @opt is used to store stretches, + * defined as "a match followed by N literals". + * Note how this is different from a Sequence, which is "N literals followed by a match". + * Storing stretches allows us to store different match predecessors + * for each literal position part of a literals run. */ - /* Don't compute the checksum for chunks, since we compute it externally, - * but write it in the header. - */ - if (job->jobID != 0) jobParams.fParams.checksumFlag = 0; - /* Don't run LDM for the chunks, since we handle it externally */ - jobParams.ldmParams.enableLdm = 0; - /* Correct nbWorkers to 0. */ - jobParams.nbWorkers = 0; + /* initialize opt[0] */ + opt[0].mlen = 0; /* there are only literals so far */ + opt[0].litlen = litlen; + /* No need to include the actual price of the literals before the first match + * because it is static for the duration of the forward pass, and is included + * in every subsequent price. But, we include the literal length because + * the cost variation of litlen depends on the value of litlen. + */ + opt[0].price = LL_PRICE(litlen); + ZSTD_STATIC_ASSERT(sizeof(opt[0].rep[0]) == sizeof(rep[0])); + ZSTD_memcpy(&opt[0].rep, rep, sizeof(opt[0].rep)); + /* large match -> immediate encoding */ + { U32 const maxML = matches[nbMatches-1].len; + U32 const maxOffBase = matches[nbMatches-1].off; + DEBUGLOG(6, "found %u matches of maxLength=%u and maxOffBase=%u at cPos=%u => start new series", + nbMatches, maxML, maxOffBase, (U32)(ip-prefixStart)); - /* init */ - if (job->cdict) { - size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, job->cdict, &jobParams, job->fullFrameSize); - assert(job->firstJob); /* only allowed for first job */ - if (ZSTD_isError(initError)) JOB_ERROR(initError); - } else { /* srcStart points at reloaded section */ - U64 const pledgedSrcSize = job->firstJob ? job->fullFrameSize : job->src.size; - { size_t const forceWindowError = ZSTD_CCtxParams_setParameter(&jobParams, ZSTD_c_forceMaxWindow, !job->firstJob); - if (ZSTD_isError(forceWindowError)) JOB_ERROR(forceWindowError); + if (maxML > sufficient_len) { + lastStretch.litlen = 0; + lastStretch.mlen = maxML; + lastStretch.off = maxOffBase; + DEBUGLOG(6, "large match (%u>%u) => immediate encoding", + maxML, sufficient_len); + cur = 0; + last_pos = maxML; + goto _shortestPath; + } } + + /* set prices for first matches starting position == 0 */ + assert(opt[0].price >= 0); + { U32 pos; + U32 matchNb; + for (pos = 1; pos < minMatch; pos++) { + opt[pos].price = ZSTD_MAX_PRICE; + opt[pos].mlen = 0; + opt[pos].litlen = litlen + pos; + } + for (matchNb = 0; matchNb < nbMatches; matchNb++) { + U32 const offBase = matches[matchNb].off; + U32 const end = matches[matchNb].len; + for ( ; pos <= end ; pos++ ) { + int const matchPrice = (int)ZSTD_getMatchPrice(offBase, pos, optStatePtr, optLevel); + int const sequencePrice = opt[0].price + matchPrice; + DEBUGLOG(7, "rPos:%u => set initial price : %.2f", + pos, ZSTD_fCost(sequencePrice)); + opt[pos].mlen = pos; + opt[pos].off = offBase; + opt[pos].litlen = 0; /* end of match */ + opt[pos].price = sequencePrice + LL_PRICE(0); + } + } + last_pos = pos-1; + opt[pos].price = ZSTD_MAX_PRICE; + } } - { size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, - job->prefix.start, job->prefix.size, ZSTD_dct_rawContent, /* load dictionary in "content-only" mode (no header analysis) */ - ZSTD_dtlm_fast, - NULL, /*cdict*/ - &jobParams, pledgedSrcSize); - if (ZSTD_isError(initError)) JOB_ERROR(initError); - } } - /* Perform serial step as early as possible, but after CCtx initialization */ - ZSTDMT_serialState_update(job->serial, cctx, rawSeqStore, job->src, job->jobID); + /* check further positions */ + for (cur = 1; cur <= last_pos; cur++) { + const BYTE* const inr = ip + cur; + assert(cur <= ZSTD_OPT_NUM); + DEBUGLOG(7, "cPos:%i==rPos:%u", (int)(inr-istart), cur); - if (!job->firstJob) { /* flush and overwrite frame header when it's not first job */ - size_t const hSize = ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.capacity, job->src.start, 0); - if (ZSTD_isError(hSize)) JOB_ERROR(hSize); - DEBUGLOG(5, "ZSTDMT_compressionJob: flush and overwrite %u bytes of frame header (not first job)", (U32)hSize); - ZSTD_invalidateRepCodes(cctx); - } + /* Fix current position with one literal if cheaper */ + { U32 const litlen = opt[cur-1].litlen + 1; + int const price = opt[cur-1].price + + LIT_PRICE(ip+cur-1) + + LL_INCPRICE(litlen); + assert(price < 1000000000); /* overflow check */ + if (price <= opt[cur].price) { + ZSTD_optimal_t const prevMatch = opt[cur]; + DEBUGLOG(7, "cPos:%i==rPos:%u : better price (%.2f<=%.2f) using literal (ll==%u) (hist:%u,%u,%u)", + (int)(inr-istart), cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price), litlen, + opt[cur-1].rep[0], opt[cur-1].rep[1], opt[cur-1].rep[2]); + opt[cur] = opt[cur-1]; + opt[cur].litlen = litlen; + opt[cur].price = price; + if ( (optLevel >= 1) /* additional check only for higher modes */ + && (prevMatch.litlen == 0) /* replace a match */ + && (LL_INCPRICE(1) < 0) /* ll1 is cheaper than ll0 */ + && LIKELY(ip + cur < iend) + ) { + /* check next position, in case it would be cheaper */ + int with1literal = prevMatch.price + LIT_PRICE(ip+cur) + LL_INCPRICE(1); + int withMoreLiterals = price + LIT_PRICE(ip+cur) + LL_INCPRICE(litlen+1); + DEBUGLOG(7, "then at next rPos %u : match+1lit %.2f vs %ulits %.2f", + cur+1, ZSTD_fCost(with1literal), litlen+1, ZSTD_fCost(withMoreLiterals)); + if ( (with1literal < withMoreLiterals) + && (with1literal < opt[cur+1].price) ) { + /* update offset history - before it disappears */ + U32 const prev = cur - prevMatch.mlen; + Repcodes_t const newReps = ZSTD_newRep(opt[prev].rep, prevMatch.off, opt[prev].litlen==0); + assert(cur >= prevMatch.mlen); + DEBUGLOG(7, "==> match+1lit is cheaper (%.2f < %.2f) (hist:%u,%u,%u) !", + ZSTD_fCost(with1literal), ZSTD_fCost(withMoreLiterals), + newReps.rep[0], newReps.rep[1], newReps.rep[2] ); + opt[cur+1] = prevMatch; /* mlen & offbase */ + ZSTD_memcpy(opt[cur+1].rep, &newReps, sizeof(Repcodes_t)); + opt[cur+1].litlen = 1; + opt[cur+1].price = with1literal; + if (last_pos < cur+1) last_pos = cur+1; + } + } + } else { + DEBUGLOG(7, "cPos:%i==rPos:%u : literal would cost more (%.2f>%.2f)", + (int)(inr-istart), cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price)); + } + } - /* compress */ - { size_t const chunkSize = 4*ZSTD_BLOCKSIZE_MAX; - int const nbChunks = (int)((job->src.size + (chunkSize-1)) / chunkSize); - const BYTE* ip = (const BYTE*) job->src.start; - BYTE* const ostart = (BYTE*)dstBuff.start; - BYTE* op = ostart; - BYTE* oend = op + dstBuff.capacity; - int chunkNb; - if (sizeof(size_t) > sizeof(int)) assert(job->src.size < ((size_t)INT_MAX) * chunkSize); /* check overflow */ - DEBUGLOG(5, "ZSTDMT_compressionJob: compress %u bytes in %i blocks", (U32)job->src.size, nbChunks); - assert(job->cSize == 0); - for (chunkNb = 1; chunkNb < nbChunks; chunkNb++) { - size_t const cSize = ZSTD_compressContinue(cctx, op, oend-op, ip, chunkSize); - if (ZSTD_isError(cSize)) JOB_ERROR(cSize); - ip += chunkSize; - op += cSize; assert(op < oend); - /* stats */ - ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex); - job->cSize += cSize; - job->consumed = chunkSize * chunkNb; - DEBUGLOG(5, "ZSTDMT_compressionJob: compress new block : cSize==%u bytes (total: %u)", - (U32)cSize, (U32)job->cSize); - ZSTD_pthread_cond_signal(&job->job_cond); /* warns some more data is ready to be flushed */ - ZSTD_pthread_mutex_unlock(&job->job_mutex); - } - /* last block */ - assert(chunkSize > 0); - assert((chunkSize & (chunkSize - 1)) == 0); /* chunkSize must be power of 2 for mask==(chunkSize-1) to work */ - if ((nbChunks > 0) | job->lastJob /*must output a "last block" flag*/ ) { - size_t const lastBlockSize1 = job->src.size & (chunkSize-1); - size_t const lastBlockSize = ((lastBlockSize1==0) & (job->src.size>=chunkSize)) ? chunkSize : lastBlockSize1; - size_t const cSize = (job->lastJob) ? - ZSTD_compressEnd (cctx, op, oend-op, ip, lastBlockSize) : - ZSTD_compressContinue(cctx, op, oend-op, ip, lastBlockSize); - if (ZSTD_isError(cSize)) JOB_ERROR(cSize); - lastCBlockSize = cSize; - } } - ZSTD_CCtx_trace(cctx, 0); + /* Offset history is not updated during match comparison. + * Do it here, now that the match is selected and confirmed. + */ + ZSTD_STATIC_ASSERT(sizeof(opt[cur].rep) == sizeof(Repcodes_t)); + assert(cur >= opt[cur].mlen); + if (opt[cur].litlen == 0) { + /* just finished a match => alter offset history */ + U32 const prev = cur - opt[cur].mlen; + Repcodes_t const newReps = ZSTD_newRep(opt[prev].rep, opt[cur].off, opt[prev].litlen==0); + ZSTD_memcpy(opt[cur].rep, &newReps, sizeof(Repcodes_t)); + } + + /* last match must start at a minimum distance of 8 from oend */ + if (inr > ilimit) continue; + + if (cur == last_pos) break; + + if ( (optLevel==0) /*static_test*/ + && (opt[cur+1].price <= opt[cur].price + (BITCOST_MULTIPLIER/2)) ) { + DEBUGLOG(7, "skip current position : next rPos(%u) price is cheaper", cur+1); + continue; /* skip unpromising positions; about ~+6% speed, -0.01 ratio */ + } + + assert(opt[cur].price >= 0); + { U32 const ll0 = (opt[cur].litlen == 0); + int const previousPrice = opt[cur].price; + int const basePrice = previousPrice + LL_PRICE(0); + U32 nbMatches = getAllMatches(matches, ms, &nextToUpdate3, inr, iend, opt[cur].rep, ll0, minMatch); + U32 matchNb; -_endJob: - ZSTDMT_serialState_ensureFinished(job->serial, job->jobID, job->cSize); - if (job->prefix.size > 0) - DEBUGLOG(5, "Finished with prefix: %zx", (size_t)job->prefix.start); - DEBUGLOG(5, "Finished with source: %zx", (size_t)job->src.start); - /* release resources */ - ZSTDMT_releaseSeq(job->seqPool, rawSeqStore); - ZSTDMT_releaseCCtx(job->cctxPool, cctx); - /* report */ - ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex); - if (ZSTD_isError(job->cSize)) assert(lastCBlockSize == 0); - job->cSize += lastCBlockSize; - job->consumed = job->src.size; /* when job->consumed == job->src.size , compression job is presumed completed */ - ZSTD_pthread_cond_signal(&job->job_cond); - ZSTD_pthread_mutex_unlock(&job->job_mutex); -} + ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches, + (U32)(inr-istart), (U32)(iend-inr), + minMatch); + if (!nbMatches) { + DEBUGLOG(7, "rPos:%u : no match found", cur); + continue; + } -/* ------------------------------------------ */ -/* ===== Multi-threaded compression ===== */ -/* ------------------------------------------ */ + { U32 const longestML = matches[nbMatches-1].len; + DEBUGLOG(7, "cPos:%i==rPos:%u, found %u matches, of longest ML=%u", + (int)(inr-istart), cur, nbMatches, longestML); + + if ( (longestML > sufficient_len) + || (cur + longestML >= ZSTD_OPT_NUM) + || (ip + cur + longestML >= iend) ) { + lastStretch.mlen = longestML; + lastStretch.off = matches[nbMatches-1].off; + lastStretch.litlen = 0; + last_pos = cur + longestML; + goto _shortestPath; + } } -typedef struct { - range_t prefix; /* read-only non-owned prefix buffer */ - buffer_t buffer; - size_t filled; -} inBuff_t; + /* set prices using matches found at position == cur */ + for (matchNb = 0; matchNb < nbMatches; matchNb++) { + U32 const offset = matches[matchNb].off; + U32 const lastML = matches[matchNb].len; + U32 const startML = (matchNb>0) ? matches[matchNb-1].len+1 : minMatch; + U32 mlen; -typedef struct { - BYTE* buffer; /* The round input buffer. All jobs get references - * to pieces of the buffer. ZSTDMT_tryGetInputRange() - * handles handing out job input buffers, and makes - * sure it doesn't overlap with any pieces still in use. - */ - size_t capacity; /* The capacity of buffer. */ - size_t pos; /* The position of the current inBuff in the round - * buffer. Updated past the end if the inBuff once - * the inBuff is sent to the worker thread. - * pos <= capacity. - */ -} roundBuff_t; + DEBUGLOG(7, "testing match %u => offBase=%4u, mlen=%2u, llen=%2u", + matchNb, matches[matchNb].off, lastML, opt[cur].litlen); -static const roundBuff_t kNullRoundBuff = {NULL, 0, 0}; + for (mlen = lastML; mlen >= startML; mlen--) { /* scan downward */ + U32 const pos = cur + mlen; + int const price = basePrice + (int)ZSTD_getMatchPrice(offset, mlen, optStatePtr, optLevel); -#define RSYNC_LENGTH 32 + if ((pos > last_pos) || (price < opt[pos].price)) { + DEBUGLOG(7, "rPos:%u (ml=%2u) => new better price (%.2f<%.2f)", + pos, mlen, ZSTD_fCost(price), ZSTD_fCost(opt[pos].price)); + while (last_pos < pos) { + /* fill empty positions, for future comparisons */ + last_pos++; + opt[last_pos].price = ZSTD_MAX_PRICE; + opt[last_pos].litlen = !0; /* just needs to be != 0, to mean "not an end of match" */ + } + opt[pos].mlen = mlen; + opt[pos].off = offset; + opt[pos].litlen = 0; + opt[pos].price = price; + } else { + DEBUGLOG(7, "rPos:%u (ml=%2u) => new price is worse (%.2f>=%.2f)", + pos, mlen, ZSTD_fCost(price), ZSTD_fCost(opt[pos].price)); + if (optLevel==0) break; /* early update abort; gets ~+10% speed for about -0.01 ratio loss */ + } + } } } + opt[last_pos+1].price = ZSTD_MAX_PRICE; + } /* for (cur = 1; cur <= last_pos; cur++) */ -typedef struct { - U64 hash; - U64 hitMask; - U64 primePower; -} rsyncState_t; + lastStretch = opt[last_pos]; + assert(cur >= lastStretch.mlen); + cur = last_pos - lastStretch.mlen; -struct ZSTDMT_CCtx_s { - POOL_ctx* factory; - ZSTDMT_jobDescription* jobs; - ZSTDMT_bufferPool* bufPool; - ZSTDMT_CCtxPool* cctxPool; - ZSTDMT_seqPool* seqPool; - ZSTD_CCtx_params params; - size_t targetSectionSize; - size_t targetPrefixSize; - int jobReady; /* 1 => one job is already prepared, but pool has shortage of workers. Don't create a new job. */ - inBuff_t inBuff; - roundBuff_t roundBuff; - serialState_t serial; - rsyncState_t rsync; - unsigned jobIDMask; - unsigned doneJobID; - unsigned nextJobID; - unsigned frameEnded; - unsigned allJobsCompleted; - unsigned long long frameContentSize; - unsigned long long consumed; - unsigned long long produced; - ZSTD_customMem cMem; - ZSTD_CDict* cdictLocal; - const ZSTD_CDict* cdict; - unsigned providedFactory: 1; -}; +_shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ + assert(opt[0].mlen == 0); + assert(last_pos >= lastStretch.mlen); + assert(cur == last_pos - lastStretch.mlen); -static void ZSTDMT_freeJobsTable(ZSTDMT_jobDescription* jobTable, U32 nbJobs, ZSTD_customMem cMem) -{ - U32 jobNb; - if (jobTable == NULL) return; - for (jobNb=0; jobNb 0); + + /* Update offset history */ + if (lastStretch.litlen == 0) { + /* finishing on a match : update offset history */ + Repcodes_t const reps = ZSTD_newRep(opt[cur].rep, lastStretch.off, opt[cur].litlen==0); + ZSTD_memcpy(rep, &reps, sizeof(Repcodes_t)); + } else { + ZSTD_memcpy(rep, lastStretch.rep, sizeof(Repcodes_t)); + assert(cur >= lastStretch.litlen); + cur -= lastStretch.litlen; + } + + /* Let's write the shortest path solution. + * It is stored in @opt in reverse order, + * starting from @storeEnd (==cur+2), + * effectively partially @opt overwriting. + * Content is changed too: + * - So far, @opt stored stretches, aka a match followed by literals + * - Now, it will store sequences, aka literals followed by a match + */ + { U32 const storeEnd = cur + 2; + U32 storeStart = storeEnd; + U32 stretchPos = cur; + + DEBUGLOG(6, "start reverse traversal (last_pos:%u, cur:%u)", + last_pos, cur); (void)last_pos; + assert(storeEnd < ZSTD_OPT_SIZE); + DEBUGLOG(6, "last stretch copied into pos=%u (llen=%u,mlen=%u,ofc=%u)", + storeEnd, lastStretch.litlen, lastStretch.mlen, lastStretch.off); + if (lastStretch.litlen > 0) { + /* last "sequence" is unfinished: just a bunch of literals */ + opt[storeEnd].litlen = lastStretch.litlen; + opt[storeEnd].mlen = 0; + storeStart = storeEnd-1; + opt[storeStart] = lastStretch; + } { + opt[storeEnd] = lastStretch; /* note: litlen will be fixed */ + storeStart = storeEnd; + } + while (1) { + ZSTD_optimal_t nextStretch = opt[stretchPos]; + opt[storeStart].litlen = nextStretch.litlen; + DEBUGLOG(6, "selected sequence (llen=%u,mlen=%u,ofc=%u)", + opt[storeStart].litlen, opt[storeStart].mlen, opt[storeStart].off); + if (nextStretch.mlen == 0) { + /* reaching beginning of segment */ + break; + } + storeStart--; + opt[storeStart] = nextStretch; /* note: litlen will be fixed */ + assert(nextStretch.litlen + nextStretch.mlen <= stretchPos); + stretchPos -= nextStretch.litlen + nextStretch.mlen; + } + + /* save sequences */ + DEBUGLOG(6, "sending selected sequences into seqStore"); + { U32 storePos; + for (storePos=storeStart; storePos <= storeEnd; storePos++) { + U32 const llen = opt[storePos].litlen; + U32 const mlen = opt[storePos].mlen; + U32 const offBase = opt[storePos].off; + U32 const advance = llen + mlen; + DEBUGLOG(6, "considering seq starting at %i, llen=%u, mlen=%u", + (int)(anchor - istart), (unsigned)llen, (unsigned)mlen); + + if (mlen==0) { /* only literals => must be last "sequence", actually starting a new stream of sequences */ + assert(storePos == storeEnd); /* must be last sequence */ + ip = anchor + llen; /* last "sequence" is a bunch of literals => don't progress anchor */ + continue; /* will finish */ + } + + assert(anchor + llen <= iend); + ZSTD_updateStats(optStatePtr, llen, anchor, offBase, mlen); + ZSTD_storeSeq(seqStore, llen, anchor, iend, offBase, mlen); + anchor += advance; + ip = anchor; + } } + DEBUGLOG(7, "new offset history : %u, %u, %u", rep[0], rep[1], rep[2]); + + /* update all costs */ + ZSTD_setBasePrices(optStatePtr, optLevel); + } + } /* while (ip < ilimit) */ + + /* Return the last literals size */ + return (size_t)(iend - anchor); } +#endif /* build exclusions */ -/* ZSTDMT_allocJobsTable() - * allocate and init a job table. - * update *nbJobsPtr to next power of 2 value, as size of table */ -static ZSTDMT_jobDescription* ZSTDMT_createJobsTable(U32* nbJobsPtr, ZSTD_customMem cMem) +#ifndef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR +static size_t ZSTD_compressBlock_opt0( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize, const ZSTD_dictMode_e dictMode) { - U32 const nbJobsLog2 = ZSTD_highbit32(*nbJobsPtr) + 1; - U32 const nbJobs = 1 << nbJobsLog2; - U32 jobNb; - ZSTDMT_jobDescription* const jobTable = (ZSTDMT_jobDescription*) - ZSTD_customCalloc(nbJobs * sizeof(ZSTDMT_jobDescription), cMem); - int initError = 0; - if (jobTable==NULL) return NULL; - *nbJobsPtr = nbJobs; - for (jobNb=0; jobNb mtctx->jobIDMask+1) { /* need more job capacity */ - ZSTDMT_freeJobsTable(mtctx->jobs, mtctx->jobIDMask+1, mtctx->cMem); - mtctx->jobIDMask = 0; - mtctx->jobs = ZSTDMT_createJobsTable(&nbJobs, mtctx->cMem); - if (mtctx->jobs==NULL) return ERROR(memory_allocation); - assert((nbJobs != 0) && ((nbJobs & (nbJobs - 1)) == 0)); /* ensure nbJobs is a power of 2 */ - mtctx->jobIDMask = nbJobs - 1; - } - return 0; +#ifndef ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR +static size_t ZSTD_compressBlock_opt2( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize, const ZSTD_dictMode_e dictMode) +{ + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /* optLevel */, dictMode); } +#endif - -/* ZSTDMT_CCtxParam_setNbWorkers(): - * Internal use only */ -static size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorkers) +#ifndef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_btopt( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) { - return ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, (int)nbWorkers); + DEBUGLOG(5, "ZSTD_compressBlock_btopt"); + return ZSTD_compressBlock_opt0(ms, seqStore, rep, src, srcSize, ZSTD_noDict); } +#endif -MEM_STATIC ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced_internal(unsigned nbWorkers, ZSTD_customMem cMem, ZSTD_threadPool* pool) -{ - ZSTDMT_CCtx* mtctx; - U32 nbJobs = nbWorkers + 2; - int initError; - DEBUGLOG(3, "ZSTDMT_createCCtx_advanced (nbWorkers = %u)", nbWorkers); - if (nbWorkers < 1) return NULL; - nbWorkers = MIN(nbWorkers , ZSTDMT_NBWORKERS_MAX); - if ((cMem.customAlloc!=NULL) ^ (cMem.customFree!=NULL)) - /* invalid custom allocator */ - return NULL; - mtctx = (ZSTDMT_CCtx*) ZSTD_customCalloc(sizeof(ZSTDMT_CCtx), cMem); - if (!mtctx) return NULL; - ZSTDMT_CCtxParam_setNbWorkers(&mtctx->params, nbWorkers); - mtctx->cMem = cMem; - mtctx->allJobsCompleted = 1; - if (pool != NULL) { - mtctx->factory = pool; - mtctx->providedFactory = 1; - } - else { - mtctx->factory = POOL_create_advanced(nbWorkers, 0, cMem); - mtctx->providedFactory = 0; - } - mtctx->jobs = ZSTDMT_createJobsTable(&nbJobs, cMem); - assert(nbJobs > 0); assert((nbJobs & (nbJobs - 1)) == 0); /* ensure nbJobs is a power of 2 */ - mtctx->jobIDMask = nbJobs - 1; - mtctx->bufPool = ZSTDMT_createBufferPool(nbWorkers, cMem); - mtctx->cctxPool = ZSTDMT_createCCtxPool(nbWorkers, cMem); - mtctx->seqPool = ZSTDMT_createSeqPool(nbWorkers, cMem); - initError = ZSTDMT_serialState_init(&mtctx->serial); - mtctx->roundBuff = kNullRoundBuff; - if (!mtctx->factory | !mtctx->jobs | !mtctx->bufPool | !mtctx->cctxPool | !mtctx->seqPool | initError) { - ZSTDMT_freeCCtx(mtctx); - return NULL; - } - DEBUGLOG(3, "mt_cctx created, for %u threads", nbWorkers); - return mtctx; -} -ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, ZSTD_customMem cMem, ZSTD_threadPool* pool) +#ifndef ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR +/* ZSTD_initStats_ultra(): + * make a first compression pass, just to seed stats with more accurate starting values. + * only works on first block, with no dictionary and no ldm. + * this function cannot error out, its narrow contract must be respected. + */ +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +void ZSTD_initStats_ultra(ZSTD_MatchState_t* ms, + SeqStore_t* seqStore, + U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) { -#ifdef ZSTD_MULTITHREAD - return ZSTDMT_createCCtx_advanced_internal(nbWorkers, cMem, pool); -#else - (void)nbWorkers; - (void)cMem; - (void)pool; - return NULL; -#endif -} + U32 tmpRep[ZSTD_REP_NUM]; /* updated rep codes will sink here */ + ZSTD_memcpy(tmpRep, rep, sizeof(tmpRep)); + + DEBUGLOG(4, "ZSTD_initStats_ultra (srcSize=%zu)", srcSize); + assert(ms->opt.litLengthSum == 0); /* first block */ + assert(seqStore->sequences == seqStore->sequencesStart); /* no ldm */ + assert(ms->window.dictLimit == ms->window.lowLimit); /* no dictionary */ + assert(ms->window.dictLimit - ms->nextToUpdate <= 1); /* no prefix (note: intentional overflow, defined as 2-complement) */ + ZSTD_compressBlock_opt2(ms, seqStore, tmpRep, src, srcSize, ZSTD_noDict); /* generate stats into ms->opt*/ -/* ZSTDMT_releaseAllJobResources() : - * note : ensure all workers are killed first ! */ -static void ZSTDMT_releaseAllJobResources(ZSTDMT_CCtx* mtctx) -{ - unsigned jobID; - DEBUGLOG(3, "ZSTDMT_releaseAllJobResources"); - for (jobID=0; jobID <= mtctx->jobIDMask; jobID++) { - /* Copy the mutex/cond out */ - ZSTD_pthread_mutex_t const mutex = mtctx->jobs[jobID].job_mutex; - ZSTD_pthread_cond_t const cond = mtctx->jobs[jobID].job_cond; + /* invalidate first scan from history, only keep entropy stats */ + ZSTD_resetSeqStore(seqStore); + ms->window.base -= srcSize; + ms->window.dictLimit += (U32)srcSize; + ms->window.lowLimit = ms->window.dictLimit; + ms->nextToUpdate = ms->window.dictLimit; - DEBUGLOG(4, "job%02u: release dst address %08X", jobID, (U32)(size_t)mtctx->jobs[jobID].dstBuff.start); - ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[jobID].dstBuff); +} - /* Clear the job description, but keep the mutex/cond */ - ZSTD_memset(&mtctx->jobs[jobID], 0, sizeof(mtctx->jobs[jobID])); - mtctx->jobs[jobID].job_mutex = mutex; - mtctx->jobs[jobID].job_cond = cond; - } - mtctx->inBuff.buffer = g_nullBuffer; - mtctx->inBuff.filled = 0; - mtctx->allJobsCompleted = 1; +size_t ZSTD_compressBlock_btultra( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_compressBlock_btultra (srcSize=%zu)", srcSize); + return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_noDict); } -static void ZSTDMT_waitForAllJobsCompleted(ZSTDMT_CCtx* mtctx) +size_t ZSTD_compressBlock_btultra2( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) { - DEBUGLOG(4, "ZSTDMT_waitForAllJobsCompleted"); - while (mtctx->doneJobID < mtctx->nextJobID) { - unsigned const jobID = mtctx->doneJobID & mtctx->jobIDMask; - ZSTD_PTHREAD_MUTEX_LOCK(&mtctx->jobs[jobID].job_mutex); - while (mtctx->jobs[jobID].consumed < mtctx->jobs[jobID].src.size) { - DEBUGLOG(4, "waiting for jobCompleted signal from job %u", mtctx->doneJobID); /* we want to block when waiting for data to flush */ - ZSTD_pthread_cond_wait(&mtctx->jobs[jobID].job_cond, &mtctx->jobs[jobID].job_mutex); - } - ZSTD_pthread_mutex_unlock(&mtctx->jobs[jobID].job_mutex); - mtctx->doneJobID++; + U32 const curr = (U32)((const BYTE*)src - ms->window.base); + DEBUGLOG(5, "ZSTD_compressBlock_btultra2 (srcSize=%zu)", srcSize); + + /* 2-passes strategy: + * this strategy makes a first pass over first block to collect statistics + * in order to seed next round's statistics with it. + * After 1st pass, function forgets history, and starts a new block. + * Consequently, this can only work if no data has been previously loaded in tables, + * aka, no dictionary, no prefix, no ldm preprocessing. + * The compression ratio gain is generally small (~0.5% on first block), + * the cost is 2x cpu time on first block. */ + assert(srcSize <= ZSTD_BLOCKSIZE_MAX); + if ( (ms->opt.litLengthSum==0) /* first block */ + && (seqStore->sequences == seqStore->sequencesStart) /* no ldm */ + && (ms->window.dictLimit == ms->window.lowLimit) /* no dictionary */ + && (curr == ms->window.dictLimit) /* start of frame, nothing already loaded nor skipped */ + && (srcSize > ZSTD_PREDEF_THRESHOLD) /* input large enough to not employ default stats */ + ) { + ZSTD_initStats_ultra(ms, seqStore, rep, src, srcSize); } + + return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_noDict); } +#endif -size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx) +#ifndef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_btopt_dictMatchState( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) { - if (mtctx==NULL) return 0; /* compatible with free on NULL */ - if (!mtctx->providedFactory) - POOL_free(mtctx->factory); /* stop and free worker threads */ - ZSTDMT_releaseAllJobResources(mtctx); /* release job resources into pools first */ - ZSTDMT_freeJobsTable(mtctx->jobs, mtctx->jobIDMask+1, mtctx->cMem); - ZSTDMT_freeBufferPool(mtctx->bufPool); - ZSTDMT_freeCCtxPool(mtctx->cctxPool); - ZSTDMT_freeSeqPool(mtctx->seqPool); - ZSTDMT_serialState_free(&mtctx->serial); - ZSTD_freeCDict(mtctx->cdictLocal); - if (mtctx->roundBuff.buffer) - ZSTD_customFree(mtctx->roundBuff.buffer, mtctx->cMem); - ZSTD_customFree(mtctx, mtctx->cMem); - return 0; + return ZSTD_compressBlock_opt0(ms, seqStore, rep, src, srcSize, ZSTD_dictMatchState); } -size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx) +size_t ZSTD_compressBlock_btopt_extDict( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) { - if (mtctx == NULL) return 0; /* supports sizeof NULL */ - return sizeof(*mtctx) - + POOL_sizeof(mtctx->factory) - + ZSTDMT_sizeof_bufferPool(mtctx->bufPool) - + (mtctx->jobIDMask+1) * sizeof(ZSTDMT_jobDescription) - + ZSTDMT_sizeof_CCtxPool(mtctx->cctxPool) - + ZSTDMT_sizeof_seqPool(mtctx->seqPool) - + ZSTD_sizeof_CDict(mtctx->cdictLocal) - + mtctx->roundBuff.capacity; + return ZSTD_compressBlock_opt0(ms, seqStore, rep, src, srcSize, ZSTD_extDict); } +#endif +#ifndef ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_btultra_dictMatchState( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_dictMatchState); +} -/* ZSTDMT_resize() : - * @return : error code if fails, 0 on success */ -static size_t ZSTDMT_resize(ZSTDMT_CCtx* mtctx, unsigned nbWorkers) +size_t ZSTD_compressBlock_btultra_extDict( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) { - if (POOL_resize(mtctx->factory, nbWorkers)) return ERROR(memory_allocation); - FORWARD_IF_ERROR( ZSTDMT_expandJobsTable(mtctx, nbWorkers) , ""); - mtctx->bufPool = ZSTDMT_expandBufferPool(mtctx->bufPool, nbWorkers); - if (mtctx->bufPool == NULL) return ERROR(memory_allocation); - mtctx->cctxPool = ZSTDMT_expandCCtxPool(mtctx->cctxPool, nbWorkers); - if (mtctx->cctxPool == NULL) return ERROR(memory_allocation); - mtctx->seqPool = ZSTDMT_expandSeqPool(mtctx->seqPool, nbWorkers); - if (mtctx->seqPool == NULL) return ERROR(memory_allocation); - ZSTDMT_CCtxParam_setNbWorkers(&mtctx->params, nbWorkers); - return 0; + return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_extDict); } +#endif +/* note : no btultra2 variant for extDict nor dictMatchState, + * because btultra2 is not meant to work with dictionaries + * and is only specific for the first block (no prefix) */ +/**** ended inlining compress/zstd_opt.c ****/ +#ifdef ZSTD_MULTITHREAD +/**** start inlining compress/zstdmt_compress.c ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ -/*! ZSTDMT_updateCParams_whileCompressing() : - * Updates a selected set of compression parameters, remaining compatible with currently active frame. - * New parameters will be applied to next compression job. */ -void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_params* cctxParams) + +/* ====== Compiler specifics ====== */ +#if defined(_MSC_VER) +# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ +#endif + + +/* ====== Dependencies ====== */ +/**** skipping file: ../common/allocations.h ****/ +/**** skipping file: ../common/zstd_deps.h ****/ +/**** skipping file: ../common/mem.h ****/ +/**** skipping file: ../common/pool.h ****/ +/**** skipping file: ../common/threading.h ****/ +/**** skipping file: zstd_compress_internal.h ****/ +/**** skipping file: zstd_ldm.h ****/ +/**** skipping file: zstdmt_compress.h ****/ + +/* Guards code to support resizing the SeqPool. + * We will want to resize the SeqPool to save memory in the future. + * Until then, comment the code out since it is unused. + */ +#define ZSTD_RESIZE_SEQPOOL 0 + +/* ====== Debug ====== */ +#if defined(DEBUGLEVEL) && (DEBUGLEVEL>=2) \ + && !defined(_MSC_VER) \ + && !defined(__MINGW32__) + +# include +# include +# include + +# define DEBUG_PRINTHEX(l,p,n) \ + do { \ + unsigned debug_u; \ + for (debug_u=0; debug_u<(n); debug_u++) \ + RAWLOG(l, "%02X ", ((const unsigned char*)(p))[debug_u]); \ + RAWLOG(l, " \n"); \ + } while (0) + +static unsigned long long GetCurrentClockTimeMicroseconds(void) { - U32 const saved_wlog = mtctx->params.cParams.windowLog; /* Do not modify windowLog while compressing */ - int const compressionLevel = cctxParams->compressionLevel; - DEBUGLOG(5, "ZSTDMT_updateCParams_whileCompressing (level:%i)", - compressionLevel); - mtctx->params.compressionLevel = compressionLevel; - { ZSTD_compressionParameters cParams = ZSTD_getCParamsFromCCtxParams(cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); - cParams.windowLog = saved_wlog; - mtctx->params.cParams = cParams; - } -} + static clock_t _ticksPerSecond = 0; + if (_ticksPerSecond <= 0) _ticksPerSecond = sysconf(_SC_CLK_TCK); -/* ZSTDMT_getFrameProgression(): - * tells how much data has been consumed (input) and produced (output) for current frame. - * able to count progression inside worker threads. - * Note : mutex will be acquired during statistics collection inside workers. */ -ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx) + { struct tms junk; clock_t newTicks = (clock_t) times(&junk); + return ((((unsigned long long)newTicks)*(1000000))/_ticksPerSecond); +} } + +#define MUTEX_WAIT_TIME_DLEVEL 6 +#define ZSTD_PTHREAD_MUTEX_LOCK(mutex) \ + do { \ + if (DEBUGLEVEL >= MUTEX_WAIT_TIME_DLEVEL) { \ + unsigned long long const beforeTime = GetCurrentClockTimeMicroseconds(); \ + ZSTD_pthread_mutex_lock(mutex); \ + { unsigned long long const afterTime = GetCurrentClockTimeMicroseconds(); \ + unsigned long long const elapsedTime = (afterTime-beforeTime); \ + if (elapsedTime > 1000) { \ + /* or whatever threshold you like; I'm using 1 millisecond here */ \ + DEBUGLOG(MUTEX_WAIT_TIME_DLEVEL, \ + "Thread took %llu microseconds to acquire mutex %s \n", \ + elapsedTime, #mutex); \ + } } \ + } else { \ + ZSTD_pthread_mutex_lock(mutex); \ + } \ + } while (0) + +#else + +# define ZSTD_PTHREAD_MUTEX_LOCK(m) ZSTD_pthread_mutex_lock(m) +# define DEBUG_PRINTHEX(l,p,n) do { } while (0) + +#endif + + +/* ===== Buffer Pool ===== */ +/* a single Buffer Pool can be invoked from multiple threads in parallel */ + +typedef struct buffer_s { + void* start; + size_t capacity; +} Buffer; + +static const Buffer g_nullBuffer = { NULL, 0 }; + +typedef struct ZSTDMT_bufferPool_s { + ZSTD_pthread_mutex_t poolMutex; + size_t bufferSize; + unsigned totalBuffers; + unsigned nbBuffers; + ZSTD_customMem cMem; + Buffer* buffers; +} ZSTDMT_bufferPool; + +static void ZSTDMT_freeBufferPool(ZSTDMT_bufferPool* bufPool) { - ZSTD_frameProgression fps; - DEBUGLOG(5, "ZSTDMT_getFrameProgression"); - fps.ingested = mtctx->consumed + mtctx->inBuff.filled; - fps.consumed = mtctx->consumed; - fps.produced = fps.flushed = mtctx->produced; - fps.currentJobID = mtctx->nextJobID; - fps.nbActiveWorkers = 0; - { unsigned jobNb; - unsigned lastJobNb = mtctx->nextJobID + mtctx->jobReady; assert(mtctx->jobReady <= 1); - DEBUGLOG(6, "ZSTDMT_getFrameProgression: jobs: from %u to <%u (jobReady:%u)", - mtctx->doneJobID, lastJobNb, mtctx->jobReady) - for (jobNb = mtctx->doneJobID ; jobNb < lastJobNb ; jobNb++) { - unsigned const wJobID = jobNb & mtctx->jobIDMask; - ZSTDMT_jobDescription* jobPtr = &mtctx->jobs[wJobID]; - ZSTD_pthread_mutex_lock(&jobPtr->job_mutex); - { size_t const cResult = jobPtr->cSize; - size_t const produced = ZSTD_isError(cResult) ? 0 : cResult; - size_t const flushed = ZSTD_isError(cResult) ? 0 : jobPtr->dstFlushed; - assert(flushed <= produced); - fps.ingested += jobPtr->src.size; - fps.consumed += jobPtr->consumed; - fps.produced += produced; - fps.flushed += flushed; - fps.nbActiveWorkers += (jobPtr->consumed < jobPtr->src.size); - } - ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex); + DEBUGLOG(3, "ZSTDMT_freeBufferPool (address:%08X)", (U32)(size_t)bufPool); + if (!bufPool) return; /* compatibility with free on NULL */ + if (bufPool->buffers) { + unsigned u; + for (u=0; utotalBuffers; u++) { + DEBUGLOG(4, "free buffer %2u (address:%08X)", u, (U32)(size_t)bufPool->buffers[u].start); + ZSTD_customFree(bufPool->buffers[u].start, bufPool->cMem); } + ZSTD_customFree(bufPool->buffers, bufPool->cMem); } - return fps; + ZSTD_pthread_mutex_destroy(&bufPool->poolMutex); + ZSTD_customFree(bufPool, bufPool->cMem); } - -size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx) +static ZSTDMT_bufferPool* ZSTDMT_createBufferPool(unsigned maxNbBuffers, ZSTD_customMem cMem) { - size_t toFlush; - unsigned const jobID = mtctx->doneJobID; - assert(jobID <= mtctx->nextJobID); - if (jobID == mtctx->nextJobID) return 0; /* no active job => nothing to flush */ - - /* look into oldest non-fully-flushed job */ - { unsigned const wJobID = jobID & mtctx->jobIDMask; - ZSTDMT_jobDescription* const jobPtr = &mtctx->jobs[wJobID]; - ZSTD_pthread_mutex_lock(&jobPtr->job_mutex); - { size_t const cResult = jobPtr->cSize; - size_t const produced = ZSTD_isError(cResult) ? 0 : cResult; - size_t const flushed = ZSTD_isError(cResult) ? 0 : jobPtr->dstFlushed; - assert(flushed <= produced); - assert(jobPtr->consumed <= jobPtr->src.size); - toFlush = produced - flushed; - /* if toFlush==0, nothing is available to flush. - * However, jobID is expected to still be active: - * if jobID was already completed and fully flushed, - * ZSTDMT_flushProduced() should have already moved onto next job. - * Therefore, some input has not yet been consumed. */ - if (toFlush==0) { - assert(jobPtr->consumed < jobPtr->src.size); - } - } - ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex); + ZSTDMT_bufferPool* const bufPool = + (ZSTDMT_bufferPool*)ZSTD_customCalloc(sizeof(ZSTDMT_bufferPool), cMem); + if (bufPool==NULL) return NULL; + if (ZSTD_pthread_mutex_init(&bufPool->poolMutex, NULL)) { + ZSTD_customFree(bufPool, cMem); + return NULL; } - - return toFlush; + bufPool->buffers = (Buffer*)ZSTD_customCalloc(maxNbBuffers * sizeof(Buffer), cMem); + if (bufPool->buffers==NULL) { + ZSTDMT_freeBufferPool(bufPool); + return NULL; + } + bufPool->bufferSize = 64 KB; + bufPool->totalBuffers = maxNbBuffers; + bufPool->nbBuffers = 0; + bufPool->cMem = cMem; + return bufPool; } +/* only works at initialization, not during compression */ +static size_t ZSTDMT_sizeof_bufferPool(ZSTDMT_bufferPool* bufPool) +{ + size_t const poolSize = sizeof(*bufPool); + size_t const arraySize = bufPool->totalBuffers * sizeof(Buffer); + unsigned u; + size_t totalBufferSize = 0; + ZSTD_pthread_mutex_lock(&bufPool->poolMutex); + for (u=0; utotalBuffers; u++) + totalBufferSize += bufPool->buffers[u].capacity; + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); -/* ------------------------------------------ */ -/* ===== Multi-threaded compression ===== */ -/* ------------------------------------------ */ + return poolSize + arraySize + totalBufferSize; +} -static unsigned ZSTDMT_computeTargetJobLog(const ZSTD_CCtx_params* params) +/* ZSTDMT_setBufferSize() : + * all future buffers provided by this buffer pool will have _at least_ this size + * note : it's better for all buffers to have same size, + * as they become freely interchangeable, reducing malloc/free usages and memory fragmentation */ +static void ZSTDMT_setBufferSize(ZSTDMT_bufferPool* const bufPool, size_t const bSize) { - unsigned jobLog; - if (params->ldmParams.enableLdm) { - /* In Long Range Mode, the windowLog is typically oversized. - * In which case, it's preferable to determine the jobSize - * based on cycleLog instead. */ - jobLog = MAX(21, ZSTD_cycleLog(params->cParams.chainLog, params->cParams.strategy) + 3); - } else { - jobLog = MAX(20, params->cParams.windowLog + 2); - } - return MIN(jobLog, (unsigned)ZSTDMT_JOBLOG_MAX); + ZSTD_pthread_mutex_lock(&bufPool->poolMutex); + DEBUGLOG(4, "ZSTDMT_setBufferSize: bSize = %u", (U32)bSize); + bufPool->bufferSize = bSize; + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); } -static int ZSTDMT_overlapLog_default(ZSTD_strategy strat) + +static ZSTDMT_bufferPool* ZSTDMT_expandBufferPool(ZSTDMT_bufferPool* srcBufPool, unsigned maxNbBuffers) { - switch(strat) - { - case ZSTD_btultra2: - return 9; - case ZSTD_btultra: - case ZSTD_btopt: - return 8; - case ZSTD_btlazy2: - case ZSTD_lazy2: - return 7; - case ZSTD_lazy: - case ZSTD_greedy: - case ZSTD_dfast: - case ZSTD_fast: - default:; + if (srcBufPool==NULL) return NULL; + if (srcBufPool->totalBuffers >= maxNbBuffers) /* good enough */ + return srcBufPool; + /* need a larger buffer pool */ + { ZSTD_customMem const cMem = srcBufPool->cMem; + size_t const bSize = srcBufPool->bufferSize; /* forward parameters */ + ZSTDMT_bufferPool* newBufPool; + ZSTDMT_freeBufferPool(srcBufPool); + newBufPool = ZSTDMT_createBufferPool(maxNbBuffers, cMem); + if (newBufPool==NULL) return newBufPool; + ZSTDMT_setBufferSize(newBufPool, bSize); + return newBufPool; } - return 6; } -static int ZSTDMT_overlapLog(int ovlog, ZSTD_strategy strat) +/** ZSTDMT_getBuffer() : + * assumption : bufPool must be valid + * @return : a buffer, with start pointer and size + * note: allocation may fail, in this case, start==NULL and size==0 */ +static Buffer ZSTDMT_getBuffer(ZSTDMT_bufferPool* bufPool) { - assert(0 <= ovlog && ovlog <= 9); - if (ovlog == 0) return ZSTDMT_overlapLog_default(strat); - return ovlog; + size_t const bSize = bufPool->bufferSize; + DEBUGLOG(5, "ZSTDMT_getBuffer: bSize = %u", (U32)bufPool->bufferSize); + ZSTD_pthread_mutex_lock(&bufPool->poolMutex); + if (bufPool->nbBuffers) { /* try to use an existing buffer */ + Buffer const buf = bufPool->buffers[--(bufPool->nbBuffers)]; + size_t const availBufferSize = buf.capacity; + bufPool->buffers[bufPool->nbBuffers] = g_nullBuffer; + if ((availBufferSize >= bSize) & ((availBufferSize>>3) <= bSize)) { + /* large enough, but not too much */ + DEBUGLOG(5, "ZSTDMT_getBuffer: provide buffer %u of size %u", + bufPool->nbBuffers, (U32)buf.capacity); + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); + return buf; + } + /* size conditions not respected : scratch this buffer, create new one */ + DEBUGLOG(5, "ZSTDMT_getBuffer: existing buffer does not meet size conditions => freeing"); + ZSTD_customFree(buf.start, bufPool->cMem); + } + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); + /* create new buffer */ + DEBUGLOG(5, "ZSTDMT_getBuffer: create a new buffer"); + { Buffer buffer; + void* const start = ZSTD_customMalloc(bSize, bufPool->cMem); + buffer.start = start; /* note : start can be NULL if malloc fails ! */ + buffer.capacity = (start==NULL) ? 0 : bSize; + if (start==NULL) { + DEBUGLOG(5, "ZSTDMT_getBuffer: buffer allocation failure !!"); + } else { + DEBUGLOG(5, "ZSTDMT_getBuffer: created buffer of size %u", (U32)bSize); + } + return buffer; + } } -static size_t ZSTDMT_computeOverlapSize(const ZSTD_CCtx_params* params) +#if ZSTD_RESIZE_SEQPOOL +/** ZSTDMT_resizeBuffer() : + * assumption : bufPool must be valid + * @return : a buffer that is at least the buffer pool buffer size. + * If a reallocation happens, the data in the input buffer is copied. + */ +static Buffer ZSTDMT_resizeBuffer(ZSTDMT_bufferPool* bufPool, Buffer buffer) { - int const overlapRLog = 9 - ZSTDMT_overlapLog(params->overlapLog, params->cParams.strategy); - int ovLog = (overlapRLog >= 8) ? 0 : (params->cParams.windowLog - overlapRLog); - assert(0 <= overlapRLog && overlapRLog <= 8); - if (params->ldmParams.enableLdm) { - /* In Long Range Mode, the windowLog is typically oversized. - * In which case, it's preferable to determine the jobSize - * based on chainLog instead. - * Then, ovLog becomes a fraction of the jobSize, rather than windowSize */ - ovLog = MIN(params->cParams.windowLog, ZSTDMT_computeTargetJobLog(params) - 2) - - overlapRLog; + size_t const bSize = bufPool->bufferSize; + if (buffer.capacity < bSize) { + void* const start = ZSTD_customMalloc(bSize, bufPool->cMem); + Buffer newBuffer; + newBuffer.start = start; + newBuffer.capacity = start == NULL ? 0 : bSize; + if (start != NULL) { + assert(newBuffer.capacity >= buffer.capacity); + ZSTD_memcpy(newBuffer.start, buffer.start, buffer.capacity); + DEBUGLOG(5, "ZSTDMT_resizeBuffer: created buffer of size %u", (U32)bSize); + return newBuffer; + } + DEBUGLOG(5, "ZSTDMT_resizeBuffer: buffer allocation failure !!"); } - assert(0 <= ovLog && ovLog <= ZSTD_WINDOWLOG_MAX); - DEBUGLOG(4, "overlapLog : %i", params->overlapLog); - DEBUGLOG(4, "overlap size : %i", 1 << ovLog); - return (ovLog==0) ? 0 : (size_t)1 << ovLog; + return buffer; } +#endif -/* ====================================== */ -/* ======= Streaming API ======= */ -/* ====================================== */ - -size_t ZSTDMT_initCStream_internal( - ZSTDMT_CCtx* mtctx, - const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType, - const ZSTD_CDict* cdict, ZSTD_CCtx_params params, - unsigned long long pledgedSrcSize) +/* store buffer for later re-use, up to pool capacity */ +static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* bufPool, Buffer buf) { - DEBUGLOG(4, "ZSTDMT_initCStream_internal (pledgedSrcSize=%u, nbWorkers=%u, cctxPool=%u)", - (U32)pledgedSrcSize, params.nbWorkers, mtctx->cctxPool->totalCCtx); - - /* params supposed partially fully validated at this point */ - assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); - assert(!((dict) && (cdict))); /* either dict or cdict, not both */ - - /* init */ - if (params.nbWorkers != mtctx->params.nbWorkers) - FORWARD_IF_ERROR( ZSTDMT_resize(mtctx, params.nbWorkers) , ""); - - if (params.jobSize != 0 && params.jobSize < ZSTDMT_JOBSIZE_MIN) params.jobSize = ZSTDMT_JOBSIZE_MIN; - if (params.jobSize > (size_t)ZSTDMT_JOBSIZE_MAX) params.jobSize = (size_t)ZSTDMT_JOBSIZE_MAX; + DEBUGLOG(5, "ZSTDMT_releaseBuffer"); + if (buf.start == NULL) return; /* compatible with release on NULL */ + ZSTD_pthread_mutex_lock(&bufPool->poolMutex); + if (bufPool->nbBuffers < bufPool->totalBuffers) { + bufPool->buffers[bufPool->nbBuffers++] = buf; /* stored for later use */ + DEBUGLOG(5, "ZSTDMT_releaseBuffer: stored buffer of size %u in slot %u", + (U32)buf.capacity, (U32)(bufPool->nbBuffers-1)); + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); + return; + } + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); + /* Reached bufferPool capacity (note: should not happen) */ + DEBUGLOG(5, "ZSTDMT_releaseBuffer: pool capacity reached => freeing "); + ZSTD_customFree(buf.start, bufPool->cMem); +} - DEBUGLOG(4, "ZSTDMT_initCStream_internal: %u workers", params.nbWorkers); +/* We need 2 output buffers per worker since each dstBuff must be flushed after it is released. + * The 3 additional buffers are as follows: + * 1 buffer for input loading + * 1 buffer for "next input" when submitting current one + * 1 buffer stuck in queue */ +#define BUF_POOL_MAX_NB_BUFFERS(nbWorkers) (2*(nbWorkers) + 3) - if (mtctx->allJobsCompleted == 0) { /* previous compression not correctly finished */ - ZSTDMT_waitForAllJobsCompleted(mtctx); - ZSTDMT_releaseAllJobResources(mtctx); - mtctx->allJobsCompleted = 1; - } +/* After a worker releases its rawSeqStore, it is immediately ready for reuse. + * So we only need one seq buffer per worker. */ +#define SEQ_POOL_MAX_NB_BUFFERS(nbWorkers) (nbWorkers) - mtctx->params = params; - mtctx->frameContentSize = pledgedSrcSize; - if (dict) { - ZSTD_freeCDict(mtctx->cdictLocal); - mtctx->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, - ZSTD_dlm_byCopy, dictContentType, /* note : a loadPrefix becomes an internal CDict */ - params.cParams, mtctx->cMem); - mtctx->cdict = mtctx->cdictLocal; - if (mtctx->cdictLocal == NULL) return ERROR(memory_allocation); - } else { - ZSTD_freeCDict(mtctx->cdictLocal); - mtctx->cdictLocal = NULL; - mtctx->cdict = cdict; - } +/* ===== Seq Pool Wrapper ====== */ - mtctx->targetPrefixSize = ZSTDMT_computeOverlapSize(¶ms); - DEBUGLOG(4, "overlapLog=%i => %u KB", params.overlapLog, (U32)(mtctx->targetPrefixSize>>10)); - mtctx->targetSectionSize = params.jobSize; - if (mtctx->targetSectionSize == 0) { - mtctx->targetSectionSize = 1ULL << ZSTDMT_computeTargetJobLog(¶ms); - } - assert(mtctx->targetSectionSize <= (size_t)ZSTDMT_JOBSIZE_MAX); +typedef ZSTDMT_bufferPool ZSTDMT_seqPool; - if (params.rsyncable) { - /* Aim for the targetsectionSize as the average job size. */ - U32 const jobSizeMB = (U32)(mtctx->targetSectionSize >> 20); - U32 const rsyncBits = ZSTD_highbit32(jobSizeMB) + 20; - assert(jobSizeMB >= 1); - DEBUGLOG(4, "rsyncLog = %u", rsyncBits); - mtctx->rsync.hash = 0; - mtctx->rsync.hitMask = (1ULL << rsyncBits) - 1; - mtctx->rsync.primePower = ZSTD_rollingHash_primePower(RSYNC_LENGTH); - } - if (mtctx->targetSectionSize < mtctx->targetPrefixSize) mtctx->targetSectionSize = mtctx->targetPrefixSize; /* job size must be >= overlap size */ - DEBUGLOG(4, "Job Size : %u KB (note : set to %u)", (U32)(mtctx->targetSectionSize>>10), (U32)params.jobSize); - DEBUGLOG(4, "inBuff Size : %u KB", (U32)(mtctx->targetSectionSize>>10)); - ZSTDMT_setBufferSize(mtctx->bufPool, ZSTD_compressBound(mtctx->targetSectionSize)); - { - /* If ldm is enabled we need windowSize space. */ - size_t const windowSize = mtctx->params.ldmParams.enableLdm ? (1U << mtctx->params.cParams.windowLog) : 0; - /* Two buffers of slack, plus extra space for the overlap - * This is the minimum slack that LDM works with. One extra because - * flush might waste up to targetSectionSize-1 bytes. Another extra - * for the overlap (if > 0), then one to fill which doesn't overlap - * with the LDM window. - */ - size_t const nbSlackBuffers = 2 + (mtctx->targetPrefixSize > 0); - size_t const slackSize = mtctx->targetSectionSize * nbSlackBuffers; - /* Compute the total size, and always have enough slack */ - size_t const nbWorkers = MAX(mtctx->params.nbWorkers, 1); - size_t const sectionsSize = mtctx->targetSectionSize * nbWorkers; - size_t const capacity = MAX(windowSize, sectionsSize) + slackSize; - if (mtctx->roundBuff.capacity < capacity) { - if (mtctx->roundBuff.buffer) - ZSTD_customFree(mtctx->roundBuff.buffer, mtctx->cMem); - mtctx->roundBuff.buffer = (BYTE*)ZSTD_customMalloc(capacity, mtctx->cMem); - if (mtctx->roundBuff.buffer == NULL) { - mtctx->roundBuff.capacity = 0; - return ERROR(memory_allocation); - } - mtctx->roundBuff.capacity = capacity; - } - } - DEBUGLOG(4, "roundBuff capacity : %u KB", (U32)(mtctx->roundBuff.capacity>>10)); - mtctx->roundBuff.pos = 0; - mtctx->inBuff.buffer = g_nullBuffer; - mtctx->inBuff.filled = 0; - mtctx->inBuff.prefix = kNullRange; - mtctx->doneJobID = 0; - mtctx->nextJobID = 0; - mtctx->frameEnded = 0; - mtctx->allJobsCompleted = 0; - mtctx->consumed = 0; - mtctx->produced = 0; - if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, mtctx->targetSectionSize, - dict, dictSize, dictContentType)) - return ERROR(memory_allocation); - return 0; +static size_t ZSTDMT_sizeof_seqPool(ZSTDMT_seqPool* seqPool) +{ + return ZSTDMT_sizeof_bufferPool(seqPool); } - -/* ZSTDMT_writeLastEmptyBlock() - * Write a single empty block with an end-of-frame to finish a frame. - * Job must be created from streaming variant. - * This function is always successful if expected conditions are fulfilled. - */ -static void ZSTDMT_writeLastEmptyBlock(ZSTDMT_jobDescription* job) +static RawSeqStore_t bufferToSeq(Buffer buffer) { - assert(job->lastJob == 1); - assert(job->src.size == 0); /* last job is empty -> will be simplified into a last empty block */ - assert(job->firstJob == 0); /* cannot be first job, as it also needs to create frame header */ - assert(job->dstBuff.start == NULL); /* invoked from streaming variant only (otherwise, dstBuff might be user's output) */ - job->dstBuff = ZSTDMT_getBuffer(job->bufPool); - if (job->dstBuff.start == NULL) { - job->cSize = ERROR(memory_allocation); - return; - } - assert(job->dstBuff.capacity >= ZSTD_blockHeaderSize); /* no buffer should ever be that small */ - job->src = kNullRange; - job->cSize = ZSTD_writeLastEmptyBlock(job->dstBuff.start, job->dstBuff.capacity); - assert(!ZSTD_isError(job->cSize)); - assert(job->consumed == 0); + RawSeqStore_t seq = kNullRawSeqStore; + seq.seq = (rawSeq*)buffer.start; + seq.capacity = buffer.capacity / sizeof(rawSeq); + return seq; } -static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* mtctx, size_t srcSize, ZSTD_EndDirective endOp) +static Buffer seqToBuffer(RawSeqStore_t seq) { - unsigned const jobID = mtctx->nextJobID & mtctx->jobIDMask; - int const endFrame = (endOp == ZSTD_e_end); + Buffer buffer; + buffer.start = seq.seq; + buffer.capacity = seq.capacity * sizeof(rawSeq); + return buffer; +} - if (mtctx->nextJobID > mtctx->doneJobID + mtctx->jobIDMask) { - DEBUGLOG(5, "ZSTDMT_createCompressionJob: will not create new job : table is full"); - assert((mtctx->nextJobID & mtctx->jobIDMask) == (mtctx->doneJobID & mtctx->jobIDMask)); - return 0; +static RawSeqStore_t ZSTDMT_getSeq(ZSTDMT_seqPool* seqPool) +{ + if (seqPool->bufferSize == 0) { + return kNullRawSeqStore; } + return bufferToSeq(ZSTDMT_getBuffer(seqPool)); +} - if (!mtctx->jobReady) { - BYTE const* src = (BYTE const*)mtctx->inBuff.buffer.start; - DEBUGLOG(5, "ZSTDMT_createCompressionJob: preparing job %u to compress %u bytes with %u preload ", - mtctx->nextJobID, (U32)srcSize, (U32)mtctx->inBuff.prefix.size); - mtctx->jobs[jobID].src.start = src; - mtctx->jobs[jobID].src.size = srcSize; - assert(mtctx->inBuff.filled >= srcSize); - mtctx->jobs[jobID].prefix = mtctx->inBuff.prefix; - mtctx->jobs[jobID].consumed = 0; - mtctx->jobs[jobID].cSize = 0; - mtctx->jobs[jobID].params = mtctx->params; - mtctx->jobs[jobID].cdict = mtctx->nextJobID==0 ? mtctx->cdict : NULL; - mtctx->jobs[jobID].fullFrameSize = mtctx->frameContentSize; - mtctx->jobs[jobID].dstBuff = g_nullBuffer; - mtctx->jobs[jobID].cctxPool = mtctx->cctxPool; - mtctx->jobs[jobID].bufPool = mtctx->bufPool; - mtctx->jobs[jobID].seqPool = mtctx->seqPool; - mtctx->jobs[jobID].serial = &mtctx->serial; - mtctx->jobs[jobID].jobID = mtctx->nextJobID; - mtctx->jobs[jobID].firstJob = (mtctx->nextJobID==0); - mtctx->jobs[jobID].lastJob = endFrame; - mtctx->jobs[jobID].frameChecksumNeeded = mtctx->params.fParams.checksumFlag && endFrame && (mtctx->nextJobID>0); - mtctx->jobs[jobID].dstFlushed = 0; +#if ZSTD_RESIZE_SEQPOOL +static RawSeqStore_t ZSTDMT_resizeSeq(ZSTDMT_seqPool* seqPool, RawSeqStore_t seq) +{ + return bufferToSeq(ZSTDMT_resizeBuffer(seqPool, seqToBuffer(seq))); +} +#endif - /* Update the round buffer pos and clear the input buffer to be reset */ - mtctx->roundBuff.pos += srcSize; - mtctx->inBuff.buffer = g_nullBuffer; - mtctx->inBuff.filled = 0; - /* Set the prefix */ - if (!endFrame) { - size_t const newPrefixSize = MIN(srcSize, mtctx->targetPrefixSize); - mtctx->inBuff.prefix.start = src + srcSize - newPrefixSize; - mtctx->inBuff.prefix.size = newPrefixSize; - } else { /* endFrame==1 => no need for another input buffer */ - mtctx->inBuff.prefix = kNullRange; - mtctx->frameEnded = endFrame; - if (mtctx->nextJobID == 0) { - /* single job exception : checksum is already calculated directly within worker thread */ - mtctx->params.fParams.checksumFlag = 0; - } } +static void ZSTDMT_releaseSeq(ZSTDMT_seqPool* seqPool, RawSeqStore_t seq) +{ + ZSTDMT_releaseBuffer(seqPool, seqToBuffer(seq)); +} - if ( (srcSize == 0) - && (mtctx->nextJobID>0)/*single job must also write frame header*/ ) { - DEBUGLOG(5, "ZSTDMT_createCompressionJob: creating a last empty block to end frame"); - assert(endOp == ZSTD_e_end); /* only possible case : need to end the frame with an empty last block */ - ZSTDMT_writeLastEmptyBlock(mtctx->jobs + jobID); - mtctx->nextJobID++; - return 0; - } - } +static void ZSTDMT_setNbSeq(ZSTDMT_seqPool* const seqPool, size_t const nbSeq) +{ + ZSTDMT_setBufferSize(seqPool, nbSeq * sizeof(rawSeq)); +} - DEBUGLOG(5, "ZSTDMT_createCompressionJob: posting job %u : %u bytes (end:%u, jobNb == %u (mod:%u))", - mtctx->nextJobID, - (U32)mtctx->jobs[jobID].src.size, - mtctx->jobs[jobID].lastJob, - mtctx->nextJobID, - jobID); - if (POOL_tryAdd(mtctx->factory, ZSTDMT_compressionJob, &mtctx->jobs[jobID])) { - mtctx->nextJobID++; - mtctx->jobReady = 0; - } else { - DEBUGLOG(5, "ZSTDMT_createCompressionJob: no worker available for job %u", mtctx->nextJobID); - mtctx->jobReady = 1; - } - return 0; +static ZSTDMT_seqPool* ZSTDMT_createSeqPool(unsigned nbWorkers, ZSTD_customMem cMem) +{ + ZSTDMT_seqPool* const seqPool = ZSTDMT_createBufferPool(SEQ_POOL_MAX_NB_BUFFERS(nbWorkers), cMem); + if (seqPool == NULL) return NULL; + ZSTDMT_setNbSeq(seqPool, 0); + return seqPool; } +static void ZSTDMT_freeSeqPool(ZSTDMT_seqPool* seqPool) +{ + ZSTDMT_freeBufferPool(seqPool); +} -/*! ZSTDMT_flushProduced() : - * flush whatever data has been produced but not yet flushed in current job. - * move to next job if current one is fully flushed. - * `output` : `pos` will be updated with amount of data flushed . - * `blockToFlush` : if >0, the function will block and wait if there is no data available to flush . - * @return : amount of data remaining within internal buffer, 0 if no more, 1 if unknown but > 0, or an error code */ -static size_t ZSTDMT_flushProduced(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, unsigned blockToFlush, ZSTD_EndDirective end) +static ZSTDMT_seqPool* ZSTDMT_expandSeqPool(ZSTDMT_seqPool* pool, U32 nbWorkers) { - unsigned const wJobID = mtctx->doneJobID & mtctx->jobIDMask; - DEBUGLOG(5, "ZSTDMT_flushProduced (blocking:%u , job %u <= %u)", - blockToFlush, mtctx->doneJobID, mtctx->nextJobID); - assert(output->size >= output->pos); + return ZSTDMT_expandBufferPool(pool, SEQ_POOL_MAX_NB_BUFFERS(nbWorkers)); +} - ZSTD_PTHREAD_MUTEX_LOCK(&mtctx->jobs[wJobID].job_mutex); - if ( blockToFlush - && (mtctx->doneJobID < mtctx->nextJobID) ) { - assert(mtctx->jobs[wJobID].dstFlushed <= mtctx->jobs[wJobID].cSize); - while (mtctx->jobs[wJobID].dstFlushed == mtctx->jobs[wJobID].cSize) { /* nothing to flush */ - if (mtctx->jobs[wJobID].consumed == mtctx->jobs[wJobID].src.size) { - DEBUGLOG(5, "job %u is completely consumed (%u == %u) => don't wait for cond, there will be none", - mtctx->doneJobID, (U32)mtctx->jobs[wJobID].consumed, (U32)mtctx->jobs[wJobID].src.size); - break; - } - DEBUGLOG(5, "waiting for something to flush from job %u (currently flushed: %u bytes)", - mtctx->doneJobID, (U32)mtctx->jobs[wJobID].dstFlushed); - ZSTD_pthread_cond_wait(&mtctx->jobs[wJobID].job_cond, &mtctx->jobs[wJobID].job_mutex); /* block when nothing to flush but some to come */ - } } - /* try to flush something */ - { size_t cSize = mtctx->jobs[wJobID].cSize; /* shared */ - size_t const srcConsumed = mtctx->jobs[wJobID].consumed; /* shared */ - size_t const srcSize = mtctx->jobs[wJobID].src.size; /* read-only, could be done after mutex lock, but no-declaration-after-statement */ - ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex); - if (ZSTD_isError(cSize)) { - DEBUGLOG(5, "ZSTDMT_flushProduced: job %u : compression error detected : %s", - mtctx->doneJobID, ZSTD_getErrorName(cSize)); - ZSTDMT_waitForAllJobsCompleted(mtctx); - ZSTDMT_releaseAllJobResources(mtctx); - return cSize; - } - /* add frame checksum if necessary (can only happen once) */ - assert(srcConsumed <= srcSize); - if ( (srcConsumed == srcSize) /* job completed -> worker no longer active */ - && mtctx->jobs[wJobID].frameChecksumNeeded ) { - U32 const checksum = (U32)XXH64_digest(&mtctx->serial.xxhState); - DEBUGLOG(4, "ZSTDMT_flushProduced: writing checksum : %08X \n", checksum); - MEM_writeLE32((char*)mtctx->jobs[wJobID].dstBuff.start + mtctx->jobs[wJobID].cSize, checksum); - cSize += 4; - mtctx->jobs[wJobID].cSize += 4; /* can write this shared value, as worker is no longer active */ - mtctx->jobs[wJobID].frameChecksumNeeded = 0; - } +/* ===== CCtx Pool ===== */ +/* a single CCtx Pool can be invoked from multiple threads in parallel */ - if (cSize > 0) { /* compression is ongoing or completed */ - size_t const toFlush = MIN(cSize - mtctx->jobs[wJobID].dstFlushed, output->size - output->pos); - DEBUGLOG(5, "ZSTDMT_flushProduced: Flushing %u bytes from job %u (completion:%u/%u, generated:%u)", - (U32)toFlush, mtctx->doneJobID, (U32)srcConsumed, (U32)srcSize, (U32)cSize); - assert(mtctx->doneJobID < mtctx->nextJobID); - assert(cSize >= mtctx->jobs[wJobID].dstFlushed); - assert(mtctx->jobs[wJobID].dstBuff.start != NULL); - if (toFlush > 0) { - ZSTD_memcpy((char*)output->dst + output->pos, - (const char*)mtctx->jobs[wJobID].dstBuff.start + mtctx->jobs[wJobID].dstFlushed, - toFlush); - } - output->pos += toFlush; - mtctx->jobs[wJobID].dstFlushed += toFlush; /* can write : this value is only used by mtctx */ +typedef struct { + ZSTD_pthread_mutex_t poolMutex; + int totalCCtx; + int availCCtx; + ZSTD_customMem cMem; + ZSTD_CCtx** cctxs; +} ZSTDMT_CCtxPool; - if ( (srcConsumed == srcSize) /* job is completed */ - && (mtctx->jobs[wJobID].dstFlushed == cSize) ) { /* output buffer fully flushed => free this job position */ - DEBUGLOG(5, "Job %u completed (%u bytes), moving to next one", - mtctx->doneJobID, (U32)mtctx->jobs[wJobID].dstFlushed); - ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[wJobID].dstBuff); - DEBUGLOG(5, "dstBuffer released"); - mtctx->jobs[wJobID].dstBuff = g_nullBuffer; - mtctx->jobs[wJobID].cSize = 0; /* ensure this job slot is considered "not started" in future check */ - mtctx->consumed += srcSize; - mtctx->produced += cSize; - mtctx->doneJobID++; - } } +/* note : all CCtx borrowed from the pool must be reverted back to the pool _before_ freeing the pool */ +static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool) +{ + if (!pool) return; + ZSTD_pthread_mutex_destroy(&pool->poolMutex); + if (pool->cctxs) { + int cid; + for (cid=0; cidtotalCCtx; cid++) + ZSTD_freeCCtx(pool->cctxs[cid]); /* free compatible with NULL */ + ZSTD_customFree(pool->cctxs, pool->cMem); + } + ZSTD_customFree(pool, pool->cMem); +} - /* return value : how many bytes left in buffer ; fake it to 1 when unknown but >0 */ - if (cSize > mtctx->jobs[wJobID].dstFlushed) return (cSize - mtctx->jobs[wJobID].dstFlushed); - if (srcSize > srcConsumed) return 1; /* current job not completely compressed */ +/* ZSTDMT_createCCtxPool() : + * implies nbWorkers >= 1 , checked by caller ZSTDMT_createCCtx() */ +static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(int nbWorkers, + ZSTD_customMem cMem) +{ + ZSTDMT_CCtxPool* const cctxPool = + (ZSTDMT_CCtxPool*) ZSTD_customCalloc(sizeof(ZSTDMT_CCtxPool), cMem); + assert(nbWorkers > 0); + if (!cctxPool) return NULL; + if (ZSTD_pthread_mutex_init(&cctxPool->poolMutex, NULL)) { + ZSTD_customFree(cctxPool, cMem); + return NULL; + } + cctxPool->totalCCtx = nbWorkers; + cctxPool->cctxs = (ZSTD_CCtx**)ZSTD_customCalloc(nbWorkers * sizeof(ZSTD_CCtx*), cMem); + if (!cctxPool->cctxs) { + ZSTDMT_freeCCtxPool(cctxPool); + return NULL; + } + cctxPool->cMem = cMem; + cctxPool->cctxs[0] = ZSTD_createCCtx_advanced(cMem); + if (!cctxPool->cctxs[0]) { ZSTDMT_freeCCtxPool(cctxPool); return NULL; } + cctxPool->availCCtx = 1; /* at least one cctx for single-thread mode */ + DEBUGLOG(3, "cctxPool created, with %u workers", nbWorkers); + return cctxPool; +} + +static ZSTDMT_CCtxPool* ZSTDMT_expandCCtxPool(ZSTDMT_CCtxPool* srcPool, + int nbWorkers) +{ + if (srcPool==NULL) return NULL; + if (nbWorkers <= srcPool->totalCCtx) return srcPool; /* good enough */ + /* need a larger cctx pool */ + { ZSTD_customMem const cMem = srcPool->cMem; + ZSTDMT_freeCCtxPool(srcPool); + return ZSTDMT_createCCtxPool(nbWorkers, cMem); + } +} + +/* only works during initialization phase, not during compression */ +static size_t ZSTDMT_sizeof_CCtxPool(ZSTDMT_CCtxPool* cctxPool) +{ + ZSTD_pthread_mutex_lock(&cctxPool->poolMutex); + { unsigned const nbWorkers = cctxPool->totalCCtx; + size_t const poolSize = sizeof(*cctxPool); + size_t const arraySize = cctxPool->totalCCtx * sizeof(ZSTD_CCtx*); + size_t totalCCtxSize = 0; + unsigned u; + for (u=0; ucctxs[u]); + } + ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex); + assert(nbWorkers > 0); + return poolSize + arraySize + totalCCtxSize; } - if (mtctx->doneJobID < mtctx->nextJobID) return 1; /* some more jobs ongoing */ - if (mtctx->jobReady) return 1; /* one job is ready to push, just not yet in the list */ - if (mtctx->inBuff.filled > 0) return 1; /* input is not empty, and still needs to be converted into a job */ - mtctx->allJobsCompleted = mtctx->frameEnded; /* all jobs are entirely flushed => if this one is last one, frame is completed */ - if (end == ZSTD_e_end) return !mtctx->frameEnded; /* for ZSTD_e_end, question becomes : is frame completed ? instead of : are internal buffers fully flushed ? */ - return 0; /* internal buffers fully flushed */ } -/** - * Returns the range of data used by the earliest job that is not yet complete. - * If the data of the first job is broken up into two segments, we cover both - * sections. - */ -static range_t ZSTDMT_getInputDataInUse(ZSTDMT_CCtx* mtctx) +static ZSTD_CCtx* ZSTDMT_getCCtx(ZSTDMT_CCtxPool* cctxPool) { - unsigned const firstJobID = mtctx->doneJobID; - unsigned const lastJobID = mtctx->nextJobID; - unsigned jobID; + DEBUGLOG(5, "ZSTDMT_getCCtx"); + ZSTD_pthread_mutex_lock(&cctxPool->poolMutex); + if (cctxPool->availCCtx) { + cctxPool->availCCtx--; + { ZSTD_CCtx* const cctx = cctxPool->cctxs[cctxPool->availCCtx]; + ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex); + return cctx; + } } + ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex); + DEBUGLOG(5, "create one more CCtx"); + return ZSTD_createCCtx_advanced(cctxPool->cMem); /* note : can be NULL, when creation fails ! */ +} - for (jobID = firstJobID; jobID < lastJobID; ++jobID) { - unsigned const wJobID = jobID & mtctx->jobIDMask; - size_t consumed; +static void ZSTDMT_releaseCCtx(ZSTDMT_CCtxPool* pool, ZSTD_CCtx* cctx) +{ + if (cctx==NULL) return; /* compatibility with release on NULL */ + ZSTD_pthread_mutex_lock(&pool->poolMutex); + if (pool->availCCtx < pool->totalCCtx) + pool->cctxs[pool->availCCtx++] = cctx; + else { + /* pool overflow : should not happen, since totalCCtx==nbWorkers */ + DEBUGLOG(4, "CCtx pool overflow : free cctx"); + ZSTD_freeCCtx(cctx); + } + ZSTD_pthread_mutex_unlock(&pool->poolMutex); +} - ZSTD_PTHREAD_MUTEX_LOCK(&mtctx->jobs[wJobID].job_mutex); - consumed = mtctx->jobs[wJobID].consumed; - ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex); +/* ==== Serial State ==== */ - if (consumed < mtctx->jobs[wJobID].src.size) { - range_t range = mtctx->jobs[wJobID].prefix; - if (range.size == 0) { - /* Empty prefix */ - range = mtctx->jobs[wJobID].src; +typedef struct { + void const* start; + size_t size; +} Range; + +typedef struct { + /* All variables in the struct are protected by mutex. */ + ZSTD_pthread_mutex_t mutex; + ZSTD_pthread_cond_t cond; + ZSTD_CCtx_params params; + ldmState_t ldmState; + XXH64_state_t xxhState; + unsigned nextJobID; + /* Protects ldmWindow. + * Must be acquired after the main mutex when acquiring both. + */ + ZSTD_pthread_mutex_t ldmWindowMutex; + ZSTD_pthread_cond_t ldmWindowCond; /* Signaled when ldmWindow is updated */ + ZSTD_window_t ldmWindow; /* A thread-safe copy of ldmState.window */ +} SerialState; + +static int +ZSTDMT_serialState_reset(SerialState* serialState, + ZSTDMT_seqPool* seqPool, + ZSTD_CCtx_params params, + size_t jobSize, + const void* dict, size_t const dictSize, + ZSTD_dictContentType_e dictContentType) +{ + /* Adjust parameters */ + if (params.ldmParams.enableLdm == ZSTD_ps_enable) { + DEBUGLOG(4, "LDM window size = %u KB", (1U << params.cParams.windowLog) >> 10); + ZSTD_ldm_adjustParameters(¶ms.ldmParams, ¶ms.cParams); + assert(params.ldmParams.hashLog >= params.ldmParams.bucketSizeLog); + assert(params.ldmParams.hashRateLog < 32); + } else { + ZSTD_memset(¶ms.ldmParams, 0, sizeof(params.ldmParams)); + } + serialState->nextJobID = 0; + if (params.fParams.checksumFlag) + XXH64_reset(&serialState->xxhState, 0); + if (params.ldmParams.enableLdm == ZSTD_ps_enable) { + ZSTD_customMem cMem = params.customMem; + unsigned const hashLog = params.ldmParams.hashLog; + size_t const hashSize = ((size_t)1 << hashLog) * sizeof(ldmEntry_t); + unsigned const bucketLog = + params.ldmParams.hashLog - params.ldmParams.bucketSizeLog; + unsigned const prevBucketLog = + serialState->params.ldmParams.hashLog - + serialState->params.ldmParams.bucketSizeLog; + size_t const numBuckets = (size_t)1 << bucketLog; + /* Size the seq pool tables */ + ZSTDMT_setNbSeq(seqPool, ZSTD_ldm_getMaxNbSeq(params.ldmParams, jobSize)); + /* Reset the window */ + ZSTD_window_init(&serialState->ldmState.window); + /* Resize tables and output space if necessary. */ + if (serialState->ldmState.hashTable == NULL || serialState->params.ldmParams.hashLog < hashLog) { + ZSTD_customFree(serialState->ldmState.hashTable, cMem); + serialState->ldmState.hashTable = (ldmEntry_t*)ZSTD_customMalloc(hashSize, cMem); + } + if (serialState->ldmState.bucketOffsets == NULL || prevBucketLog < bucketLog) { + ZSTD_customFree(serialState->ldmState.bucketOffsets, cMem); + serialState->ldmState.bucketOffsets = (BYTE*)ZSTD_customMalloc(numBuckets, cMem); + } + if (!serialState->ldmState.hashTable || !serialState->ldmState.bucketOffsets) + return 1; + /* Zero the tables */ + ZSTD_memset(serialState->ldmState.hashTable, 0, hashSize); + ZSTD_memset(serialState->ldmState.bucketOffsets, 0, numBuckets); + + /* Update window state and fill hash table with dict */ + serialState->ldmState.loadedDictEnd = 0; + if (dictSize > 0) { + if (dictContentType == ZSTD_dct_rawContent) { + BYTE const* const dictEnd = (const BYTE*)dict + dictSize; + ZSTD_window_update(&serialState->ldmState.window, dict, dictSize, /* forceNonContiguous */ 0); + ZSTD_ldm_fillHashTable(&serialState->ldmState, (const BYTE*)dict, dictEnd, ¶ms.ldmParams); + serialState->ldmState.loadedDictEnd = params.forceWindow ? 0 : (U32)(dictEnd - serialState->ldmState.window.base); + } else { + /* don't even load anything */ } - /* Job source in multiple segments not supported yet */ - assert(range.start <= mtctx->jobs[wJobID].src.start); - return range; } + + /* Initialize serialState's copy of ldmWindow. */ + serialState->ldmWindow = serialState->ldmState.window; } - return kNullRange; + + serialState->params = params; + serialState->params.jobSize = (U32)jobSize; + return 0; } -/** - * Returns non-zero iff buffer and range overlap. - */ -static int ZSTDMT_isOverlapped(buffer_t buffer, range_t range) +static int ZSTDMT_serialState_init(SerialState* serialState) { - BYTE const* const bufferStart = (BYTE const*)buffer.start; - BYTE const* const bufferEnd = bufferStart + buffer.capacity; - BYTE const* const rangeStart = (BYTE const*)range.start; - BYTE const* const rangeEnd = range.size != 0 ? rangeStart + range.size : rangeStart; - - if (rangeStart == NULL || bufferStart == NULL) - return 0; - /* Empty ranges cannot overlap */ - if (bufferStart == bufferEnd || rangeStart == rangeEnd) - return 0; - - return bufferStart < rangeEnd && rangeStart < bufferEnd; + int initError = 0; + ZSTD_memset(serialState, 0, sizeof(*serialState)); + initError |= ZSTD_pthread_mutex_init(&serialState->mutex, NULL); + initError |= ZSTD_pthread_cond_init(&serialState->cond, NULL); + initError |= ZSTD_pthread_mutex_init(&serialState->ldmWindowMutex, NULL); + initError |= ZSTD_pthread_cond_init(&serialState->ldmWindowCond, NULL); + return initError; } -static int ZSTDMT_doesOverlapWindow(buffer_t buffer, ZSTD_window_t window) +static void ZSTDMT_serialState_free(SerialState* serialState) { - range_t extDict; - range_t prefix; - - DEBUGLOG(5, "ZSTDMT_doesOverlapWindow"); - extDict.start = window.dictBase + window.lowLimit; - extDict.size = window.dictLimit - window.lowLimit; - - prefix.start = window.base + window.dictLimit; - prefix.size = window.nextSrc - (window.base + window.dictLimit); - DEBUGLOG(5, "extDict [0x%zx, 0x%zx)", - (size_t)extDict.start, - (size_t)extDict.start + extDict.size); - DEBUGLOG(5, "prefix [0x%zx, 0x%zx)", - (size_t)prefix.start, - (size_t)prefix.start + prefix.size); - - return ZSTDMT_isOverlapped(buffer, extDict) - || ZSTDMT_isOverlapped(buffer, prefix); + ZSTD_customMem cMem = serialState->params.customMem; + ZSTD_pthread_mutex_destroy(&serialState->mutex); + ZSTD_pthread_cond_destroy(&serialState->cond); + ZSTD_pthread_mutex_destroy(&serialState->ldmWindowMutex); + ZSTD_pthread_cond_destroy(&serialState->ldmWindowCond); + ZSTD_customFree(serialState->ldmState.hashTable, cMem); + ZSTD_customFree(serialState->ldmState.bucketOffsets, cMem); } -static void ZSTDMT_waitForLdmComplete(ZSTDMT_CCtx* mtctx, buffer_t buffer) +static void +ZSTDMT_serialState_genSequences(SerialState* serialState, + RawSeqStore_t* seqStore, + Range src, unsigned jobID) { - if (mtctx->params.ldmParams.enableLdm) { - ZSTD_pthread_mutex_t* mutex = &mtctx->serial.ldmWindowMutex; - DEBUGLOG(5, "ZSTDMT_waitForLdmComplete"); - DEBUGLOG(5, "source [0x%zx, 0x%zx)", - (size_t)buffer.start, - (size_t)buffer.start + buffer.capacity); - ZSTD_PTHREAD_MUTEX_LOCK(mutex); - while (ZSTDMT_doesOverlapWindow(buffer, mtctx->serial.ldmWindow)) { - DEBUGLOG(5, "Waiting for LDM to finish..."); - ZSTD_pthread_cond_wait(&mtctx->serial.ldmWindowCond, mutex); + /* Wait for our turn */ + ZSTD_PTHREAD_MUTEX_LOCK(&serialState->mutex); + while (serialState->nextJobID < jobID) { + DEBUGLOG(5, "wait for serialState->cond"); + ZSTD_pthread_cond_wait(&serialState->cond, &serialState->mutex); + } + /* A future job may error and skip our job */ + if (serialState->nextJobID == jobID) { + /* It is now our turn, do any processing necessary */ + if (serialState->params.ldmParams.enableLdm == ZSTD_ps_enable) { + size_t error; + DEBUGLOG(6, "ZSTDMT_serialState_genSequences: LDM update"); + assert(seqStore->seq != NULL && seqStore->pos == 0 && + seqStore->size == 0 && seqStore->capacity > 0); + assert(src.size <= serialState->params.jobSize); + ZSTD_window_update(&serialState->ldmState.window, src.start, src.size, /* forceNonContiguous */ 0); + error = ZSTD_ldm_generateSequences( + &serialState->ldmState, seqStore, + &serialState->params.ldmParams, src.start, src.size); + /* We provide a large enough buffer to never fail. */ + assert(!ZSTD_isError(error)); (void)error; + /* Update ldmWindow to match the ldmState.window and signal the main + * thread if it is waiting for a buffer. + */ + ZSTD_PTHREAD_MUTEX_LOCK(&serialState->ldmWindowMutex); + serialState->ldmWindow = serialState->ldmState.window; + ZSTD_pthread_cond_signal(&serialState->ldmWindowCond); + ZSTD_pthread_mutex_unlock(&serialState->ldmWindowMutex); } - DEBUGLOG(6, "Done waiting for LDM to finish"); - ZSTD_pthread_mutex_unlock(mutex); + if (serialState->params.fParams.checksumFlag && src.size > 0) + XXH64_update(&serialState->xxhState, src.start, src.size); } + /* Now it is the next jobs turn */ + serialState->nextJobID++; + ZSTD_pthread_cond_broadcast(&serialState->cond); + ZSTD_pthread_mutex_unlock(&serialState->mutex); } -/** - * Attempts to set the inBuff to the next section to fill. - * If any part of the new section is still in use we give up. - * Returns non-zero if the buffer is filled. - */ -static int ZSTDMT_tryGetInputRange(ZSTDMT_CCtx* mtctx) +static void +ZSTDMT_serialState_applySequences(const SerialState* serialState, /* just for an assert() check */ + ZSTD_CCtx* jobCCtx, + const RawSeqStore_t* seqStore) { - range_t const inUse = ZSTDMT_getInputDataInUse(mtctx); - size_t const spaceLeft = mtctx->roundBuff.capacity - mtctx->roundBuff.pos; - size_t const target = mtctx->targetSectionSize; - buffer_t buffer; - - DEBUGLOG(5, "ZSTDMT_tryGetInputRange"); - assert(mtctx->inBuff.buffer.start == NULL); - assert(mtctx->roundBuff.capacity >= target); - - if (spaceLeft < target) { - /* ZSTD_invalidateRepCodes() doesn't work for extDict variants. - * Simply copy the prefix to the beginning in that case. - */ - BYTE* const start = (BYTE*)mtctx->roundBuff.buffer; - size_t const prefixSize = mtctx->inBuff.prefix.size; - - buffer.start = start; - buffer.capacity = prefixSize; - if (ZSTDMT_isOverlapped(buffer, inUse)) { - DEBUGLOG(5, "Waiting for buffer..."); - return 0; - } - ZSTDMT_waitForLdmComplete(mtctx, buffer); - ZSTD_memmove(start, mtctx->inBuff.prefix.start, prefixSize); - mtctx->inBuff.prefix.start = start; - mtctx->roundBuff.pos = prefixSize; + if (seqStore->size > 0) { + DEBUGLOG(5, "ZSTDMT_serialState_applySequences: uploading %u external sequences", (unsigned)seqStore->size); + assert(serialState->params.ldmParams.enableLdm == ZSTD_ps_enable); (void)serialState; + assert(jobCCtx); + ZSTD_referenceExternalSequences(jobCCtx, seqStore->seq, seqStore->size); } - buffer.start = mtctx->roundBuff.buffer + mtctx->roundBuff.pos; - buffer.capacity = target; +} + +static void ZSTDMT_serialState_ensureFinished(SerialState* serialState, + unsigned jobID, size_t cSize) +{ + ZSTD_PTHREAD_MUTEX_LOCK(&serialState->mutex); + if (serialState->nextJobID <= jobID) { + assert(ZSTD_isError(cSize)); (void)cSize; + DEBUGLOG(5, "Skipping past job %u because of error", jobID); + serialState->nextJobID = jobID + 1; + ZSTD_pthread_cond_broadcast(&serialState->cond); - if (ZSTDMT_isOverlapped(buffer, inUse)) { - DEBUGLOG(5, "Waiting for buffer..."); - return 0; + ZSTD_PTHREAD_MUTEX_LOCK(&serialState->ldmWindowMutex); + ZSTD_window_clear(&serialState->ldmWindow); + ZSTD_pthread_cond_signal(&serialState->ldmWindowCond); + ZSTD_pthread_mutex_unlock(&serialState->ldmWindowMutex); } - assert(!ZSTDMT_isOverlapped(buffer, mtctx->inBuff.prefix)); + ZSTD_pthread_mutex_unlock(&serialState->mutex); - ZSTDMT_waitForLdmComplete(mtctx, buffer); +} - DEBUGLOG(5, "Using prefix range [%zx, %zx)", - (size_t)mtctx->inBuff.prefix.start, - (size_t)mtctx->inBuff.prefix.start + mtctx->inBuff.prefix.size); - DEBUGLOG(5, "Using source range [%zx, %zx)", - (size_t)buffer.start, - (size_t)buffer.start + buffer.capacity); +/* ------------------------------------------ */ +/* ===== Worker thread ===== */ +/* ------------------------------------------ */ - mtctx->inBuff.buffer = buffer; - mtctx->inBuff.filled = 0; - assert(mtctx->roundBuff.pos + buffer.capacity <= mtctx->roundBuff.capacity); - return 1; -} +static const Range kNullRange = { NULL, 0 }; typedef struct { - size_t toLoad; /* The number of bytes to load from the input. */ - int flush; /* Boolean declaring if we must flush because we found a synchronization point. */ -} syncPoint_t; + size_t consumed; /* SHARED - set0 by mtctx, then modified by worker AND read by mtctx */ + size_t cSize; /* SHARED - set0 by mtctx, then modified by worker AND read by mtctx, then set0 by mtctx */ + ZSTD_pthread_mutex_t job_mutex; /* Thread-safe - used by mtctx and worker */ + ZSTD_pthread_cond_t job_cond; /* Thread-safe - used by mtctx and worker */ + ZSTDMT_CCtxPool* cctxPool; /* Thread-safe - used by mtctx and (all) workers */ + ZSTDMT_bufferPool* bufPool; /* Thread-safe - used by mtctx and (all) workers */ + ZSTDMT_seqPool* seqPool; /* Thread-safe - used by mtctx and (all) workers */ + SerialState* serial; /* Thread-safe - used by mtctx and (all) workers */ + Buffer dstBuff; /* set by worker (or mtctx), then read by worker & mtctx, then modified by mtctx => no barrier */ + Range prefix; /* set by mtctx, then read by worker & mtctx => no barrier */ + Range src; /* set by mtctx, then read by worker & mtctx => no barrier */ + unsigned jobID; /* set by mtctx, then read by worker => no barrier */ + unsigned firstJob; /* set by mtctx, then read by worker => no barrier */ + unsigned lastJob; /* set by mtctx, then read by worker => no barrier */ + ZSTD_CCtx_params params; /* set by mtctx, then read by worker => no barrier */ + const ZSTD_CDict* cdict; /* set by mtctx, then read by worker => no barrier */ + unsigned long long fullFrameSize; /* set by mtctx, then read by worker => no barrier */ + size_t dstFlushed; /* used only by mtctx */ + unsigned frameChecksumNeeded; /* used only by mtctx */ +} ZSTDMT_jobDescription; -/** - * Searches through the input for a synchronization point. If one is found, we - * will instruct the caller to flush, and return the number of bytes to load. - * Otherwise, we will load as many bytes as possible and instruct the caller - * to continue as normal. - */ -static syncPoint_t -findSynchronizationPoint(ZSTDMT_CCtx const* mtctx, ZSTD_inBuffer const input) -{ - BYTE const* const istart = (BYTE const*)input.src + input.pos; - U64 const primePower = mtctx->rsync.primePower; - U64 const hitMask = mtctx->rsync.hitMask; +#define JOB_ERROR(e) \ + do { \ + ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex); \ + job->cSize = e; \ + ZSTD_pthread_mutex_unlock(&job->job_mutex); \ + goto _endJob; \ + } while (0) - syncPoint_t syncPoint; - U64 hash; - BYTE const* prev; - size_t pos; +/* ZSTDMT_compressionJob() is a POOL_function type */ +static void ZSTDMT_compressionJob(void* jobDescription) +{ + ZSTDMT_jobDescription* const job = (ZSTDMT_jobDescription*)jobDescription; + ZSTD_CCtx_params jobParams = job->params; /* do not modify job->params ! copy it, modify the copy */ + ZSTD_CCtx* const cctx = ZSTDMT_getCCtx(job->cctxPool); + RawSeqStore_t rawSeqStore = ZSTDMT_getSeq(job->seqPool); + Buffer dstBuff = job->dstBuff; + size_t lastCBlockSize = 0; - syncPoint.toLoad = MIN(input.size - input.pos, mtctx->targetSectionSize - mtctx->inBuff.filled); - syncPoint.flush = 0; - if (!mtctx->params.rsyncable) - /* Rsync is disabled. */ - return syncPoint; - if (mtctx->inBuff.filled + syncPoint.toLoad < RSYNC_LENGTH) - /* Not enough to compute the hash. - * We will miss any synchronization points in this RSYNC_LENGTH byte - * window. However, since it depends only in the internal buffers, if the - * state is already synchronized, we will remain synchronized. - * Additionally, the probability that we miss a synchronization point is - * low: RSYNC_LENGTH / targetSectionSize. - */ - return syncPoint; - /* Initialize the loop variables. */ - if (mtctx->inBuff.filled >= RSYNC_LENGTH) { - /* We have enough bytes buffered to initialize the hash. - * Start scanning at the beginning of the input. - */ - pos = 0; - prev = (BYTE const*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled - RSYNC_LENGTH; - hash = ZSTD_rollingHash_compute(prev, RSYNC_LENGTH); - if ((hash & hitMask) == hitMask) { - /* We're already at a sync point so don't load any more until - * we're able to flush this sync point. - * This likely happened because the job table was full so we - * couldn't add our job. - */ - syncPoint.toLoad = 0; - syncPoint.flush = 1; - return syncPoint; - } - } else { - /* We don't have enough bytes buffered to initialize the hash, but - * we know we have at least RSYNC_LENGTH bytes total. - * Start scanning after the first RSYNC_LENGTH bytes less the bytes - * already buffered. - */ - pos = RSYNC_LENGTH - mtctx->inBuff.filled; - prev = (BYTE const*)mtctx->inBuff.buffer.start - pos; - hash = ZSTD_rollingHash_compute(mtctx->inBuff.buffer.start, mtctx->inBuff.filled); - hash = ZSTD_rollingHash_append(hash, istart, pos); + DEBUGLOG(5, "ZSTDMT_compressionJob: job %u", job->jobID); + /* resources */ + if (cctx==NULL) JOB_ERROR(ERROR(memory_allocation)); + if (dstBuff.start == NULL) { /* streaming job : doesn't provide a dstBuffer */ + dstBuff = ZSTDMT_getBuffer(job->bufPool); + if (dstBuff.start==NULL) JOB_ERROR(ERROR(memory_allocation)); + job->dstBuff = dstBuff; /* this value can be read in ZSTDMT_flush, when it copies the whole job */ } - /* Starting with the hash of the previous RSYNC_LENGTH bytes, roll - * through the input. If we hit a synchronization point, then cut the - * job off, and tell the compressor to flush the job. Otherwise, load - * all the bytes and continue as normal. - * If we go too long without a synchronization point (targetSectionSize) - * then a block will be emitted anyways, but this is okay, since if we - * are already synchronized we will remain synchronized. + if (jobParams.ldmParams.enableLdm == ZSTD_ps_enable && rawSeqStore.seq == NULL) + JOB_ERROR(ERROR(memory_allocation)); + + /* Don't compute the checksum for chunks, since we compute it externally, + * but write it in the header. */ - for (; pos < syncPoint.toLoad; ++pos) { - BYTE const toRemove = pos < RSYNC_LENGTH ? prev[pos] : istart[pos - RSYNC_LENGTH]; - /* if (pos >= RSYNC_LENGTH) assert(ZSTD_rollingHash_compute(istart + pos - RSYNC_LENGTH, RSYNC_LENGTH) == hash); */ - hash = ZSTD_rollingHash_rotate(hash, toRemove, istart[pos], primePower); - if ((hash & hitMask) == hitMask) { - syncPoint.toLoad = pos + 1; - syncPoint.flush = 1; - break; - } - } - return syncPoint; -} + if (job->jobID != 0) jobParams.fParams.checksumFlag = 0; + /* Don't run LDM for the chunks, since we handle it externally */ + jobParams.ldmParams.enableLdm = ZSTD_ps_disable; + /* Correct nbWorkers to 0. */ + jobParams.nbWorkers = 0; -size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx) -{ - size_t hintInSize = mtctx->targetSectionSize - mtctx->inBuff.filled; - if (hintInSize==0) hintInSize = mtctx->targetSectionSize; - return hintInSize; -} -/** ZSTDMT_compressStream_generic() : - * internal use only - exposed to be invoked from zstd_compress.c - * assumption : output and input are valid (pos <= size) - * @return : minimum amount of data remaining to flush, 0 if none */ -size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, - ZSTD_outBuffer* output, - ZSTD_inBuffer* input, - ZSTD_EndDirective endOp) -{ - unsigned forwardInputProgress = 0; - DEBUGLOG(5, "ZSTDMT_compressStream_generic (endOp=%u, srcSize=%u)", - (U32)endOp, (U32)(input->size - input->pos)); - assert(output->pos <= output->size); - assert(input->pos <= input->size); + /* init */ - if ((mtctx->frameEnded) && (endOp==ZSTD_e_continue)) { - /* current frame being ended. Only flush/end are allowed */ - return ERROR(stage_wrong); - } + /* Perform serial step as early as possible */ + ZSTDMT_serialState_genSequences(job->serial, &rawSeqStore, job->src, job->jobID); - /* fill input buffer */ - if ( (!mtctx->jobReady) - && (input->size > input->pos) ) { /* support NULL input */ - if (mtctx->inBuff.buffer.start == NULL) { - assert(mtctx->inBuff.filled == 0); /* Can't fill an empty buffer */ - if (!ZSTDMT_tryGetInputRange(mtctx)) { - /* It is only possible for this operation to fail if there are - * still compression jobs ongoing. - */ - DEBUGLOG(5, "ZSTDMT_tryGetInputRange failed"); - assert(mtctx->doneJobID != mtctx->nextJobID); - } else - DEBUGLOG(5, "ZSTDMT_tryGetInputRange completed successfully : mtctx->inBuff.buffer.start = %p", mtctx->inBuff.buffer.start); + if (job->cdict) { + size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, job->cdict, &jobParams, job->fullFrameSize); + assert(job->firstJob); /* only allowed for first job */ + if (ZSTD_isError(initError)) JOB_ERROR(initError); + } else { + U64 const pledgedSrcSize = job->firstJob ? job->fullFrameSize : job->src.size; + { size_t const forceWindowError = ZSTD_CCtxParams_setParameter(&jobParams, ZSTD_c_forceMaxWindow, !job->firstJob); + if (ZSTD_isError(forceWindowError)) JOB_ERROR(forceWindowError); } - if (mtctx->inBuff.buffer.start != NULL) { - syncPoint_t const syncPoint = findSynchronizationPoint(mtctx, *input); - if (syncPoint.flush && endOp == ZSTD_e_continue) { - endOp = ZSTD_e_flush; - } - assert(mtctx->inBuff.buffer.capacity >= mtctx->targetSectionSize); - DEBUGLOG(5, "ZSTDMT_compressStream_generic: adding %u bytes on top of %u to buffer of size %u", - (U32)syncPoint.toLoad, (U32)mtctx->inBuff.filled, (U32)mtctx->targetSectionSize); - ZSTD_memcpy((char*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled, (const char*)input->src + input->pos, syncPoint.toLoad); - input->pos += syncPoint.toLoad; - mtctx->inBuff.filled += syncPoint.toLoad; - forwardInputProgress = syncPoint.toLoad>0; + if (!job->firstJob) { + size_t const err = ZSTD_CCtxParams_setParameter(&jobParams, ZSTD_c_deterministicRefPrefix, 0); + if (ZSTD_isError(err)) JOB_ERROR(err); } - } - if ((input->pos < input->size) && (endOp == ZSTD_e_end)) { - /* Can't end yet because the input is not fully consumed. - * We are in one of these cases: - * - mtctx->inBuff is NULL & empty: we couldn't get an input buffer so don't create a new job. - * - We filled the input buffer: flush this job but don't end the frame. - * - We hit a synchronization point: flush this job but don't end the frame. - */ - assert(mtctx->inBuff.filled == 0 || mtctx->inBuff.filled == mtctx->targetSectionSize || mtctx->params.rsyncable); - endOp = ZSTD_e_flush; - } + DEBUGLOG(6, "ZSTDMT_compressionJob: job %u: loading prefix of size %zu", job->jobID, job->prefix.size); + { size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, + job->prefix.start, job->prefix.size, ZSTD_dct_rawContent, + ZSTD_dtlm_fast, + NULL, /*cdict*/ + &jobParams, pledgedSrcSize); + if (ZSTD_isError(initError)) JOB_ERROR(initError); + } } - if ( (mtctx->jobReady) - || (mtctx->inBuff.filled >= mtctx->targetSectionSize) /* filled enough : let's compress */ - || ((endOp != ZSTD_e_continue) && (mtctx->inBuff.filled > 0)) /* something to flush : let's go */ - || ((endOp == ZSTD_e_end) && (!mtctx->frameEnded)) ) { /* must finish the frame with a zero-size block */ - size_t const jobSize = mtctx->inBuff.filled; - assert(mtctx->inBuff.filled <= mtctx->targetSectionSize); - FORWARD_IF_ERROR( ZSTDMT_createCompressionJob(mtctx, jobSize, endOp) , ""); + /* External Sequences can only be applied after CCtx initialization */ + ZSTDMT_serialState_applySequences(job->serial, cctx, &rawSeqStore); + + if (!job->firstJob) { /* flush and overwrite frame header when it's not first job */ + size_t const hSize = ZSTD_compressContinue_public(cctx, dstBuff.start, dstBuff.capacity, job->src.start, 0); + if (ZSTD_isError(hSize)) JOB_ERROR(hSize); + DEBUGLOG(5, "ZSTDMT_compressionJob: flush and overwrite %u bytes of frame header (not first job)", (U32)hSize); + ZSTD_invalidateRepCodes(cctx); } - /* check for potential compressed data ready to be flushed */ - { size_t const remainingToFlush = ZSTDMT_flushProduced(mtctx, output, !forwardInputProgress, endOp); /* block if there was no forward input progress */ - if (input->pos < input->size) return MAX(remainingToFlush, 1); /* input not consumed : do not end flush yet */ - DEBUGLOG(5, "end of ZSTDMT_compressStream_generic: remainingToFlush = %u", (U32)remainingToFlush); - return remainingToFlush; + /* compress the entire job by smaller chunks, for better granularity */ + { size_t const chunkSize = 4*ZSTD_BLOCKSIZE_MAX; + int const nbChunks = (int)((job->src.size + (chunkSize-1)) / chunkSize); + const BYTE* ip = (const BYTE*) job->src.start; + BYTE* const ostart = (BYTE*)dstBuff.start; + BYTE* op = ostart; + BYTE* oend = op + dstBuff.capacity; + int chunkNb; + if (sizeof(size_t) > sizeof(int)) assert(job->src.size < ((size_t)INT_MAX) * chunkSize); /* check overflow */ + DEBUGLOG(5, "ZSTDMT_compressionJob: compress %u bytes in %i blocks", (U32)job->src.size, nbChunks); + assert(job->cSize == 0); + for (chunkNb = 1; chunkNb < nbChunks; chunkNb++) { + size_t const cSize = ZSTD_compressContinue_public(cctx, op, oend-op, ip, chunkSize); + if (ZSTD_isError(cSize)) JOB_ERROR(cSize); + ip += chunkSize; + op += cSize; assert(op < oend); + /* stats */ + ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex); + job->cSize += cSize; + job->consumed = chunkSize * chunkNb; + DEBUGLOG(5, "ZSTDMT_compressionJob: compress new block : cSize==%u bytes (total: %u)", + (U32)cSize, (U32)job->cSize); + ZSTD_pthread_cond_signal(&job->job_cond); /* warns some more data is ready to be flushed */ + ZSTD_pthread_mutex_unlock(&job->job_mutex); + } + /* last block */ + assert(chunkSize > 0); + assert((chunkSize & (chunkSize - 1)) == 0); /* chunkSize must be power of 2 for mask==(chunkSize-1) to work */ + if ((nbChunks > 0) | job->lastJob /*must output a "last block" flag*/ ) { + size_t const lastBlockSize1 = job->src.size & (chunkSize-1); + size_t const lastBlockSize = ((lastBlockSize1==0) & (job->src.size>=chunkSize)) ? chunkSize : lastBlockSize1; + size_t const cSize = (job->lastJob) ? + ZSTD_compressEnd_public(cctx, op, oend-op, ip, lastBlockSize) : + ZSTD_compressContinue_public(cctx, op, oend-op, ip, lastBlockSize); + if (ZSTD_isError(cSize)) JOB_ERROR(cSize); + lastCBlockSize = cSize; + } } + if (!job->firstJob) { + /* Double check that we don't have an ext-dict, because then our + * repcode invalidation doesn't work. + */ + assert(!ZSTD_window_hasExtDict(cctx->blockState.matchState.window)); } + ZSTD_CCtx_trace(cctx, 0); + +_endJob: + ZSTDMT_serialState_ensureFinished(job->serial, job->jobID, job->cSize); + if (job->prefix.size > 0) + DEBUGLOG(5, "Finished with prefix: %zx", (size_t)job->prefix.start); + DEBUGLOG(5, "Finished with source: %zx", (size_t)job->src.start); + /* release resources */ + ZSTDMT_releaseSeq(job->seqPool, rawSeqStore); + ZSTDMT_releaseCCtx(job->cctxPool, cctx); + /* report */ + ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex); + if (ZSTD_isError(job->cSize)) assert(lastCBlockSize == 0); + job->cSize += lastCBlockSize; + job->consumed = job->src.size; /* when job->consumed == job->src.size , compression job is presumed completed */ + ZSTD_pthread_cond_signal(&job->job_cond); + ZSTD_pthread_mutex_unlock(&job->job_mutex); } -/**** ended inlining compress/zstdmt_compress.c ****/ -#endif -/**** start inlining decompress/huf_decompress.c ****/ -/* ****************************************************************** - * huff0 huffman decoder, - * part of Finite State Entropy library - * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc. - * - * You can contact the author at : - * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. -****************************************************************** */ -/* ************************************************************** -* Dependencies -****************************************************************/ -/**** skipping file: ../common/zstd_deps.h ****/ -/**** skipping file: ../common/compiler.h ****/ -/**** skipping file: ../common/bitstream.h ****/ -/**** skipping file: ../common/fse.h ****/ -#define HUF_STATIC_LINKING_ONLY -/**** skipping file: ../common/huf.h ****/ -/**** skipping file: ../common/error_private.h ****/ +/* ------------------------------------------ */ +/* ===== Multi-threaded compression ===== */ +/* ------------------------------------------ */ -/* ************************************************************** -* Macros -****************************************************************/ +typedef struct { + Range prefix; /* read-only non-owned prefix buffer */ + Buffer buffer; + size_t filled; +} InBuff_t; -/* These two optional macros force the use one way or another of the two - * Huffman decompression implementations. You can't force in both directions - * at the same time. - */ -#if defined(HUF_FORCE_DECOMPRESS_X1) && \ - defined(HUF_FORCE_DECOMPRESS_X2) -#error "Cannot force the use of the X1 and X2 decoders at the same time!" -#endif +typedef struct { + BYTE* buffer; /* The round input buffer. All jobs get references + * to pieces of the buffer. ZSTDMT_tryGetInputRange() + * handles handing out job input buffers, and makes + * sure it doesn't overlap with any pieces still in use. + */ + size_t capacity; /* The capacity of buffer. */ + size_t pos; /* The position of the current inBuff in the round + * buffer. Updated past the end if the inBuff once + * the inBuff is sent to the worker thread. + * pos <= capacity. + */ +} RoundBuff_t; +static const RoundBuff_t kNullRoundBuff = {NULL, 0, 0}; -/* ************************************************************** -* Error Management -****************************************************************/ -#define HUF_isError ERR_isError +#define RSYNC_LENGTH 32 +/* Don't create chunks smaller than the zstd block size. + * This stops us from regressing compression ratio too much, + * and ensures our output fits in ZSTD_compressBound(). + * + * If this is shrunk < ZSTD_BLOCKSIZELOG_MIN then + * ZSTD_COMPRESSBOUND() will need to be updated. + */ +#define RSYNC_MIN_BLOCK_LOG ZSTD_BLOCKSIZELOG_MAX +#define RSYNC_MIN_BLOCK_SIZE (1< one job is already prepared, but pool has shortage of workers. Don't create a new job. */ + InBuff_t inBuff; + RoundBuff_t roundBuff; + SerialState serial; + RSyncState_t rsync; + unsigned jobIDMask; + unsigned doneJobID; + unsigned nextJobID; + unsigned frameEnded; + unsigned allJobsCompleted; + unsigned long long frameContentSize; + unsigned long long consumed; + unsigned long long produced; + ZSTD_customMem cMem; + ZSTD_CDict* cdictLocal; + const ZSTD_CDict* cdict; + unsigned providedFactory: 1; +}; +static void ZSTDMT_freeJobsTable(ZSTDMT_jobDescription* jobTable, U32 nbJobs, ZSTD_customMem cMem) +{ + U32 jobNb; + if (jobTable == NULL) return; + for (jobNb=0; jobNb mtctx->jobIDMask+1) { /* need more job capacity */ + ZSTDMT_freeJobsTable(mtctx->jobs, mtctx->jobIDMask+1, mtctx->cMem); + mtctx->jobIDMask = 0; + mtctx->jobs = ZSTDMT_createJobsTable(&nbJobs, mtctx->cMem); + if (mtctx->jobs==NULL) return ERROR(memory_allocation); + assert((nbJobs != 0) && ((nbJobs & (nbJobs - 1)) == 0)); /* ensure nbJobs is a power of 2 */ + mtctx->jobIDMask = nbJobs - 1; } + return 0; +} -#else -#define HUF_DGEN(fn) \ - static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ - size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \ - { \ - (void)bmi2; \ - return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ - } +/* ZSTDMT_CCtxParam_setNbWorkers(): + * Internal use only */ +static size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorkers) +{ + return ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, (int)nbWorkers); +} -#endif +MEM_STATIC ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced_internal(unsigned nbWorkers, ZSTD_customMem cMem, ZSTD_threadPool* pool) +{ + ZSTDMT_CCtx* mtctx; + U32 nbJobs = nbWorkers + 2; + int initError; + DEBUGLOG(3, "ZSTDMT_createCCtx_advanced (nbWorkers = %u)", nbWorkers); + if (nbWorkers < 1) return NULL; + nbWorkers = MIN(nbWorkers , ZSTDMT_NBWORKERS_MAX); + if ((cMem.customAlloc!=NULL) ^ (cMem.customFree!=NULL)) + /* invalid custom allocator */ + return NULL; -/*-***************************/ -/* generic DTableDesc */ -/*-***************************/ -typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc; + mtctx = (ZSTDMT_CCtx*) ZSTD_customCalloc(sizeof(ZSTDMT_CCtx), cMem); + if (!mtctx) return NULL; + ZSTDMT_CCtxParam_setNbWorkers(&mtctx->params, nbWorkers); + mtctx->cMem = cMem; + mtctx->allJobsCompleted = 1; + if (pool != NULL) { + mtctx->factory = pool; + mtctx->providedFactory = 1; + } + else { + mtctx->factory = POOL_create_advanced(nbWorkers, 0, cMem); + mtctx->providedFactory = 0; + } + mtctx->jobs = ZSTDMT_createJobsTable(&nbJobs, cMem); + assert(nbJobs > 0); assert((nbJobs & (nbJobs - 1)) == 0); /* ensure nbJobs is a power of 2 */ + mtctx->jobIDMask = nbJobs - 1; + mtctx->bufPool = ZSTDMT_createBufferPool(BUF_POOL_MAX_NB_BUFFERS(nbWorkers), cMem); + mtctx->cctxPool = ZSTDMT_createCCtxPool(nbWorkers, cMem); + mtctx->seqPool = ZSTDMT_createSeqPool(nbWorkers, cMem); + initError = ZSTDMT_serialState_init(&mtctx->serial); + mtctx->roundBuff = kNullRoundBuff; + if (!mtctx->factory | !mtctx->jobs | !mtctx->bufPool | !mtctx->cctxPool | !mtctx->seqPool | initError) { + ZSTDMT_freeCCtx(mtctx); + return NULL; + } + DEBUGLOG(3, "mt_cctx created, for %u threads", nbWorkers); + return mtctx; +} -static DTableDesc HUF_getDTableDesc(const HUF_DTable* table) +ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, ZSTD_customMem cMem, ZSTD_threadPool* pool) { - DTableDesc dtd; - ZSTD_memcpy(&dtd, table, sizeof(dtd)); - return dtd; +#ifdef ZSTD_MULTITHREAD + return ZSTDMT_createCCtx_advanced_internal(nbWorkers, cMem, pool); +#else + (void)nbWorkers; + (void)cMem; + (void)pool; + return NULL; +#endif } -#ifndef HUF_FORCE_DECOMPRESS_X2 +/* ZSTDMT_releaseAllJobResources() : + * note : ensure all workers are killed first ! */ +static void ZSTDMT_releaseAllJobResources(ZSTDMT_CCtx* mtctx) +{ + unsigned jobID; + DEBUGLOG(3, "ZSTDMT_releaseAllJobResources"); + for (jobID=0; jobID <= mtctx->jobIDMask; jobID++) { + /* Copy the mutex/cond out */ + ZSTD_pthread_mutex_t const mutex = mtctx->jobs[jobID].job_mutex; + ZSTD_pthread_cond_t const cond = mtctx->jobs[jobID].job_cond; -/*-***************************/ -/* single-symbol decoding */ -/*-***************************/ -typedef struct { BYTE byte; BYTE nbBits; } HUF_DEltX1; /* single-symbol decoding */ + DEBUGLOG(4, "job%02u: release dst address %08X", jobID, (U32)(size_t)mtctx->jobs[jobID].dstBuff.start); + ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[jobID].dstBuff); -/** - * Packs 4 HUF_DEltX1 structs into a U64. This is used to lay down 4 entries at - * a time. - */ -static U64 HUF_DEltX1_set4(BYTE symbol, BYTE nbBits) { - U64 D4; - if (MEM_isLittleEndian()) { - D4 = symbol + (nbBits << 8); - } else { - D4 = (symbol << 8) + nbBits; + /* Clear the job description, but keep the mutex/cond */ + ZSTD_memset(&mtctx->jobs[jobID], 0, sizeof(mtctx->jobs[jobID])); + mtctx->jobs[jobID].job_mutex = mutex; + mtctx->jobs[jobID].job_cond = cond; } - D4 *= 0x0001000100010001ULL; - return D4; + mtctx->inBuff.buffer = g_nullBuffer; + mtctx->inBuff.filled = 0; + mtctx->allJobsCompleted = 1; } -typedef struct { - U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; - U32 rankStart[HUF_TABLELOG_ABSOLUTEMAX + 1]; - U32 statsWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; - BYTE symbols[HUF_SYMBOLVALUE_MAX + 1]; - BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; -} HUF_ReadDTableX1_Workspace; - - -size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize) +static void ZSTDMT_waitForAllJobsCompleted(ZSTDMT_CCtx* mtctx) { - return HUF_readDTableX1_wksp_bmi2(DTable, src, srcSize, workSpace, wkspSize, /* bmi2 */ 0); + DEBUGLOG(4, "ZSTDMT_waitForAllJobsCompleted"); + while (mtctx->doneJobID < mtctx->nextJobID) { + unsigned const jobID = mtctx->doneJobID & mtctx->jobIDMask; + ZSTD_PTHREAD_MUTEX_LOCK(&mtctx->jobs[jobID].job_mutex); + while (mtctx->jobs[jobID].consumed < mtctx->jobs[jobID].src.size) { + DEBUGLOG(4, "waiting for jobCompleted signal from job %u", mtctx->doneJobID); /* we want to block when waiting for data to flush */ + ZSTD_pthread_cond_wait(&mtctx->jobs[jobID].job_cond, &mtctx->jobs[jobID].job_mutex); + } + ZSTD_pthread_mutex_unlock(&mtctx->jobs[jobID].job_mutex); + mtctx->doneJobID++; + } } -size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2) +size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx) { - U32 tableLog = 0; - U32 nbSymbols = 0; - size_t iSize; - void* const dtPtr = DTable + 1; - HUF_DEltX1* const dt = (HUF_DEltX1*)dtPtr; - HUF_ReadDTableX1_Workspace* wksp = (HUF_ReadDTableX1_Workspace*)workSpace; + if (mtctx==NULL) return 0; /* compatible with free on NULL */ + if (!mtctx->providedFactory) + POOL_free(mtctx->factory); /* stop and free worker threads */ + ZSTDMT_releaseAllJobResources(mtctx); /* release job resources into pools first */ + ZSTDMT_freeJobsTable(mtctx->jobs, mtctx->jobIDMask+1, mtctx->cMem); + ZSTDMT_freeBufferPool(mtctx->bufPool); + ZSTDMT_freeCCtxPool(mtctx->cctxPool); + ZSTDMT_freeSeqPool(mtctx->seqPool); + ZSTDMT_serialState_free(&mtctx->serial); + ZSTD_freeCDict(mtctx->cdictLocal); + if (mtctx->roundBuff.buffer) + ZSTD_customFree(mtctx->roundBuff.buffer, mtctx->cMem); + ZSTD_customFree(mtctx, mtctx->cMem); + return 0; +} - DEBUG_STATIC_ASSERT(HUF_DECOMPRESS_WORKSPACE_SIZE >= sizeof(*wksp)); - if (sizeof(*wksp) > wkspSize) return ERROR(tableLog_tooLarge); +size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx) +{ + if (mtctx == NULL) return 0; /* supports sizeof NULL */ + return sizeof(*mtctx) + + POOL_sizeof(mtctx->factory) + + ZSTDMT_sizeof_bufferPool(mtctx->bufPool) + + (mtctx->jobIDMask+1) * sizeof(ZSTDMT_jobDescription) + + ZSTDMT_sizeof_CCtxPool(mtctx->cctxPool) + + ZSTDMT_sizeof_seqPool(mtctx->seqPool) + + ZSTD_sizeof_CDict(mtctx->cdictLocal) + + mtctx->roundBuff.capacity; +} - DEBUG_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable)); - /* ZSTD_memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */ - iSize = HUF_readStats_wksp(wksp->huffWeight, HUF_SYMBOLVALUE_MAX + 1, wksp->rankVal, &nbSymbols, &tableLog, src, srcSize, wksp->statsWksp, sizeof(wksp->statsWksp), bmi2); - if (HUF_isError(iSize)) return iSize; +/* ZSTDMT_resize() : + * @return : error code if fails, 0 on success */ +static size_t ZSTDMT_resize(ZSTDMT_CCtx* mtctx, unsigned nbWorkers) +{ + if (POOL_resize(mtctx->factory, nbWorkers)) return ERROR(memory_allocation); + FORWARD_IF_ERROR( ZSTDMT_expandJobsTable(mtctx, nbWorkers) , ""); + mtctx->bufPool = ZSTDMT_expandBufferPool(mtctx->bufPool, BUF_POOL_MAX_NB_BUFFERS(nbWorkers)); + if (mtctx->bufPool == NULL) return ERROR(memory_allocation); + mtctx->cctxPool = ZSTDMT_expandCCtxPool(mtctx->cctxPool, nbWorkers); + if (mtctx->cctxPool == NULL) return ERROR(memory_allocation); + mtctx->seqPool = ZSTDMT_expandSeqPool(mtctx->seqPool, nbWorkers); + if (mtctx->seqPool == NULL) return ERROR(memory_allocation); + ZSTDMT_CCtxParam_setNbWorkers(&mtctx->params, nbWorkers); + return 0; +} - /* Table header */ - { DTableDesc dtd = HUF_getDTableDesc(DTable); - if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, Huffman tree cannot fit in */ - dtd.tableType = 0; - dtd.tableLog = (BYTE)tableLog; - ZSTD_memcpy(DTable, &dtd, sizeof(dtd)); - } - /* Compute symbols and rankStart given rankVal: - * - * rankVal already contains the number of values of each weight. - * - * symbols contains the symbols ordered by weight. First are the rankVal[0] - * weight 0 symbols, followed by the rankVal[1] weight 1 symbols, and so on. - * symbols[0] is filled (but unused) to avoid a branch. - * - * rankStart contains the offset where each rank belongs in the DTable. - * rankStart[0] is not filled because there are no entries in the table for - * weight 0. - */ - { - int n; - int nextRankStart = 0; - int const unroll = 4; - int const nLimit = (int)nbSymbols - unroll + 1; - for (n=0; n<(int)tableLog+1; n++) { - U32 const curr = nextRankStart; - nextRankStart += wksp->rankVal[n]; - wksp->rankStart[n] = curr; - } - for (n=0; n < nLimit; n += unroll) { - int u; - for (u=0; u < unroll; ++u) { - size_t const w = wksp->huffWeight[n+u]; - wksp->symbols[wksp->rankStart[w]++] = (BYTE)(n+u); - } - } - for (; n < (int)nbSymbols; ++n) { - size_t const w = wksp->huffWeight[n]; - wksp->symbols[wksp->rankStart[w]++] = (BYTE)n; - } +/*! ZSTDMT_updateCParams_whileCompressing() : + * Updates a selected set of compression parameters, remaining compatible with currently active frame. + * New parameters will be applied to next compression job. */ +void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_params* cctxParams) +{ + U32 const saved_wlog = mtctx->params.cParams.windowLog; /* Do not modify windowLog while compressing */ + int const compressionLevel = cctxParams->compressionLevel; + DEBUGLOG(5, "ZSTDMT_updateCParams_whileCompressing (level:%i)", + compressionLevel); + mtctx->params.compressionLevel = compressionLevel; + { ZSTD_compressionParameters cParams = ZSTD_getCParamsFromCCtxParams(cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); + cParams.windowLog = saved_wlog; + mtctx->params.cParams = cParams; } +} - /* fill DTable - * We fill all entries of each weight in order. - * That way length is a constant for each iteration of the outter loop. - * We can switch based on the length to a different inner loop which is - * optimized for that particular case. - */ - { - U32 w; - int symbol=wksp->rankVal[0]; - int rankStart=0; - for (w=1; wrankVal[w]; - int const length = (1 << w) >> 1; - int uStart = rankStart; - BYTE const nbBits = (BYTE)(tableLog + 1 - w); - int s; - int u; - switch (length) { - case 1: - for (s=0; ssymbols[symbol + s]; - D.nbBits = nbBits; - dt[uStart] = D; - uStart += 1; - } - break; - case 2: - for (s=0; ssymbols[symbol + s]; - D.nbBits = nbBits; - dt[uStart+0] = D; - dt[uStart+1] = D; - uStart += 2; - } - break; - case 4: - for (s=0; ssymbols[symbol + s], nbBits); - MEM_write64(dt + uStart, D4); - uStart += 4; - } - break; - case 8: - for (s=0; ssymbols[symbol + s], nbBits); - MEM_write64(dt + uStart, D4); - MEM_write64(dt + uStart + 4, D4); - uStart += 8; - } - break; - default: - for (s=0; ssymbols[symbol + s], nbBits); - for (u=0; u < length; u += 16) { - MEM_write64(dt + uStart + u + 0, D4); - MEM_write64(dt + uStart + u + 4, D4); - MEM_write64(dt + uStart + u + 8, D4); - MEM_write64(dt + uStart + u + 12, D4); - } - assert(u == length); - uStart += length; - } - break; +/* ZSTDMT_getFrameProgression(): + * tells how much data has been consumed (input) and produced (output) for current frame. + * able to count progression inside worker threads. + * Note : mutex will be acquired during statistics collection inside workers. */ +ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx) +{ + ZSTD_frameProgression fps; + DEBUGLOG(5, "ZSTDMT_getFrameProgression"); + fps.ingested = mtctx->consumed + mtctx->inBuff.filled; + fps.consumed = mtctx->consumed; + fps.produced = fps.flushed = mtctx->produced; + fps.currentJobID = mtctx->nextJobID; + fps.nbActiveWorkers = 0; + { unsigned jobNb; + unsigned lastJobNb = mtctx->nextJobID + mtctx->jobReady; assert(mtctx->jobReady <= 1); + DEBUGLOG(6, "ZSTDMT_getFrameProgression: jobs: from %u to <%u (jobReady:%u)", + mtctx->doneJobID, lastJobNb, mtctx->jobReady); + for (jobNb = mtctx->doneJobID ; jobNb < lastJobNb ; jobNb++) { + unsigned const wJobID = jobNb & mtctx->jobIDMask; + ZSTDMT_jobDescription* jobPtr = &mtctx->jobs[wJobID]; + ZSTD_pthread_mutex_lock(&jobPtr->job_mutex); + { size_t const cResult = jobPtr->cSize; + size_t const produced = ZSTD_isError(cResult) ? 0 : cResult; + size_t const flushed = ZSTD_isError(cResult) ? 0 : jobPtr->dstFlushed; + assert(flushed <= produced); + fps.ingested += jobPtr->src.size; + fps.consumed += jobPtr->consumed; + fps.produced += produced; + fps.flushed += flushed; + fps.nbActiveWorkers += (jobPtr->consumed < jobPtr->src.size); } - symbol += symbolCount; - rankStart += symbolCount * length; + ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex); } } - return iSize; + return fps; } -FORCE_INLINE_TEMPLATE BYTE -HUF_decodeSymbolX1(BIT_DStream_t* Dstream, const HUF_DEltX1* dt, const U32 dtLog) + +size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx) { - size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */ - BYTE const c = dt[val].byte; - BIT_skipBits(Dstream, dt[val].nbBits); - return c; -} + size_t toFlush; + unsigned const jobID = mtctx->doneJobID; + assert(jobID <= mtctx->nextJobID); + if (jobID == mtctx->nextJobID) return 0; /* no active job => nothing to flush */ + + /* look into oldest non-fully-flushed job */ + { unsigned const wJobID = jobID & mtctx->jobIDMask; + ZSTDMT_jobDescription* const jobPtr = &mtctx->jobs[wJobID]; + ZSTD_pthread_mutex_lock(&jobPtr->job_mutex); + { size_t const cResult = jobPtr->cSize; + size_t const produced = ZSTD_isError(cResult) ? 0 : cResult; + size_t const flushed = ZSTD_isError(cResult) ? 0 : jobPtr->dstFlushed; + assert(flushed <= produced); + assert(jobPtr->consumed <= jobPtr->src.size); + toFlush = produced - flushed; + /* if toFlush==0, nothing is available to flush. + * However, jobID is expected to still be active: + * if jobID was already completed and fully flushed, + * ZSTDMT_flushProduced() should have already moved onto next job. + * Therefore, some input has not yet been consumed. */ + if (toFlush==0) { + assert(jobPtr->consumed < jobPtr->src.size); + } + } + ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex); + } -#define HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) \ - *ptr++ = HUF_decodeSymbolX1(DStreamPtr, dt, dtLog) + return toFlush; +} -#define HUF_DECODE_SYMBOLX1_1(ptr, DStreamPtr) \ - if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ - HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) -#define HUF_DECODE_SYMBOLX1_2(ptr, DStreamPtr) \ - if (MEM_64bits()) \ - HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) +/* ------------------------------------------ */ +/* ===== Multi-threaded compression ===== */ +/* ------------------------------------------ */ -HINT_INLINE size_t -HUF_decodeStreamX1(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX1* const dt, const U32 dtLog) +static unsigned ZSTDMT_computeTargetJobLog(const ZSTD_CCtx_params* params) { - BYTE* const pStart = p; - - /* up to 4 symbols at a time */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-3)) { - HUF_DECODE_SYMBOLX1_2(p, bitDPtr); - HUF_DECODE_SYMBOLX1_1(p, bitDPtr); - HUF_DECODE_SYMBOLX1_2(p, bitDPtr); - HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + unsigned jobLog; + if (params->ldmParams.enableLdm == ZSTD_ps_enable) { + /* In Long Range Mode, the windowLog is typically oversized. + * In which case, it's preferable to determine the jobSize + * based on cycleLog instead. */ + jobLog = MAX(21, ZSTD_cycleLog(params->cParams.chainLog, params->cParams.strategy) + 3); + } else { + jobLog = MAX(20, params->cParams.windowLog + 2); } - - /* [0-3] symbols remaining */ - if (MEM_32bits()) - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd)) - HUF_DECODE_SYMBOLX1_0(p, bitDPtr); - - /* no more data to retrieve from bitstream, no need to reload */ - while (p < pEnd) - HUF_DECODE_SYMBOLX1_0(p, bitDPtr); - - return pEnd-pStart; + return MIN(jobLog, (unsigned)ZSTDMT_JOBLOG_MAX); } -FORCE_INLINE_TEMPLATE size_t -HUF_decompress1X1_usingDTable_internal_body( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) +static int ZSTDMT_overlapLog_default(ZSTD_strategy strat) { - BYTE* op = (BYTE*)dst; - BYTE* const oend = op + dstSize; - const void* dtPtr = DTable + 1; - const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr; - BIT_DStream_t bitD; - DTableDesc const dtd = HUF_getDTableDesc(DTable); - U32 const dtLog = dtd.tableLog; - - CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) ); - - HUF_decodeStreamX1(op, &bitD, oend, dt, dtLog); - - if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); - - return dstSize; + switch(strat) + { + case ZSTD_btultra2: + return 9; + case ZSTD_btultra: + case ZSTD_btopt: + return 8; + case ZSTD_btlazy2: + case ZSTD_lazy2: + return 7; + case ZSTD_lazy: + case ZSTD_greedy: + case ZSTD_dfast: + case ZSTD_fast: + default:; + } + return 6; } -FORCE_INLINE_TEMPLATE size_t -HUF_decompress4X1_usingDTable_internal_body( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) +static int ZSTDMT_overlapLog(int ovlog, ZSTD_strategy strat) { - /* Check */ - if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ - - { const BYTE* const istart = (const BYTE*) cSrc; - BYTE* const ostart = (BYTE*) dst; - BYTE* const oend = ostart + dstSize; - BYTE* const olimit = oend - 3; - const void* const dtPtr = DTable + 1; - const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr; + assert(0 <= ovlog && ovlog <= 9); + if (ovlog == 0) return ZSTDMT_overlapLog_default(strat); + return ovlog; +} - /* Init */ - BIT_DStream_t bitD1; - BIT_DStream_t bitD2; - BIT_DStream_t bitD3; - BIT_DStream_t bitD4; - size_t const length1 = MEM_readLE16(istart); - size_t const length2 = MEM_readLE16(istart+2); - size_t const length3 = MEM_readLE16(istart+4); - size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); - const BYTE* const istart1 = istart + 6; /* jumpTable */ - const BYTE* const istart2 = istart1 + length1; - const BYTE* const istart3 = istart2 + length2; - const BYTE* const istart4 = istart3 + length3; - const size_t segmentSize = (dstSize+3) / 4; - BYTE* const opStart2 = ostart + segmentSize; - BYTE* const opStart3 = opStart2 + segmentSize; - BYTE* const opStart4 = opStart3 + segmentSize; - BYTE* op1 = ostart; - BYTE* op2 = opStart2; - BYTE* op3 = opStart3; - BYTE* op4 = opStart4; - DTableDesc const dtd = HUF_getDTableDesc(DTable); - U32 const dtLog = dtd.tableLog; - U32 endSignal = 1; +static size_t ZSTDMT_computeOverlapSize(const ZSTD_CCtx_params* params) +{ + int const overlapRLog = 9 - ZSTDMT_overlapLog(params->overlapLog, params->cParams.strategy); + int ovLog = (overlapRLog >= 8) ? 0 : (params->cParams.windowLog - overlapRLog); + assert(0 <= overlapRLog && overlapRLog <= 8); + if (params->ldmParams.enableLdm == ZSTD_ps_enable) { + /* In Long Range Mode, the windowLog is typically oversized. + * In which case, it's preferable to determine the jobSize + * based on chainLog instead. + * Then, ovLog becomes a fraction of the jobSize, rather than windowSize */ + ovLog = MIN(params->cParams.windowLog, ZSTDMT_computeTargetJobLog(params) - 2) + - overlapRLog; + } + assert(0 <= ovLog && ovLog <= ZSTD_WINDOWLOG_MAX); + DEBUGLOG(4, "overlapLog : %i", params->overlapLog); + DEBUGLOG(4, "overlap size : %i", 1 << ovLog); + return (ovLog==0) ? 0 : (size_t)1 << ovLog; +} - if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ - CHECK_F( BIT_initDStream(&bitD1, istart1, length1) ); - CHECK_F( BIT_initDStream(&bitD2, istart2, length2) ); - CHECK_F( BIT_initDStream(&bitD3, istart3, length3) ); - CHECK_F( BIT_initDStream(&bitD4, istart4, length4) ); +/* ====================================== */ +/* ======= Streaming API ======= */ +/* ====================================== */ - /* up to 16 symbols per loop (4 symbols per stream) in 64-bit mode */ - for ( ; (endSignal) & (op4 < olimit) ; ) { - HUF_DECODE_SYMBOLX1_2(op1, &bitD1); - HUF_DECODE_SYMBOLX1_2(op2, &bitD2); - HUF_DECODE_SYMBOLX1_2(op3, &bitD3); - HUF_DECODE_SYMBOLX1_2(op4, &bitD4); - HUF_DECODE_SYMBOLX1_1(op1, &bitD1); - HUF_DECODE_SYMBOLX1_1(op2, &bitD2); - HUF_DECODE_SYMBOLX1_1(op3, &bitD3); - HUF_DECODE_SYMBOLX1_1(op4, &bitD4); - HUF_DECODE_SYMBOLX1_2(op1, &bitD1); - HUF_DECODE_SYMBOLX1_2(op2, &bitD2); - HUF_DECODE_SYMBOLX1_2(op3, &bitD3); - HUF_DECODE_SYMBOLX1_2(op4, &bitD4); - HUF_DECODE_SYMBOLX1_0(op1, &bitD1); - HUF_DECODE_SYMBOLX1_0(op2, &bitD2); - HUF_DECODE_SYMBOLX1_0(op3, &bitD3); - HUF_DECODE_SYMBOLX1_0(op4, &bitD4); - endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished; - endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished; - endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished; - endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished; - } +size_t ZSTDMT_initCStream_internal( + ZSTDMT_CCtx* mtctx, + const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType, + const ZSTD_CDict* cdict, ZSTD_CCtx_params params, + unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTDMT_initCStream_internal (pledgedSrcSize=%u, nbWorkers=%u, cctxPool=%u)", + (U32)pledgedSrcSize, params.nbWorkers, mtctx->cctxPool->totalCCtx); - /* check corruption */ - /* note : should not be necessary : op# advance in lock step, and we control op4. - * but curiously, binary generated by gcc 7.2 & 7.3 with -mbmi2 runs faster when >=1 test is present */ - if (op1 > opStart2) return ERROR(corruption_detected); - if (op2 > opStart3) return ERROR(corruption_detected); - if (op3 > opStart4) return ERROR(corruption_detected); - /* note : op4 supposed already verified within main loop */ + /* params supposed partially fully validated at this point */ + assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + assert(!((dict) && (cdict))); /* either dict or cdict, not both */ - /* finish bitStreams one by one */ - HUF_decodeStreamX1(op1, &bitD1, opStart2, dt, dtLog); - HUF_decodeStreamX1(op2, &bitD2, opStart3, dt, dtLog); - HUF_decodeStreamX1(op3, &bitD3, opStart4, dt, dtLog); - HUF_decodeStreamX1(op4, &bitD4, oend, dt, dtLog); + /* init */ + if (params.nbWorkers != mtctx->params.nbWorkers) + FORWARD_IF_ERROR( ZSTDMT_resize(mtctx, (unsigned)params.nbWorkers) , ""); - /* check */ - { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); - if (!endCheck) return ERROR(corruption_detected); } + if (params.jobSize != 0 && params.jobSize < ZSTDMT_JOBSIZE_MIN) params.jobSize = ZSTDMT_JOBSIZE_MIN; + if (params.jobSize > (size_t)ZSTDMT_JOBSIZE_MAX) params.jobSize = (size_t)ZSTDMT_JOBSIZE_MAX; - /* decoded size */ - return dstSize; + if (mtctx->allJobsCompleted == 0) { /* previous compression not correctly finished */ + ZSTDMT_waitForAllJobsCompleted(mtctx); + ZSTDMT_releaseAllJobResources(mtctx); + mtctx->allJobsCompleted = 1; } -} - - -typedef size_t (*HUF_decompress_usingDTable_t)(void *dst, size_t dstSize, - const void *cSrc, - size_t cSrcSize, - const HUF_DTable *DTable); - -HUF_DGEN(HUF_decompress1X1_usingDTable_internal) -HUF_DGEN(HUF_decompress4X1_usingDTable_internal) - + mtctx->params = params; + mtctx->frameContentSize = pledgedSrcSize; + ZSTD_freeCDict(mtctx->cdictLocal); + if (dict) { + mtctx->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, + ZSTD_dlm_byCopy, dictContentType, /* note : a loadPrefix becomes an internal CDict */ + params.cParams, mtctx->cMem); + mtctx->cdict = mtctx->cdictLocal; + if (mtctx->cdictLocal == NULL) return ERROR(memory_allocation); + } else { + mtctx->cdictLocal = NULL; + mtctx->cdict = cdict; + } -size_t HUF_decompress1X1_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (dtd.tableType != 0) return ERROR(GENERIC); - return HUF_decompress1X1_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -} + mtctx->targetPrefixSize = ZSTDMT_computeOverlapSize(¶ms); + DEBUGLOG(4, "overlapLog=%i => %u KB", params.overlapLog, (U32)(mtctx->targetPrefixSize>>10)); + mtctx->targetSectionSize = params.jobSize; + if (mtctx->targetSectionSize == 0) { + mtctx->targetSectionSize = 1ULL << ZSTDMT_computeTargetJobLog(¶ms); + } + assert(mtctx->targetSectionSize <= (size_t)ZSTDMT_JOBSIZE_MAX); -size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize) -{ - const BYTE* ip = (const BYTE*) cSrc; + if (params.rsyncable) { + /* Aim for the targetsectionSize as the average job size. */ + U32 const jobSizeKB = (U32)(mtctx->targetSectionSize >> 10); + U32 const rsyncBits = (assert(jobSizeKB >= 1), ZSTD_highbit32(jobSizeKB) + 10); + /* We refuse to create jobs < RSYNC_MIN_BLOCK_SIZE bytes, so make sure our + * expected job size is at least 4x larger. */ + assert(rsyncBits >= RSYNC_MIN_BLOCK_LOG + 2); + DEBUGLOG(4, "rsyncLog = %u", rsyncBits); + mtctx->rsync.hash = 0; + mtctx->rsync.hitMask = (1ULL << rsyncBits) - 1; + mtctx->rsync.primePower = ZSTD_rollingHash_primePower(RSYNC_LENGTH); + } + if (mtctx->targetSectionSize < mtctx->targetPrefixSize) mtctx->targetSectionSize = mtctx->targetPrefixSize; /* job size must be >= overlap size */ + DEBUGLOG(4, "Job Size : %u KB (note : set to %u)", (U32)(mtctx->targetSectionSize>>10), (U32)params.jobSize); + DEBUGLOG(4, "inBuff Size : %u KB", (U32)(mtctx->targetSectionSize>>10)); + ZSTDMT_setBufferSize(mtctx->bufPool, ZSTD_compressBound(mtctx->targetSectionSize)); + { + /* If ldm is enabled we need windowSize space. */ + size_t const windowSize = mtctx->params.ldmParams.enableLdm == ZSTD_ps_enable ? (1U << mtctx->params.cParams.windowLog) : 0; + /* Two buffers of slack, plus extra space for the overlap + * This is the minimum slack that LDM works with. One extra because + * flush might waste up to targetSectionSize-1 bytes. Another extra + * for the overlap (if > 0), then one to fill which doesn't overlap + * with the LDM window. + */ + size_t const nbSlackBuffers = 2 + (mtctx->targetPrefixSize > 0); + size_t const slackSize = mtctx->targetSectionSize * nbSlackBuffers; + /* Compute the total size, and always have enough slack */ + size_t const nbWorkers = MAX(mtctx->params.nbWorkers, 1); + size_t const sectionsSize = mtctx->targetSectionSize * nbWorkers; + size_t const capacity = MAX(windowSize, sectionsSize) + slackSize; + if (mtctx->roundBuff.capacity < capacity) { + if (mtctx->roundBuff.buffer) + ZSTD_customFree(mtctx->roundBuff.buffer, mtctx->cMem); + mtctx->roundBuff.buffer = (BYTE*)ZSTD_customMalloc(capacity, mtctx->cMem); + if (mtctx->roundBuff.buffer == NULL) { + mtctx->roundBuff.capacity = 0; + return ERROR(memory_allocation); + } + mtctx->roundBuff.capacity = capacity; + } + } + DEBUGLOG(4, "roundBuff capacity : %u KB", (U32)(mtctx->roundBuff.capacity>>10)); + mtctx->roundBuff.pos = 0; + mtctx->inBuff.buffer = g_nullBuffer; + mtctx->inBuff.filled = 0; + mtctx->inBuff.prefix = kNullRange; + mtctx->doneJobID = 0; + mtctx->nextJobID = 0; + mtctx->frameEnded = 0; + mtctx->allJobsCompleted = 0; + mtctx->consumed = 0; + mtctx->produced = 0; - size_t const hSize = HUF_readDTableX1_wksp(DCtx, cSrc, cSrcSize, workSpace, wkspSize); - if (HUF_isError(hSize)) return hSize; - if (hSize >= cSrcSize) return ERROR(srcSize_wrong); - ip += hSize; cSrcSize -= hSize; + /* update dictionary */ + ZSTD_freeCDict(mtctx->cdictLocal); + mtctx->cdictLocal = NULL; + mtctx->cdict = NULL; + if (dict) { + if (dictContentType == ZSTD_dct_rawContent) { + mtctx->inBuff.prefix.start = (const BYTE*)dict; + mtctx->inBuff.prefix.size = dictSize; + } else { + /* note : a loadPrefix becomes an internal CDict */ + mtctx->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, + ZSTD_dlm_byRef, dictContentType, + params.cParams, mtctx->cMem); + mtctx->cdict = mtctx->cdictLocal; + if (mtctx->cdictLocal == NULL) return ERROR(memory_allocation); + } + } else { + mtctx->cdict = cdict; + } - return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, /* bmi2 */ 0); -} + if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, mtctx->targetSectionSize, + dict, dictSize, dictContentType)) + return ERROR(memory_allocation); -size_t HUF_decompress4X1_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (dtd.tableType != 0) return ERROR(GENERIC); - return HUF_decompress4X1_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); + return 0; } -static size_t HUF_decompress4X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize, int bmi2) -{ - const BYTE* ip = (const BYTE*) cSrc; - - size_t const hSize = HUF_readDTableX1_wksp_bmi2(dctx, cSrc, cSrcSize, workSpace, wkspSize, bmi2); - if (HUF_isError(hSize)) return hSize; - if (hSize >= cSrcSize) return ERROR(srcSize_wrong); - ip += hSize; cSrcSize -= hSize; - - return HUF_decompress4X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2); -} -size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize) +/* ZSTDMT_writeLastEmptyBlock() + * Write a single empty block with an end-of-frame to finish a frame. + * Job must be created from streaming variant. + * This function is always successful if expected conditions are fulfilled. + */ +static void ZSTDMT_writeLastEmptyBlock(ZSTDMT_jobDescription* job) { - return HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, 0); + assert(job->lastJob == 1); + assert(job->src.size == 0); /* last job is empty -> will be simplified into a last empty block */ + assert(job->firstJob == 0); /* cannot be first job, as it also needs to create frame header */ + assert(job->dstBuff.start == NULL); /* invoked from streaming variant only (otherwise, dstBuff might be user's output) */ + job->dstBuff = ZSTDMT_getBuffer(job->bufPool); + if (job->dstBuff.start == NULL) { + job->cSize = ERROR(memory_allocation); + return; + } + assert(job->dstBuff.capacity >= ZSTD_blockHeaderSize); /* no buffer should ever be that small */ + job->src = kNullRange; + job->cSize = ZSTD_writeLastEmptyBlock(job->dstBuff.start, job->dstBuff.capacity); + assert(!ZSTD_isError(job->cSize)); + assert(job->consumed == 0); } - -#endif /* HUF_FORCE_DECOMPRESS_X2 */ - - -#ifndef HUF_FORCE_DECOMPRESS_X1 - -/* *************************/ -/* double-symbols decoding */ -/* *************************/ - -typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX2; /* double-symbols decoding */ -typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t; -typedef U32 rankValCol_t[HUF_TABLELOG_MAX + 1]; -typedef rankValCol_t rankVal_t[HUF_TABLELOG_MAX]; - - -/* HUF_fillDTableX2Level2() : - * `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */ -static void HUF_fillDTableX2Level2(HUF_DEltX2* DTable, U32 sizeLog, const U32 consumed, - const U32* rankValOrigin, const int minWeight, - const sortedSymbol_t* sortedSymbols, const U32 sortedListSize, - U32 nbBitsBaseline, U16 baseSeq) +static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* mtctx, size_t srcSize, ZSTD_EndDirective endOp) { - HUF_DEltX2 DElt; - U32 rankVal[HUF_TABLELOG_MAX + 1]; - - /* get pre-calculated rankVal */ - ZSTD_memcpy(rankVal, rankValOrigin, sizeof(rankVal)); - - /* fill skipped values */ - if (minWeight>1) { - U32 i, skipSize = rankVal[minWeight]; - MEM_writeLE16(&(DElt.sequence), baseSeq); - DElt.nbBits = (BYTE)(consumed); - DElt.length = 1; - for (i = 0; i < skipSize; i++) - DTable[i] = DElt; - } - - /* fill DTable */ - { U32 s; for (s=0; s= 1 */ - - rankVal[weight] += length; - } } -} + unsigned const jobID = mtctx->nextJobID & mtctx->jobIDMask; + int const endFrame = (endOp == ZSTD_e_end); + if (mtctx->nextJobID > mtctx->doneJobID + mtctx->jobIDMask) { + DEBUGLOG(5, "ZSTDMT_createCompressionJob: will not create new job : table is full"); + assert((mtctx->nextJobID & mtctx->jobIDMask) == (mtctx->doneJobID & mtctx->jobIDMask)); + return 0; + } -static void HUF_fillDTableX2(HUF_DEltX2* DTable, const U32 targetLog, - const sortedSymbol_t* sortedList, const U32 sortedListSize, - const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight, - const U32 nbBitsBaseline) -{ - U32 rankVal[HUF_TABLELOG_MAX + 1]; - const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */ - const U32 minBits = nbBitsBaseline - maxWeight; - U32 s; + if (!mtctx->jobReady) { + BYTE const* src = (BYTE const*)mtctx->inBuff.buffer.start; + DEBUGLOG(5, "ZSTDMT_createCompressionJob: preparing job %u to compress %u bytes with %u preload ", + mtctx->nextJobID, (U32)srcSize, (U32)mtctx->inBuff.prefix.size); + mtctx->jobs[jobID].src.start = src; + mtctx->jobs[jobID].src.size = srcSize; + assert(mtctx->inBuff.filled >= srcSize); + mtctx->jobs[jobID].prefix = mtctx->inBuff.prefix; + mtctx->jobs[jobID].consumed = 0; + mtctx->jobs[jobID].cSize = 0; + mtctx->jobs[jobID].params = mtctx->params; + mtctx->jobs[jobID].cdict = mtctx->nextJobID==0 ? mtctx->cdict : NULL; + mtctx->jobs[jobID].fullFrameSize = mtctx->frameContentSize; + mtctx->jobs[jobID].dstBuff = g_nullBuffer; + mtctx->jobs[jobID].cctxPool = mtctx->cctxPool; + mtctx->jobs[jobID].bufPool = mtctx->bufPool; + mtctx->jobs[jobID].seqPool = mtctx->seqPool; + mtctx->jobs[jobID].serial = &mtctx->serial; + mtctx->jobs[jobID].jobID = mtctx->nextJobID; + mtctx->jobs[jobID].firstJob = (mtctx->nextJobID==0); + mtctx->jobs[jobID].lastJob = endFrame; + mtctx->jobs[jobID].frameChecksumNeeded = mtctx->params.fParams.checksumFlag && endFrame && (mtctx->nextJobID>0); + mtctx->jobs[jobID].dstFlushed = 0; - ZSTD_memcpy(rankVal, rankValOrigin, sizeof(rankVal)); + /* Update the round buffer pos and clear the input buffer to be reset */ + mtctx->roundBuff.pos += srcSize; + mtctx->inBuff.buffer = g_nullBuffer; + mtctx->inBuff.filled = 0; + /* Set the prefix for next job */ + if (!endFrame) { + size_t const newPrefixSize = MIN(srcSize, mtctx->targetPrefixSize); + mtctx->inBuff.prefix.start = src + srcSize - newPrefixSize; + mtctx->inBuff.prefix.size = newPrefixSize; + } else { /* endFrame==1 => no need for another input buffer */ + mtctx->inBuff.prefix = kNullRange; + mtctx->frameEnded = endFrame; + if (mtctx->nextJobID == 0) { + /* single job exception : checksum is already calculated directly within worker thread */ + mtctx->params.fParams.checksumFlag = 0; + } } - /* fill DTable */ - for (s=0; snextJobID>0)/*single job must also write frame header*/ ) { + DEBUGLOG(5, "ZSTDMT_createCompressionJob: creating a last empty block to end frame"); + assert(endOp == ZSTD_e_end); /* only possible case : need to end the frame with an empty last block */ + ZSTDMT_writeLastEmptyBlock(mtctx->jobs + jobID); + mtctx->nextJobID++; + return 0; + } + } - if (targetLog-nbBits >= minBits) { /* enough room for a second symbol */ - U32 sortedRank; - int minWeight = nbBits + scaleLog; - if (minWeight < 1) minWeight = 1; - sortedRank = rankStart[minWeight]; - HUF_fillDTableX2Level2(DTable+start, targetLog-nbBits, nbBits, - rankValOrigin[nbBits], minWeight, - sortedList+sortedRank, sortedListSize-sortedRank, - nbBitsBaseline, symbol); - } else { - HUF_DEltX2 DElt; - MEM_writeLE16(&(DElt.sequence), symbol); - DElt.nbBits = (BYTE)(nbBits); - DElt.length = 1; - { U32 const end = start + length; - U32 u; - for (u = start; u < end; u++) DTable[u] = DElt; - } } - rankVal[weight] += length; + DEBUGLOG(5, "ZSTDMT_createCompressionJob: posting job %u : %u bytes (end:%u, jobNb == %u (mod:%u))", + mtctx->nextJobID, + (U32)mtctx->jobs[jobID].src.size, + mtctx->jobs[jobID].lastJob, + mtctx->nextJobID, + jobID); + if (POOL_tryAdd(mtctx->factory, ZSTDMT_compressionJob, &mtctx->jobs[jobID])) { + mtctx->nextJobID++; + mtctx->jobReady = 0; + } else { + DEBUGLOG(5, "ZSTDMT_createCompressionJob: no worker available for job %u", mtctx->nextJobID); + mtctx->jobReady = 1; } + return 0; } -size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, - const void* src, size_t srcSize, - void* workSpace, size_t wkspSize) -{ - U32 tableLog, maxW, sizeOfSort, nbSymbols; - DTableDesc dtd = HUF_getDTableDesc(DTable); - U32 const maxTableLog = dtd.maxTableLog; - size_t iSize; - void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */ - HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr; - U32 *rankStart; - - rankValCol_t* rankVal; - U32* rankStats; - U32* rankStart0; - sortedSymbol_t* sortedSymbol; - BYTE* weightList; - size_t spaceUsed32 = 0; - - rankVal = (rankValCol_t *)((U32 *)workSpace + spaceUsed32); - spaceUsed32 += (sizeof(rankValCol_t) * HUF_TABLELOG_MAX) >> 2; - rankStats = (U32 *)workSpace + spaceUsed32; - spaceUsed32 += HUF_TABLELOG_MAX + 1; - rankStart0 = (U32 *)workSpace + spaceUsed32; - spaceUsed32 += HUF_TABLELOG_MAX + 2; - sortedSymbol = (sortedSymbol_t *)workSpace + (spaceUsed32 * sizeof(U32)) / sizeof(sortedSymbol_t); - spaceUsed32 += HUF_ALIGN(sizeof(sortedSymbol_t) * (HUF_SYMBOLVALUE_MAX + 1), sizeof(U32)) >> 2; - weightList = (BYTE *)((U32 *)workSpace + spaceUsed32); - spaceUsed32 += HUF_ALIGN(HUF_SYMBOLVALUE_MAX + 1, sizeof(U32)) >> 2; - - if ((spaceUsed32 << 2) > wkspSize) return ERROR(tableLog_tooLarge); - - rankStart = rankStart0 + 1; - ZSTD_memset(rankStats, 0, sizeof(U32) * (2 * HUF_TABLELOG_MAX + 2 + 1)); - - DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(HUF_DTable)); /* if compiler fails here, assertion is wrong */ - if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); - /* ZSTD_memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */ - - iSize = HUF_readStats(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize); - if (HUF_isError(iSize)) return iSize; - - /* check result */ - if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ - /* find maxWeight */ - for (maxW = tableLog; rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */ +/*! ZSTDMT_flushProduced() : + * flush whatever data has been produced but not yet flushed in current job. + * move to next job if current one is fully flushed. + * `output` : `pos` will be updated with amount of data flushed . + * `blockToFlush` : if >0, the function will block and wait if there is no data available to flush . + * @return : amount of data remaining within internal buffer, 0 if no more, 1 if unknown but > 0, or an error code */ +static size_t ZSTDMT_flushProduced(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, unsigned blockToFlush, ZSTD_EndDirective end) +{ + unsigned const wJobID = mtctx->doneJobID & mtctx->jobIDMask; + DEBUGLOG(5, "ZSTDMT_flushProduced (blocking:%u , job %u <= %u)", + blockToFlush, mtctx->doneJobID, mtctx->nextJobID); + assert(output->size >= output->pos); - /* Get start index of each weight */ - { U32 w, nextRankStart = 0; - for (w=1; wjobs[wJobID].job_mutex); + if ( blockToFlush + && (mtctx->doneJobID < mtctx->nextJobID) ) { + assert(mtctx->jobs[wJobID].dstFlushed <= mtctx->jobs[wJobID].cSize); + while (mtctx->jobs[wJobID].dstFlushed == mtctx->jobs[wJobID].cSize) { /* nothing to flush */ + if (mtctx->jobs[wJobID].consumed == mtctx->jobs[wJobID].src.size) { + DEBUGLOG(5, "job %u is completely consumed (%u == %u) => don't wait for cond, there will be none", + mtctx->doneJobID, (U32)mtctx->jobs[wJobID].consumed, (U32)mtctx->jobs[wJobID].src.size); + break; + } + DEBUGLOG(5, "waiting for something to flush from job %u (currently flushed: %u bytes)", + mtctx->doneJobID, (U32)mtctx->jobs[wJobID].dstFlushed); + ZSTD_pthread_cond_wait(&mtctx->jobs[wJobID].job_cond, &mtctx->jobs[wJobID].job_mutex); /* block when nothing to flush but some to come */ + } } - /* sort symbols by weight */ - { U32 s; - for (s=0; sjobs[wJobID].cSize; /* shared */ + size_t const srcConsumed = mtctx->jobs[wJobID].consumed; /* shared */ + size_t const srcSize = mtctx->jobs[wJobID].src.size; /* read-only, could be done after mutex lock, but no-declaration-after-statement */ + ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex); + if (ZSTD_isError(cSize)) { + DEBUGLOG(5, "ZSTDMT_flushProduced: job %u : compression error detected : %s", + mtctx->doneJobID, ZSTD_getErrorName(cSize)); + ZSTDMT_waitForAllJobsCompleted(mtctx); + ZSTDMT_releaseAllJobResources(mtctx); + return cSize; + } + /* add frame checksum if necessary (can only happen once) */ + assert(srcConsumed <= srcSize); + if ( (srcConsumed == srcSize) /* job completed -> worker no longer active */ + && mtctx->jobs[wJobID].frameChecksumNeeded ) { + U32 const checksum = (U32)XXH64_digest(&mtctx->serial.xxhState); + DEBUGLOG(4, "ZSTDMT_flushProduced: writing checksum : %08X \n", checksum); + MEM_writeLE32((char*)mtctx->jobs[wJobID].dstBuff.start + mtctx->jobs[wJobID].cSize, checksum); + cSize += 4; + mtctx->jobs[wJobID].cSize += 4; /* can write this shared value, as worker is no longer active */ + mtctx->jobs[wJobID].frameChecksumNeeded = 0; } - rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */ - } - - /* Build rankVal */ - { U32* const rankVal0 = rankVal[0]; - { int const rescale = (maxTableLog-tableLog) - 1; /* tableLog <= maxTableLog */ - U32 nextRankVal = 0; - U32 w; - for (w=1; w> consumed; - } } } } - - HUF_fillDTableX2(dt, maxTableLog, - sortedSymbol, sizeOfSort, - rankStart0, rankVal, maxW, - tableLog+1); - dtd.tableLog = (BYTE)maxTableLog; - dtd.tableType = 1; - ZSTD_memcpy(DTable, &dtd, sizeof(dtd)); - return iSize; -} + if (cSize > 0) { /* compression is ongoing or completed */ + size_t const toFlush = MIN(cSize - mtctx->jobs[wJobID].dstFlushed, output->size - output->pos); + DEBUGLOG(5, "ZSTDMT_flushProduced: Flushing %u bytes from job %u (completion:%u/%u, generated:%u)", + (U32)toFlush, mtctx->doneJobID, (U32)srcConsumed, (U32)srcSize, (U32)cSize); + assert(mtctx->doneJobID < mtctx->nextJobID); + assert(cSize >= mtctx->jobs[wJobID].dstFlushed); + assert(mtctx->jobs[wJobID].dstBuff.start != NULL); + if (toFlush > 0) { + ZSTD_memcpy((char*)output->dst + output->pos, + (const char*)mtctx->jobs[wJobID].dstBuff.start + mtctx->jobs[wJobID].dstFlushed, + toFlush); + } + output->pos += toFlush; + mtctx->jobs[wJobID].dstFlushed += toFlush; /* can write : this value is only used by mtctx */ + if ( (srcConsumed == srcSize) /* job is completed */ + && (mtctx->jobs[wJobID].dstFlushed == cSize) ) { /* output buffer fully flushed => free this job position */ + DEBUGLOG(5, "Job %u completed (%u bytes), moving to next one", + mtctx->doneJobID, (U32)mtctx->jobs[wJobID].dstFlushed); + ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[wJobID].dstBuff); + DEBUGLOG(5, "dstBuffer released"); + mtctx->jobs[wJobID].dstBuff = g_nullBuffer; + mtctx->jobs[wJobID].cSize = 0; /* ensure this job slot is considered "not started" in future check */ + mtctx->consumed += srcSize; + mtctx->produced += cSize; + mtctx->doneJobID++; + } } -FORCE_INLINE_TEMPLATE U32 -HUF_decodeSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog) -{ - size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ - ZSTD_memcpy(op, dt+val, 2); - BIT_skipBits(DStream, dt[val].nbBits); - return dt[val].length; + /* return value : how many bytes left in buffer ; fake it to 1 when unknown but >0 */ + if (cSize > mtctx->jobs[wJobID].dstFlushed) return (cSize - mtctx->jobs[wJobID].dstFlushed); + if (srcSize > srcConsumed) return 1; /* current job not completely compressed */ + } + if (mtctx->doneJobID < mtctx->nextJobID) return 1; /* some more jobs ongoing */ + if (mtctx->jobReady) return 1; /* one job is ready to push, just not yet in the list */ + if (mtctx->inBuff.filled > 0) return 1; /* input is not empty, and still needs to be converted into a job */ + mtctx->allJobsCompleted = mtctx->frameEnded; /* all jobs are entirely flushed => if this one is last one, frame is completed */ + if (end == ZSTD_e_end) return !mtctx->frameEnded; /* for ZSTD_e_end, question becomes : is frame completed ? instead of : are internal buffers fully flushed ? */ + return 0; /* internal buffers fully flushed */ } -FORCE_INLINE_TEMPLATE U32 -HUF_decodeLastSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog) +/** + * Returns the range of data used by the earliest job that is not yet complete. + * If the data of the first job is broken up into two segments, we cover both + * sections. + */ +static Range ZSTDMT_getInputDataInUse(ZSTDMT_CCtx* mtctx) { - size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ - ZSTD_memcpy(op, dt+val, 1); - if (dt[val].length==1) BIT_skipBits(DStream, dt[val].nbBits); - else { - if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) { - BIT_skipBits(DStream, dt[val].nbBits); - if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8)) - /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ - DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); - } } - return 1; -} - -#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \ - ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog) + unsigned const firstJobID = mtctx->doneJobID; + unsigned const lastJobID = mtctx->nextJobID; + unsigned jobID; -#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ - if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ - ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog) + /* no need to check during first round */ + size_t roundBuffCapacity = mtctx->roundBuff.capacity; + size_t nbJobs1stRoundMin = roundBuffCapacity / mtctx->targetSectionSize; + if (lastJobID < nbJobs1stRoundMin) return kNullRange; -#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ - if (MEM_64bits()) \ - ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog) + for (jobID = firstJobID; jobID < lastJobID; ++jobID) { + unsigned const wJobID = jobID & mtctx->jobIDMask; + size_t consumed; -HINT_INLINE size_t -HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, - const HUF_DEltX2* const dt, const U32 dtLog) -{ - BYTE* const pStart = p; + ZSTD_PTHREAD_MUTEX_LOCK(&mtctx->jobs[wJobID].job_mutex); + consumed = mtctx->jobs[wJobID].consumed; + ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex); - /* up to 8 symbols at a time */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-(sizeof(bitDPtr->bitContainer)-1))) { - HUF_DECODE_SYMBOLX2_2(p, bitDPtr); - HUF_DECODE_SYMBOLX2_1(p, bitDPtr); - HUF_DECODE_SYMBOLX2_2(p, bitDPtr); - HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + if (consumed < mtctx->jobs[wJobID].src.size) { + Range range = mtctx->jobs[wJobID].prefix; + if (range.size == 0) { + /* Empty prefix */ + range = mtctx->jobs[wJobID].src; + } + /* Job source in multiple segments not supported yet */ + assert(range.start <= mtctx->jobs[wJobID].src.start); + return range; + } } - - /* closer to end : up to 2 symbols at a time */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd-2)) - HUF_DECODE_SYMBOLX2_0(p, bitDPtr); - - while (p <= pEnd-2) - HUF_DECODE_SYMBOLX2_0(p, bitDPtr); /* no need to reload : reached the end of DStream */ - - if (p < pEnd) - p += HUF_decodeLastSymbolX2(p, bitDPtr, dt, dtLog); - - return p-pStart; + return kNullRange; } -FORCE_INLINE_TEMPLATE size_t -HUF_decompress1X2_usingDTable_internal_body( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) +/** + * Returns non-zero iff buffer and range overlap. + */ +static int ZSTDMT_isOverlapped(Buffer buffer, Range range) { - BIT_DStream_t bitD; + BYTE const* const bufferStart = (BYTE const*)buffer.start; + BYTE const* const rangeStart = (BYTE const*)range.start; - /* Init */ - CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) ); + if (rangeStart == NULL || bufferStart == NULL) + return 0; - /* decode */ - { BYTE* const ostart = (BYTE*) dst; - BYTE* const oend = ostart + dstSize; - const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */ - const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; - DTableDesc const dtd = HUF_getDTableDesc(DTable); - HUF_decodeStreamX2(ostart, &bitD, oend, dt, dtd.tableLog); - } + { + BYTE const* const bufferEnd = bufferStart + buffer.capacity; + BYTE const* const rangeEnd = rangeStart + range.size; - /* check */ - if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); + /* Empty ranges cannot overlap */ + if (bufferStart == bufferEnd || rangeStart == rangeEnd) + return 0; - /* decoded size */ - return dstSize; + return bufferStart < rangeEnd && rangeStart < bufferEnd; + } } -FORCE_INLINE_TEMPLATE size_t -HUF_decompress4X2_usingDTable_internal_body( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) +static int ZSTDMT_doesOverlapWindow(Buffer buffer, ZSTD_window_t window) { - if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + Range extDict; + Range prefix; - { const BYTE* const istart = (const BYTE*) cSrc; - BYTE* const ostart = (BYTE*) dst; - BYTE* const oend = ostart + dstSize; - BYTE* const olimit = oend - (sizeof(size_t)-1); - const void* const dtPtr = DTable+1; - const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; + DEBUGLOG(5, "ZSTDMT_doesOverlapWindow"); + extDict.start = window.dictBase + window.lowLimit; + extDict.size = window.dictLimit - window.lowLimit; - /* Init */ - BIT_DStream_t bitD1; - BIT_DStream_t bitD2; - BIT_DStream_t bitD3; - BIT_DStream_t bitD4; - size_t const length1 = MEM_readLE16(istart); - size_t const length2 = MEM_readLE16(istart+2); - size_t const length3 = MEM_readLE16(istart+4); - size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); - const BYTE* const istart1 = istart + 6; /* jumpTable */ - const BYTE* const istart2 = istart1 + length1; - const BYTE* const istart3 = istart2 + length2; - const BYTE* const istart4 = istart3 + length3; - size_t const segmentSize = (dstSize+3) / 4; - BYTE* const opStart2 = ostart + segmentSize; - BYTE* const opStart3 = opStart2 + segmentSize; - BYTE* const opStart4 = opStart3 + segmentSize; - BYTE* op1 = ostart; - BYTE* op2 = opStart2; - BYTE* op3 = opStart3; - BYTE* op4 = opStart4; - U32 endSignal = 1; - DTableDesc const dtd = HUF_getDTableDesc(DTable); - U32 const dtLog = dtd.tableLog; + prefix.start = window.base + window.dictLimit; + prefix.size = window.nextSrc - (window.base + window.dictLimit); + DEBUGLOG(5, "extDict [0x%zx, 0x%zx)", + (size_t)extDict.start, + (size_t)extDict.start + extDict.size); + DEBUGLOG(5, "prefix [0x%zx, 0x%zx)", + (size_t)prefix.start, + (size_t)prefix.start + prefix.size); - if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ - CHECK_F( BIT_initDStream(&bitD1, istart1, length1) ); - CHECK_F( BIT_initDStream(&bitD2, istart2, length2) ); - CHECK_F( BIT_initDStream(&bitD3, istart3, length3) ); - CHECK_F( BIT_initDStream(&bitD4, istart4, length4) ); + return ZSTDMT_isOverlapped(buffer, extDict) + || ZSTDMT_isOverlapped(buffer, prefix); +} - /* 16-32 symbols per loop (4-8 symbols per stream) */ - for ( ; (endSignal) & (op4 < olimit); ) { -#if defined(__clang__) && (defined(__x86_64__) || defined(__i386__)) - HUF_DECODE_SYMBOLX2_2(op1, &bitD1); - HUF_DECODE_SYMBOLX2_1(op1, &bitD1); - HUF_DECODE_SYMBOLX2_2(op1, &bitD1); - HUF_DECODE_SYMBOLX2_0(op1, &bitD1); - HUF_DECODE_SYMBOLX2_2(op2, &bitD2); - HUF_DECODE_SYMBOLX2_1(op2, &bitD2); - HUF_DECODE_SYMBOLX2_2(op2, &bitD2); - HUF_DECODE_SYMBOLX2_0(op2, &bitD2); - endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished; - endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished; - HUF_DECODE_SYMBOLX2_2(op3, &bitD3); - HUF_DECODE_SYMBOLX2_1(op3, &bitD3); - HUF_DECODE_SYMBOLX2_2(op3, &bitD3); - HUF_DECODE_SYMBOLX2_0(op3, &bitD3); - HUF_DECODE_SYMBOLX2_2(op4, &bitD4); - HUF_DECODE_SYMBOLX2_1(op4, &bitD4); - HUF_DECODE_SYMBOLX2_2(op4, &bitD4); - HUF_DECODE_SYMBOLX2_0(op4, &bitD4); - endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished; - endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished; -#else - HUF_DECODE_SYMBOLX2_2(op1, &bitD1); - HUF_DECODE_SYMBOLX2_2(op2, &bitD2); - HUF_DECODE_SYMBOLX2_2(op3, &bitD3); - HUF_DECODE_SYMBOLX2_2(op4, &bitD4); - HUF_DECODE_SYMBOLX2_1(op1, &bitD1); - HUF_DECODE_SYMBOLX2_1(op2, &bitD2); - HUF_DECODE_SYMBOLX2_1(op3, &bitD3); - HUF_DECODE_SYMBOLX2_1(op4, &bitD4); - HUF_DECODE_SYMBOLX2_2(op1, &bitD1); - HUF_DECODE_SYMBOLX2_2(op2, &bitD2); - HUF_DECODE_SYMBOLX2_2(op3, &bitD3); - HUF_DECODE_SYMBOLX2_2(op4, &bitD4); - HUF_DECODE_SYMBOLX2_0(op1, &bitD1); - HUF_DECODE_SYMBOLX2_0(op2, &bitD2); - HUF_DECODE_SYMBOLX2_0(op3, &bitD3); - HUF_DECODE_SYMBOLX2_0(op4, &bitD4); - endSignal = (U32)LIKELY( - (BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished) - & (BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished) - & (BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished) - & (BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished)); -#endif +static void ZSTDMT_waitForLdmComplete(ZSTDMT_CCtx* mtctx, Buffer buffer) +{ + if (mtctx->params.ldmParams.enableLdm == ZSTD_ps_enable) { + ZSTD_pthread_mutex_t* mutex = &mtctx->serial.ldmWindowMutex; + DEBUGLOG(5, "ZSTDMT_waitForLdmComplete"); + DEBUGLOG(5, "source [0x%zx, 0x%zx)", + (size_t)buffer.start, + (size_t)buffer.start + buffer.capacity); + ZSTD_PTHREAD_MUTEX_LOCK(mutex); + while (ZSTDMT_doesOverlapWindow(buffer, mtctx->serial.ldmWindow)) { + DEBUGLOG(5, "Waiting for LDM to finish..."); + ZSTD_pthread_cond_wait(&mtctx->serial.ldmWindowCond, mutex); } + DEBUGLOG(6, "Done waiting for LDM to finish"); + ZSTD_pthread_mutex_unlock(mutex); + } +} - /* check corruption */ - if (op1 > opStart2) return ERROR(corruption_detected); - if (op2 > opStart3) return ERROR(corruption_detected); - if (op3 > opStart4) return ERROR(corruption_detected); - /* note : op4 already verified within main loop */ +/** + * Attempts to set the inBuff to the next section to fill. + * If any part of the new section is still in use we give up. + * Returns non-zero if the buffer is filled. + */ +static int ZSTDMT_tryGetInputRange(ZSTDMT_CCtx* mtctx) +{ + Range const inUse = ZSTDMT_getInputDataInUse(mtctx); + size_t const spaceLeft = mtctx->roundBuff.capacity - mtctx->roundBuff.pos; + size_t const spaceNeeded = mtctx->targetSectionSize; + Buffer buffer; - /* finish bitStreams one by one */ - HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog); - HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog); - HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog); - HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog); + DEBUGLOG(5, "ZSTDMT_tryGetInputRange"); + assert(mtctx->inBuff.buffer.start == NULL); + assert(mtctx->roundBuff.capacity >= spaceNeeded); - /* check */ - { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); - if (!endCheck) return ERROR(corruption_detected); } + if (spaceLeft < spaceNeeded) { + /* ZSTD_invalidateRepCodes() doesn't work for extDict variants. + * Simply copy the prefix to the beginning in that case. + */ + BYTE* const start = (BYTE*)mtctx->roundBuff.buffer; + size_t const prefixSize = mtctx->inBuff.prefix.size; - /* decoded size */ - return dstSize; + buffer.start = start; + buffer.capacity = prefixSize; + if (ZSTDMT_isOverlapped(buffer, inUse)) { + DEBUGLOG(5, "Waiting for buffer..."); + return 0; + } + ZSTDMT_waitForLdmComplete(mtctx, buffer); + ZSTD_memmove(start, mtctx->inBuff.prefix.start, prefixSize); + mtctx->inBuff.prefix.start = start; + mtctx->roundBuff.pos = prefixSize; } -} - -HUF_DGEN(HUF_decompress1X2_usingDTable_internal) -HUF_DGEN(HUF_decompress4X2_usingDTable_internal) - -size_t HUF_decompress1X2_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (dtd.tableType != 1) return ERROR(GENERIC); - return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -} + buffer.start = mtctx->roundBuff.buffer + mtctx->roundBuff.pos; + buffer.capacity = spaceNeeded; -size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize) -{ - const BYTE* ip = (const BYTE*) cSrc; + if (ZSTDMT_isOverlapped(buffer, inUse)) { + DEBUGLOG(5, "Waiting for buffer..."); + return 0; + } + assert(!ZSTDMT_isOverlapped(buffer, mtctx->inBuff.prefix)); - size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize, - workSpace, wkspSize); - if (HUF_isError(hSize)) return hSize; - if (hSize >= cSrcSize) return ERROR(srcSize_wrong); - ip += hSize; cSrcSize -= hSize; + ZSTDMT_waitForLdmComplete(mtctx, buffer); - return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, /* bmi2 */ 0); -} + DEBUGLOG(5, "Using prefix range [%zx, %zx)", + (size_t)mtctx->inBuff.prefix.start, + (size_t)mtctx->inBuff.prefix.start + mtctx->inBuff.prefix.size); + DEBUGLOG(5, "Using source range [%zx, %zx)", + (size_t)buffer.start, + (size_t)buffer.start + buffer.capacity); -size_t HUF_decompress4X2_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (dtd.tableType != 1) return ERROR(GENERIC); - return HUF_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); + mtctx->inBuff.buffer = buffer; + mtctx->inBuff.filled = 0; + assert(mtctx->roundBuff.pos + buffer.capacity <= mtctx->roundBuff.capacity); + return 1; } -static size_t HUF_decompress4X2_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize, int bmi2) -{ - const BYTE* ip = (const BYTE*) cSrc; - - size_t hSize = HUF_readDTableX2_wksp(dctx, cSrc, cSrcSize, - workSpace, wkspSize); - if (HUF_isError(hSize)) return hSize; - if (hSize >= cSrcSize) return ERROR(srcSize_wrong); - ip += hSize; cSrcSize -= hSize; - - return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2); -} +typedef struct { + size_t toLoad; /* The number of bytes to load from the input. */ + int flush; /* Boolean declaring if we must flush because we found a synchronization point. */ +} SyncPoint; -size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize) +/** + * Searches through the input for a synchronization point. If one is found, we + * will instruct the caller to flush, and return the number of bytes to load. + * Otherwise, we will load as many bytes as possible and instruct the caller + * to continue as normal. + */ +static SyncPoint +findSynchronizationPoint(ZSTDMT_CCtx const* mtctx, ZSTD_inBuffer const input) { - return HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, /* bmi2 */ 0); -} - - -#endif /* HUF_FORCE_DECOMPRESS_X1 */ - + BYTE const* const istart = (BYTE const*)input.src + input.pos; + U64 const primePower = mtctx->rsync.primePower; + U64 const hitMask = mtctx->rsync.hitMask; -/* ***********************************/ -/* Universal decompression selectors */ -/* ***********************************/ + SyncPoint syncPoint; + U64 hash; + BYTE const* prev; + size_t pos; -size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc const dtd = HUF_getDTableDesc(DTable); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)dtd; - assert(dtd.tableType == 0); - return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)dtd; - assert(dtd.tableType == 1); - return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -#else - return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) : - HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -#endif + syncPoint.toLoad = MIN(input.size - input.pos, mtctx->targetSectionSize - mtctx->inBuff.filled); + syncPoint.flush = 0; + if (!mtctx->params.rsyncable) + /* Rsync is disabled. */ + return syncPoint; + if (mtctx->inBuff.filled + input.size - input.pos < RSYNC_MIN_BLOCK_SIZE) + /* We don't emit synchronization points if it would produce too small blocks. + * We don't have enough input to find a synchronization point, so don't look. + */ + return syncPoint; + if (mtctx->inBuff.filled + syncPoint.toLoad < RSYNC_LENGTH) + /* Not enough to compute the hash. + * We will miss any synchronization points in this RSYNC_LENGTH byte + * window. However, since it depends only in the internal buffers, if the + * state is already synchronized, we will remain synchronized. + * Additionally, the probability that we miss a synchronization point is + * low: RSYNC_LENGTH / targetSectionSize. + */ + return syncPoint; + /* Initialize the loop variables. */ + if (mtctx->inBuff.filled < RSYNC_MIN_BLOCK_SIZE) { + /* We don't need to scan the first RSYNC_MIN_BLOCK_SIZE positions + * because they can't possibly be a sync point. So we can start + * part way through the input buffer. + */ + pos = RSYNC_MIN_BLOCK_SIZE - mtctx->inBuff.filled; + if (pos >= RSYNC_LENGTH) { + prev = istart + pos - RSYNC_LENGTH; + hash = ZSTD_rollingHash_compute(prev, RSYNC_LENGTH); + } else { + assert(mtctx->inBuff.filled >= RSYNC_LENGTH); + prev = (BYTE const*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled - RSYNC_LENGTH; + hash = ZSTD_rollingHash_compute(prev + pos, (RSYNC_LENGTH - pos)); + hash = ZSTD_rollingHash_append(hash, istart, pos); + } + } else { + /* We have enough bytes buffered to initialize the hash, + * and have processed enough bytes to find a sync point. + * Start scanning at the beginning of the input. + */ + assert(mtctx->inBuff.filled >= RSYNC_MIN_BLOCK_SIZE); + assert(RSYNC_MIN_BLOCK_SIZE >= RSYNC_LENGTH); + pos = 0; + prev = (BYTE const*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled - RSYNC_LENGTH; + hash = ZSTD_rollingHash_compute(prev, RSYNC_LENGTH); + if ((hash & hitMask) == hitMask) { + /* We're already at a sync point so don't load any more until + * we're able to flush this sync point. + * This likely happened because the job table was full so we + * couldn't add our job. + */ + syncPoint.toLoad = 0; + syncPoint.flush = 1; + return syncPoint; + } + } + /* Starting with the hash of the previous RSYNC_LENGTH bytes, roll + * through the input. If we hit a synchronization point, then cut the + * job off, and tell the compressor to flush the job. Otherwise, load + * all the bytes and continue as normal. + * If we go too long without a synchronization point (targetSectionSize) + * then a block will be emitted anyways, but this is okay, since if we + * are already synchronized we will remain synchronized. + */ + assert(pos < RSYNC_LENGTH || ZSTD_rollingHash_compute(istart + pos - RSYNC_LENGTH, RSYNC_LENGTH) == hash); + for (; pos < syncPoint.toLoad; ++pos) { + BYTE const toRemove = pos < RSYNC_LENGTH ? prev[pos] : istart[pos - RSYNC_LENGTH]; + /* This assert is very expensive, and Debian compiles with asserts enabled. + * So disable it for now. We can get similar coverage by checking it at the + * beginning & end of the loop. + * assert(pos < RSYNC_LENGTH || ZSTD_rollingHash_compute(istart + pos - RSYNC_LENGTH, RSYNC_LENGTH) == hash); + */ + hash = ZSTD_rollingHash_rotate(hash, toRemove, istart[pos], primePower); + assert(mtctx->inBuff.filled + pos >= RSYNC_MIN_BLOCK_SIZE); + if ((hash & hitMask) == hitMask) { + syncPoint.toLoad = pos + 1; + syncPoint.flush = 1; + ++pos; /* for assert */ + break; + } + } + assert(pos < RSYNC_LENGTH || ZSTD_rollingHash_compute(istart + pos - RSYNC_LENGTH, RSYNC_LENGTH) == hash); + return syncPoint; } -size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) +size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx) { - DTableDesc const dtd = HUF_getDTableDesc(DTable); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)dtd; - assert(dtd.tableType == 0); - return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)dtd; - assert(dtd.tableType == 1); - return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -#else - return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) : - HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -#endif + size_t hintInSize = mtctx->targetSectionSize - mtctx->inBuff.filled; + if (hintInSize==0) hintInSize = mtctx->targetSectionSize; + return hintInSize; } - -#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2) -typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t; -static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] = +/** ZSTDMT_compressStream_generic() : + * internal use only - exposed to be invoked from zstd_compress.c + * assumption : output and input are valid (pos <= size) + * @return : minimum amount of data remaining to flush, 0 if none */ +size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective endOp) { - /* single, double, quad */ - {{0,0}, {1,1}, {2,2}}, /* Q==0 : impossible */ - {{0,0}, {1,1}, {2,2}}, /* Q==1 : impossible */ - {{ 38,130}, {1313, 74}, {2151, 38}}, /* Q == 2 : 12-18% */ - {{ 448,128}, {1353, 74}, {2238, 41}}, /* Q == 3 : 18-25% */ - {{ 556,128}, {1353, 74}, {2238, 47}}, /* Q == 4 : 25-32% */ - {{ 714,128}, {1418, 74}, {2436, 53}}, /* Q == 5 : 32-38% */ - {{ 883,128}, {1437, 74}, {2464, 61}}, /* Q == 6 : 38-44% */ - {{ 897,128}, {1515, 75}, {2622, 68}}, /* Q == 7 : 44-50% */ - {{ 926,128}, {1613, 75}, {2730, 75}}, /* Q == 8 : 50-56% */ - {{ 947,128}, {1729, 77}, {3359, 77}}, /* Q == 9 : 56-62% */ - {{1107,128}, {2083, 81}, {4006, 84}}, /* Q ==10 : 62-69% */ - {{1177,128}, {2379, 87}, {4785, 88}}, /* Q ==11 : 69-75% */ - {{1242,128}, {2415, 93}, {5155, 84}}, /* Q ==12 : 75-81% */ - {{1349,128}, {2644,106}, {5260,106}}, /* Q ==13 : 81-87% */ - {{1455,128}, {2422,124}, {4174,124}}, /* Q ==14 : 87-93% */ - {{ 722,128}, {1891,145}, {1936,146}}, /* Q ==15 : 93-99% */ -}; -#endif + unsigned forwardInputProgress = 0; + DEBUGLOG(5, "ZSTDMT_compressStream_generic (endOp=%u, srcSize=%u)", + (U32)endOp, (U32)(input->size - input->pos)); + assert(output->pos <= output->size); + assert(input->pos <= input->size); -/** HUF_selectDecoder() : - * Tells which decoder is likely to decode faster, - * based on a set of pre-computed metrics. - * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 . - * Assumption : 0 < dstSize <= 128 KB */ -U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize) -{ - assert(dstSize > 0); - assert(dstSize <= 128*1024); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)dstSize; - (void)cSrcSize; - return 0; -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)dstSize; - (void)cSrcSize; - return 1; -#else - /* decoder timing evaluation */ - { U32 const Q = (cSrcSize >= dstSize) ? 15 : (U32)(cSrcSize * 16 / dstSize); /* Q < 16 */ - U32 const D256 = (U32)(dstSize >> 8); - U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256); - U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256); - DTime1 += DTime1 >> 3; /* advantage to algorithm using less memory, to reduce cache eviction */ - return DTime1 < DTime0; + if ((mtctx->frameEnded) && (endOp==ZSTD_e_continue)) { + /* current frame being ended. Only flush/end are allowed */ + return ERROR(stage_wrong); } -#endif -} - -size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, - size_t dstSize, const void* cSrc, - size_t cSrcSize, void* workSpace, - size_t wkspSize) -{ - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize == 0) return ERROR(corruption_detected); - - { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)algoNb; - assert(algoNb == 0); - return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)algoNb; - assert(algoNb == 1); - return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize); -#else - return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, - cSrcSize, workSpace, wkspSize): - HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize); -#endif + /* fill input buffer */ + if ( (!mtctx->jobReady) + && (input->size > input->pos) ) { /* support NULL input */ + if (mtctx->inBuff.buffer.start == NULL) { + assert(mtctx->inBuff.filled == 0); /* Can't fill an empty buffer */ + if (!ZSTDMT_tryGetInputRange(mtctx)) { + /* It is only possible for this operation to fail if there are + * still compression jobs ongoing. + */ + DEBUGLOG(5, "ZSTDMT_tryGetInputRange failed"); + assert(mtctx->doneJobID != mtctx->nextJobID); + } else + DEBUGLOG(5, "ZSTDMT_tryGetInputRange completed successfully : mtctx->inBuff.buffer.start = %p", mtctx->inBuff.buffer.start); + } + if (mtctx->inBuff.buffer.start != NULL) { + SyncPoint const syncPoint = findSynchronizationPoint(mtctx, *input); + if (syncPoint.flush && endOp == ZSTD_e_continue) { + endOp = ZSTD_e_flush; + } + assert(mtctx->inBuff.buffer.capacity >= mtctx->targetSectionSize); + DEBUGLOG(5, "ZSTDMT_compressStream_generic: adding %u bytes on top of %u to buffer of size %u", + (U32)syncPoint.toLoad, (U32)mtctx->inBuff.filled, (U32)mtctx->targetSectionSize); + ZSTD_memcpy((char*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled, (const char*)input->src + input->pos, syncPoint.toLoad); + input->pos += syncPoint.toLoad; + mtctx->inBuff.filled += syncPoint.toLoad; + forwardInputProgress = syncPoint.toLoad>0; + } + } + if ((input->pos < input->size) && (endOp == ZSTD_e_end)) { + /* Can't end yet because the input is not fully consumed. + * We are in one of these cases: + * - mtctx->inBuff is NULL & empty: we couldn't get an input buffer so don't create a new job. + * - We filled the input buffer: flush this job but don't end the frame. + * - We hit a synchronization point: flush this job but don't end the frame. + */ + assert(mtctx->inBuff.filled == 0 || mtctx->inBuff.filled == mtctx->targetSectionSize || mtctx->params.rsyncable); + endOp = ZSTD_e_flush; } -} -size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize) -{ - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ - if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ - if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ + if ( (mtctx->jobReady) + || (mtctx->inBuff.filled >= mtctx->targetSectionSize) /* filled enough : let's compress */ + || ((endOp != ZSTD_e_continue) && (mtctx->inBuff.filled > 0)) /* something to flush : let's go */ + || ((endOp == ZSTD_e_end) && (!mtctx->frameEnded)) ) { /* must finish the frame with a zero-size block */ + size_t const jobSize = mtctx->inBuff.filled; + assert(mtctx->inBuff.filled <= mtctx->targetSectionSize); + FORWARD_IF_ERROR( ZSTDMT_createCompressionJob(mtctx, jobSize, endOp) , ""); + } - { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)algoNb; - assert(algoNb == 0); - return HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, - cSrcSize, workSpace, wkspSize); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)algoNb; - assert(algoNb == 1); - return HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, - cSrcSize, workSpace, wkspSize); -#else - return algoNb ? HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, - cSrcSize, workSpace, wkspSize): - HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, - cSrcSize, workSpace, wkspSize); -#endif + /* check for potential compressed data ready to be flushed */ + { size_t const remainingToFlush = ZSTDMT_flushProduced(mtctx, output, !forwardInputProgress, endOp); /* block if there was no forward input progress */ + if (input->pos < input->size) return MAX(remainingToFlush, 1); /* input not consumed : do not end flush yet */ + DEBUGLOG(5, "end of ZSTDMT_compressStream_generic: remainingToFlush = %u", (U32)remainingToFlush); + return remainingToFlush; } } +/**** ended inlining compress/zstdmt_compress.c ****/ +#endif +/**** start inlining decompress/huf_decompress.c ****/ +/* ****************************************************************** + * huff0 huffman decoder, + * part of Finite State Entropy library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ -size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2) -{ - DTableDesc const dtd = HUF_getDTableDesc(DTable); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)dtd; - assert(dtd.tableType == 0); - return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)dtd; - assert(dtd.tableType == 1); - return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); -#else - return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) : - HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); -#endif -} +/* ************************************************************** +* Dependencies +****************************************************************/ +/**** skipping file: ../common/zstd_deps.h ****/ +/**** skipping file: ../common/compiler.h ****/ +/**** skipping file: ../common/bitstream.h ****/ +/**** skipping file: ../common/fse.h ****/ +/**** skipping file: ../common/huf.h ****/ +/**** skipping file: ../common/error_private.h ****/ +/**** skipping file: ../common/zstd_internal.h ****/ +/**** skipping file: ../common/bits.h ****/ -#ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2) -{ - const BYTE* ip = (const BYTE*) cSrc; +/* ************************************************************** +* Constants +****************************************************************/ - size_t const hSize = HUF_readDTableX1_wksp_bmi2(dctx, cSrc, cSrcSize, workSpace, wkspSize, bmi2); - if (HUF_isError(hSize)) return hSize; - if (hSize >= cSrcSize) return ERROR(srcSize_wrong); - ip += hSize; cSrcSize -= hSize; +#define HUF_DECODER_FAST_TABLELOG 11 - return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2); -} -#endif +/* ************************************************************** +* Macros +****************************************************************/ -size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2) -{ - DTableDesc const dtd = HUF_getDTableDesc(DTable); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)dtd; - assert(dtd.tableType == 0); - return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)dtd; - assert(dtd.tableType == 1); - return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); +#ifdef HUF_DISABLE_FAST_DECODE +# define HUF_ENABLE_FAST_DECODE 0 #else - return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) : - HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); +# define HUF_ENABLE_FAST_DECODE 1 #endif -} -size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2) -{ - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize == 0) return ERROR(corruption_detected); +/* These two optional macros force the use one way or another of the two + * Huffman decompression implementations. You can't force in both directions + * at the same time. + */ +#if defined(HUF_FORCE_DECOMPRESS_X1) && \ + defined(HUF_FORCE_DECOMPRESS_X2) +#error "Cannot force the use of the X1 and X2 decoders at the same time!" +#endif - { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)algoNb; - assert(algoNb == 0); - return HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)algoNb; - assert(algoNb == 1); - return HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2); +/* When DYNAMIC_BMI2 is enabled, fast decoders are only called when bmi2 is + * supported at runtime, so we can add the BMI2 target attribute. + * When it is disabled, we will still get BMI2 if it is enabled statically. + */ +#if DYNAMIC_BMI2 +# define HUF_FAST_BMI2_ATTRS BMI2_TARGET_ATTRIBUTE #else - return algoNb ? HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2) : - HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2); +# define HUF_FAST_BMI2_ATTRS #endif - } -} - -#ifndef ZSTD_NO_UNUSED_FUNCTIONS -#ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_readDTableX1(HUF_DTable* DTable, const void* src, size_t srcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_readDTableX1_wksp(DTable, src, srcSize, - workSpace, sizeof(workSpace)); -} - -size_t HUF_decompress1X1_DCtx(HUF_DTable* DCtx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_decompress1X1_DCtx_wksp(DCtx, dst, dstSize, cSrc, cSrcSize, - workSpace, sizeof(workSpace)); -} -size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUF_CREATE_STATIC_DTABLEX1(DTable, HUF_TABLELOG_MAX); - return HUF_decompress1X1_DCtx (DTable, dst, dstSize, cSrc, cSrcSize); -} +#ifdef __cplusplus +# define HUF_EXTERN_C extern "C" +#else +# define HUF_EXTERN_C #endif +#define HUF_ASM_DECL HUF_EXTERN_C -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_readDTableX2(HUF_DTable* DTable, const void* src, size_t srcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_readDTableX2_wksp(DTable, src, srcSize, - workSpace, sizeof(workSpace)); -} - -size_t HUF_decompress1X2_DCtx(HUF_DTable* DCtx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_decompress1X2_DCtx_wksp(DCtx, dst, dstSize, cSrc, cSrcSize, - workSpace, sizeof(workSpace)); -} - -size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX); - return HUF_decompress1X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); -} +#if DYNAMIC_BMI2 +# define HUF_NEED_BMI2_FUNCTION 1 +#else +# define HUF_NEED_BMI2_FUNCTION 0 #endif -#ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_decompress4X1_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, - workSpace, sizeof(workSpace)); -} -size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUF_CREATE_STATIC_DTABLEX1(DTable, HUF_TABLELOG_MAX); - return HUF_decompress4X1_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); -} -#endif +/* ************************************************************** +* Error Management +****************************************************************/ +#define HUF_isError ERR_isError -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, - workSpace, sizeof(workSpace)); -} -size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX); - return HUF_decompress4X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); -} -#endif +/* ************************************************************** +* Byte alignment for workSpace management +****************************************************************/ +#define HUF_ALIGN(x, a) HUF_ALIGN_MASK((x), (a) - 1) +#define HUF_ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask)) -typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); -size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ -#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2) - static const decompressionAlgo decompress[2] = { HUF_decompress4X1, HUF_decompress4X2 }; -#endif +/* ************************************************************** +* BMI2 Variant Wrappers +****************************************************************/ +typedef size_t (*HUF_DecompressUsingDTableFn)(void *dst, size_t dstSize, + const void *cSrc, + size_t cSrcSize, + const HUF_DTable *DTable); - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ - if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ - if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ +#if DYNAMIC_BMI2 - { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)algoNb; - assert(algoNb == 0); - return HUF_decompress4X1(dst, dstSize, cSrc, cSrcSize); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)algoNb; - assert(algoNb == 1); - return HUF_decompress4X2(dst, dstSize, cSrc, cSrcSize); -#else - return decompress[algoNb](dst, dstSize, cSrc, cSrcSize); -#endif +#define HUF_DGEN(fn) \ + \ + static size_t fn##_default( \ + void* dst, size_t dstSize, \ + const void* cSrc, size_t cSrcSize, \ + const HUF_DTable* DTable) \ + { \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + \ + static BMI2_TARGET_ATTRIBUTE size_t fn##_bmi2( \ + void* dst, size_t dstSize, \ + const void* cSrc, size_t cSrcSize, \ + const HUF_DTable* DTable) \ + { \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + \ + static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ + size_t cSrcSize, HUF_DTable const* DTable, int flags) \ + { \ + if (flags & HUF_flags_bmi2) { \ + return fn##_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + return fn##_default(dst, dstSize, cSrc, cSrcSize, DTable); \ } -} -size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ - if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ - if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ - - { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)algoNb; - assert(algoNb == 0); - return HUF_decompress4X1_DCtx(dctx, dst, dstSize, cSrc, cSrcSize); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)algoNb; - assert(algoNb == 1); - return HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize); #else - return algoNb ? HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : - HUF_decompress4X1_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; -#endif - } -} -size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_decompress4X_hufOnly_wksp(dctx, dst, dstSize, cSrc, cSrcSize, - workSpace, sizeof(workSpace)); -} +#define HUF_DGEN(fn) \ + static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ + size_t cSrcSize, HUF_DTable const* DTable, int flags) \ + { \ + (void)flags; \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } -size_t HUF_decompress1X_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_decompress1X_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, - workSpace, sizeof(workSpace)); -} #endif -/**** ended inlining decompress/huf_decompress.c ****/ -/**** start inlining decompress/zstd_ddict.c ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ -/* zstd_ddict.c : - * concentrates all logic that needs to know the internals of ZSTD_DDict object */ -/*-******************************************************* -* Dependencies -*********************************************************/ -/**** skipping file: ../common/zstd_deps.h ****/ -/**** skipping file: ../common/cpu.h ****/ -/**** skipping file: ../common/mem.h ****/ -#define FSE_STATIC_LINKING_ONLY -/**** skipping file: ../common/fse.h ****/ -#define HUF_STATIC_LINKING_ONLY -/**** skipping file: ../common/huf.h ****/ -/**** start inlining zstd_decompress_internal.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. +/*-***************************/ +/* generic DTableDesc */ +/*-***************************/ +typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc; + +static DTableDesc HUF_getDTableDesc(const HUF_DTable* table) +{ + DTableDesc dtd; + ZSTD_memcpy(&dtd, table, sizeof(dtd)); + return dtd; +} + +static size_t HUF_initFastDStream(BYTE const* ip) { + BYTE const lastByte = ip[7]; + size_t const bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0; + size_t const value = MEM_readLEST(ip) | 1; + assert(bitsConsumed <= 8); + assert(sizeof(size_t) == 8); + return value << bitsConsumed; +} + + +/** + * The input/output arguments to the Huffman fast decoding loop: + * + * ip [in/out] - The input pointers, must be updated to reflect what is consumed. + * op [in/out] - The output pointers, must be updated to reflect what is written. + * bits [in/out] - The bitstream containers, must be updated to reflect the current state. + * dt [in] - The decoding table. + * ilowest [in] - The beginning of the valid range of the input. Decoders may read + * down to this pointer. It may be below iend[0]. + * oend [in] - The end of the output stream. op[3] must not cross oend. + * iend [in] - The end of each input stream. ip[i] may cross iend[i], + * as long as it is above ilowest, but that indicates corruption. */ +typedef struct { + BYTE const* ip[4]; + BYTE* op[4]; + U64 bits[4]; + void const* dt; + BYTE const* ilowest; + BYTE* oend; + BYTE const* iend[4]; +} HUF_DecompressFastArgs; +typedef void (*HUF_DecompressFastLoopFn)(HUF_DecompressFastArgs*); -/* zstd_decompress_internal: - * objects and definitions shared within lib/decompress modules */ +/** + * Initializes args for the fast decoding loop. + * @returns 1 on success + * 0 if the fallback implementation should be used. + * Or an error code on failure. + */ +static size_t HUF_DecompressFastArgs_init(HUF_DecompressFastArgs* args, void* dst, size_t dstSize, void const* src, size_t srcSize, const HUF_DTable* DTable) +{ + void const* dt = DTable + 1; + U32 const dtLog = HUF_getDTableDesc(DTable).tableLog; - #ifndef ZSTD_DECOMPRESS_INTERNAL_H - #define ZSTD_DECOMPRESS_INTERNAL_H + const BYTE* const istart = (const BYTE*)src; + BYTE* const oend = ZSTD_maybeNullPtrAdd((BYTE*)dst, dstSize); -/*-******************************************************* - * Dependencies - *********************************************************/ -/**** skipping file: ../common/mem.h ****/ -/**** skipping file: ../common/zstd_internal.h ****/ -/**** skipping file: ../common/zstd_trace.h ****/ + /* The fast decoding loop assumes 64-bit little-endian. + * This condition is false on x32. + */ + if (!MEM_isLittleEndian() || MEM_32bits()) + return 0; + /* Avoid nullptr addition */ + if (dstSize == 0) + return 0; + assert(dst != NULL); + /* strict minimum : jump table + 1 byte per stream */ + if (srcSize < 10) + return ERROR(corruption_detected); -/*-******************************************************* - * Constants - *********************************************************/ -static UNUSED_ATTR const U32 LL_base[MaxLL+1] = { - 0, 1, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, - 16, 18, 20, 22, 24, 28, 32, 40, - 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, - 0x2000, 0x4000, 0x8000, 0x10000 }; + /* Must have at least 8 bytes per stream because we don't handle initializing smaller bit containers. + * If table log is not correct at this point, fallback to the old decoder. + * On small inputs we don't have enough data to trigger the fast loop, so use the old decoder. + */ + if (dtLog != HUF_DECODER_FAST_TABLELOG) + return 0; -static UNUSED_ATTR const U32 OF_base[MaxOff+1] = { - 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, - 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, - 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, - 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD }; + /* Read the jump table. */ + { + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = srcSize - (length1 + length2 + length3 + 6); + args->iend[0] = istart + 6; /* jumpTable */ + args->iend[1] = args->iend[0] + length1; + args->iend[2] = args->iend[1] + length2; + args->iend[3] = args->iend[2] + length3; + + /* HUF_initFastDStream() requires this, and this small of an input + * won't benefit from the ASM loop anyways. + */ + if (length1 < 8 || length2 < 8 || length3 < 8 || length4 < 8) + return 0; + if (length4 > srcSize) return ERROR(corruption_detected); /* overflow */ + } + /* ip[] contains the position that is currently loaded into bits[]. */ + args->ip[0] = args->iend[1] - sizeof(U64); + args->ip[1] = args->iend[2] - sizeof(U64); + args->ip[2] = args->iend[3] - sizeof(U64); + args->ip[3] = (BYTE const*)src + srcSize - sizeof(U64); + + /* op[] contains the output pointers. */ + args->op[0] = (BYTE*)dst; + args->op[1] = args->op[0] + (dstSize+3)/4; + args->op[2] = args->op[1] + (dstSize+3)/4; + args->op[3] = args->op[2] + (dstSize+3)/4; + + /* No point to call the ASM loop for tiny outputs. */ + if (args->op[3] >= oend) + return 0; -static UNUSED_ATTR const U32 OF_bits[MaxOff+1] = { - 0, 1, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 21, 22, 23, - 24, 25, 26, 27, 28, 29, 30, 31 }; + /* bits[] is the bit container. + * It is read from the MSB down to the LSB. + * It is shifted left as it is read, and zeros are + * shifted in. After the lowest valid bit a 1 is + * set, so that CountTrailingZeros(bits[]) can be used + * to count how many bits we've consumed. + */ + args->bits[0] = HUF_initFastDStream(args->ip[0]); + args->bits[1] = HUF_initFastDStream(args->ip[1]); + args->bits[2] = HUF_initFastDStream(args->ip[2]); + args->bits[3] = HUF_initFastDStream(args->ip[3]); + + /* The decoders must be sure to never read beyond ilowest. + * This is lower than iend[0], but allowing decoders to read + * down to ilowest can allow an extra iteration or two in the + * fast loop. + */ + args->ilowest = istart; -static UNUSED_ATTR const U32 ML_base[MaxML+1] = { - 3, 4, 5, 6, 7, 8, 9, 10, - 11, 12, 13, 14, 15, 16, 17, 18, - 19, 20, 21, 22, 23, 24, 25, 26, - 27, 28, 29, 30, 31, 32, 33, 34, - 35, 37, 39, 41, 43, 47, 51, 59, - 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, - 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 }; + args->oend = oend; + args->dt = dt; + return 1; +} -/*-******************************************************* - * Decompression types - *********************************************************/ - typedef struct { - U32 fastMode; - U32 tableLog; - } ZSTD_seqSymbol_header; +static size_t HUF_initRemainingDStream(BIT_DStream_t* bit, HUF_DecompressFastArgs const* args, int stream, BYTE* segmentEnd) +{ + /* Validate that we haven't overwritten. */ + if (args->op[stream] > segmentEnd) + return ERROR(corruption_detected); + /* Validate that we haven't read beyond iend[]. + * Note that ip[] may be < iend[] because the MSB is + * the next bit to read, and we may have consumed 100% + * of the stream, so down to iend[i] - 8 is valid. + */ + if (args->ip[stream] < args->iend[stream] - 8) + return ERROR(corruption_detected); - typedef struct { - U16 nextState; - BYTE nbAdditionalBits; - BYTE nbBits; - U32 baseValue; - } ZSTD_seqSymbol; + /* Construct the BIT_DStream_t. */ + assert(sizeof(size_t) == 8); + bit->bitContainer = MEM_readLEST(args->ip[stream]); + bit->bitsConsumed = ZSTD_countTrailingZeros64(args->bits[stream]); + bit->start = (const char*)args->ilowest; + bit->limitPtr = bit->start + sizeof(size_t); + bit->ptr = (const char*)args->ip[stream]; - #define SEQSYMBOL_TABLE_SIZE(log) (1 + (1 << (log))) + return 0; +} -#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE (sizeof(S16) * (MaxSeq + 1) + (1u << MaxFSELog) + sizeof(U64)) -#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32 ((ZSTD_BUILD_FSE_TABLE_WKSP_SIZE + sizeof(U32) - 1) / sizeof(U32)) +/* Calls X(N) for each stream 0, 1, 2, 3. */ +#define HUF_4X_FOR_EACH_STREAM(X) \ + do { \ + X(0); \ + X(1); \ + X(2); \ + X(3); \ + } while (0) -typedef struct { - ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]; /* Note : Space reserved for FSE Tables */ - ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]; /* is also used as temporary workspace while building hufTable during DDict creation */ - ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]; /* and therefore must be at least HUF_DECOMPRESS_WORKSPACE_SIZE large */ - HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */ - U32 rep[ZSTD_REP_NUM]; - U32 workspace[ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32]; -} ZSTD_entropyDTables_t; +/* Calls X(N, var) for each stream 0, 1, 2, 3. */ +#define HUF_4X_FOR_EACH_STREAM_WITH_VAR(X, var) \ + do { \ + X(0, (var)); \ + X(1, (var)); \ + X(2, (var)); \ + X(3, (var)); \ + } while (0) -typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, - ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock, - ZSTDds_decompressLastBlock, ZSTDds_checkChecksum, - ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage; -typedef enum { zdss_init=0, zdss_loadHeader, - zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage; +#ifndef HUF_FORCE_DECOMPRESS_X2 -typedef enum { - ZSTD_use_indefinitely = -1, /* Use the dictionary indefinitely */ - ZSTD_dont_use = 0, /* Do not use the dictionary (if one exists free it) */ - ZSTD_use_once = 1 /* Use the dictionary once and set to ZSTD_dont_use */ -} ZSTD_dictUses_e; +/*-***************************/ +/* single-symbol decoding */ +/*-***************************/ +typedef struct { BYTE nbBits; BYTE byte; } HUF_DEltX1; /* single-symbol decoding */ -/* Hashset for storing references to multiple ZSTD_DDict within ZSTD_DCtx */ -typedef struct { - const ZSTD_DDict** ddictPtrTable; - size_t ddictPtrTableSize; - size_t ddictPtrCount; -} ZSTD_DDictHashSet; +/** + * Packs 4 HUF_DEltX1 structs into a U64. This is used to lay down 4 entries at + * a time. + */ +static U64 HUF_DEltX1_set4(BYTE symbol, BYTE nbBits) { + U64 D4; + if (MEM_isLittleEndian()) { + D4 = (U64)((symbol << 8) + nbBits); + } else { + D4 = (U64)(symbol + (nbBits << 8)); + } + assert(D4 < (1U << 16)); + D4 *= 0x0001000100010001ULL; + return D4; +} -struct ZSTD_DCtx_s +/** + * Increase the tableLog to targetTableLog and rescales the stats. + * If tableLog > targetTableLog this is a no-op. + * @returns New tableLog + */ +static U32 HUF_rescaleStats(BYTE* huffWeight, U32* rankVal, U32 nbSymbols, U32 tableLog, U32 targetTableLog) { - const ZSTD_seqSymbol* LLTptr; - const ZSTD_seqSymbol* MLTptr; - const ZSTD_seqSymbol* OFTptr; - const HUF_DTable* HUFptr; - ZSTD_entropyDTables_t entropy; - U32 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; /* space needed when building huffman tables */ - const void* previousDstEnd; /* detect continuity */ - const void* prefixStart; /* start of current segment */ - const void* virtualStart; /* virtual start of previous segment if it was just before current one */ - const void* dictEnd; /* end of previous segment */ - size_t expected; - ZSTD_frameHeader fParams; - U64 processedCSize; - U64 decodedSize; - blockType_e bType; /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block decompression stages */ - ZSTD_dStage stage; - U32 litEntropy; - U32 fseEntropy; - XXH64_state_t xxhState; - size_t headerSize; - ZSTD_format_e format; - ZSTD_forceIgnoreChecksum_e forceIgnoreChecksum; /* User specified: if == 1, will ignore checksums in compressed frame. Default == 0 */ - U32 validateChecksum; /* if == 1, will validate checksum. Is == 1 if (fParams.checksumFlag == 1) and (forceIgnoreChecksum == 0). */ - const BYTE* litPtr; - ZSTD_customMem customMem; - size_t litSize; - size_t rleSize; - size_t staticSize; - int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */ + if (tableLog > targetTableLog) + return tableLog; + if (tableLog < targetTableLog) { + U32 const scale = targetTableLog - tableLog; + U32 s; + /* Increase the weight for all non-zero probability symbols by scale. */ + for (s = 0; s < nbSymbols; ++s) { + huffWeight[s] += (BYTE)((huffWeight[s] == 0) ? 0 : scale); + } + /* Update rankVal to reflect the new weights. + * All weights except 0 get moved to weight + scale. + * Weights [1, scale] are empty. + */ + for (s = targetTableLog; s > scale; --s) { + rankVal[s] = rankVal[s - scale]; + } + for (s = scale; s > 0; --s) { + rankVal[s] = 0; + } + } + return targetTableLog; +} - /* dictionary */ - ZSTD_DDict* ddictLocal; - const ZSTD_DDict* ddict; /* set by ZSTD_initDStream_usingDDict(), or ZSTD_DCtx_refDDict() */ - U32 dictID; - int ddictIsCold; /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */ - ZSTD_dictUses_e dictUses; - ZSTD_DDictHashSet* ddictSet; /* Hash set for multiple ddicts */ - ZSTD_refMultipleDDicts_e refMultipleDDicts; /* User specified: if == 1, will allow references to multiple DDicts. Default == 0 (disabled) */ +typedef struct { + U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; + U32 rankStart[HUF_TABLELOG_ABSOLUTEMAX + 1]; + U32 statsWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; + BYTE symbols[HUF_SYMBOLVALUE_MAX + 1]; + BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; +} HUF_ReadDTableX1_Workspace; - /* streaming */ - ZSTD_dStreamStage streamStage; - char* inBuff; - size_t inBuffSize; - size_t inPos; - size_t maxWindowSize; - char* outBuff; - size_t outBuffSize; - size_t outStart; - size_t outEnd; - size_t lhSize; - void* legacyContext; - U32 previousLegacyVersion; - U32 legacyVersion; - U32 hostageByte; - int noForwardProgress; - ZSTD_bufferMode_e outBufferMode; - ZSTD_outBuffer expectedOutBuffer; +size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags) +{ + U32 tableLog = 0; + U32 nbSymbols = 0; + size_t iSize; + void* const dtPtr = DTable + 1; + HUF_DEltX1* const dt = (HUF_DEltX1*)dtPtr; + HUF_ReadDTableX1_Workspace* wksp = (HUF_ReadDTableX1_Workspace*)workSpace; - /* workspace */ - BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH]; - BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; + DEBUG_STATIC_ASSERT(HUF_DECOMPRESS_WORKSPACE_SIZE >= sizeof(*wksp)); + if (sizeof(*wksp) > wkspSize) return ERROR(tableLog_tooLarge); - size_t oversizedDuration; + DEBUG_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable)); + /* ZSTD_memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */ -#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION - void const* dictContentBeginForFuzzing; - void const* dictContentEndForFuzzing; -#endif + iSize = HUF_readStats_wksp(wksp->huffWeight, HUF_SYMBOLVALUE_MAX + 1, wksp->rankVal, &nbSymbols, &tableLog, src, srcSize, wksp->statsWksp, sizeof(wksp->statsWksp), flags); + if (HUF_isError(iSize)) return iSize; - /* Tracing */ -#if ZSTD_TRACE - ZSTD_TraceCtx traceCtx; -#endif -}; /* typedef'd to ZSTD_DCtx within "zstd.h" */ + /* Table header */ + { DTableDesc dtd = HUF_getDTableDesc(DTable); + U32 const maxTableLog = dtd.maxTableLog + 1; + U32 const targetTableLog = MIN(maxTableLog, HUF_DECODER_FAST_TABLELOG); + tableLog = HUF_rescaleStats(wksp->huffWeight, wksp->rankVal, nbSymbols, tableLog, targetTableLog); + if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, Huffman tree cannot fit in */ + dtd.tableType = 0; + dtd.tableLog = (BYTE)tableLog; + ZSTD_memcpy(DTable, &dtd, sizeof(dtd)); + } -/*-******************************************************* - * Shared internal functions - *********************************************************/ + /* Compute symbols and rankStart given rankVal: + * + * rankVal already contains the number of values of each weight. + * + * symbols contains the symbols ordered by weight. First are the rankVal[0] + * weight 0 symbols, followed by the rankVal[1] weight 1 symbols, and so on. + * symbols[0] is filled (but unused) to avoid a branch. + * + * rankStart contains the offset where each rank belongs in the DTable. + * rankStart[0] is not filled because there are no entries in the table for + * weight 0. + */ + { int n; + U32 nextRankStart = 0; + int const unroll = 4; + int const nLimit = (int)nbSymbols - unroll + 1; + for (n=0; n<(int)tableLog+1; n++) { + U32 const curr = nextRankStart; + nextRankStart += wksp->rankVal[n]; + wksp->rankStart[n] = curr; + } + for (n=0; n < nLimit; n += unroll) { + int u; + for (u=0; u < unroll; ++u) { + size_t const w = wksp->huffWeight[n+u]; + wksp->symbols[wksp->rankStart[w]++] = (BYTE)(n+u); + } + } + for (; n < (int)nbSymbols; ++n) { + size_t const w = wksp->huffWeight[n]; + wksp->symbols[wksp->rankStart[w]++] = (BYTE)n; + } + } -/*! ZSTD_loadDEntropy() : - * dict : must point at beginning of a valid zstd dictionary. - * @return : size of dictionary header (size of magic number + dict ID + entropy tables) */ -size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, - const void* const dict, size_t const dictSize); + /* fill DTable + * We fill all entries of each weight in order. + * That way length is a constant for each iteration of the outer loop. + * We can switch based on the length to a different inner loop which is + * optimized for that particular case. + */ + { U32 w; + int symbol = wksp->rankVal[0]; + int rankStart = 0; + for (w=1; wrankVal[w]; + int const length = (1 << w) >> 1; + int uStart = rankStart; + BYTE const nbBits = (BYTE)(tableLog + 1 - w); + int s; + int u; + switch (length) { + case 1: + for (s=0; ssymbols[symbol + s]; + D.nbBits = nbBits; + dt[uStart] = D; + uStart += 1; + } + break; + case 2: + for (s=0; ssymbols[symbol + s]; + D.nbBits = nbBits; + dt[uStart+0] = D; + dt[uStart+1] = D; + uStart += 2; + } + break; + case 4: + for (s=0; ssymbols[symbol + s], nbBits); + MEM_write64(dt + uStart, D4); + uStart += 4; + } + break; + case 8: + for (s=0; ssymbols[symbol + s], nbBits); + MEM_write64(dt + uStart, D4); + MEM_write64(dt + uStart + 4, D4); + uStart += 8; + } + break; + default: + for (s=0; ssymbols[symbol + s], nbBits); + for (u=0; u < length; u += 16) { + MEM_write64(dt + uStart + u + 0, D4); + MEM_write64(dt + uStart + u + 4, D4); + MEM_write64(dt + uStart + u + 8, D4); + MEM_write64(dt + uStart + u + 12, D4); + } + assert(u == length); + uStart += length; + } + break; + } + symbol += symbolCount; + rankStart += symbolCount * length; + } + } + return iSize; +} -/*! ZSTD_checkContinuity() : - * check if next `dst` follows previous position, where decompression ended. - * If yes, do nothing (continue on current segment). - * If not, classify previous segment as "external dictionary", and start a new segment. - * This function cannot fail. */ -void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize); +FORCE_INLINE_TEMPLATE BYTE +HUF_decodeSymbolX1(BIT_DStream_t* Dstream, const HUF_DEltX1* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */ + BYTE const c = dt[val].byte; + BIT_skipBits(Dstream, dt[val].nbBits); + return c; +} +#define HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) \ + do { *ptr++ = HUF_decodeSymbolX1(DStreamPtr, dt, dtLog); } while (0) -#endif /* ZSTD_DECOMPRESS_INTERNAL_H */ -/**** ended inlining zstd_decompress_internal.h ****/ -/**** start inlining zstd_ddict.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ +#define HUF_DECODE_SYMBOLX1_1(ptr, DStreamPtr) \ + do { \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ + HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr); \ + } while (0) +#define HUF_DECODE_SYMBOLX1_2(ptr, DStreamPtr) \ + do { \ + if (MEM_64bits()) \ + HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr); \ + } while (0) -#ifndef ZSTD_DDICT_H -#define ZSTD_DDICT_H +HINT_INLINE size_t +HUF_decodeStreamX1(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX1* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; -/*-******************************************************* - * Dependencies - *********************************************************/ -/**** skipping file: ../common/zstd_deps.h ****/ -/**** skipping file: ../zstd.h ****/ + /* up to 4 symbols at a time */ + if ((pEnd - p) > 3) { + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-3)) { + HUF_DECODE_SYMBOLX1_2(p, bitDPtr); + HUF_DECODE_SYMBOLX1_1(p, bitDPtr); + HUF_DECODE_SYMBOLX1_2(p, bitDPtr); + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + } + } else { + BIT_reloadDStream(bitDPtr); + } + /* [0-3] symbols remaining */ + if (MEM_32bits()) + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd)) + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); -/*-******************************************************* - * Interface - *********************************************************/ + /* no more data to retrieve from bitstream, no need to reload */ + while (p < pEnd) + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); -/* note: several prototypes are already published in `zstd.h` : - * ZSTD_createDDict() - * ZSTD_createDDict_byReference() - * ZSTD_createDDict_advanced() - * ZSTD_freeDDict() - * ZSTD_initStaticDDict() - * ZSTD_sizeof_DDict() - * ZSTD_estimateDDictSize() - * ZSTD_getDictID_fromDict() - */ + return (size_t)(pEnd-pStart); +} -const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict); -size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict); +FORCE_INLINE_TEMPLATE size_t +HUF_decompress1X1_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + BYTE* op = (BYTE*)dst; + BYTE* const oend = ZSTD_maybeNullPtrAdd(op, dstSize); + const void* dtPtr = DTable + 1; + const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr; + BIT_DStream_t bitD; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; -void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) ); + HUF_decodeStreamX1(op, &bitD, oend, dt, dtLog); + if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); -#endif /* ZSTD_DDICT_H */ -/**** ended inlining zstd_ddict.h ****/ + return dstSize; +} -#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) -/**** start inlining ../legacy/zstd_legacy.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. +/* HUF_decompress4X1_usingDTable_internal_body(): + * Conditions : + * @dstSize >= 6 */ +FORCE_INLINE_TEMPLATE size_t +HUF_decompress4X1_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + /* Check */ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + if (dstSize < 6) return ERROR(corruption_detected); /* stream 4-split doesn't work */ -#ifndef ZSTD_LEGACY_H -#define ZSTD_LEGACY_H - -#if defined (__cplusplus) -extern "C" { -#endif - -/* ************************************* -* Includes -***************************************/ -/**** skipping file: ../common/mem.h ****/ -/**** skipping file: ../common/error_private.h ****/ -/**** skipping file: ../common/zstd_internal.h ****/ - -#if !defined (ZSTD_LEGACY_SUPPORT) || (ZSTD_LEGACY_SUPPORT == 0) -# undef ZSTD_LEGACY_SUPPORT -# define ZSTD_LEGACY_SUPPORT 8 -#endif + { const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + BYTE* const olimit = oend - 3; + const void* const dtPtr = DTable + 1; + const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr; -#if (ZSTD_LEGACY_SUPPORT <= 1) -/**** start inlining zstd_v01.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + const size_t segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + U32 endSignal = 1; -#ifndef ZSTD_V01_H_28739879432 -#define ZSTD_V01_H_28739879432 + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + if (opStart4 > oend) return ERROR(corruption_detected); /* overflow */ + assert(dstSize >= 6); /* validated above */ + CHECK_F( BIT_initDStream(&bitD1, istart1, length1) ); + CHECK_F( BIT_initDStream(&bitD2, istart2, length2) ); + CHECK_F( BIT_initDStream(&bitD3, istart3, length3) ); + CHECK_F( BIT_initDStream(&bitD4, istart4, length4) ); -#if defined (__cplusplus) -extern "C" { -#endif + /* up to 16 symbols per loop (4 symbols per stream) in 64-bit mode */ + if ((size_t)(oend - op4) >= sizeof(size_t)) { + for ( ; (endSignal) & (op4 < olimit) ; ) { + HUF_DECODE_SYMBOLX1_2(op1, &bitD1); + HUF_DECODE_SYMBOLX1_2(op2, &bitD2); + HUF_DECODE_SYMBOLX1_2(op3, &bitD3); + HUF_DECODE_SYMBOLX1_2(op4, &bitD4); + HUF_DECODE_SYMBOLX1_1(op1, &bitD1); + HUF_DECODE_SYMBOLX1_1(op2, &bitD2); + HUF_DECODE_SYMBOLX1_1(op3, &bitD3); + HUF_DECODE_SYMBOLX1_1(op4, &bitD4); + HUF_DECODE_SYMBOLX1_2(op1, &bitD1); + HUF_DECODE_SYMBOLX1_2(op2, &bitD2); + HUF_DECODE_SYMBOLX1_2(op3, &bitD3); + HUF_DECODE_SYMBOLX1_2(op4, &bitD4); + HUF_DECODE_SYMBOLX1_0(op1, &bitD1); + HUF_DECODE_SYMBOLX1_0(op2, &bitD2); + HUF_DECODE_SYMBOLX1_0(op3, &bitD3); + HUF_DECODE_SYMBOLX1_0(op4, &bitD4); + endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished; + } + } -/* ************************************* -* Includes -***************************************/ -#include /* size_t */ + /* check corruption */ + /* note : should not be necessary : op# advance in lock step, and we control op4. + * but curiously, binary generated by gcc 7.2 & 7.3 with -mbmi2 runs faster when >=1 test is present */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 supposed already verified within main loop */ + /* finish bitStreams one by one */ + HUF_decodeStreamX1(op1, &bitD1, opStart2, dt, dtLog); + HUF_decodeStreamX1(op2, &bitD2, opStart3, dt, dtLog); + HUF_decodeStreamX1(op3, &bitD3, opStart4, dt, dtLog); + HUF_decodeStreamX1(op4, &bitD4, oend, dt, dtLog); -/* ************************************* -* Simple one-step function -***************************************/ -/** -ZSTDv01_decompress() : decompress ZSTD frames compliant with v0.1.x format - compressedSize : is the exact source size - maxOriginalSize : is the size of the 'dst' buffer, which must be already allocated. - It must be equal or larger than originalSize, otherwise decompression will fail. - return : the number of bytes decompressed into destination buffer (originalSize) - or an errorCode if it fails (which can be tested using ZSTDv01_isError()) -*/ -size_t ZSTDv01_decompress( void* dst, size_t maxOriginalSize, - const void* src, size_t compressedSize); + /* check */ + { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endCheck) return ERROR(corruption_detected); } - /** - ZSTDv01_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.1.x format - srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' - cSize (output parameter) : the number of bytes that would be read to decompress this frame - or an error code if it fails (which can be tested using ZSTDv01_isError()) - dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame - or ZSTD_CONTENTSIZE_ERROR if an error occurs + /* decoded size */ + return dstSize; + } +} - note : assumes `cSize` and `dBound` are _not_ NULL. - */ -void ZSTDv01_findFrameSizeInfoLegacy(const void *src, size_t srcSize, - size_t* cSize, unsigned long long* dBound); +#if HUF_NEED_BMI2_FUNCTION +static BMI2_TARGET_ATTRIBUTE +size_t HUF_decompress4X1_usingDTable_internal_bmi2(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} +#endif -/** -ZSTDv01_isError() : tells if the result of ZSTDv01_decompress() is an error -*/ -unsigned ZSTDv01_isError(size_t code); +static +size_t HUF_decompress4X1_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} +#if ZSTD_ENABLE_ASM_X86_64_BMI2 -/* ************************************* -* Advanced functions -***************************************/ -typedef struct ZSTDv01_Dctx_s ZSTDv01_Dctx; -ZSTDv01_Dctx* ZSTDv01_createDCtx(void); -size_t ZSTDv01_freeDCtx(ZSTDv01_Dctx* dctx); +HUF_ASM_DECL void HUF_decompress4X1_usingDTable_internal_fast_asm_loop(HUF_DecompressFastArgs* args) ZSTDLIB_HIDDEN; -size_t ZSTDv01_decompressDCtx(void* ctx, - void* dst, size_t maxOriginalSize, - const void* src, size_t compressedSize); +#endif -/* ************************************* -* Streaming functions -***************************************/ -size_t ZSTDv01_resetDCtx(ZSTDv01_Dctx* dctx); +static HUF_FAST_BMI2_ATTRS +void HUF_decompress4X1_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs* args) +{ + U64 bits[4]; + BYTE const* ip[4]; + BYTE* op[4]; + U16 const* const dtable = (U16 const*)args->dt; + BYTE* const oend = args->oend; + BYTE const* const ilowest = args->ilowest; -size_t ZSTDv01_nextSrcSizeToDecompress(ZSTDv01_Dctx* dctx); -size_t ZSTDv01_decompressContinue(ZSTDv01_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); -/** - Use above functions alternatively. - ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue(). - ZSTD_decompressContinue() will use previous data blocks to improve compression if they are located prior to current block. - Result is the number of bytes regenerated within 'dst'. - It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header. -*/ + /* Copy the arguments to local variables */ + ZSTD_memcpy(&bits, &args->bits, sizeof(bits)); + ZSTD_memcpy((void*)(&ip), &args->ip, sizeof(ip)); + ZSTD_memcpy(&op, &args->op, sizeof(op)); -/* ************************************* -* Prefix - version detection -***************************************/ -#define ZSTDv01_magicNumber 0xFD2FB51E /* Big Endian version */ -#define ZSTDv01_magicNumberLE 0x1EB52FFD /* Little Endian version */ + assert(MEM_isLittleEndian()); + assert(!MEM_32bits()); + for (;;) { + BYTE* olimit; + int stream; -#if defined (__cplusplus) -} + /* Assert loop preconditions */ +#ifndef NDEBUG + for (stream = 0; stream < 4; ++stream) { + assert(op[stream] <= (stream == 3 ? oend : op[stream + 1])); + assert(ip[stream] >= ilowest); + } #endif + /* Compute olimit */ + { + /* Each iteration produces 5 output symbols per stream */ + size_t const oiters = (size_t)(oend - op[3]) / 5; + /* Each iteration consumes up to 11 bits * 5 = 55 bits < 7 bytes + * per stream. + */ + size_t const iiters = (size_t)(ip[0] - ilowest) / 7; + /* We can safely run iters iterations before running bounds checks */ + size_t const iters = MIN(oiters, iiters); + size_t const symbols = iters * 5; + + /* We can simply check that op[3] < olimit, instead of checking all + * of our bounds, since we can't hit the other bounds until we've run + * iters iterations, which only happens when op[3] == olimit. + */ + olimit = op[3] + symbols; -#endif /* ZSTD_V01_H_28739879432 */ -/**** ended inlining zstd_v01.h ****/ -#endif -#if (ZSTD_LEGACY_SUPPORT <= 2) -/**** start inlining zstd_v02.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ + /* Exit fast decoding loop once we reach the end. */ + if (op[3] == olimit) + break; -#ifndef ZSTD_V02_H_4174539423 -#define ZSTD_V02_H_4174539423 + /* Exit the decoding loop if any input pointer has crossed the + * previous one. This indicates corruption, and a precondition + * to our loop is that ip[i] >= ip[0]. + */ + for (stream = 1; stream < 4; ++stream) { + if (ip[stream] < ip[stream - 1]) + goto _out; + } + } -#if defined (__cplusplus) -extern "C" { +#ifndef NDEBUG + for (stream = 1; stream < 4; ++stream) { + assert(ip[stream] >= ip[stream - 1]); + } #endif -/* ************************************* -* Includes -***************************************/ -#include /* size_t */ +#define HUF_4X1_DECODE_SYMBOL(_stream, _symbol) \ + do { \ + int const index = (int)(bits[(_stream)] >> 53); \ + int const entry = (int)dtable[index]; \ + bits[(_stream)] <<= (entry & 0x3F); \ + op[(_stream)][(_symbol)] = (BYTE)((entry >> 8) & 0xFF); \ + } while (0) +#define HUF_4X1_RELOAD_STREAM(_stream) \ + do { \ + int const ctz = ZSTD_countTrailingZeros64(bits[(_stream)]); \ + int const nbBits = ctz & 7; \ + int const nbBytes = ctz >> 3; \ + op[(_stream)] += 5; \ + ip[(_stream)] -= nbBytes; \ + bits[(_stream)] = MEM_read64(ip[(_stream)]) | 1; \ + bits[(_stream)] <<= nbBits; \ + } while (0) -/* ************************************* -* Simple one-step function -***************************************/ -/** -ZSTDv02_decompress() : decompress ZSTD frames compliant with v0.2.x format - compressedSize : is the exact source size - maxOriginalSize : is the size of the 'dst' buffer, which must be already allocated. - It must be equal or larger than originalSize, otherwise decompression will fail. - return : the number of bytes decompressed into destination buffer (originalSize) - or an errorCode if it fails (which can be tested using ZSTDv01_isError()) -*/ -size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize, - const void* src, size_t compressedSize); + /* Manually unroll the loop because compilers don't consistently + * unroll the inner loops, which destroys performance. + */ + do { + /* Decode 5 symbols in each of the 4 streams */ + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 0); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 1); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 2); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 3); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 4); - /** - ZSTDv02_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.2.x format - srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' - cSize (output parameter) : the number of bytes that would be read to decompress this frame - or an error code if it fails (which can be tested using ZSTDv01_isError()) - dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame - or ZSTD_CONTENTSIZE_ERROR if an error occurs + /* Reload each of the 4 the bitstreams */ + HUF_4X_FOR_EACH_STREAM(HUF_4X1_RELOAD_STREAM); + } while (op[3] < olimit); - note : assumes `cSize` and `dBound` are _not_ NULL. - */ -void ZSTDv02_findFrameSizeInfoLegacy(const void *src, size_t srcSize, - size_t* cSize, unsigned long long* dBound); +#undef HUF_4X1_DECODE_SYMBOL +#undef HUF_4X1_RELOAD_STREAM + } -/** -ZSTDv02_isError() : tells if the result of ZSTDv02_decompress() is an error -*/ -unsigned ZSTDv02_isError(size_t code); +_out: + /* Save the final values of each of the state variables back to args. */ + ZSTD_memcpy(&args->bits, &bits, sizeof(bits)); + ZSTD_memcpy((void*)(&args->ip), &ip, sizeof(ip)); + ZSTD_memcpy(&args->op, &op, sizeof(op)); +} -/* ************************************* -* Advanced functions -***************************************/ -typedef struct ZSTDv02_Dctx_s ZSTDv02_Dctx; -ZSTDv02_Dctx* ZSTDv02_createDCtx(void); -size_t ZSTDv02_freeDCtx(ZSTDv02_Dctx* dctx); +/** + * @returns @p dstSize on success (>= 6) + * 0 if the fallback implementation should be used + * An error if an error occurred + */ +static HUF_FAST_BMI2_ATTRS +size_t +HUF_decompress4X1_usingDTable_internal_fast( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable, + HUF_DecompressFastLoopFn loopFn) +{ + void const* dt = DTable + 1; + BYTE const* const ilowest = (BYTE const*)cSrc; + BYTE* const oend = ZSTD_maybeNullPtrAdd((BYTE*)dst, dstSize); + HUF_DecompressFastArgs args; + { size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable); + FORWARD_IF_ERROR(ret, "Failed to init fast loop args"); + if (ret == 0) + return 0; + } -size_t ZSTDv02_decompressDCtx(void* ctx, - void* dst, size_t maxOriginalSize, - const void* src, size_t compressedSize); + assert(args.ip[0] >= args.ilowest); + loopFn(&args); -/* ************************************* -* Streaming functions -***************************************/ -size_t ZSTDv02_resetDCtx(ZSTDv02_Dctx* dctx); + /* Our loop guarantees that ip[] >= ilowest and that we haven't + * overwritten any op[]. + */ + assert(args.ip[0] >= ilowest); + assert(args.ip[0] >= ilowest); + assert(args.ip[1] >= ilowest); + assert(args.ip[2] >= ilowest); + assert(args.ip[3] >= ilowest); + assert(args.op[3] <= oend); + + assert(ilowest == args.ilowest); + assert(ilowest + 6 == args.iend[0]); + (void)ilowest; + + /* finish bit streams one by one. */ + { size_t const segmentSize = (dstSize+3) / 4; + BYTE* segmentEnd = (BYTE*)dst; + int i; + for (i = 0; i < 4; ++i) { + BIT_DStream_t bit; + if (segmentSize <= (size_t)(oend - segmentEnd)) + segmentEnd += segmentSize; + else + segmentEnd = oend; + FORWARD_IF_ERROR(HUF_initRemainingDStream(&bit, &args, i, segmentEnd), "corruption"); + /* Decompress and validate that we've produced exactly the expected length. */ + args.op[i] += HUF_decodeStreamX1(args.op[i], &bit, segmentEnd, (HUF_DEltX1 const*)dt, HUF_DECODER_FAST_TABLELOG); + if (args.op[i] != segmentEnd) return ERROR(corruption_detected); + } + } -size_t ZSTDv02_nextSrcSizeToDecompress(ZSTDv02_Dctx* dctx); -size_t ZSTDv02_decompressContinue(ZSTDv02_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); -/** - Use above functions alternatively. - ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue(). - ZSTD_decompressContinue() will use previous data blocks to improve compression if they are located prior to current block. - Result is the number of bytes regenerated within 'dst'. - It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header. -*/ + /* decoded size */ + assert(dstSize != 0); + return dstSize; +} -/* ************************************* -* Prefix - version detection -***************************************/ -#define ZSTDv02_magicNumber 0xFD2FB522 /* v0.2 */ +HUF_DGEN(HUF_decompress1X1_usingDTable_internal) +static size_t HUF_decompress4X1_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable, int flags) +{ + HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X1_usingDTable_internal_default; + HUF_DecompressFastLoopFn loopFn = HUF_decompress4X1_usingDTable_internal_fast_c_loop; -#if defined (__cplusplus) -} +#if DYNAMIC_BMI2 + if (flags & HUF_flags_bmi2) { + fallbackFn = HUF_decompress4X1_usingDTable_internal_bmi2; +# if ZSTD_ENABLE_ASM_X86_64_BMI2 + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop; + } +# endif + } else { + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); + } #endif -#endif /* ZSTD_V02_H_4174539423 */ -/**** ended inlining zstd_v02.h ****/ +#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__) + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop; + } #endif -#if (ZSTD_LEGACY_SUPPORT <= 3) -/**** start inlining zstd_v03.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ -#ifndef ZSTD_V03_H_298734209782 -#define ZSTD_V03_H_298734209782 + if (HUF_ENABLE_FAST_DECODE && !(flags & HUF_flags_disableFast)) { + size_t const ret = HUF_decompress4X1_usingDTable_internal_fast(dst, dstSize, cSrc, cSrcSize, DTable, loopFn); + if (ret != 0) + return ret; + } + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); +} -#if defined (__cplusplus) -extern "C" { -#endif +static size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int flags) +{ + const BYTE* ip = (const BYTE*) cSrc; -/* ************************************* -* Includes -***************************************/ -#include /* size_t */ + size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize, flags); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + return HUF_decompress4X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags); +} -/* ************************************* -* Simple one-step function -***************************************/ -/** -ZSTDv03_decompress() : decompress ZSTD frames compliant with v0.3.x format - compressedSize : is the exact source size - maxOriginalSize : is the size of the 'dst' buffer, which must be already allocated. - It must be equal or larger than originalSize, otherwise decompression will fail. - return : the number of bytes decompressed into destination buffer (originalSize) - or an errorCode if it fails (which can be tested using ZSTDv01_isError()) -*/ -size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize, - const void* src, size_t compressedSize); +#endif /* HUF_FORCE_DECOMPRESS_X2 */ - /** - ZSTDv03_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.3.x format - srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' - cSize (output parameter) : the number of bytes that would be read to decompress this frame - or an error code if it fails (which can be tested using ZSTDv01_isError()) - dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame - or ZSTD_CONTENTSIZE_ERROR if an error occurs - note : assumes `cSize` and `dBound` are _not_ NULL. - */ - void ZSTDv03_findFrameSizeInfoLegacy(const void *src, size_t srcSize, - size_t* cSize, unsigned long long* dBound); +#ifndef HUF_FORCE_DECOMPRESS_X1 - /** -ZSTDv03_isError() : tells if the result of ZSTDv03_decompress() is an error -*/ -unsigned ZSTDv03_isError(size_t code); +/* *************************/ +/* double-symbols decoding */ +/* *************************/ +typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX2; /* double-symbols decoding */ +typedef struct { BYTE symbol; } sortedSymbol_t; +typedef U32 rankValCol_t[HUF_TABLELOG_MAX + 1]; +typedef rankValCol_t rankVal_t[HUF_TABLELOG_MAX]; -/* ************************************* -* Advanced functions -***************************************/ -typedef struct ZSTDv03_Dctx_s ZSTDv03_Dctx; -ZSTDv03_Dctx* ZSTDv03_createDCtx(void); -size_t ZSTDv03_freeDCtx(ZSTDv03_Dctx* dctx); +/** + * Constructs a HUF_DEltX2 in a U32. + */ +static U32 HUF_buildDEltX2U32(U32 symbol, U32 nbBits, U32 baseSeq, int level) +{ + U32 seq; + DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, sequence) == 0); + DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, nbBits) == 2); + DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, length) == 3); + DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(U32)); + if (MEM_isLittleEndian()) { + seq = level == 1 ? symbol : (baseSeq + (symbol << 8)); + return seq + (nbBits << 16) + ((U32)level << 24); + } else { + seq = level == 1 ? (symbol << 8) : ((baseSeq << 8) + symbol); + return (seq << 16) + (nbBits << 8) + (U32)level; + } +} -size_t ZSTDv03_decompressDCtx(void* ctx, - void* dst, size_t maxOriginalSize, - const void* src, size_t compressedSize); +/** + * Constructs a HUF_DEltX2. + */ +static HUF_DEltX2 HUF_buildDEltX2(U32 symbol, U32 nbBits, U32 baseSeq, int level) +{ + HUF_DEltX2 DElt; + U32 const val = HUF_buildDEltX2U32(symbol, nbBits, baseSeq, level); + DEBUG_STATIC_ASSERT(sizeof(DElt) == sizeof(val)); + ZSTD_memcpy(&DElt, &val, sizeof(val)); + return DElt; +} -/* ************************************* -* Streaming functions -***************************************/ -size_t ZSTDv03_resetDCtx(ZSTDv03_Dctx* dctx); +/** + * Constructs 2 HUF_DEltX2s and packs them into a U64. + */ +static U64 HUF_buildDEltX2U64(U32 symbol, U32 nbBits, U16 baseSeq, int level) +{ + U32 DElt = HUF_buildDEltX2U32(symbol, nbBits, baseSeq, level); + return (U64)DElt + ((U64)DElt << 32); +} -size_t ZSTDv03_nextSrcSizeToDecompress(ZSTDv03_Dctx* dctx); -size_t ZSTDv03_decompressContinue(ZSTDv03_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); /** - Use above functions alternatively. - ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue(). - ZSTD_decompressContinue() will use previous data blocks to improve compression if they are located prior to current block. - Result is the number of bytes regenerated within 'dst'. - It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header. -*/ + * Fills the DTable rank with all the symbols from [begin, end) that are each + * nbBits long. + * + * @param DTableRank The start of the rank in the DTable. + * @param begin The first symbol to fill (inclusive). + * @param end The last symbol to fill (exclusive). + * @param nbBits Each symbol is nbBits long. + * @param tableLog The table log. + * @param baseSeq If level == 1 { 0 } else { the first level symbol } + * @param level The level in the table. Must be 1 or 2. + */ +static void HUF_fillDTableX2ForWeight( + HUF_DEltX2* DTableRank, + sortedSymbol_t const* begin, sortedSymbol_t const* end, + U32 nbBits, U32 tableLog, + U16 baseSeq, int const level) +{ + U32 const length = 1U << ((tableLog - nbBits) & 0x1F /* quiet static-analyzer */); + const sortedSymbol_t* ptr; + assert(level >= 1 && level <= 2); + switch (length) { + case 1: + for (ptr = begin; ptr != end; ++ptr) { + HUF_DEltX2 const DElt = HUF_buildDEltX2(ptr->symbol, nbBits, baseSeq, level); + *DTableRank++ = DElt; + } + break; + case 2: + for (ptr = begin; ptr != end; ++ptr) { + HUF_DEltX2 const DElt = HUF_buildDEltX2(ptr->symbol, nbBits, baseSeq, level); + DTableRank[0] = DElt; + DTableRank[1] = DElt; + DTableRank += 2; + } + break; + case 4: + for (ptr = begin; ptr != end; ++ptr) { + U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level); + ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2)); + DTableRank += 4; + } + break; + case 8: + for (ptr = begin; ptr != end; ++ptr) { + U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level); + ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 4, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 6, &DEltX2, sizeof(DEltX2)); + DTableRank += 8; + } + break; + default: + for (ptr = begin; ptr != end; ++ptr) { + U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level); + HUF_DEltX2* const DTableRankEnd = DTableRank + length; + for (; DTableRank != DTableRankEnd; DTableRank += 8) { + ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 4, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 6, &DEltX2, sizeof(DEltX2)); + } + } + break; + } +} -/* ************************************* -* Prefix - version detection -***************************************/ -#define ZSTDv03_magicNumber 0xFD2FB523 /* v0.3 */ +/* HUF_fillDTableX2Level2() : + * `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */ +static void HUF_fillDTableX2Level2(HUF_DEltX2* DTable, U32 targetLog, const U32 consumedBits, + const U32* rankVal, const int minWeight, const int maxWeight1, + const sortedSymbol_t* sortedSymbols, U32 const* rankStart, + U32 nbBitsBaseline, U16 baseSeq) +{ + /* Fill skipped values (all positions up to rankVal[minWeight]). + * These are positions only get a single symbol because the combined weight + * is too large. + */ + if (minWeight>1) { + U32 const length = 1U << ((targetLog - consumedBits) & 0x1F /* quiet static-analyzer */); + U64 const DEltX2 = HUF_buildDEltX2U64(baseSeq, consumedBits, /* baseSeq */ 0, /* level */ 1); + int const skipSize = rankVal[minWeight]; + assert(length > 1); + assert((U32)skipSize < length); + switch (length) { + case 2: + assert(skipSize == 1); + ZSTD_memcpy(DTable, &DEltX2, sizeof(DEltX2)); + break; + case 4: + assert(skipSize <= 4); + ZSTD_memcpy(DTable + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + 2, &DEltX2, sizeof(DEltX2)); + break; + default: + { + int i; + for (i = 0; i < skipSize; i += 8) { + ZSTD_memcpy(DTable + i + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + i + 2, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + i + 4, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + i + 6, &DEltX2, sizeof(DEltX2)); + } + } + } + } + /* Fill each of the second level symbols by weight. */ + { + int w; + for (w = minWeight; w < maxWeight1; ++w) { + int const begin = rankStart[w]; + int const end = rankStart[w+1]; + U32 const nbBits = nbBitsBaseline - w; + U32 const totalBits = nbBits + consumedBits; + HUF_fillDTableX2ForWeight( + DTable + rankVal[w], + sortedSymbols + begin, sortedSymbols + end, + totalBits, targetLog, + baseSeq, /* level */ 2); + } + } +} -#if defined (__cplusplus) +static void HUF_fillDTableX2(HUF_DEltX2* DTable, const U32 targetLog, + const sortedSymbol_t* sortedList, + const U32* rankStart, rankValCol_t* rankValOrigin, const U32 maxWeight, + const U32 nbBitsBaseline) +{ + U32* const rankVal = rankValOrigin[0]; + const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */ + const U32 minBits = nbBitsBaseline - maxWeight; + int w; + int const wEnd = (int)maxWeight + 1; + + /* Fill DTable in order of weight. */ + for (w = 1; w < wEnd; ++w) { + int const begin = (int)rankStart[w]; + int const end = (int)rankStart[w+1]; + U32 const nbBits = nbBitsBaseline - w; + + if (targetLog-nbBits >= minBits) { + /* Enough room for a second symbol. */ + int start = rankVal[w]; + U32 const length = 1U << ((targetLog - nbBits) & 0x1F /* quiet static-analyzer */); + int minWeight = nbBits + scaleLog; + int s; + if (minWeight < 1) minWeight = 1; + /* Fill the DTable for every symbol of weight w. + * These symbols get at least 1 second symbol. + */ + for (s = begin; s != end; ++s) { + HUF_fillDTableX2Level2( + DTable + start, targetLog, nbBits, + rankValOrigin[nbBits], minWeight, wEnd, + sortedList, rankStart, + nbBitsBaseline, sortedList[s].symbol); + start += length; + } + } else { + /* Only a single symbol. */ + HUF_fillDTableX2ForWeight( + DTable + rankVal[w], + sortedList + begin, sortedList + end, + nbBits, targetLog, + /* baseSeq */ 0, /* level */ 1); + } + } } -#endif -#endif /* ZSTD_V03_H_298734209782 */ -/**** ended inlining zstd_v03.h ****/ -#endif -#if (ZSTD_LEGACY_SUPPORT <= 4) -/**** start inlining zstd_v04.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ +typedef struct { + rankValCol_t rankVal[HUF_TABLELOG_MAX]; + U32 rankStats[HUF_TABLELOG_MAX + 1]; + U32 rankStart0[HUF_TABLELOG_MAX + 3]; + sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1]; + BYTE weightList[HUF_SYMBOLVALUE_MAX + 1]; + U32 calleeWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; +} HUF_ReadDTableX2_Workspace; -#ifndef ZSTD_V04_H_91868324769238 -#define ZSTD_V04_H_91868324769238 +size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize, int flags) +{ + U32 tableLog, maxW, nbSymbols; + DTableDesc dtd = HUF_getDTableDesc(DTable); + U32 maxTableLog = dtd.maxTableLog; + size_t iSize; + void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */ + HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr; + U32 *rankStart; -#if defined (__cplusplus) -extern "C" { -#endif + HUF_ReadDTableX2_Workspace* const wksp = (HUF_ReadDTableX2_Workspace*)workSpace; -/* ************************************* -* Includes -***************************************/ -#include /* size_t */ + if (sizeof(*wksp) > wkspSize) return ERROR(GENERIC); + rankStart = wksp->rankStart0 + 1; + ZSTD_memset(wksp->rankStats, 0, sizeof(wksp->rankStats)); + ZSTD_memset(wksp->rankStart0, 0, sizeof(wksp->rankStart0)); -/* ************************************* -* Simple one-step function -***************************************/ -/** -ZSTDv04_decompress() : decompress ZSTD frames compliant with v0.4.x format - compressedSize : is the exact source size - maxOriginalSize : is the size of the 'dst' buffer, which must be already allocated. - It must be equal or larger than originalSize, otherwise decompression will fail. - return : the number of bytes decompressed into destination buffer (originalSize) - or an errorCode if it fails (which can be tested using ZSTDv01_isError()) -*/ -size_t ZSTDv04_decompress( void* dst, size_t maxOriginalSize, - const void* src, size_t compressedSize); + DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(HUF_DTable)); /* if compiler fails here, assertion is wrong */ + if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); + /* ZSTD_memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */ - /** - ZSTDv04_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.4.x format - srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' - cSize (output parameter) : the number of bytes that would be read to decompress this frame - or an error code if it fails (which can be tested using ZSTDv01_isError()) - dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame - or ZSTD_CONTENTSIZE_ERROR if an error occurs + iSize = HUF_readStats_wksp(wksp->weightList, HUF_SYMBOLVALUE_MAX + 1, wksp->rankStats, &nbSymbols, &tableLog, src, srcSize, wksp->calleeWksp, sizeof(wksp->calleeWksp), flags); + if (HUF_isError(iSize)) return iSize; - note : assumes `cSize` and `dBound` are _not_ NULL. - */ - void ZSTDv04_findFrameSizeInfoLegacy(const void *src, size_t srcSize, - size_t* cSize, unsigned long long* dBound); + /* check result */ + if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ + if (tableLog <= HUF_DECODER_FAST_TABLELOG && maxTableLog > HUF_DECODER_FAST_TABLELOG) maxTableLog = HUF_DECODER_FAST_TABLELOG; -/** -ZSTDv04_isError() : tells if the result of ZSTDv04_decompress() is an error -*/ -unsigned ZSTDv04_isError(size_t code); + /* find maxWeight */ + for (maxW = tableLog; wksp->rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */ + /* Get start index of each weight */ + { U32 w, nextRankStart = 0; + for (w=1; wrankStats[w]; + rankStart[w] = curr; + } + rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/ + rankStart[maxW+1] = nextRankStart; + } -/* ************************************* -* Advanced functions -***************************************/ -typedef struct ZSTDv04_Dctx_s ZSTDv04_Dctx; -ZSTDv04_Dctx* ZSTDv04_createDCtx(void); -size_t ZSTDv04_freeDCtx(ZSTDv04_Dctx* dctx); + /* sort symbols by weight */ + { U32 s; + for (s=0; sweightList[s]; + U32 const r = rankStart[w]++; + wksp->sortedSymbol[r].symbol = (BYTE)s; + } + rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */ + } -size_t ZSTDv04_decompressDCtx(ZSTDv04_Dctx* dctx, - void* dst, size_t maxOriginalSize, - const void* src, size_t compressedSize); + /* Build rankVal */ + { U32* const rankVal0 = wksp->rankVal[0]; + { int const rescale = (maxTableLog-tableLog) - 1; /* tableLog <= maxTableLog */ + U32 nextRankVal = 0; + U32 w; + for (w=1; wrankStats[w] << (w+rescale); + rankVal0[w] = curr; + } } + { U32 const minBits = tableLog+1 - maxW; + U32 consumed; + for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) { + U32* const rankValPtr = wksp->rankVal[consumed]; + U32 w; + for (w = 1; w < maxW+1; w++) { + rankValPtr[w] = rankVal0[w] >> consumed; + } } } } + HUF_fillDTableX2(dt, maxTableLog, + wksp->sortedSymbol, + wksp->rankStart0, wksp->rankVal, maxW, + tableLog+1); -/* ************************************* -* Direct Streaming -***************************************/ -size_t ZSTDv04_resetDCtx(ZSTDv04_Dctx* dctx); + dtd.tableLog = (BYTE)maxTableLog; + dtd.tableType = 1; + ZSTD_memcpy(DTable, &dtd, sizeof(dtd)); + return iSize; +} -size_t ZSTDv04_nextSrcSizeToDecompress(ZSTDv04_Dctx* dctx); -size_t ZSTDv04_decompressContinue(ZSTDv04_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); -/** - Use above functions alternatively. - ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue(). - ZSTD_decompressContinue() will use previous data blocks to improve compression if they are located prior to current block. - Result is the number of bytes regenerated within 'dst'. - It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header. -*/ +FORCE_INLINE_TEMPLATE U32 +HUF_decodeSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + ZSTD_memcpy(op, &dt[val].sequence, 2); + BIT_skipBits(DStream, dt[val].nbBits); + return dt[val].length; +} -/* ************************************* -* Buffered Streaming -***************************************/ -typedef struct ZBUFFv04_DCtx_s ZBUFFv04_DCtx; -ZBUFFv04_DCtx* ZBUFFv04_createDCtx(void); -size_t ZBUFFv04_freeDCtx(ZBUFFv04_DCtx* dctx); +FORCE_INLINE_TEMPLATE U32 +HUF_decodeLastSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + ZSTD_memcpy(op, &dt[val].sequence, 1); + if (dt[val].length==1) { + BIT_skipBits(DStream, dt[val].nbBits); + } else { + if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) { + BIT_skipBits(DStream, dt[val].nbBits); + if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8)) + /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ + DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); + } + } + return 1; +} -size_t ZBUFFv04_decompressInit(ZBUFFv04_DCtx* dctx); -size_t ZBUFFv04_decompressWithDictionary(ZBUFFv04_DCtx* dctx, const void* dict, size_t dictSize); +#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \ + do { ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog); } while (0) -size_t ZBUFFv04_decompressContinue(ZBUFFv04_DCtx* dctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr); +#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ + do { \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ + ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog); \ + } while (0) -/** ************************************************ -* Streaming decompression -* -* A ZBUFF_DCtx object is required to track streaming operation. -* Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources. -* Use ZBUFF_decompressInit() to start a new decompression operation. -* ZBUFF_DCtx objects can be reused multiple times. -* -* Optionally, a reference to a static dictionary can be set, using ZBUFF_decompressWithDictionary() -* It must be the same content as the one set during compression phase. -* Dictionary content must remain accessible during the decompression process. -* -* Use ZBUFF_decompressContinue() repetitively to consume your input. -* *srcSizePtr and *maxDstSizePtr can be any size. -* The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr. -* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. -* The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst. -* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency) -* or 0 when a frame is completely decoded -* or an error code, which can be tested using ZBUFF_isError(). -* -* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize / ZBUFF_recommendedDOutSize -* output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when it's decoded. -* input : ZBUFF_recommendedDInSize==128Kb+3; just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . -* **************************************************/ -unsigned ZBUFFv04_isError(size_t errorCode); -const char* ZBUFFv04_getErrorName(size_t errorCode); +#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ + do { \ + if (MEM_64bits()) \ + ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog); \ + } while (0) +HINT_INLINE size_t +HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, + const HUF_DEltX2* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; -/** The below functions provide recommended buffer sizes for Compression or Decompression operations. -* These sizes are not compulsory, they just tend to offer better latency */ -size_t ZBUFFv04_recommendedDInSize(void); -size_t ZBUFFv04_recommendedDOutSize(void); + /* up to 8 symbols at a time */ + if ((size_t)(pEnd - p) >= sizeof(bitDPtr->bitContainer)) { + if (dtLog <= 11 && MEM_64bits()) { + /* up to 10 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-9)) { + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + } + } else { + /* up to 8 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-(sizeof(bitDPtr->bitContainer)-1))) { + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_1(p, bitDPtr); + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + } + } + } else { + BIT_reloadDStream(bitDPtr); + } + /* closer to end : up to 2 symbols at a time */ + if ((size_t)(pEnd - p) >= 2) { + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd-2)) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); -/* ************************************* -* Prefix - version detection -***************************************/ -#define ZSTDv04_magicNumber 0xFD2FB524 /* v0.4 */ + while (p <= pEnd-2) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); /* no need to reload : reached the end of DStream */ + } + if (p < pEnd) + p += HUF_decodeLastSymbolX2(p, bitDPtr, dt, dtLog); -#if defined (__cplusplus) + return p-pStart; } -#endif - -#endif /* ZSTD_V04_H_91868324769238 */ -/**** ended inlining zstd_v04.h ****/ -#endif -#if (ZSTD_LEGACY_SUPPORT <= 5) -/**** start inlining zstd_v05.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ -#ifndef ZSTDv05_H -#define ZSTDv05_H - -#if defined (__cplusplus) -extern "C" { -#endif +FORCE_INLINE_TEMPLATE size_t +HUF_decompress1X2_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + BIT_DStream_t bitD; -/*-************************************* -* Dependencies -***************************************/ -#include /* size_t */ -/**** skipping file: ../common/mem.h ****/ + /* Init */ + CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) ); + /* decode */ + { BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ZSTD_maybeNullPtrAdd(ostart, dstSize); + const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */ + const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + HUF_decodeStreamX2(ostart, &bitD, oend, dt, dtd.tableLog); + } -/* ************************************* -* Simple functions -***************************************/ -/*! ZSTDv05_decompress() : - `compressedSize` : is the _exact_ size of the compressed blob, otherwise decompression will fail. - `dstCapacity` must be large enough, equal or larger than originalSize. - @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), - or an errorCode if it fails (which can be tested using ZSTDv05_isError()) */ -size_t ZSTDv05_decompress( void* dst, size_t dstCapacity, - const void* src, size_t compressedSize); + /* check */ + if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); - /** - ZSTDv05_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.5.x format - srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' - cSize (output parameter) : the number of bytes that would be read to decompress this frame - or an error code if it fails (which can be tested using ZSTDv01_isError()) - dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame - or ZSTD_CONTENTSIZE_ERROR if an error occurs + /* decoded size */ + return dstSize; +} - note : assumes `cSize` and `dBound` are _not_ NULL. +/* HUF_decompress4X2_usingDTable_internal_body(): + * Conditions: + * @dstSize >= 6 */ -void ZSTDv05_findFrameSizeInfoLegacy(const void *src, size_t srcSize, - size_t* cSize, unsigned long long* dBound); +FORCE_INLINE_TEMPLATE size_t +HUF_decompress4X2_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + if (dstSize < 6) return ERROR(corruption_detected); /* stream 4-split doesn't work */ -/* ************************************* -* Helper functions -***************************************/ -/* Error Management */ -unsigned ZSTDv05_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ -const char* ZSTDv05_getErrorName(size_t code); /*!< provides readable string for an error code */ + { const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + BYTE* const olimit = oend - (sizeof(size_t)-1); + const void* const dtPtr = DTable+1; + const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + size_t const segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + U32 endSignal = 1; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; -/* ************************************* -* Explicit memory management -***************************************/ -/** Decompression context */ -typedef struct ZSTDv05_DCtx_s ZSTDv05_DCtx; -ZSTDv05_DCtx* ZSTDv05_createDCtx(void); -size_t ZSTDv05_freeDCtx(ZSTDv05_DCtx* dctx); /*!< @return : errorCode */ + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + if (opStart4 > oend) return ERROR(corruption_detected); /* overflow */ + assert(dstSize >= 6 /* validated above */); + CHECK_F( BIT_initDStream(&bitD1, istart1, length1) ); + CHECK_F( BIT_initDStream(&bitD2, istart2, length2) ); + CHECK_F( BIT_initDStream(&bitD3, istart3, length3) ); + CHECK_F( BIT_initDStream(&bitD4, istart4, length4) ); -/** ZSTDv05_decompressDCtx() : -* Same as ZSTDv05_decompress(), but requires an already allocated ZSTDv05_DCtx (see ZSTDv05_createDCtx()) */ -size_t ZSTDv05_decompressDCtx(ZSTDv05_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + /* 16-32 symbols per loop (4-8 symbols per stream) */ + if ((size_t)(oend - op4) >= sizeof(size_t)) { + for ( ; (endSignal) & (op4 < olimit); ) { +#if defined(__clang__) && (defined(__x86_64__) || defined(__i386__)) + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_1(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_0(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_1(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_0(op2, &bitD2); + endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished; + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_1(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_0(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_1(op4, &bitD4); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_0(op4, &bitD4); + endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished; +#else + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_1(op1, &bitD1); + HUF_DECODE_SYMBOLX2_1(op2, &bitD2); + HUF_DECODE_SYMBOLX2_1(op3, &bitD3); + HUF_DECODE_SYMBOLX2_1(op4, &bitD4); + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_0(op1, &bitD1); + HUF_DECODE_SYMBOLX2_0(op2, &bitD2); + HUF_DECODE_SYMBOLX2_0(op3, &bitD3); + HUF_DECODE_SYMBOLX2_0(op4, &bitD4); + endSignal = (U32)LIKELY((U32) + (BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished)); +#endif + } + } + /* check corruption */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 already verified within main loop */ -/*-*********************** -* Simple Dictionary API -*************************/ -/*! ZSTDv05_decompress_usingDict() : -* Decompression using a pre-defined Dictionary content (see dictBuilder). -* Dictionary must be identical to the one used during compression, otherwise regenerated data will be corrupted. -* Note : dict can be NULL, in which case, it's equivalent to ZSTDv05_decompressDCtx() */ -size_t ZSTDv05_decompress_usingDict(ZSTDv05_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize); + /* finish bitStreams one by one */ + HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog); + HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog); + HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog); + HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog); -/*-************************ -* Advanced Streaming API -***************************/ -typedef enum { ZSTDv05_fast, ZSTDv05_greedy, ZSTDv05_lazy, ZSTDv05_lazy2, ZSTDv05_btlazy2, ZSTDv05_opt, ZSTDv05_btopt } ZSTDv05_strategy; -typedef struct { - U64 srcSize; - U32 windowLog; /* the only useful information to retrieve */ - U32 contentLog; U32 hashLog; U32 searchLog; U32 searchLength; U32 targetLength; ZSTDv05_strategy strategy; -} ZSTDv05_parameters; -size_t ZSTDv05_getFrameParams(ZSTDv05_parameters* params, const void* src, size_t srcSize); + /* check */ + { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endCheck) return ERROR(corruption_detected); } -size_t ZSTDv05_decompressBegin_usingDict(ZSTDv05_DCtx* dctx, const void* dict, size_t dictSize); -void ZSTDv05_copyDCtx(ZSTDv05_DCtx* dstDCtx, const ZSTDv05_DCtx* srcDCtx); -size_t ZSTDv05_nextSrcSizeToDecompress(ZSTDv05_DCtx* dctx); -size_t ZSTDv05_decompressContinue(ZSTDv05_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + /* decoded size */ + return dstSize; + } +} +#if HUF_NEED_BMI2_FUNCTION +static BMI2_TARGET_ATTRIBUTE +size_t HUF_decompress4X2_usingDTable_internal_bmi2(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} +#endif -/*-*********************** -* ZBUFF API -*************************/ -typedef struct ZBUFFv05_DCtx_s ZBUFFv05_DCtx; -ZBUFFv05_DCtx* ZBUFFv05_createDCtx(void); -size_t ZBUFFv05_freeDCtx(ZBUFFv05_DCtx* dctx); +static +size_t HUF_decompress4X2_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} -size_t ZBUFFv05_decompressInit(ZBUFFv05_DCtx* dctx); -size_t ZBUFFv05_decompressInitDictionary(ZBUFFv05_DCtx* dctx, const void* dict, size_t dictSize); +#if ZSTD_ENABLE_ASM_X86_64_BMI2 -size_t ZBUFFv05_decompressContinue(ZBUFFv05_DCtx* dctx, - void* dst, size_t* dstCapacityPtr, - const void* src, size_t* srcSizePtr); +HUF_ASM_DECL void HUF_decompress4X2_usingDTable_internal_fast_asm_loop(HUF_DecompressFastArgs* args) ZSTDLIB_HIDDEN; -/*-*************************************************************************** -* Streaming decompression -* -* A ZBUFFv05_DCtx object is required to track streaming operations. -* Use ZBUFFv05_createDCtx() and ZBUFFv05_freeDCtx() to create/release resources. -* Use ZBUFFv05_decompressInit() to start a new decompression operation, -* or ZBUFFv05_decompressInitDictionary() if decompression requires a dictionary. -* Note that ZBUFFv05_DCtx objects can be reused multiple times. -* -* Use ZBUFFv05_decompressContinue() repetitively to consume your input. -* *srcSizePtr and *dstCapacityPtr can be any size. -* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. -* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. -* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters or change @dst. -* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency) -* or 0 when a frame is completely decoded -* or an error code, which can be tested using ZBUFFv05_isError(). -* -* Hint : recommended buffer sizes (not compulsory) : ZBUFFv05_recommendedDInSize() / ZBUFFv05_recommendedDOutSize() -* output : ZBUFFv05_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. -* input : ZBUFFv05_recommendedDInSize==128Kb+3; just follow indications from ZBUFFv05_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . -* *******************************************************************************/ +#endif +static HUF_FAST_BMI2_ATTRS +void HUF_decompress4X2_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs* args) +{ + U64 bits[4]; + BYTE const* ip[4]; + BYTE* op[4]; + BYTE* oend[4]; + HUF_DEltX2 const* const dtable = (HUF_DEltX2 const*)args->dt; + BYTE const* const ilowest = args->ilowest; -/* ************************************* -* Tool functions -***************************************/ -unsigned ZBUFFv05_isError(size_t errorCode); -const char* ZBUFFv05_getErrorName(size_t errorCode); + /* Copy the arguments to local registers. */ + ZSTD_memcpy(&bits, &args->bits, sizeof(bits)); + ZSTD_memcpy((void*)(&ip), &args->ip, sizeof(ip)); + ZSTD_memcpy(&op, &args->op, sizeof(op)); -/** Functions below provide recommended buffer sizes for Compression or Decompression operations. -* These sizes are just hints, and tend to offer better latency */ -size_t ZBUFFv05_recommendedDInSize(void); -size_t ZBUFFv05_recommendedDOutSize(void); + oend[0] = op[1]; + oend[1] = op[2]; + oend[2] = op[3]; + oend[3] = args->oend; + assert(MEM_isLittleEndian()); + assert(!MEM_32bits()); + for (;;) { + BYTE* olimit; + int stream; -/*-************************************* -* Constants -***************************************/ -#define ZSTDv05_MAGICNUMBER 0xFD2FB525 /* v0.5 */ + /* Assert loop preconditions */ +#ifndef NDEBUG + for (stream = 0; stream < 4; ++stream) { + assert(op[stream] <= oend[stream]); + assert(ip[stream] >= ilowest); + } +#endif + /* Compute olimit */ + { + /* Each loop does 5 table lookups for each of the 4 streams. + * Each table lookup consumes up to 11 bits of input, and produces + * up to 2 bytes of output. + */ + /* We can consume up to 7 bytes of input per iteration per stream. + * We also know that each input pointer is >= ip[0]. So we can run + * iters loops before running out of input. + */ + size_t iters = (size_t)(ip[0] - ilowest) / 7; + /* Each iteration can produce up to 10 bytes of output per stream. + * Each output stream my advance at different rates. So take the + * minimum number of safe iterations among all the output streams. + */ + for (stream = 0; stream < 4; ++stream) { + size_t const oiters = (size_t)(oend[stream] - op[stream]) / 10; + iters = MIN(iters, oiters); + } + /* Each iteration produces at least 5 output symbols. So until + * op[3] crosses olimit, we know we haven't executed iters + * iterations yet. This saves us maintaining an iters counter, + * at the expense of computing the remaining # of iterations + * more frequently. + */ + olimit = op[3] + (iters * 5); + /* Exit the fast decoding loop once we reach the end. */ + if (op[3] == olimit) + break; + /* Exit the decoding loop if any input pointer has crossed the + * previous one. This indicates corruption, and a precondition + * to our loop is that ip[i] >= ip[0]. + */ + for (stream = 1; stream < 4; ++stream) { + if (ip[stream] < ip[stream - 1]) + goto _out; + } + } -#if defined (__cplusplus) -} +#ifndef NDEBUG + for (stream = 1; stream < 4; ++stream) { + assert(ip[stream] >= ip[stream - 1]); + } #endif -#endif /* ZSTDv0505_H */ -/**** ended inlining zstd_v05.h ****/ -#endif -#if (ZSTD_LEGACY_SUPPORT <= 6) -/**** start inlining zstd_v06.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ +#define HUF_4X2_DECODE_SYMBOL(_stream, _decode3) \ + do { \ + if ((_decode3) || (_stream) != 3) { \ + int const index = (int)(bits[(_stream)] >> 53); \ + HUF_DEltX2 const entry = dtable[index]; \ + MEM_write16(op[(_stream)], entry.sequence); \ + bits[(_stream)] <<= (entry.nbBits) & 0x3F; \ + op[(_stream)] += (entry.length); \ + } \ + } while (0) + +#define HUF_4X2_RELOAD_STREAM(_stream) \ + do { \ + HUF_4X2_DECODE_SYMBOL(3, 1); \ + { \ + int const ctz = ZSTD_countTrailingZeros64(bits[(_stream)]); \ + int const nbBits = ctz & 7; \ + int const nbBytes = ctz >> 3; \ + ip[(_stream)] -= nbBytes; \ + bits[(_stream)] = MEM_read64(ip[(_stream)]) | 1; \ + bits[(_stream)] <<= nbBits; \ + } \ + } while (0) -#ifndef ZSTDv06_H -#define ZSTDv06_H + /* Manually unroll the loop because compilers don't consistently + * unroll the inner loops, which destroys performance. + */ + do { + /* Decode 5 symbols from each of the first 3 streams. + * The final stream will be decoded during the reload phase + * to reduce register pressure. + */ + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); + + /* Decode one symbol from the final stream */ + HUF_4X2_DECODE_SYMBOL(3, 1); + + /* Decode 4 symbols from the final stream & reload bitstreams. + * The final stream is reloaded last, meaning that all 5 symbols + * are decoded from the final stream before it is reloaded. + */ + HUF_4X_FOR_EACH_STREAM(HUF_4X2_RELOAD_STREAM); + } while (op[3] < olimit); + } -#if defined (__cplusplus) -extern "C" { -#endif +#undef HUF_4X2_DECODE_SYMBOL +#undef HUF_4X2_RELOAD_STREAM -/*====== Dependency ======*/ -#include /* size_t */ +_out: + /* Save the final values of each of the state variables back to args. */ + ZSTD_memcpy(&args->bits, &bits, sizeof(bits)); + ZSTD_memcpy((void*)(&args->ip), &ip, sizeof(ip)); + ZSTD_memcpy(&args->op, &op, sizeof(op)); +} -/*====== Export for Windows ======*/ -/*! -* ZSTDv06_DLL_EXPORT : -* Enable exporting of functions when building a Windows DLL -*/ -#if defined(_WIN32) && defined(ZSTDv06_DLL_EXPORT) && (ZSTDv06_DLL_EXPORT==1) -# define ZSTDLIBv06_API __declspec(dllexport) -#else -# define ZSTDLIBv06_API -#endif +static HUF_FAST_BMI2_ATTRS size_t +HUF_decompress4X2_usingDTable_internal_fast( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable, + HUF_DecompressFastLoopFn loopFn) { + void const* dt = DTable + 1; + const BYTE* const ilowest = (const BYTE*)cSrc; + BYTE* const oend = ZSTD_maybeNullPtrAdd((BYTE*)dst, dstSize); + HUF_DecompressFastArgs args; + { + size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable); + FORWARD_IF_ERROR(ret, "Failed to init asm args"); + if (ret == 0) + return 0; + } -/* ************************************* -* Simple functions -***************************************/ -/*! ZSTDv06_decompress() : - `compressedSize` : is the _exact_ size of the compressed blob, otherwise decompression will fail. - `dstCapacity` must be large enough, equal or larger than originalSize. - @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), - or an errorCode if it fails (which can be tested using ZSTDv06_isError()) */ -ZSTDLIBv06_API size_t ZSTDv06_decompress( void* dst, size_t dstCapacity, - const void* src, size_t compressedSize); + assert(args.ip[0] >= args.ilowest); + loopFn(&args); -/** -ZSTDv06_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.6.x format - srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' - cSize (output parameter) : the number of bytes that would be read to decompress this frame - or an error code if it fails (which can be tested using ZSTDv01_isError()) - dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame - or ZSTD_CONTENTSIZE_ERROR if an error occurs - - note : assumes `cSize` and `dBound` are _not_ NULL. -*/ -void ZSTDv06_findFrameSizeInfoLegacy(const void *src, size_t srcSize, - size_t* cSize, unsigned long long* dBound); + /* note : op4 already verified within main loop */ + assert(args.ip[0] >= ilowest); + assert(args.ip[1] >= ilowest); + assert(args.ip[2] >= ilowest); + assert(args.ip[3] >= ilowest); + assert(args.op[3] <= oend); -/* ************************************* -* Helper functions -***************************************/ -ZSTDLIBv06_API size_t ZSTDv06_compressBound(size_t srcSize); /*!< maximum compressed size (worst case scenario) */ + assert(ilowest == args.ilowest); + assert(ilowest + 6 == args.iend[0]); + (void)ilowest; -/* Error Management */ -ZSTDLIBv06_API unsigned ZSTDv06_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ -ZSTDLIBv06_API const char* ZSTDv06_getErrorName(size_t code); /*!< provides readable string for an error code */ + /* finish bitStreams one by one */ + { + size_t const segmentSize = (dstSize+3) / 4; + BYTE* segmentEnd = (BYTE*)dst; + int i; + for (i = 0; i < 4; ++i) { + BIT_DStream_t bit; + if (segmentSize <= (size_t)(oend - segmentEnd)) + segmentEnd += segmentSize; + else + segmentEnd = oend; + FORWARD_IF_ERROR(HUF_initRemainingDStream(&bit, &args, i, segmentEnd), "corruption"); + args.op[i] += HUF_decodeStreamX2(args.op[i], &bit, segmentEnd, (HUF_DEltX2 const*)dt, HUF_DECODER_FAST_TABLELOG); + if (args.op[i] != segmentEnd) + return ERROR(corruption_detected); + } + } + /* decoded size */ + return dstSize; +} -/* ************************************* -* Explicit memory management -***************************************/ -/** Decompression context */ -typedef struct ZSTDv06_DCtx_s ZSTDv06_DCtx; -ZSTDLIBv06_API ZSTDv06_DCtx* ZSTDv06_createDCtx(void); -ZSTDLIBv06_API size_t ZSTDv06_freeDCtx(ZSTDv06_DCtx* dctx); /*!< @return : errorCode */ +static size_t HUF_decompress4X2_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable, int flags) +{ + HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X2_usingDTable_internal_default; + HUF_DecompressFastLoopFn loopFn = HUF_decompress4X2_usingDTable_internal_fast_c_loop; -/** ZSTDv06_decompressDCtx() : -* Same as ZSTDv06_decompress(), but requires an already allocated ZSTDv06_DCtx (see ZSTDv06_createDCtx()) */ -ZSTDLIBv06_API size_t ZSTDv06_decompressDCtx(ZSTDv06_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +#if DYNAMIC_BMI2 + if (flags & HUF_flags_bmi2) { + fallbackFn = HUF_decompress4X2_usingDTable_internal_bmi2; +# if ZSTD_ENABLE_ASM_X86_64_BMI2 + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop; + } +# endif + } else { + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); + } +#endif +#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__) + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop; + } +#endif -/*-*********************** -* Dictionary API -*************************/ -/*! ZSTDv06_decompress_usingDict() : -* Decompression using a pre-defined Dictionary content (see dictBuilder). -* Dictionary must be identical to the one used during compression, otherwise regenerated data will be corrupted. -* Note : dict can be NULL, in which case, it's equivalent to ZSTDv06_decompressDCtx() */ -ZSTDLIBv06_API size_t ZSTDv06_decompress_usingDict(ZSTDv06_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize); + if (HUF_ENABLE_FAST_DECODE && !(flags & HUF_flags_disableFast)) { + size_t const ret = HUF_decompress4X2_usingDTable_internal_fast(dst, dstSize, cSrc, cSrcSize, DTable, loopFn); + if (ret != 0) + return ret; + } + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); +} + +HUF_DGEN(HUF_decompress1X2_usingDTable_internal) +size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int flags) +{ + const BYTE* ip = (const BYTE*) cSrc; -/*-************************ -* Advanced Streaming API -***************************/ -struct ZSTDv06_frameParams_s { unsigned long long frameContentSize; unsigned windowLog; }; -typedef struct ZSTDv06_frameParams_s ZSTDv06_frameParams; + size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize, + workSpace, wkspSize, flags); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; -ZSTDLIBv06_API size_t ZSTDv06_getFrameParams(ZSTDv06_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */ -ZSTDLIBv06_API size_t ZSTDv06_decompressBegin_usingDict(ZSTDv06_DCtx* dctx, const void* dict, size_t dictSize); -ZSTDLIBv06_API void ZSTDv06_copyDCtx(ZSTDv06_DCtx* dctx, const ZSTDv06_DCtx* preparedDCtx); + return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, flags); +} -ZSTDLIBv06_API size_t ZSTDv06_nextSrcSizeToDecompress(ZSTDv06_DCtx* dctx); -ZSTDLIBv06_API size_t ZSTDv06_decompressContinue(ZSTDv06_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +static size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int flags) +{ + const BYTE* ip = (const BYTE*) cSrc; + size_t hSize = HUF_readDTableX2_wksp(dctx, cSrc, cSrcSize, + workSpace, wkspSize, flags); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags); +} -/* ************************************* -* ZBUFF API -***************************************/ +#endif /* HUF_FORCE_DECOMPRESS_X1 */ -typedef struct ZBUFFv06_DCtx_s ZBUFFv06_DCtx; -ZSTDLIBv06_API ZBUFFv06_DCtx* ZBUFFv06_createDCtx(void); -ZSTDLIBv06_API size_t ZBUFFv06_freeDCtx(ZBUFFv06_DCtx* dctx); -ZSTDLIBv06_API size_t ZBUFFv06_decompressInit(ZBUFFv06_DCtx* dctx); -ZSTDLIBv06_API size_t ZBUFFv06_decompressInitDictionary(ZBUFFv06_DCtx* dctx, const void* dict, size_t dictSize); +/* ***********************************/ +/* Universal decompression selectors */ +/* ***********************************/ -ZSTDLIBv06_API size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* dctx, - void* dst, size_t* dstCapacityPtr, - const void* src, size_t* srcSizePtr); -/*-*************************************************************************** -* Streaming decompression howto -* -* A ZBUFFv06_DCtx object is required to track streaming operations. -* Use ZBUFFv06_createDCtx() and ZBUFFv06_freeDCtx() to create/release resources. -* Use ZBUFFv06_decompressInit() to start a new decompression operation, -* or ZBUFFv06_decompressInitDictionary() if decompression requires a dictionary. -* Note that ZBUFFv06_DCtx objects can be re-init multiple times. -* -* Use ZBUFFv06_decompressContinue() repetitively to consume your input. -* *srcSizePtr and *dstCapacityPtr can be any size. -* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. -* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. -* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change `dst`. -* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency), -* or 0 when a frame is completely decoded, -* or an error code, which can be tested using ZBUFFv06_isError(). -* -* Hint : recommended buffer sizes (not compulsory) : ZBUFFv06_recommendedDInSize() and ZBUFFv06_recommendedDOutSize() -* output : ZBUFFv06_recommendedDOutSize== 128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. -* input : ZBUFFv06_recommendedDInSize == 128KB + 3; -* just follow indications from ZBUFFv06_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . -* *******************************************************************************/ +#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2) +typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t; +static const algo_time_t algoTime[16 /* Quantization */][2 /* single, double */] = +{ + /* single, double, quad */ + {{0,0}, {1,1}}, /* Q==0 : impossible */ + {{0,0}, {1,1}}, /* Q==1 : impossible */ + {{ 150,216}, { 381,119}}, /* Q == 2 : 12-18% */ + {{ 170,205}, { 514,112}}, /* Q == 3 : 18-25% */ + {{ 177,199}, { 539,110}}, /* Q == 4 : 25-32% */ + {{ 197,194}, { 644,107}}, /* Q == 5 : 32-38% */ + {{ 221,192}, { 735,107}}, /* Q == 6 : 38-44% */ + {{ 256,189}, { 881,106}}, /* Q == 7 : 44-50% */ + {{ 359,188}, {1167,109}}, /* Q == 8 : 50-56% */ + {{ 582,187}, {1570,114}}, /* Q == 9 : 56-62% */ + {{ 688,187}, {1712,122}}, /* Q ==10 : 62-69% */ + {{ 825,186}, {1965,136}}, /* Q ==11 : 69-75% */ + {{ 976,185}, {2131,150}}, /* Q ==12 : 75-81% */ + {{1180,186}, {2070,175}}, /* Q ==13 : 81-87% */ + {{1377,185}, {1731,202}}, /* Q ==14 : 87-93% */ + {{1412,185}, {1695,202}}, /* Q ==15 : 93-99% */ +}; +#endif +/** HUF_selectDecoder() : + * Tells which decoder is likely to decode faster, + * based on a set of pre-computed metrics. + * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 . + * Assumption : 0 < dstSize <= 128 KB */ +U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize) +{ + assert(dstSize > 0); + assert(dstSize <= 128*1024); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dstSize; + (void)cSrcSize; + return 0; +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dstSize; + (void)cSrcSize; + return 1; +#else + /* decoder timing evaluation */ + { U32 const Q = (cSrcSize >= dstSize) ? 15 : (U32)(cSrcSize * 16 / dstSize); /* Q < 16 */ + U32 const D256 = (U32)(dstSize >> 8); + U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256); + U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256); + DTime1 += DTime1 >> 5; /* small advantage to algorithm using less memory, to reduce cache eviction */ + return DTime1 < DTime0; + } +#endif +} -/* ************************************* -* Tool functions -***************************************/ -ZSTDLIBv06_API unsigned ZBUFFv06_isError(size_t errorCode); -ZSTDLIBv06_API const char* ZBUFFv06_getErrorName(size_t errorCode); +size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int flags) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ + if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ -/** Functions below provide recommended buffer sizes for Compression or Decompression operations. -* These sizes are just hints, they tend to offer better latency */ -ZSTDLIBv06_API size_t ZBUFFv06_recommendedDInSize(void); -ZSTDLIBv06_API size_t ZBUFFv06_recommendedDOutSize(void); + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize, flags); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize, flags); +#else + return algoNb ? HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize, flags): + HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize, flags); +#endif + } +} -/*-************************************* -* Constants -***************************************/ -#define ZSTDv06_MAGICNUMBER 0xFD2FB526 /* v0.6 */ +size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dtd; + assert(dtd.tableType == 0); + return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dtd; + assert(dtd.tableType == 1); + return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#else + return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags) : + HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#endif +} +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags) +{ + const BYTE* ip = (const BYTE*) cSrc; + size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize, flags); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; -#if defined (__cplusplus) + return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags); } #endif -#endif /* ZSTDv06_BUFFERED_H */ -/**** ended inlining zstd_v06.h ****/ +size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dtd; + assert(dtd.tableType == 0); + return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dtd; + assert(dtd.tableType == 1); + return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#else + return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags) : + HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#endif +} + +size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize == 0) return ERROR(corruption_detected); + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags); +#else + return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags) : + HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags); #endif -#if (ZSTD_LEGACY_SUPPORT <= 7) -/**** start inlining zstd_v07.h ****/ + } +} +/**** ended inlining decompress/huf_decompress.c ****/ +/**** start inlining decompress/zstd_ddict.c ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -28590,551 +38022,310 @@ ZSTDLIBv06_API size_t ZBUFFv06_recommendedDOutSize(void); * You may select, at your option, one of the above-listed licenses. */ -#ifndef ZSTDv07_H_235446 -#define ZSTDv07_H_235446 +/* zstd_ddict.c : + * concentrates all logic that needs to know the internals of ZSTD_DDict object */ -#if defined (__cplusplus) -extern "C" { -#endif +/*-******************************************************* +* Dependencies +*********************************************************/ +/**** skipping file: ../common/allocations.h ****/ +/**** skipping file: ../common/zstd_deps.h ****/ +/**** skipping file: ../common/cpu.h ****/ +/**** skipping file: ../common/mem.h ****/ +#define FSE_STATIC_LINKING_ONLY +/**** skipping file: ../common/fse.h ****/ +/**** skipping file: ../common/huf.h ****/ +/**** start inlining zstd_decompress_internal.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ -/*====== Dependency ======*/ -#include /* size_t */ +/* zstd_decompress_internal: + * objects and definitions shared within lib/decompress modules */ -/*====== Export for Windows ======*/ -/*! -* ZSTDv07_DLL_EXPORT : -* Enable exporting of functions when building a Windows DLL -*/ -#if defined(_WIN32) && defined(ZSTDv07_DLL_EXPORT) && (ZSTDv07_DLL_EXPORT==1) -# define ZSTDLIBv07_API __declspec(dllexport) -#else -# define ZSTDLIBv07_API -#endif + #ifndef ZSTD_DECOMPRESS_INTERNAL_H + #define ZSTD_DECOMPRESS_INTERNAL_H -/* ************************************* -* Simple API -***************************************/ -/*! ZSTDv07_getDecompressedSize() : -* @return : decompressed size if known, 0 otherwise. - note 1 : if `0`, follow up with ZSTDv07_getFrameParams() to know precise failure cause. - note 2 : decompressed size could be wrong or intentionally modified ! - always ensure results fit within application's authorized limits */ -unsigned long long ZSTDv07_getDecompressedSize(const void* src, size_t srcSize); - -/*! ZSTDv07_decompress() : - `compressedSize` : must be _exact_ size of compressed input, otherwise decompression will fail. - `dstCapacity` must be equal or larger than originalSize. - @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), - or an errorCode if it fails (which can be tested using ZSTDv07_isError()) */ -ZSTDLIBv07_API size_t ZSTDv07_decompress( void* dst, size_t dstCapacity, - const void* src, size_t compressedSize); +/*-******************************************************* + * Dependencies + *********************************************************/ +/**** skipping file: ../common/mem.h ****/ +/**** skipping file: ../common/zstd_internal.h ****/ -/** -ZSTDv07_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.7.x format - srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' - cSize (output parameter) : the number of bytes that would be read to decompress this frame - or an error code if it fails (which can be tested using ZSTDv01_isError()) - dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame - or ZSTD_CONTENTSIZE_ERROR if an error occurs - - note : assumes `cSize` and `dBound` are _not_ NULL. -*/ -void ZSTDv07_findFrameSizeInfoLegacy(const void *src, size_t srcSize, - size_t* cSize, unsigned long long* dBound); -/*====== Helper functions ======*/ -ZSTDLIBv07_API unsigned ZSTDv07_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ -ZSTDLIBv07_API const char* ZSTDv07_getErrorName(size_t code); /*!< provides readable string from an error code */ +/*-******************************************************* + * Constants + *********************************************************/ +static UNUSED_ATTR const U32 LL_base[MaxLL+1] = { + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 18, 20, 22, 24, 28, 32, 40, + 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, + 0x2000, 0x4000, 0x8000, 0x10000 }; -/*-************************************* -* Explicit memory management -***************************************/ -/** Decompression context */ -typedef struct ZSTDv07_DCtx_s ZSTDv07_DCtx; -ZSTDLIBv07_API ZSTDv07_DCtx* ZSTDv07_createDCtx(void); -ZSTDLIBv07_API size_t ZSTDv07_freeDCtx(ZSTDv07_DCtx* dctx); /*!< @return : errorCode */ - -/** ZSTDv07_decompressDCtx() : -* Same as ZSTDv07_decompress(), requires an allocated ZSTDv07_DCtx (see ZSTDv07_createDCtx()) */ -ZSTDLIBv07_API size_t ZSTDv07_decompressDCtx(ZSTDv07_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); - - -/*-************************ -* Simple dictionary API -***************************/ -/*! ZSTDv07_decompress_usingDict() : -* Decompression using a pre-defined Dictionary content (see dictBuilder). -* Dictionary must be identical to the one used during compression. -* Note : This function load the dictionary, resulting in a significant startup time */ -ZSTDLIBv07_API size_t ZSTDv07_decompress_usingDict(ZSTDv07_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize); - - -/*-************************** -* Advanced Dictionary API -****************************/ -/*! ZSTDv07_createDDict() : -* Create a digested dictionary, ready to start decompression operation without startup delay. -* `dict` can be released after creation */ -typedef struct ZSTDv07_DDict_s ZSTDv07_DDict; -ZSTDLIBv07_API ZSTDv07_DDict* ZSTDv07_createDDict(const void* dict, size_t dictSize); -ZSTDLIBv07_API size_t ZSTDv07_freeDDict(ZSTDv07_DDict* ddict); - -/*! ZSTDv07_decompress_usingDDict() : -* Decompression using a pre-digested Dictionary -* Faster startup than ZSTDv07_decompress_usingDict(), recommended when same dictionary is used multiple times. */ -ZSTDLIBv07_API size_t ZSTDv07_decompress_usingDDict(ZSTDv07_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTDv07_DDict* ddict); +static UNUSED_ATTR const U32 OF_base[MaxOff+1] = { + 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, + 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, + 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, + 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD }; -typedef struct { - unsigned long long frameContentSize; - unsigned windowSize; - unsigned dictID; - unsigned checksumFlag; -} ZSTDv07_frameParams; +static UNUSED_ATTR const U8 OF_bits[MaxOff+1] = { + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31 }; + +static UNUSED_ATTR const U32 ML_base[MaxML+1] = { + 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, + 27, 28, 29, 30, 31, 32, 33, 34, + 35, 37, 39, 41, 43, 47, 51, 59, + 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, + 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 }; -ZSTDLIBv07_API size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */ +/*-******************************************************* + * Decompression types + *********************************************************/ + typedef struct { + U32 fastMode; + U32 tableLog; + } ZSTD_seqSymbol_header; + typedef struct { + U16 nextState; + BYTE nbAdditionalBits; + BYTE nbBits; + U32 baseValue; + } ZSTD_seqSymbol; + #define SEQSYMBOL_TABLE_SIZE(log) (1 + (1 << (log))) -/* ************************************* -* Streaming functions -***************************************/ -typedef struct ZBUFFv07_DCtx_s ZBUFFv07_DCtx; -ZSTDLIBv07_API ZBUFFv07_DCtx* ZBUFFv07_createDCtx(void); -ZSTDLIBv07_API size_t ZBUFFv07_freeDCtx(ZBUFFv07_DCtx* dctx); +#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE (sizeof(S16) * (MaxSeq + 1) + (1u << MaxFSELog) + sizeof(U64)) +#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32 ((ZSTD_BUILD_FSE_TABLE_WKSP_SIZE + sizeof(U32) - 1) / sizeof(U32)) +#define ZSTD_HUFFDTABLE_CAPACITY_LOG 12 -ZSTDLIBv07_API size_t ZBUFFv07_decompressInit(ZBUFFv07_DCtx* dctx); -ZSTDLIBv07_API size_t ZBUFFv07_decompressInitDictionary(ZBUFFv07_DCtx* dctx, const void* dict, size_t dictSize); +typedef struct { + ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]; /* Note : Space reserved for FSE Tables */ + ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]; /* is also used as temporary workspace while building hufTable during DDict creation */ + ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]; /* and therefore must be at least HUF_DECOMPRESS_WORKSPACE_SIZE large */ + HUF_DTable hufTable[HUF_DTABLE_SIZE(ZSTD_HUFFDTABLE_CAPACITY_LOG)]; /* can accommodate HUF_decompress4X */ + U32 rep[ZSTD_REP_NUM]; + U32 workspace[ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32]; +} ZSTD_entropyDTables_t; -ZSTDLIBv07_API size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* dctx, - void* dst, size_t* dstCapacityPtr, - const void* src, size_t* srcSizePtr); +typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, + ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock, + ZSTDds_decompressLastBlock, ZSTDds_checkChecksum, + ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage; -/*-*************************************************************************** -* Streaming decompression howto -* -* A ZBUFFv07_DCtx object is required to track streaming operations. -* Use ZBUFFv07_createDCtx() and ZBUFFv07_freeDCtx() to create/release resources. -* Use ZBUFFv07_decompressInit() to start a new decompression operation, -* or ZBUFFv07_decompressInitDictionary() if decompression requires a dictionary. -* Note that ZBUFFv07_DCtx objects can be re-init multiple times. -* -* Use ZBUFFv07_decompressContinue() repetitively to consume your input. -* *srcSizePtr and *dstCapacityPtr can be any size. -* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. -* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. -* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change `dst`. -* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency), -* or 0 when a frame is completely decoded, -* or an error code, which can be tested using ZBUFFv07_isError(). -* -* Hint : recommended buffer sizes (not compulsory) : ZBUFFv07_recommendedDInSize() and ZBUFFv07_recommendedDOutSize() -* output : ZBUFFv07_recommendedDOutSize== 128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. -* input : ZBUFFv07_recommendedDInSize == 128KB + 3; -* just follow indications from ZBUFFv07_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . -* *******************************************************************************/ +typedef enum { zdss_init=0, zdss_loadHeader, + zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage; +typedef enum { + ZSTD_use_indefinitely = -1, /* Use the dictionary indefinitely */ + ZSTD_dont_use = 0, /* Do not use the dictionary (if one exists free it) */ + ZSTD_use_once = 1 /* Use the dictionary once and set to ZSTD_dont_use */ +} ZSTD_dictUses_e; -/* ************************************* -* Tool functions -***************************************/ -ZSTDLIBv07_API unsigned ZBUFFv07_isError(size_t errorCode); -ZSTDLIBv07_API const char* ZBUFFv07_getErrorName(size_t errorCode); +/* Hashset for storing references to multiple ZSTD_DDict within ZSTD_DCtx */ +typedef struct { + const ZSTD_DDict** ddictPtrTable; + size_t ddictPtrTableSize; + size_t ddictPtrCount; +} ZSTD_DDictHashSet; -/** Functions below provide recommended buffer sizes for Compression or Decompression operations. -* These sizes are just hints, they tend to offer better latency */ -ZSTDLIBv07_API size_t ZBUFFv07_recommendedDInSize(void); -ZSTDLIBv07_API size_t ZBUFFv07_recommendedDOutSize(void); +#ifndef ZSTD_DECODER_INTERNAL_BUFFER +# define ZSTD_DECODER_INTERNAL_BUFFER (1 << 16) +#endif +#define ZSTD_LBMIN 64 +#define ZSTD_LBMAX (128 << 10) -/*-************************************* -* Constants -***************************************/ -#define ZSTDv07_MAGICNUMBER 0xFD2FB527 /* v0.7 */ +/* extra buffer, compensates when dst is not large enough to store litBuffer */ +#define ZSTD_LITBUFFEREXTRASIZE BOUNDED(ZSTD_LBMIN, ZSTD_DECODER_INTERNAL_BUFFER, ZSTD_LBMAX) +typedef enum { + ZSTD_not_in_dst = 0, /* Stored entirely within litExtraBuffer */ + ZSTD_in_dst = 1, /* Stored entirely within dst (in memory after current output write) */ + ZSTD_split = 2 /* Split between litExtraBuffer and dst */ +} ZSTD_litLocation_e; -#if defined (__cplusplus) -} +struct ZSTD_DCtx_s +{ + const ZSTD_seqSymbol* LLTptr; + const ZSTD_seqSymbol* MLTptr; + const ZSTD_seqSymbol* OFTptr; + const HUF_DTable* HUFptr; + ZSTD_entropyDTables_t entropy; + U32 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; /* space needed when building huffman tables */ + const void* previousDstEnd; /* detect continuity */ + const void* prefixStart; /* start of current segment */ + const void* virtualStart; /* virtual start of previous segment if it was just before current one */ + const void* dictEnd; /* end of previous segment */ + size_t expected; + ZSTD_FrameHeader fParams; + U64 processedCSize; + U64 decodedSize; + blockType_e bType; /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block decompression stages */ + ZSTD_dStage stage; + U32 litEntropy; + U32 fseEntropy; + XXH64_state_t xxhState; + size_t headerSize; + ZSTD_format_e format; + ZSTD_forceIgnoreChecksum_e forceIgnoreChecksum; /* User specified: if == 1, will ignore checksums in compressed frame. Default == 0 */ + U32 validateChecksum; /* if == 1, will validate checksum. Is == 1 if (fParams.checksumFlag == 1) and (forceIgnoreChecksum == 0). */ + const BYTE* litPtr; + ZSTD_customMem customMem; + size_t litSize; + size_t rleSize; + size_t staticSize; + int isFrameDecompression; +#if DYNAMIC_BMI2 + int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */ #endif -#endif /* ZSTDv07_H_235446 */ -/**** ended inlining zstd_v07.h ****/ -#endif + /* dictionary */ + ZSTD_DDict* ddictLocal; + const ZSTD_DDict* ddict; /* set by ZSTD_initDStream_usingDDict(), or ZSTD_DCtx_refDDict() */ + U32 dictID; + int ddictIsCold; /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */ + ZSTD_dictUses_e dictUses; + ZSTD_DDictHashSet* ddictSet; /* Hash set for multiple ddicts */ + ZSTD_refMultipleDDicts_e refMultipleDDicts; /* User specified: if == 1, will allow references to multiple DDicts. Default == 0 (disabled) */ + int disableHufAsm; + int maxBlockSizeParam; -/** ZSTD_isLegacy() : - @return : > 0 if supported by legacy decoder. 0 otherwise. - return value is the version. -*/ -MEM_STATIC unsigned ZSTD_isLegacy(const void* src, size_t srcSize) -{ - U32 magicNumberLE; - if (srcSize<4) return 0; - magicNumberLE = MEM_readLE32(src); - switch(magicNumberLE) - { -#if (ZSTD_LEGACY_SUPPORT <= 1) - case ZSTDv01_magicNumberLE:return 1; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 2) - case ZSTDv02_magicNumber : return 2; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 3) - case ZSTDv03_magicNumber : return 3; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 4) - case ZSTDv04_magicNumber : return 4; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 5) - case ZSTDv05_MAGICNUMBER : return 5; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 6) - case ZSTDv06_MAGICNUMBER : return 6; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 7) - case ZSTDv07_MAGICNUMBER : return 7; + /* streaming */ + ZSTD_dStreamStage streamStage; + char* inBuff; + size_t inBuffSize; + size_t inPos; + size_t maxWindowSize; + char* outBuff; + size_t outBuffSize; + size_t outStart; + size_t outEnd; + size_t lhSize; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + void* legacyContext; + U32 previousLegacyVersion; + U32 legacyVersion; #endif - default : return 0; - } -} + U32 hostageByte; + int noForwardProgress; + ZSTD_bufferMode_e outBufferMode; + ZSTD_outBuffer expectedOutBuffer; + /* workspace */ + BYTE* litBuffer; + const BYTE* litBufferEnd; + ZSTD_litLocation_e litBufferLocation; + BYTE litExtraBuffer[ZSTD_LITBUFFEREXTRASIZE + WILDCOPY_OVERLENGTH]; /* literal buffer can be split between storage within dst and within this scratch buffer */ + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; -MEM_STATIC unsigned long long ZSTD_getDecompressedSize_legacy(const void* src, size_t srcSize) -{ - U32 const version = ZSTD_isLegacy(src, srcSize); - if (version < 5) return 0; /* no decompressed size in frame header, or not a legacy format */ -#if (ZSTD_LEGACY_SUPPORT <= 5) - if (version==5) { - ZSTDv05_parameters fParams; - size_t const frResult = ZSTDv05_getFrameParams(&fParams, src, srcSize); - if (frResult != 0) return 0; - return fParams.srcSize; - } + size_t oversizedDuration; + +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + void const* dictContentBeginForFuzzing; + void const* dictContentEndForFuzzing; #endif -#if (ZSTD_LEGACY_SUPPORT <= 6) - if (version==6) { - ZSTDv06_frameParams fParams; - size_t const frResult = ZSTDv06_getFrameParams(&fParams, src, srcSize); - if (frResult != 0) return 0; - return fParams.frameContentSize; - } + + /* Tracing */ +#if ZSTD_TRACE + ZSTD_TraceCtx traceCtx; #endif -#if (ZSTD_LEGACY_SUPPORT <= 7) - if (version==7) { - ZSTDv07_frameParams fParams; - size_t const frResult = ZSTDv07_getFrameParams(&fParams, src, srcSize); - if (frResult != 0) return 0; - return fParams.frameContentSize; - } +}; /* typedef'd to ZSTD_DCtx within "zstd.h" */ + +MEM_STATIC int ZSTD_DCtx_get_bmi2(const struct ZSTD_DCtx_s *dctx) { +#if DYNAMIC_BMI2 + return dctx->bmi2; +#else + (void)dctx; + return 0; #endif - return 0; /* should not be possible */ } +/*-******************************************************* + * Shared internal functions + *********************************************************/ -MEM_STATIC size_t ZSTD_decompressLegacy( - void* dst, size_t dstCapacity, - const void* src, size_t compressedSize, - const void* dict,size_t dictSize) -{ - U32 const version = ZSTD_isLegacy(src, compressedSize); - (void)dst; (void)dstCapacity; (void)dict; (void)dictSize; /* unused when ZSTD_LEGACY_SUPPORT >= 8 */ - switch(version) - { -#if (ZSTD_LEGACY_SUPPORT <= 1) - case 1 : - return ZSTDv01_decompress(dst, dstCapacity, src, compressedSize); -#endif -#if (ZSTD_LEGACY_SUPPORT <= 2) - case 2 : - return ZSTDv02_decompress(dst, dstCapacity, src, compressedSize); -#endif -#if (ZSTD_LEGACY_SUPPORT <= 3) - case 3 : - return ZSTDv03_decompress(dst, dstCapacity, src, compressedSize); -#endif -#if (ZSTD_LEGACY_SUPPORT <= 4) - case 4 : - return ZSTDv04_decompress(dst, dstCapacity, src, compressedSize); -#endif -#if (ZSTD_LEGACY_SUPPORT <= 5) - case 5 : - { size_t result; - ZSTDv05_DCtx* const zd = ZSTDv05_createDCtx(); - if (zd==NULL) return ERROR(memory_allocation); - result = ZSTDv05_decompress_usingDict(zd, dst, dstCapacity, src, compressedSize, dict, dictSize); - ZSTDv05_freeDCtx(zd); - return result; - } -#endif -#if (ZSTD_LEGACY_SUPPORT <= 6) - case 6 : - { size_t result; - ZSTDv06_DCtx* const zd = ZSTDv06_createDCtx(); - if (zd==NULL) return ERROR(memory_allocation); - result = ZSTDv06_decompress_usingDict(zd, dst, dstCapacity, src, compressedSize, dict, dictSize); - ZSTDv06_freeDCtx(zd); - return result; - } -#endif -#if (ZSTD_LEGACY_SUPPORT <= 7) - case 7 : - { size_t result; - ZSTDv07_DCtx* const zd = ZSTDv07_createDCtx(); - if (zd==NULL) return ERROR(memory_allocation); - result = ZSTDv07_decompress_usingDict(zd, dst, dstCapacity, src, compressedSize, dict, dictSize); - ZSTDv07_freeDCtx(zd); - return result; - } -#endif - default : - return ERROR(prefix_unknown); - } -} +/*! ZSTD_loadDEntropy() : + * dict : must point at beginning of a valid zstd dictionary. + * @return : size of dictionary header (size of magic number + dict ID + entropy tables) */ +size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, + const void* const dict, size_t const dictSize); -MEM_STATIC ZSTD_frameSizeInfo ZSTD_findFrameSizeInfoLegacy(const void *src, size_t srcSize) -{ - ZSTD_frameSizeInfo frameSizeInfo; - U32 const version = ZSTD_isLegacy(src, srcSize); - switch(version) - { -#if (ZSTD_LEGACY_SUPPORT <= 1) - case 1 : - ZSTDv01_findFrameSizeInfoLegacy(src, srcSize, - &frameSizeInfo.compressedSize, - &frameSizeInfo.decompressedBound); - break; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 2) - case 2 : - ZSTDv02_findFrameSizeInfoLegacy(src, srcSize, - &frameSizeInfo.compressedSize, - &frameSizeInfo.decompressedBound); - break; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 3) - case 3 : - ZSTDv03_findFrameSizeInfoLegacy(src, srcSize, - &frameSizeInfo.compressedSize, - &frameSizeInfo.decompressedBound); - break; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 4) - case 4 : - ZSTDv04_findFrameSizeInfoLegacy(src, srcSize, - &frameSizeInfo.compressedSize, - &frameSizeInfo.decompressedBound); - break; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 5) - case 5 : - ZSTDv05_findFrameSizeInfoLegacy(src, srcSize, - &frameSizeInfo.compressedSize, - &frameSizeInfo.decompressedBound); - break; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 6) - case 6 : - ZSTDv06_findFrameSizeInfoLegacy(src, srcSize, - &frameSizeInfo.compressedSize, - &frameSizeInfo.decompressedBound); - break; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 7) - case 7 : - ZSTDv07_findFrameSizeInfoLegacy(src, srcSize, - &frameSizeInfo.compressedSize, - &frameSizeInfo.decompressedBound); - break; -#endif - default : - frameSizeInfo.compressedSize = ERROR(prefix_unknown); - frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR; - break; - } - if (!ZSTD_isError(frameSizeInfo.compressedSize) && frameSizeInfo.compressedSize > srcSize) { - frameSizeInfo.compressedSize = ERROR(srcSize_wrong); - frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR; - } - return frameSizeInfo; -} +/*! ZSTD_checkContinuity() : + * check if next `dst` follows previous position, where decompression ended. + * If yes, do nothing (continue on current segment). + * If not, classify previous segment as "external dictionary", and start a new segment. + * This function cannot fail. */ +void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize); -MEM_STATIC size_t ZSTD_findFrameCompressedSizeLegacy(const void *src, size_t srcSize) -{ - ZSTD_frameSizeInfo frameSizeInfo = ZSTD_findFrameSizeInfoLegacy(src, srcSize); - return frameSizeInfo.compressedSize; -} -MEM_STATIC size_t ZSTD_freeLegacyStreamContext(void* legacyContext, U32 version) -{ - switch(version) - { - default : - case 1 : - case 2 : - case 3 : - (void)legacyContext; - return ERROR(version_unsupported); -#if (ZSTD_LEGACY_SUPPORT <= 4) - case 4 : return ZBUFFv04_freeDCtx((ZBUFFv04_DCtx*)legacyContext); -#endif -#if (ZSTD_LEGACY_SUPPORT <= 5) - case 5 : return ZBUFFv05_freeDCtx((ZBUFFv05_DCtx*)legacyContext); -#endif -#if (ZSTD_LEGACY_SUPPORT <= 6) - case 6 : return ZBUFFv06_freeDCtx((ZBUFFv06_DCtx*)legacyContext); -#endif -#if (ZSTD_LEGACY_SUPPORT <= 7) - case 7 : return ZBUFFv07_freeDCtx((ZBUFFv07_DCtx*)legacyContext); -#endif - } -} +#endif /* ZSTD_DECOMPRESS_INTERNAL_H */ +/**** ended inlining zstd_decompress_internal.h ****/ +/**** start inlining zstd_ddict.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ -MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U32 newVersion, - const void* dict, size_t dictSize) -{ - DEBUGLOG(5, "ZSTD_initLegacyStream for v0.%u", newVersion); - if (prevVersion != newVersion) ZSTD_freeLegacyStreamContext(*legacyContext, prevVersion); - switch(newVersion) - { - default : - case 1 : - case 2 : - case 3 : - (void)dict; (void)dictSize; - return 0; -#if (ZSTD_LEGACY_SUPPORT <= 4) - case 4 : - { - ZBUFFv04_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv04_createDCtx() : (ZBUFFv04_DCtx*)*legacyContext; - if (dctx==NULL) return ERROR(memory_allocation); - ZBUFFv04_decompressInit(dctx); - ZBUFFv04_decompressWithDictionary(dctx, dict, dictSize); - *legacyContext = dctx; - return 0; - } -#endif -#if (ZSTD_LEGACY_SUPPORT <= 5) - case 5 : - { - ZBUFFv05_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv05_createDCtx() : (ZBUFFv05_DCtx*)*legacyContext; - if (dctx==NULL) return ERROR(memory_allocation); - ZBUFFv05_decompressInitDictionary(dctx, dict, dictSize); - *legacyContext = dctx; - return 0; - } -#endif -#if (ZSTD_LEGACY_SUPPORT <= 6) - case 6 : - { - ZBUFFv06_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv06_createDCtx() : (ZBUFFv06_DCtx*)*legacyContext; - if (dctx==NULL) return ERROR(memory_allocation); - ZBUFFv06_decompressInitDictionary(dctx, dict, dictSize); - *legacyContext = dctx; - return 0; - } -#endif -#if (ZSTD_LEGACY_SUPPORT <= 7) - case 7 : - { - ZBUFFv07_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv07_createDCtx() : (ZBUFFv07_DCtx*)*legacyContext; - if (dctx==NULL) return ERROR(memory_allocation); - ZBUFFv07_decompressInitDictionary(dctx, dict, dictSize); - *legacyContext = dctx; - return 0; - } -#endif - } -} +#ifndef ZSTD_DDICT_H +#define ZSTD_DDICT_H +/*-******************************************************* + * Dependencies + *********************************************************/ +/**** skipping file: ../common/zstd_deps.h ****/ +/**** skipping file: ../zstd.h ****/ -MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version, - ZSTD_outBuffer* output, ZSTD_inBuffer* input) -{ - DEBUGLOG(5, "ZSTD_decompressLegacyStream for v0.%u", version); - switch(version) - { - default : - case 1 : - case 2 : - case 3 : - (void)legacyContext; (void)output; (void)input; - return ERROR(version_unsupported); -#if (ZSTD_LEGACY_SUPPORT <= 4) - case 4 : - { - ZBUFFv04_DCtx* dctx = (ZBUFFv04_DCtx*) legacyContext; - const void* src = (const char*)input->src + input->pos; - size_t readSize = input->size - input->pos; - void* dst = (char*)output->dst + output->pos; - size_t decodedSize = output->size - output->pos; - size_t const hintSize = ZBUFFv04_decompressContinue(dctx, dst, &decodedSize, src, &readSize); - output->pos += decodedSize; - input->pos += readSize; - return hintSize; - } -#endif -#if (ZSTD_LEGACY_SUPPORT <= 5) - case 5 : - { - ZBUFFv05_DCtx* dctx = (ZBUFFv05_DCtx*) legacyContext; - const void* src = (const char*)input->src + input->pos; - size_t readSize = input->size - input->pos; - void* dst = (char*)output->dst + output->pos; - size_t decodedSize = output->size - output->pos; - size_t const hintSize = ZBUFFv05_decompressContinue(dctx, dst, &decodedSize, src, &readSize); - output->pos += decodedSize; - input->pos += readSize; - return hintSize; - } -#endif -#if (ZSTD_LEGACY_SUPPORT <= 6) - case 6 : - { - ZBUFFv06_DCtx* dctx = (ZBUFFv06_DCtx*) legacyContext; - const void* src = (const char*)input->src + input->pos; - size_t readSize = input->size - input->pos; - void* dst = (char*)output->dst + output->pos; - size_t decodedSize = output->size - output->pos; - size_t const hintSize = ZBUFFv06_decompressContinue(dctx, dst, &decodedSize, src, &readSize); - output->pos += decodedSize; - input->pos += readSize; - return hintSize; - } -#endif -#if (ZSTD_LEGACY_SUPPORT <= 7) - case 7 : - { - ZBUFFv07_DCtx* dctx = (ZBUFFv07_DCtx*) legacyContext; - const void* src = (const char*)input->src + input->pos; - size_t readSize = input->size - input->pos; - void* dst = (char*)output->dst + output->pos; - size_t decodedSize = output->size - output->pos; - size_t const hintSize = ZBUFFv07_decompressContinue(dctx, dst, &decodedSize, src, &readSize); - output->pos += decodedSize; - input->pos += readSize; - return hintSize; - } -#endif - } -} +/*-******************************************************* + * Interface + *********************************************************/ + +/* note: several prototypes are already published in `zstd.h` : + * ZSTD_createDDict() + * ZSTD_createDDict_byReference() + * ZSTD_createDDict_advanced() + * ZSTD_freeDDict() + * ZSTD_initStaticDDict() + * ZSTD_sizeof_DDict() + * ZSTD_estimateDDictSize() + * ZSTD_getDictID_fromDict() + */ + +const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict); +size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict); + +void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); -#if defined (__cplusplus) -} -#endif -#endif /* ZSTD_LEGACY_H */ -/**** ended inlining ../legacy/zstd_legacy.h ****/ +#endif /* ZSTD_DDICT_H */ +/**** ended inlining zstd_ddict.h ****/ + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) +#error Using excluded file: ../legacy/zstd_legacy.h (re-amalgamate source to fix) #endif @@ -29243,7 +38434,7 @@ static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, ZSTD_memcpy(internalBuffer, dict, dictSize); } ddict->dictSize = dictSize; - ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ + ddict->entropy.hufTable[0] = (HUF_DTable)((ZSTD_HUFFDTABLE_CAPACITY_LOG)*0x1000001); /* cover both little and big endian */ /* parse dictionary content */ FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) , ""); @@ -29349,12 +38540,12 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict) unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) { if (ddict==NULL) return 0; - return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize); + return ddict->dictID; } /**** ended inlining decompress/zstd_ddict.c ****/ /**** start inlining decompress/zstd_decompress.c ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -29411,20 +38602,20 @@ unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) * Dependencies *********************************************************/ /**** skipping file: ../common/zstd_deps.h ****/ -/**** skipping file: ../common/cpu.h ****/ +/**** skipping file: ../common/allocations.h ****/ +/**** skipping file: ../common/error_private.h ****/ +/**** skipping file: ../common/zstd_internal.h ****/ /**** skipping file: ../common/mem.h ****/ -/**** skipping file: ../common/zstd_trace.h ****/ +/**** skipping file: ../common/bits.h ****/ #define FSE_STATIC_LINKING_ONLY /**** skipping file: ../common/fse.h ****/ -#define HUF_STATIC_LINKING_ONLY /**** skipping file: ../common/huf.h ****/ /**** skipping file: ../common/xxhash.h ****/ -/**** skipping file: ../common/zstd_internal.h ****/ /**** skipping file: zstd_decompress_internal.h ****/ /**** skipping file: zstd_ddict.h ****/ /**** start inlining zstd_decompress_block.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -29458,6 +38649,12 @@ unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) */ + /* Streaming state is used to inform allocation of the literal buffer */ +typedef enum { + not_streaming = 0, + is_streaming = 1 +} streaming_operation; + /* ZSTD_decompressBlock_internal() : * decompress block, starting at `src`, * into destination buffer `dst`. @@ -29466,7 +38663,7 @@ unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) */ size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, - const void* src, size_t srcSize, const int frame); + const void* src, size_t srcSize, const streaming_operation streaming); /* ZSTD_buildFSETable() : * generate FSE decoding table for one symbol (ll, ml or off) @@ -29479,16 +38676,21 @@ size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, */ void ZSTD_buildFSETable(ZSTD_seqSymbol* dt, const short* normalizedCounter, unsigned maxSymbolValue, - const U32* baseValue, const U32* nbAdditionalBits, + const U32* baseValue, const U8* nbAdditionalBits, unsigned tableLog, void* wksp, size_t wkspSize, int bmi2); +/* Internal definition of ZSTD_decompressBlock() to avoid deprecation warnings. */ +size_t ZSTD_decompressBlock_deprecated(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + #endif /* ZSTD_DEC_BLOCK_H */ /**** ended inlining zstd_decompress_block.h ****/ #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) -/**** skipping file: ../legacy/zstd_legacy.h ****/ +#error Using excluded file: ../legacy/zstd_legacy.h (re-amalgamate source to fix) #endif @@ -29498,11 +38700,11 @@ void ZSTD_buildFSETable(ZSTD_seqSymbol* dt, *************************************/ #define DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT 4 -#define DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT 3 /* These two constants represent SIZE_MULT/COUNT_MULT load factor without using a float. - * Currently, that means a 0.75 load factor. - * So, if count * COUNT_MULT / size * SIZE_MULT != 0, then we've exceeded - * the load factor of the ddict hash set. - */ +#define DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT 3 /* These two constants represent SIZE_MULT/COUNT_MULT load factor without using a float. + * Currently, that means a 0.75 load factor. + * So, if count * COUNT_MULT / size * SIZE_MULT != 0, then we've exceeded + * the load factor of the ddict hash set. + */ #define DDICT_HASHSET_TABLE_BASE_SIZE 64 #define DDICT_HASHSET_RESIZE_FACTOR 2 @@ -29596,12 +38798,15 @@ static const ZSTD_DDict* ZSTD_DDictHashSet_getDDict(ZSTD_DDictHashSet* hashSet, static ZSTD_DDictHashSet* ZSTD_createDDictHashSet(ZSTD_customMem customMem) { ZSTD_DDictHashSet* ret = (ZSTD_DDictHashSet*)ZSTD_customMalloc(sizeof(ZSTD_DDictHashSet), customMem); DEBUGLOG(4, "Allocating new hash set"); + if (!ret) + return NULL; ret->ddictPtrTable = (const ZSTD_DDict**)ZSTD_customCalloc(DDICT_HASHSET_TABLE_BASE_SIZE * sizeof(ZSTD_DDict*), customMem); - ret->ddictPtrTableSize = DDICT_HASHSET_TABLE_BASE_SIZE; - ret->ddictPtrCount = 0; - if (!ret || !ret->ddictPtrTable) { + if (!ret->ddictPtrTable) { + ZSTD_customFree(ret, customMem); return NULL; } + ret->ddictPtrTableSize = DDICT_HASHSET_TABLE_BASE_SIZE; + ret->ddictPtrCount = 0; return ret; } @@ -29660,6 +38865,8 @@ static void ZSTD_DCtx_resetParameters(ZSTD_DCtx* dctx) dctx->outBufferMode = ZSTD_bm_buffered; dctx->forceIgnoreChecksum = ZSTD_d_validateChecksum; dctx->refMultipleDDicts = ZSTD_rmd_refSingleDDict; + dctx->disableHufAsm = 0; + dctx->maxBlockSizeParam = 0; } static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx) @@ -29674,11 +38881,16 @@ static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx) dctx->inBuffSize = 0; dctx->outBuffSize = 0; dctx->streamStage = zdss_init; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) dctx->legacyContext = NULL; dctx->previousLegacyVersion = 0; +#endif dctx->noForwardProgress = 0; dctx->oversizedDuration = 0; - dctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); + dctx->isFrameDecompression = 1; +#if DYNAMIC_BMI2 + dctx->bmi2 = ZSTD_cpuSupportsBmi2(); +#endif dctx->ddictSet = NULL; ZSTD_DCtx_resetParameters(dctx); #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION @@ -29699,8 +38911,7 @@ ZSTD_DCtx* ZSTD_initStaticDCtx(void *workspace, size_t workspaceSize) return dctx; } -ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) -{ +static ZSTD_DCtx* ZSTD_createDCtx_internal(ZSTD_customMem customMem) { if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; { ZSTD_DCtx* const dctx = (ZSTD_DCtx*)ZSTD_customMalloc(sizeof(*dctx), customMem); @@ -29711,10 +38922,15 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) } } +ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) +{ + return ZSTD_createDCtx_internal(customMem); +} + ZSTD_DCtx* ZSTD_createDCtx(void) { DEBUGLOG(3, "ZSTD_createDCtx"); - return ZSTD_createDCtx_advanced(ZSTD_defaultCMem); + return ZSTD_createDCtx_internal(ZSTD_defaultCMem); } static void ZSTD_clearDict(ZSTD_DCtx* dctx) @@ -29799,6 +39015,19 @@ unsigned ZSTD_isFrame(const void* buffer, size_t size) return 0; } +/*! ZSTD_isSkippableFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame. + * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. + */ +unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size) +{ + if (size < ZSTD_FRAMEIDSIZE) return 0; + { U32 const magic = MEM_readLE32(buffer); + if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1; + } + return 0; +} + /** ZSTD_frameHeaderSize_internal() : * srcSize must be large enough to reach header size fields. * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless. @@ -29834,16 +39063,40 @@ size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless * @return : 0, `zfhPtr` is correctly filled, * >0, `srcSize` is too small, value is wanted `srcSize` amount, - * or an error code, which can be tested using ZSTD_isError() */ -size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format) +** or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_getFrameHeader_advanced(ZSTD_FrameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format) { const BYTE* ip = (const BYTE*)src; size_t const minInputSize = ZSTD_startingInputLength(format); - ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzer do not understand that zfhPtr is only going to be read only if return value is zero, since they are 2 different signals */ - if (srcSize < minInputSize) return minInputSize; - RETURN_ERROR_IF(src==NULL, GENERIC, "invalid parameter"); + DEBUGLOG(5, "ZSTD_getFrameHeader_advanced: minInputSize = %zu, srcSize = %zu", minInputSize, srcSize); + + if (srcSize > 0) { + /* note : technically could be considered an assert(), since it's an invalid entry */ + RETURN_ERROR_IF(src==NULL, GENERIC, "invalid parameter : src==NULL, but srcSize>0"); + } + if (srcSize < minInputSize) { + if (srcSize > 0 && format != ZSTD_f_zstd1_magicless) { + /* when receiving less than @minInputSize bytes, + * control these bytes at least correspond to a supported magic number + * in order to error out early if they don't. + **/ + size_t const toCopy = MIN(4, srcSize); + unsigned char hbuf[4]; MEM_writeLE32(hbuf, ZSTD_MAGICNUMBER); + assert(src != NULL); + ZSTD_memcpy(hbuf, src, toCopy); + if ( MEM_readLE32(hbuf) != ZSTD_MAGICNUMBER ) { + /* not a zstd frame : let's check if it's a skippable frame */ + MEM_writeLE32(hbuf, ZSTD_MAGIC_SKIPPABLE_START); + ZSTD_memcpy(hbuf, src, toCopy); + if ((MEM_readLE32(hbuf) & ZSTD_MAGIC_SKIPPABLE_MASK) != ZSTD_MAGIC_SKIPPABLE_START) { + RETURN_ERROR(prefix_unknown, + "first bytes don't correspond to any supported magic number"); + } } } + return minInputSize; + } + ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzers may not understand that zfhPtr will be read only if return value is zero, since they are 2 different signals */ if ( (format != ZSTD_f_zstd1_magicless) && (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) { if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { @@ -29851,8 +39104,10 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s if (srcSize < ZSTD_SKIPPABLEHEADERSIZE) return ZSTD_SKIPPABLEHEADERSIZE; /* magic number + frame length */ ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); - zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE); zfhPtr->frameType = ZSTD_skippableFrame; + zfhPtr->dictID = MEM_readLE32(src) - ZSTD_MAGIC_SKIPPABLE_START; + zfhPtr->headerSize = ZSTD_SKIPPABLEHEADERSIZE; + zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE); return 0; } RETURN_ERROR(prefix_unknown, ""); @@ -29885,7 +39140,9 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s } switch(dictIDSizeCode) { - default: assert(0); /* impossible */ + default: + assert(0); /* impossible */ + ZSTD_FALLTHROUGH; case 0 : break; case 1 : dictID = ip[pos]; pos++; break; case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break; @@ -29893,7 +39150,9 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s } switch(fcsID) { - default: assert(0); /* impossible */ + default: + assert(0); /* impossible */ + ZSTD_FALLTHROUGH; case 0 : if (singleSegment) frameContentSize = ip[pos]; break; case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break; case 2 : frameContentSize = MEM_readLE32(ip+pos); break; @@ -29917,12 +39176,11 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s * @return : 0, `zfhPtr` is correctly filled, * >0, `srcSize` is too small, value is wanted `srcSize` amount, * or an error code, which can be tested using ZSTD_isError() */ -size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize) +size_t ZSTD_getFrameHeader(ZSTD_FrameHeader* zfhPtr, const void* src, size_t srcSize) { return ZSTD_getFrameHeader_advanced(zfhPtr, src, srcSize, ZSTD_f_zstd1); } - /** ZSTD_getFrameContentSize() : * compatible with legacy mode * @return : decompressed size of the single frame pointed to be `src` if known, otherwise @@ -29936,7 +39194,7 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize) return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret; } #endif - { ZSTD_frameHeader zfh; + { ZSTD_FrameHeader zfh; if (ZSTD_getFrameHeader(&zfh, src, srcSize) != 0) return ZSTD_CONTENTSIZE_ERROR; if (zfh.frameType == ZSTD_skippableFrame) { @@ -29956,18 +39214,52 @@ static size_t readSkippableFrameSize(void const* src, size_t srcSize) sizeU32 = MEM_readLE32((BYTE const*)src + ZSTD_FRAMEIDSIZE); RETURN_ERROR_IF((U32)(sizeU32 + ZSTD_SKIPPABLEHEADERSIZE) < sizeU32, frameParameter_unsupported, ""); - { - size_t const skippableSize = skippableHeaderSize + sizeU32; + { size_t const skippableSize = skippableHeaderSize + sizeU32; RETURN_ERROR_IF(skippableSize > srcSize, srcSize_wrong, ""); return skippableSize; } } +/*! ZSTD_readSkippableFrame() : + * Retrieves content of a skippable frame, and writes it to dst buffer. + * + * The parameter magicVariant will receive the magicVariant that was supplied when the frame was written, + * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested + * in the magicVariant. + * + * Returns an error if destination buffer is not large enough, or if this is not a valid skippable frame. + * + * @return : number of bytes written or a ZSTD error. + */ +size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, + unsigned* magicVariant, /* optional, can be NULL */ + const void* src, size_t srcSize) +{ + RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong, ""); + + { U32 const magicNumber = MEM_readLE32(src); + size_t skippableFrameSize = readSkippableFrameSize(src, srcSize); + size_t skippableContentSize = skippableFrameSize - ZSTD_SKIPPABLEHEADERSIZE; + + /* check input validity */ + RETURN_ERROR_IF(!ZSTD_isSkippableFrame(src, srcSize), frameParameter_unsupported, ""); + RETURN_ERROR_IF(skippableFrameSize < ZSTD_SKIPPABLEHEADERSIZE || skippableFrameSize > srcSize, srcSize_wrong, ""); + RETURN_ERROR_IF(skippableContentSize > dstCapacity, dstSize_tooSmall, ""); + + /* deliver payload */ + if (skippableContentSize > 0 && dst != NULL) + ZSTD_memcpy(dst, (const BYTE *)src + ZSTD_SKIPPABLEHEADERSIZE, skippableContentSize); + if (magicVariant != NULL) + *magicVariant = magicNumber - ZSTD_MAGIC_SKIPPABLE_START; + return skippableContentSize; + } +} + /** ZSTD_findDecompressedSize() : - * compatible with legacy mode * `srcSize` must be the exact length of some number of ZSTD compressed and/or * skippable frames - * @return : decompressed size of the frames contained */ + * note: compatible with legacy mode + * @return : decompressed size of the frames contained */ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) { unsigned long long totalDstSize = 0; @@ -29977,9 +39269,7 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { size_t const skippableSize = readSkippableFrameSize(src, srcSize); - if (ZSTD_isError(skippableSize)) { - return ZSTD_CONTENTSIZE_ERROR; - } + if (ZSTD_isError(skippableSize)) return ZSTD_CONTENTSIZE_ERROR; assert(skippableSize <= srcSize); src = (const BYTE *)src + skippableSize; @@ -29987,17 +39277,17 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) continue; } - { unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize); - if (ret >= ZSTD_CONTENTSIZE_ERROR) return ret; + { unsigned long long const fcs = ZSTD_getFrameContentSize(src, srcSize); + if (fcs >= ZSTD_CONTENTSIZE_ERROR) return fcs; - /* check for overflow */ - if (totalDstSize + ret < totalDstSize) return ZSTD_CONTENTSIZE_ERROR; - totalDstSize += ret; + if (totalDstSize + fcs < totalDstSize) + return ZSTD_CONTENTSIZE_ERROR; /* check for overflow */ + totalDstSize += fcs; } + /* skip to next frame */ { size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize); - if (ZSTD_isError(frameSrcSize)) { - return ZSTD_CONTENTSIZE_ERROR; - } + if (ZSTD_isError(frameSrcSize)) return ZSTD_CONTENTSIZE_ERROR; + assert(frameSrcSize <= srcSize); src = (const BYTE *)src + frameSrcSize; srcSize -= frameSrcSize; @@ -30061,17 +39351,17 @@ static ZSTD_frameSizeInfo ZSTD_errorFrameSizeInfo(size_t ret) return frameSizeInfo; } -static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize) +static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize, ZSTD_format_e format) { ZSTD_frameSizeInfo frameSizeInfo; ZSTD_memset(&frameSizeInfo, 0, sizeof(ZSTD_frameSizeInfo)); #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) - if (ZSTD_isLegacy(src, srcSize)) + if (format == ZSTD_f_zstd1 && ZSTD_isLegacy(src, srcSize)) return ZSTD_findFrameSizeInfoLegacy(src, srcSize); #endif - if ((srcSize >= ZSTD_SKIPPABLEHEADERSIZE) + if (format == ZSTD_f_zstd1 && (srcSize >= ZSTD_SKIPPABLEHEADERSIZE) && (MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { frameSizeInfo.compressedSize = readSkippableFrameSize(src, srcSize); assert(ZSTD_isError(frameSizeInfo.compressedSize) || @@ -30082,10 +39372,10 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize const BYTE* const ipstart = ip; size_t remainingSize = srcSize; size_t nbBlocks = 0; - ZSTD_frameHeader zfh; + ZSTD_FrameHeader zfh; /* Extract Frame Header */ - { size_t const ret = ZSTD_getFrameHeader(&zfh, src, srcSize); + { size_t const ret = ZSTD_getFrameHeader_advanced(&zfh, src, srcSize, format); if (ZSTD_isError(ret)) return ZSTD_errorFrameSizeInfo(ret); if (ret > 0) @@ -30119,28 +39409,31 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize ip += 4; } + frameSizeInfo.nbBlocks = nbBlocks; frameSizeInfo.compressedSize = (size_t)(ip - ipstart); frameSizeInfo.decompressedBound = (zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) ? zfh.frameContentSize - : nbBlocks * zfh.blockSizeMax; + : (unsigned long long)nbBlocks * zfh.blockSizeMax; return frameSizeInfo; } } +static size_t ZSTD_findFrameCompressedSize_advanced(const void *src, size_t srcSize, ZSTD_format_e format) { + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize, format); + return frameSizeInfo.compressedSize; +} + /** ZSTD_findFrameCompressedSize() : - * compatible with legacy mode - * `src` must point to the start of a ZSTD frame, ZSTD legacy frame, or skippable frame - * `srcSize` must be at least as large as the frame contained - * @return : the compressed size of the frame starting at `src` */ + * See docs in zstd.h + * Note: compatible with legacy mode */ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) { - ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize); - return frameSizeInfo.compressedSize; + return ZSTD_findFrameCompressedSize_advanced(src, srcSize, ZSTD_f_zstd1); } /** ZSTD_decompressBound() : * compatible with legacy mode - * `src` must point to the start of a ZSTD frame or a skippeable frame + * `src` must point to the start of a ZSTD frame or a skippable frame * `srcSize` must be at least as large as the frame contained * @return : the maximum decompressed size of the compressed source */ @@ -30149,7 +39442,7 @@ unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize) unsigned long long bound = 0; /* Iterate over each frame */ while (srcSize > 0) { - ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize); + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize, ZSTD_f_zstd1); size_t const compressedSize = frameSizeInfo.compressedSize; unsigned long long const decompressedBound = frameSizeInfo.decompressedBound; if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR) @@ -30162,6 +39455,48 @@ unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize) return bound; } +size_t ZSTD_decompressionMargin(void const* src, size_t srcSize) +{ + size_t margin = 0; + unsigned maxBlockSize = 0; + + /* Iterate over each frame */ + while (srcSize > 0) { + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize, ZSTD_f_zstd1); + size_t const compressedSize = frameSizeInfo.compressedSize; + unsigned long long const decompressedBound = frameSizeInfo.decompressedBound; + ZSTD_FrameHeader zfh; + + FORWARD_IF_ERROR(ZSTD_getFrameHeader(&zfh, src, srcSize), ""); + if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR) + return ERROR(corruption_detected); + + if (zfh.frameType == ZSTD_frame) { + /* Add the frame header to our margin */ + margin += zfh.headerSize; + /* Add the checksum to our margin */ + margin += zfh.checksumFlag ? 4 : 0; + /* Add 3 bytes per block */ + margin += 3 * frameSizeInfo.nbBlocks; + + /* Compute the max block size */ + maxBlockSize = MAX(maxBlockSize, zfh.blockSizeMax); + } else { + assert(zfh.frameType == ZSTD_skippableFrame); + /* Add the entire skippable frame size to our margin. */ + margin += compressedSize; + } + + assert(srcSize >= compressedSize); + src = (const BYTE*)src + compressedSize; + srcSize -= compressedSize; + } + + /* Add the max block size back to the margin. */ + margin += maxBlockSize; + + return margin; +} /*-************************************************************* * Frame decoding @@ -30187,7 +39522,7 @@ static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity, if (srcSize == 0) return 0; RETURN_ERROR(dstBuffer_null, ""); } - ZSTD_memcpy(dst, src, srcSize); + ZSTD_memmove(dst, src, srcSize); return srcSize; } @@ -30204,10 +39539,10 @@ static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity, return regenSize; } -static void ZSTD_DCtx_trace_end(ZSTD_DCtx const* dctx, U64 uncompressedSize, U64 compressedSize, unsigned streaming) +static void ZSTD_DCtx_trace_end(ZSTD_DCtx const* dctx, U64 uncompressedSize, U64 compressedSize, int streaming) { #if ZSTD_TRACE - if (dctx->traceCtx) { + if (dctx->traceCtx && ZSTD_trace_decompress_end != NULL) { ZSTD_Trace trace; ZSTD_memset(&trace, 0, sizeof(trace)); trace.version = ZSTD_VERSION_NUMBER; @@ -30263,10 +39598,16 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, ip += frameHeaderSize; remainingSrcSize -= frameHeaderSize; } + /* Shrink the blockSizeMax if enabled */ + if (dctx->maxBlockSizeParam != 0) + dctx->fParams.blockSizeMax = MIN(dctx->fParams.blockSizeMax, (unsigned)dctx->maxBlockSizeParam); + /* Loop on each block */ while (1) { + BYTE* oBlockEnd = oend; size_t decodedSize; blockProperties_t blockProperties; + memset(&blockProperties, 0, sizeof(blockProperties)); // shut up gcc warning size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSrcSize, &blockProperties); if (ZSTD_isError(cBlockSize)) return cBlockSize; @@ -30274,27 +39615,48 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, remainingSrcSize -= ZSTD_blockHeaderSize; RETURN_ERROR_IF(cBlockSize > remainingSrcSize, srcSize_wrong, ""); + if (ip >= op && ip < oBlockEnd) { + /* We are decompressing in-place. Limit the output pointer so that we + * don't overwrite the block that we are currently reading. This will + * fail decompression if the input & output pointers aren't spaced + * far enough apart. + * + * This is important to set, even when the pointers are far enough + * apart, because ZSTD_decompressBlock_internal() can decide to store + * literals in the output buffer, after the block it is decompressing. + * Since we don't want anything to overwrite our input, we have to tell + * ZSTD_decompressBlock_internal to never write past ip. + * + * See ZSTD_allocateLiteralsBuffer() for reference. + */ + oBlockEnd = op + (ip - op); + } + switch(blockProperties.blockType) { case bt_compressed: - decodedSize = ZSTD_decompressBlock_internal(dctx, op, (size_t)(oend-op), ip, cBlockSize, /* frame */ 1); + assert(dctx->isFrameDecompression == 1); + decodedSize = ZSTD_decompressBlock_internal(dctx, op, (size_t)(oBlockEnd-op), ip, cBlockSize, not_streaming); break; case bt_raw : + /* Use oend instead of oBlockEnd because this function is safe to overlap. It uses memmove. */ decodedSize = ZSTD_copyRawBlock(op, (size_t)(oend-op), ip, cBlockSize); break; case bt_rle : - decodedSize = ZSTD_setRleBlock(op, (size_t)(oend-op), *ip, blockProperties.origSize); + decodedSize = ZSTD_setRleBlock(op, (size_t)(oBlockEnd-op), *ip, blockProperties.origSize); break; case bt_reserved : default: RETURN_ERROR(corruption_detected, "invalid block type"); } - - if (ZSTD_isError(decodedSize)) return decodedSize; - if (dctx->validateChecksum) + FORWARD_IF_ERROR(decodedSize, "Block decompression failure"); + DEBUGLOG(5, "Decompressed block of dSize = %u", (unsigned)decodedSize); + if (dctx->validateChecksum) { XXH64_update(&dctx->xxhState, op, decodedSize); - if (decodedSize != 0) + } + if (decodedSize) /* support dst = NULL,0 */ { op += decodedSize; + } assert(ip != NULL); ip += cBlockSize; remainingSrcSize -= cBlockSize; @@ -30318,12 +39680,15 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, } ZSTD_DCtx_trace_end(dctx, (U64)(op-ostart), (U64)(ip-istart), /* streaming */ 0); /* Allow caller to get size read */ + DEBUGLOG(4, "ZSTD_decompressFrame: decompressed frame of size %i, consuming %i bytes of input", (int)(op-ostart), (int)(ip - (const BYTE*)*srcPtr)); *srcPtr = ip; *srcSizePtr = remainingSrcSize; return (size_t)(op-ostart); } -static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict, size_t dictSize, @@ -30343,7 +39708,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, while (srcSize >= ZSTD_startingInputLength(dctx->format)) { #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) - if (ZSTD_isLegacy(src, srcSize)) { + if (dctx->format == ZSTD_f_zstd1 && ZSTD_isLegacy(src, srcSize)) { size_t decodedSize; size_t const frameSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize); if (ZSTD_isError(frameSize)) return frameSize; @@ -30353,6 +39718,15 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, decodedSize = ZSTD_decompressLegacy(dst, dstCapacity, src, frameSize, dict, dictSize); if (ZSTD_isError(decodedSize)) return decodedSize; + { + unsigned long long const expectedSize = ZSTD_getFrameContentSize(src, srcSize); + RETURN_ERROR_IF(expectedSize == ZSTD_CONTENTSIZE_ERROR, corruption_detected, "Corrupted frame header!"); + if (expectedSize != ZSTD_CONTENTSIZE_UNKNOWN) { + RETURN_ERROR_IF(expectedSize != decodedSize, corruption_detected, + "Frame header size does not match decoded size!"); + } + } + assert(decodedSize <= dstCapacity); dst = (BYTE*)dst + decodedSize; dstCapacity -= decodedSize; @@ -30364,17 +39738,18 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, } #endif - { U32 const magicNumber = MEM_readLE32(src); - DEBUGLOG(4, "reading magic number %08X (expecting %08X)", - (unsigned)magicNumber, ZSTD_MAGICNUMBER); + if (dctx->format == ZSTD_f_zstd1 && srcSize >= 4) { + U32 const magicNumber = MEM_readLE32(src); + DEBUGLOG(5, "reading magic number %08X", (unsigned)magicNumber); if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + /* skippable frame detected : skip it */ size_t const skippableSize = readSkippableFrameSize(src, srcSize); - FORWARD_IF_ERROR(skippableSize, "readSkippableFrameSize failed"); + FORWARD_IF_ERROR(skippableSize, "invalid skippable frame"); assert(skippableSize <= srcSize); src = (const BYTE *)src + skippableSize; srcSize -= skippableSize; - continue; + continue; /* check next frame */ } } if (ddict) { @@ -30428,7 +39803,7 @@ static ZSTD_DDict const* ZSTD_getDDict(ZSTD_DCtx* dctx) switch (dctx->dictUses) { default: assert(0 /* Impossible */); - /* fall-through */ + ZSTD_FALLTHROUGH; case ZSTD_dont_use: ZSTD_clearDict(dctx); return NULL; @@ -30450,7 +39825,7 @@ size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t sr { #if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE>=1) size_t regenSize; - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx_internal(ZSTD_defaultCMem); RETURN_ERROR_IF(dctx==NULL, memory_allocation, "NULL pointer!"); regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize); ZSTD_freeDCtx(dctx); @@ -30470,8 +39845,8 @@ size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t sr size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) { return dctx->expected; } /** - * Similar to ZSTD_nextSrcSizeToDecompress(), but when when a block input can be streamed, - * we allow taking a partial block as the input. Currently only raw uncompressed blocks can + * Similar to ZSTD_nextSrcSizeToDecompress(), but when a block input can be streamed, we + * allow taking a partial block as the input. Currently only raw uncompressed blocks can * be streamed. * * For blocks that can be streamed, this allows us to reduce the latency until we produce @@ -30484,7 +39859,7 @@ static size_t ZSTD_nextSrcSizeToDecompressWithInputSize(ZSTD_DCtx* dctx, size_t return dctx->expected; if (dctx->bType != bt_raw) return dctx->expected; - return MIN(MAX(inputSize, 1), dctx->expected); + return BOUNDED(1, inputSize, dctx->expected); } ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) { @@ -30492,7 +39867,9 @@ ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) { { default: /* should not happen */ assert(0); + ZSTD_FALLTHROUGH; case ZSTDds_getFrameHeaderSize: + ZSTD_FALLTHROUGH; case ZSTDds_decodeFrameHeader: return ZSTDnit_frameHeader; case ZSTDds_decodeBlockHeader: @@ -30504,6 +39881,7 @@ ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) { case ZSTDds_checkChecksum: return ZSTDnit_checksum; case ZSTDds_decodeSkippableHeader: + ZSTD_FALLTHROUGH; case ZSTDds_skipFrame: return ZSTDnit_skippableFrame; } @@ -30587,7 +39965,8 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c { case bt_compressed: DEBUGLOG(5, "ZSTD_decompressContinue: case bt_compressed"); - rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 1); + assert(dctx->isFrameDecompression == 1); + rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, is_streaming); dctx->expected = 0; /* Streaming not supported */ break; case bt_raw : @@ -30656,6 +40035,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c case ZSTDds_decodeSkippableHeader: assert(src != NULL); assert(srcSize <= ZSTD_SKIPPABLEHEADERSIZE); + assert(dctx->format != ZSTD_f_zstd1_magicless); ZSTD_memcpy(dctx->headerBuffer + (ZSTD_SKIPPABLEHEADERSIZE - srcSize), src, srcSize); /* complete skippable header */ dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_FRAMEIDSIZE); /* note : dctx->expected can grow seriously large, beyond local buffer size */ dctx->stage = ZSTDds_skipFrame; @@ -30668,7 +40048,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c default: assert(0); /* impossible */ - RETURN_ERROR(GENERIC, "impossible to reach"); /* some compiler require default to do something */ + RETURN_ERROR(GENERIC, "impossible to reach"); /* some compilers require default to do something */ } } @@ -30709,11 +40089,11 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, /* in minimal huffman, we always use X1 variants */ size_t const hSize = HUF_readDTableX1_wksp(entropy->hufTable, dictPtr, dictEnd - dictPtr, - workspace, workspaceSize); + workspace, workspaceSize, /* flags */ 0); #else size_t const hSize = HUF_readDTableX2_wksp(entropy->hufTable, dictPtr, (size_t)(dictEnd - dictPtr), - workspace, workspaceSize); + workspace, workspaceSize, /* flags */ 0); #endif RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted, ""); dictPtr += hSize; @@ -30802,7 +40182,7 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) { assert(dctx != NULL); #if ZSTD_TRACE - dctx->traceCtx = ZSTD_trace_decompress_begin(dctx); + dctx->traceCtx = (ZSTD_trace_decompress_begin != NULL) ? ZSTD_trace_decompress_begin(dctx) : 0; #endif dctx->expected = ZSTD_startingInputLength(dctx->format); /* dctx->format must be properly set */ dctx->stage = ZSTDds_getFrameHeaderSize; @@ -30812,10 +40192,11 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) dctx->prefixStart = NULL; dctx->virtualStart = NULL; dctx->dictEnd = NULL; - dctx->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ + dctx->entropy.hufTable[0] = (HUF_DTable)((ZSTD_HUFFDTABLE_CAPACITY_LOG)*0x1000001); /* cover both little and big endian */ dctx->litEntropy = dctx->fseEntropy = 0; dctx->dictID = 0; dctx->bType = bt_reserved; + dctx->isFrameDecompression = 1; ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue)); ZSTD_memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */ dctx->LLTptr = dctx->entropy.LLTable; @@ -30874,7 +40255,7 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize) * This could for one of the following reasons : * - The frame does not require a dictionary (most common case). * - The frame was built with dictID intentionally removed. - * Needed dictionary is a hidden information. + * Needed dictionary is a hidden piece of information. * Note : this use case also happens when using a non-conformant dictionary. * - `srcSize` is too small, and as a result, frame header could not be decoded. * Note : possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`. @@ -30883,7 +40264,7 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize) * ZSTD_getFrameHeader(), which will provide a more precise error code. */ unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize) { - ZSTD_frameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0 }; + ZSTD_FrameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0, 0, 0 }; size_t const hError = ZSTD_getFrameHeader(&zfp, src, srcSize); if (ZSTD_isError(hError)) return 0; return zfp.dictID; @@ -30912,7 +40293,7 @@ size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, ZSTD_DStream* ZSTD_createDStream(void) { DEBUGLOG(3, "ZSTD_createDStream"); - return ZSTD_createDStream_advanced(ZSTD_defaultCMem); + return ZSTD_createDCtx_internal(ZSTD_defaultCMem); } ZSTD_DStream* ZSTD_initStaticDStream(void *workspace, size_t workspaceSize) @@ -30922,7 +40303,7 @@ ZSTD_DStream* ZSTD_initStaticDStream(void *workspace, size_t workspaceSize) ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem) { - return ZSTD_createDCtx_advanced(customMem); + return ZSTD_createDCtx_internal(customMem); } size_t ZSTD_freeDStream(ZSTD_DStream* zds) @@ -30990,7 +40371,9 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di size_t ZSTD_initDStream(ZSTD_DStream* zds) { DEBUGLOG(4, "ZSTD_initDStream"); - return ZSTD_initDStream_usingDDict(zds, NULL); + FORWARD_IF_ERROR(ZSTD_DCtx_reset(zds, ZSTD_reset_session_only), ""); + FORWARD_IF_ERROR(ZSTD_DCtx_refDDict(zds, NULL), ""); + return ZSTD_startingInputLength(zds->format); } /* ZSTD_initDStream_usingDDict() : @@ -30998,6 +40381,7 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds) * this function cannot fail */ size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict) { + DEBUGLOG(4, "ZSTD_initDStream_usingDDict"); FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) , ""); FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) , ""); return ZSTD_startingInputLength(dctx->format); @@ -31008,6 +40392,7 @@ size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict) * this function cannot fail */ size_t ZSTD_resetDStream(ZSTD_DStream* dctx) { + DEBUGLOG(4, "ZSTD_resetDStream"); FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only), ""); return ZSTD_startingInputLength(dctx->format); } @@ -31079,6 +40464,15 @@ ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam) bounds.lowerBound = (int)ZSTD_rmd_refSingleDDict; bounds.upperBound = (int)ZSTD_rmd_refMultipleDDicts; return bounds; + case ZSTD_d_disableHuffmanAssembly: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + case ZSTD_d_maxBlockSize: + bounds.lowerBound = ZSTD_BLOCKSIZE_MAX_MIN; + bounds.upperBound = ZSTD_BLOCKSIZE_MAX; + return bounds; + default:; } bounds.error = ERROR(parameter_unsupported); @@ -31119,6 +40513,12 @@ size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value case ZSTD_d_refMultipleDDicts: *value = (int)dctx->refMultipleDDicts; return 0; + case ZSTD_d_disableHuffmanAssembly: + *value = (int)dctx->disableHufAsm; + return 0; + case ZSTD_d_maxBlockSize: + *value = dctx->maxBlockSizeParam; + return 0; default:; } RETURN_ERROR(parameter_unsupported, ""); @@ -31152,6 +40552,14 @@ size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value } dctx->refMultipleDDicts = (ZSTD_refMultipleDDicts_e)value; return 0; + case ZSTD_d_disableHuffmanAssembly: + CHECK_DBOUNDS(ZSTD_d_disableHuffmanAssembly, value); + dctx->disableHufAsm = value != 0; + return 0; + case ZSTD_d_maxBlockSize: + if (value != 0) CHECK_DBOUNDS(ZSTD_d_maxBlockSize, value); + dctx->maxBlockSizeParam = value; + return 0; default:; } RETURN_ERROR(parameter_unsupported, ""); @@ -31163,6 +40571,7 @@ size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset) || (reset == ZSTD_reset_session_and_parameters) ) { dctx->streamStage = zdss_init; dctx->noForwardProgress = 0; + dctx->isFrameDecompression = 1; } if ( (reset == ZSTD_reset_parameters) || (reset == ZSTD_reset_session_and_parameters) ) { @@ -31179,10 +40588,17 @@ size_t ZSTD_sizeof_DStream(const ZSTD_DStream* dctx) return ZSTD_sizeof_DCtx(dctx); } -size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize) +static size_t ZSTD_decodingBufferSize_internal(unsigned long long windowSize, unsigned long long frameContentSize, size_t blockSizeMax) { - size_t const blockSize = (size_t) MIN(windowSize, ZSTD_BLOCKSIZE_MAX); - unsigned long long const neededRBSize = windowSize + blockSize + (WILDCOPY_OVERLENGTH * 2); + size_t const blockSize = MIN((size_t)MIN(windowSize, ZSTD_BLOCKSIZE_MAX), blockSizeMax); + /* We need blockSize + WILDCOPY_OVERLENGTH worth of buffer so that if a block + * ends at windowSize + WILDCOPY_OVERLENGTH + 1 bytes, we can start writing + * the block at the beginning of the output buffer, and maintain a full window. + * + * We need another blockSize worth of buffer so that we can store split + * literals at the end of the block without overwriting the extDict window. + */ + unsigned long long const neededRBSize = windowSize + (blockSize * 2) + (WILDCOPY_OVERLENGTH * 2); unsigned long long const neededSize = MIN(frameContentSize, neededRBSize); size_t const minRBSize = (size_t) neededSize; RETURN_ERROR_IF((unsigned long long)minRBSize != neededSize, @@ -31190,6 +40606,11 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long return minRBSize; } +size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize) +{ + return ZSTD_decodingBufferSize_internal(windowSize, frameContentSize, ZSTD_BLOCKSIZE_MAX); +} + size_t ZSTD_estimateDStreamSize(size_t windowSize) { size_t const blockSize = MIN(windowSize, ZSTD_BLOCKSIZE_MAX); @@ -31201,7 +40622,7 @@ size_t ZSTD_estimateDStreamSize(size_t windowSize) size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize) { U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; /* note : should be user-selectable, but requires an additional parameter (or a dctx) */ - ZSTD_frameHeader zfh; + ZSTD_FrameHeader zfh; size_t const err = ZSTD_getFrameHeader(&zfh, src, srcSize); if (ZSTD_isError(err)) return err; RETURN_ERROR_IF(err>0, srcSize_wrong, ""); @@ -31296,6 +40717,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB U32 someMoreWork = 1; DEBUGLOG(5, "ZSTD_decompressStream"); + assert(zds != NULL); RETURN_ERROR_IF( input->pos > input->size, srcSize_wrong, @@ -31316,10 +40738,12 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB DEBUGLOG(5, "stage zdss_init => transparent reset "); zds->streamStage = zdss_loadHeader; zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) zds->legacyVersion = 0; +#endif zds->hostageByte = 0; zds->expectedOutBuffer = *output; - /* fall-through */ + ZSTD_FALLTHROUGH; case zdss_loadHeader : DEBUGLOG(5, "stage zdss_loadHeader (srcSize : %u)", (U32)(iend - ip)); @@ -31336,7 +40760,6 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB if (zds->refMultipleDDicts && zds->ddictSet) { ZSTD_DCtx_selectFrameDDict(zds); } - DEBUGLOG(5, "header size : %u", (U32)hSize); if (ZSTD_isError(hSize)) { #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart); @@ -31368,6 +40791,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB zds->lhSize += remainingInput; } input->pos = input->size; + /* check first few bytes */ + FORWARD_IF_ERROR( + ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format), + "First few bytes detected incorrect" ); + /* return hint input size */ return (MAX((size_t)ZSTD_FRAMEHEADERSIZE_MIN(zds->format), hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ } assert(ip != NULL); @@ -31379,14 +40807,15 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB if (zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN && zds->fParams.frameType != ZSTD_skippableFrame && (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) { - size_t const cSize = ZSTD_findFrameCompressedSize(istart, (size_t)(iend-istart)); + size_t const cSize = ZSTD_findFrameCompressedSize_advanced(istart, (size_t)(iend-istart), zds->format); if (cSize <= (size_t)(iend-istart)) { /* shortcut : using single-pass mode */ size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, (size_t)(oend-op), istart, cSize, ZSTD_getDDict(zds)); if (ZSTD_isError(decompressedSize)) return decompressedSize; - DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()") + DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()"); + assert(istart != NULL); ip = istart + cSize; - op += decompressedSize; + op = op ? op + decompressedSize : op; /* can occur if frameContentSize = 0 (empty frame) */ zds->expected = 0; zds->streamStage = zdss_init; someMoreWork = 0; @@ -31405,7 +40834,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB DEBUGLOG(4, "Consume header"); FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(zds, ZSTD_getDDict(zds)), ""); - if ((MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ + if (zds->format == ZSTD_f_zstd1 + && (MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE); zds->stage = ZSTDds_skipFrame; } else { @@ -31421,11 +40851,13 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); RETURN_ERROR_IF(zds->fParams.windowSize > zds->maxWindowSize, frameParameter_windowTooLarge, ""); + if (zds->maxBlockSizeParam != 0) + zds->fParams.blockSizeMax = MIN(zds->fParams.blockSizeMax, (unsigned)zds->maxBlockSizeParam); /* Adapt buffer sizes to frame header instructions */ { size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */); size_t const neededOutBuffSize = zds->outBufferMode == ZSTD_bm_buffered - ? ZSTD_decodingBufferSize_min(zds->fParams.windowSize, zds->fParams.frameContentSize) + ? ZSTD_decodingBufferSize_internal(zds->fParams.windowSize, zds->fParams.frameContentSize, zds->fParams.blockSizeMax) : 0; ZSTD_DCtx_updateOversizedDuration(zds, neededInBuffSize, neededOutBuffSize); @@ -31457,7 +40889,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB zds->outBuffSize = neededOutBuffSize; } } } zds->streamStage = zdss_read; - /* fall-through */ + ZSTD_FALLTHROUGH; case zdss_read: DEBUGLOG(5, "stage zdss_read"); @@ -31470,13 +40902,14 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB } if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, ip, neededInSize), ""); + assert(ip != NULL); ip += neededInSize; /* Function modifies the stage so we must break */ break; } } if (ip==iend) { someMoreWork = 0; break; } /* no more input */ zds->streamStage = zdss_load; - /* fall-through */ + ZSTD_FALLTHROUGH; case zdss_load: { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds); @@ -31484,7 +40917,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB int const isSkipFrame = ZSTD_isSkipFrame(zds); size_t loadedSize; /* At this point we shouldn't be decompressing a block that we can stream. */ - assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, iend - ip)); + assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, (size_t)(iend - ip))); if (isSkipFrame) { loadedSize = MIN(toLoad, (size_t)(iend-ip)); } else { @@ -31493,8 +40926,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB "should never happen"); loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, (size_t)(iend-ip)); } - ip += loadedSize; - zds->inPos += loadedSize; + if (loadedSize != 0) { + /* ip may be NULL */ + ip += loadedSize; + zds->inPos += loadedSize; + } if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */ /* decode loaded input */ @@ -31504,14 +40940,17 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB break; } case zdss_flush: - { size_t const toFlushSize = zds->outEnd - zds->outStart; + { + size_t const toFlushSize = zds->outEnd - zds->outStart; size_t const flushedSize = ZSTD_limitCopy(op, (size_t)(oend-op), zds->outBuff + zds->outStart, toFlushSize); - op += flushedSize; + + op = op ? op + flushedSize : op; + zds->outStart += flushedSize; if (flushedSize == toFlushSize) { /* flush completed */ zds->streamStage = zdss_read; if ( (zds->outBuffSize < zds->fParams.frameContentSize) - && (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) { + && (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) { DEBUGLOG(5, "restart filling outBuff from beginning (left:%i, needed:%u)", (int)(zds->outBuffSize - zds->outStart), (U32)zds->fParams.blockSizeMax); @@ -31525,7 +40964,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB default: assert(0); /* impossible */ - RETURN_ERROR(GENERIC, "impossible to reach"); /* some compiler require default to do something */ + RETURN_ERROR(GENERIC, "impossible to reach"); /* some compilers require default to do something */ } } /* result */ @@ -31538,8 +40977,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB if ((ip==istart) && (op==ostart)) { /* no forward progress */ zds->noForwardProgress ++; if (zds->noForwardProgress >= ZSTD_NO_FORWARD_PROGRESS_MAX) { - RETURN_ERROR_IF(op==oend, dstSize_tooSmall, ""); - RETURN_ERROR_IF(ip==iend, srcSize_wrong, ""); + RETURN_ERROR_IF(op==oend, noForwardProgress_destFull, ""); + RETURN_ERROR_IF(ip==iend, noForwardProgress_inputEmpty, ""); assert(0); } } else { @@ -31576,18 +41015,24 @@ size_t ZSTD_decompressStream_simpleArgs ( void* dst, size_t dstCapacity, size_t* dstPos, const void* src, size_t srcSize, size_t* srcPos) { - ZSTD_outBuffer output = { dst, dstCapacity, *dstPos }; - ZSTD_inBuffer input = { src, srcSize, *srcPos }; - /* ZSTD_compress_generic() will check validity of dstPos and srcPos */ - size_t const cErr = ZSTD_decompressStream(dctx, &output, &input); - *dstPos = output.pos; - *srcPos = input.pos; - return cErr; + ZSTD_outBuffer output; + ZSTD_inBuffer input; + output.dst = dst; + output.size = dstCapacity; + output.pos = *dstPos; + input.src = src; + input.size = srcSize; + input.pos = *srcPos; + { size_t const cErr = ZSTD_decompressStream(dctx, &output, &input); + *dstPos = output.pos; + *srcPos = input.pos; + return cErr; + } } /**** ended inlining decompress/zstd_decompress.c ****/ /**** start inlining decompress/zstd_decompress_block.c ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -31608,12 +41053,12 @@ size_t ZSTD_decompressStream_simpleArgs ( /**** skipping file: ../common/mem.h ****/ #define FSE_STATIC_LINKING_ONLY /**** skipping file: ../common/fse.h ****/ -#define HUF_STATIC_LINKING_ONLY /**** skipping file: ../common/huf.h ****/ /**** skipping file: ../common/zstd_internal.h ****/ /**** skipping file: zstd_decompress_internal.h ****/ /**** skipping file: zstd_ddict.h ****/ /**** skipping file: zstd_decompress_block.h ****/ +/**** skipping file: ../common/bits.h ****/ /*_******************************************************* * Macros @@ -31639,6 +41084,13 @@ static void ZSTD_copy4(void* dst, const void* src) { ZSTD_memcpy(dst, src, 4); } * Block decoding ***************************************************************/ +static size_t ZSTD_blockSizeMax(ZSTD_DCtx const* dctx) +{ + size_t const blockSizeMax = dctx->isFrameDecompression ? dctx->fParams.blockSizeMax : ZSTD_BLOCKSIZE_MAX; + assert(blockSizeMax <= ZSTD_BLOCKSIZE_MAX); + return blockSizeMax; +} + /*! ZSTD_getcBlockSize() : * Provides the size of compressed block from block header `src` */ size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, @@ -31657,36 +41109,90 @@ size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, } } +/* Allocate buffer for literals, either overlapping current dst, or split between dst and litExtraBuffer, or stored entirely within litExtraBuffer */ +static void ZSTD_allocateLiteralsBuffer(ZSTD_DCtx* dctx, void* const dst, const size_t dstCapacity, const size_t litSize, + const streaming_operation streaming, const size_t expectedWriteSize, const unsigned splitImmediately) +{ + size_t const blockSizeMax = ZSTD_blockSizeMax(dctx); + assert(litSize <= blockSizeMax); + assert(dctx->isFrameDecompression || streaming == not_streaming); + assert(expectedWriteSize <= blockSizeMax); + if (streaming == not_streaming && dstCapacity > blockSizeMax + WILDCOPY_OVERLENGTH + litSize + WILDCOPY_OVERLENGTH) { + /* If we aren't streaming, we can just put the literals after the output + * of the current block. We don't need to worry about overwriting the + * extDict of our window, because it doesn't exist. + * So if we have space after the end of the block, just put it there. + */ + dctx->litBuffer = (BYTE*)dst + blockSizeMax + WILDCOPY_OVERLENGTH; + dctx->litBufferEnd = dctx->litBuffer + litSize; + dctx->litBufferLocation = ZSTD_in_dst; + } else if (litSize <= ZSTD_LITBUFFEREXTRASIZE) { + /* Literals fit entirely within the extra buffer, put them there to avoid + * having to split the literals. + */ + dctx->litBuffer = dctx->litExtraBuffer; + dctx->litBufferEnd = dctx->litBuffer + litSize; + dctx->litBufferLocation = ZSTD_not_in_dst; + } else { + assert(blockSizeMax > ZSTD_LITBUFFEREXTRASIZE); + /* Literals must be split between the output block and the extra lit + * buffer. We fill the extra lit buffer with the tail of the literals, + * and put the rest of the literals at the end of the block, with + * WILDCOPY_OVERLENGTH of buffer room to allow for overreads. + * This MUST not write more than our maxBlockSize beyond dst, because in + * streaming mode, that could overwrite part of our extDict window. + */ + if (splitImmediately) { + /* won't fit in litExtraBuffer, so it will be split between end of dst and extra buffer */ + dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH; + dctx->litBufferEnd = dctx->litBuffer + litSize - ZSTD_LITBUFFEREXTRASIZE; + } else { + /* initially this will be stored entirely in dst during huffman decoding, it will partially be shifted to litExtraBuffer after */ + dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize; + dctx->litBufferEnd = (BYTE*)dst + expectedWriteSize; + } + dctx->litBufferLocation = ZSTD_split; + assert(dctx->litBufferEnd <= (BYTE*)dst + expectedWriteSize); + } +} -/* Hidden declaration for fullbench */ -size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, - const void* src, size_t srcSize); /*! ZSTD_decodeLiteralsBlock() : + * Where it is possible to do so without being stomped by the output during decompression, the literals block will be stored + * in the dstBuffer. If there is room to do so, it will be stored in full in the excess dst space after where the current + * block will be output. Otherwise it will be stored at the end of the current dst blockspace, with a small portion being + * stored in dctx->litExtraBuffer to help keep it "ahead" of the current output write. + * * @return : nb of bytes read from src (< srcSize ) * note : symbol not declared but exposed for fullbench */ -size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, - const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */ +static size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, + const void* src, size_t srcSize, /* note : srcSize < BLOCKSIZE */ + void* dst, size_t dstCapacity, const streaming_operation streaming) { DEBUGLOG(5, "ZSTD_decodeLiteralsBlock"); RETURN_ERROR_IF(srcSize < MIN_CBLOCK_SIZE, corruption_detected, ""); { const BYTE* const istart = (const BYTE*) src; - symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3); + SymbolEncodingType_e const litEncType = (SymbolEncodingType_e)(istart[0] & 3); + size_t const blockSizeMax = ZSTD_blockSizeMax(dctx); switch(litEncType) { case set_repeat: DEBUGLOG(5, "set_repeat flag : re-using stats from previous compressed literals block"); RETURN_ERROR_IF(dctx->litEntropy==0, dictionary_corrupted, ""); - /* fall-through */ + ZSTD_FALLTHROUGH; case set_compressed: - RETURN_ERROR_IF(srcSize < 5, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3"); + RETURN_ERROR_IF(srcSize < 5, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need up to 5 for case 3"); { size_t lhSize, litSize, litCSize; U32 singleStream=0; U32 const lhlCode = (istart[0] >> 2) & 3; U32 const lhc = MEM_readLE32(istart); size_t hufSuccess; + size_t expectedWriteSize = MIN(blockSizeMax, dstCapacity); + int const flags = 0 + | (ZSTD_DCtx_get_bmi2(dctx) ? HUF_flags_bmi2 : 0) + | (dctx->disableHufAsm ? HUF_flags_disableAsm : 0); switch(lhlCode) { case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */ @@ -31709,8 +41215,15 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, litCSize = (lhc >> 22) + ((size_t)istart[4] << 10); break; } - RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, ""); + RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(litSize > blockSizeMax, corruption_detected, ""); + if (!singleStream) + RETURN_ERROR_IF(litSize < MIN_LITERALS_FOR_4_STREAMS, literals_headerWrong, + "Not enough literals (%zu) for the 4-streams mode (min %u)", + litSize, MIN_LITERALS_FOR_4_STREAMS); RETURN_ERROR_IF(litCSize + lhSize > srcSize, corruption_detected, ""); + RETURN_ERROR_IF(expectedWriteSize < litSize , dstSize_tooSmall, ""); + ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 0); /* prefetch huffman table if cold */ if (dctx->ddictIsCold && (litSize > 768 /* heuristic */)) { @@ -31719,13 +41232,14 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, if (litEncType==set_repeat) { if (singleStream) { - hufSuccess = HUF_decompress1X_usingDTable_bmi2( + hufSuccess = HUF_decompress1X_usingDTable( dctx->litBuffer, litSize, istart+lhSize, litCSize, - dctx->HUFptr, dctx->bmi2); + dctx->HUFptr, flags); } else { - hufSuccess = HUF_decompress4X_usingDTable_bmi2( + assert(litSize >= MIN_LITERALS_FOR_4_STREAMS); + hufSuccess = HUF_decompress4X_usingDTable( dctx->litBuffer, litSize, istart+lhSize, litCSize, - dctx->HUFptr, dctx->bmi2); + dctx->HUFptr, flags); } } else { if (singleStream) { @@ -31733,20 +41247,29 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, hufSuccess = HUF_decompress1X_DCtx_wksp( dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->workspace, - sizeof(dctx->workspace)); + sizeof(dctx->workspace), flags); #else - hufSuccess = HUF_decompress1X1_DCtx_wksp_bmi2( + hufSuccess = HUF_decompress1X1_DCtx_wksp( dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->workspace, - sizeof(dctx->workspace), dctx->bmi2); + sizeof(dctx->workspace), flags); #endif } else { - hufSuccess = HUF_decompress4X_hufOnly_wksp_bmi2( + hufSuccess = HUF_decompress4X_hufOnly_wksp( dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->workspace, - sizeof(dctx->workspace), dctx->bmi2); + sizeof(dctx->workspace), flags); } } + if (dctx->litBufferLocation == ZSTD_split) + { + assert(litSize > ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memcpy(dctx->litExtraBuffer, dctx->litBufferEnd - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memmove(dctx->litBuffer + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH, dctx->litBuffer, litSize - ZSTD_LITBUFFEREXTRASIZE); + dctx->litBuffer += ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH; + dctx->litBufferEnd -= WILDCOPY_OVERLENGTH; + assert(dctx->litBufferEnd <= (BYTE*)dst + blockSizeMax); + } RETURN_ERROR_IF(HUF_isError(hufSuccess), corruption_detected, ""); @@ -31754,13 +41277,13 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, dctx->litSize = litSize; dctx->litEntropy = 1; if (litEncType==set_compressed) dctx->HUFptr = dctx->entropy.hufTable; - ZSTD_memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); return litCSize + lhSize; } case set_basic: { size_t litSize, lhSize; U32 const lhlCode = ((istart[0]) >> 2) & 3; + size_t expectedWriteSize = MIN(blockSizeMax, dstCapacity); switch(lhlCode) { case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ @@ -31773,27 +41296,42 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, break; case 3: lhSize = 3; + RETURN_ERROR_IF(srcSize<3, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize = 3"); litSize = MEM_readLE24(istart) >> 4; break; } + RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(litSize > blockSizeMax, corruption_detected, ""); + RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, ""); + ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1); if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */ RETURN_ERROR_IF(litSize+lhSize > srcSize, corruption_detected, ""); - ZSTD_memcpy(dctx->litBuffer, istart+lhSize, litSize); + if (dctx->litBufferLocation == ZSTD_split) + { + ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize - ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memcpy(dctx->litExtraBuffer, istart + lhSize + litSize - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE); + } + else + { + ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize); + } dctx->litPtr = dctx->litBuffer; dctx->litSize = litSize; - ZSTD_memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); return lhSize+litSize; } /* direct reference into compressed stream */ dctx->litPtr = istart+lhSize; dctx->litSize = litSize; + dctx->litBufferEnd = dctx->litPtr + litSize; + dctx->litBufferLocation = ZSTD_not_in_dst; return lhSize+litSize; } case set_rle: { U32 const lhlCode = ((istart[0]) >> 2) & 3; size_t litSize, lhSize; + size_t expectedWriteSize = MIN(blockSizeMax, dstCapacity); switch(lhlCode) { case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ @@ -31802,16 +41340,28 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, break; case 1: lhSize = 2; + RETURN_ERROR_IF(srcSize<3, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize+1 = 3"); litSize = MEM_readLE16(istart) >> 4; break; case 3: lhSize = 3; + RETURN_ERROR_IF(srcSize<4, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize+1 = 4"); litSize = MEM_readLE24(istart) >> 4; - RETURN_ERROR_IF(srcSize<4, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4"); break; } - RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, ""); - ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH); + RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(litSize > blockSizeMax, corruption_detected, ""); + RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, ""); + ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1); + if (dctx->litBufferLocation == ZSTD_split) + { + ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize - ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memset(dctx->litExtraBuffer, istart[lhSize], ZSTD_LITBUFFEREXTRASIZE); + } + else + { + ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize); + } dctx->litPtr = dctx->litBuffer; dctx->litSize = litSize; return lhSize+1; @@ -31822,6 +41372,18 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, } } +/* Hidden declaration for fullbench */ +size_t ZSTD_decodeLiteralsBlock_wrapper(ZSTD_DCtx* dctx, + const void* src, size_t srcSize, + void* dst, size_t dstCapacity); +size_t ZSTD_decodeLiteralsBlock_wrapper(ZSTD_DCtx* dctx, + const void* src, size_t srcSize, + void* dst, size_t dstCapacity) +{ + dctx->isFrameDecompression = 0; + return ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, not_streaming); +} + /* Default FSE distribution tables. * These are pre-calculated FSE decoding tables using default distributions as defined in specification : * https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#default-distributions @@ -31829,7 +41391,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, * - start from default distributions, present in /lib/common/zstd_internal.h * - generate tables normally, using ZSTD_buildFSETable() * - printout the content of tables - * - pretify output, report below, test with fuzzer to ensure it's correct */ + * - prettify output, report below, test with fuzzer to ensure it's correct */ /* Default FSE distribution table for Literal Lengths */ static const ZSTD_seqSymbol LL_defaultDTable[(1<nbBits = 0; cell->nextState = 0; assert(nbAddBits < 255); - cell->nbAdditionalBits = (BYTE)nbAddBits; + cell->nbAdditionalBits = nbAddBits; cell->baseValue = baseValue; } @@ -31955,7 +41517,7 @@ static void ZSTD_buildSeqTable_rle(ZSTD_seqSymbol* dt, U32 baseValue, U32 nbAddB FORCE_INLINE_TEMPLATE void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt, const short* normalizedCounter, unsigned maxSymbolValue, - const U32* baseValue, const U32* nbAdditionalBits, + const U32* baseValue, const U8* nbAdditionalBits, unsigned tableLog, void* wksp, size_t wkspSize) { ZSTD_seqSymbol* const tableDecode = dt+1; @@ -32018,14 +41580,15 @@ void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt, for (i = 8; i < n; i += 8) { MEM_write64(spread + pos + i, sv); } - pos += n; + assert(n>=0); + pos += (size_t)n; } } /* Now we spread those positions across the table. - * The benefit of doing it in two stages is that we avoid the the + * The benefit of doing it in two stages is that we avoid the * variable size inner loop, which caused lots of branch misses. * Now we can run through all the positions without any branch misses. - * We unroll the loop twice, since that is what emperically worked best. + * We unroll the loop twice, since that is what empirically worked best. */ { size_t position = 0; @@ -32052,7 +41615,7 @@ void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt, for (i=0; i highThreshold) position = (position + step) & tableMask; /* lowprob area */ + while (UNLIKELY(position > highThreshold)) position = (position + step) & tableMask; /* lowprob area */ } } assert(position == 0); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ } @@ -32063,10 +41626,10 @@ void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt, for (u=0; u max, corruption_detected, ""); { U32 const symbol = *(const BYTE*)src; U32 const baseline = baseValue[symbol]; - U32 const nbBits = nbAdditionalBits[symbol]; + U8 const nbBits = nbAdditionalBits[symbol]; ZSTD_buildSeqTable_rle(DTableSpace, baseline, nbBits); } *DTablePtr = DTableSpace; @@ -32176,11 +41739,6 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, /* SeqHead */ nbSeq = *ip++; - if (!nbSeq) { - *nbSeqPtr=0; - RETURN_ERROR_IF(srcSize != 1, srcSize_wrong, ""); - return 1; - } if (nbSeq > 0x7F) { if (nbSeq == 0xFF) { RETURN_ERROR_IF(ip+2 > iend, srcSize_wrong, ""); @@ -32193,11 +41751,19 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, } *nbSeqPtr = nbSeq; + if (nbSeq == 0) { + /* No sequence : section ends immediately */ + RETURN_ERROR_IF(ip != iend, corruption_detected, + "extraneous data present in the Sequences section"); + return (size_t)(ip - istart); + } + /* FSE table descriptors */ RETURN_ERROR_IF(ip+1 > iend, srcSize_wrong, ""); /* minimum possible size: 1 byte for symbol encoding types */ - { symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6); - symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3); - symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3); + RETURN_ERROR_IF(*ip & 3, corruption_detected, ""); /* The last field, Reserved, must be all-zeroes. */ + { SymbolEncodingType_e const LLtype = (SymbolEncodingType_e)(*ip >> 6); + SymbolEncodingType_e const OFtype = (SymbolEncodingType_e)((*ip >> 4) & 3); + SymbolEncodingType_e const MLtype = (SymbolEncodingType_e)((*ip >> 2) & 3); ip++; /* Build DTables */ @@ -32208,7 +41774,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, LL_defaultDTable, dctx->fseEntropy, dctx->ddictIsCold, nbSeq, dctx->workspace, sizeof(dctx->workspace), - dctx->bmi2); + ZSTD_DCtx_get_bmi2(dctx)); RETURN_ERROR_IF(ZSTD_isError(llhSize), corruption_detected, "ZSTD_buildSeqTable failed"); ip += llhSize; } @@ -32220,7 +41786,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, OF_defaultDTable, dctx->fseEntropy, dctx->ddictIsCold, nbSeq, dctx->workspace, sizeof(dctx->workspace), - dctx->bmi2); + ZSTD_DCtx_get_bmi2(dctx)); RETURN_ERROR_IF(ZSTD_isError(ofhSize), corruption_detected, "ZSTD_buildSeqTable failed"); ip += ofhSize; } @@ -32232,7 +41798,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, ML_defaultDTable, dctx->fseEntropy, dctx->ddictIsCold, nbSeq, dctx->workspace, sizeof(dctx->workspace), - dctx->bmi2); + ZSTD_DCtx_get_bmi2(dctx)); RETURN_ERROR_IF(ZSTD_isError(mlhSize), corruption_detected, "ZSTD_buildSeqTable failed"); ip += mlhSize; } @@ -32246,7 +41812,6 @@ typedef struct { size_t litLength; size_t matchLength; size_t offset; - const BYTE* match; } seq_t; typedef struct { @@ -32260,9 +41825,6 @@ typedef struct { ZSTD_fseState stateOffb; ZSTD_fseState stateML; size_t prevOffset[ZSTD_REP_NUM]; - const BYTE* prefixStart; - const BYTE* dictEnd; - size_t pos; } seqState_t; /*! ZSTD_overlapCopy8() : @@ -32305,7 +41867,7 @@ HINT_INLINE void ZSTD_overlapCopy8(BYTE** op, BYTE const** ip, size_t offset) { * - ZSTD_overlap_src_before_dst: The src and dst may overlap and may be any distance apart. * The src buffer must be before the dst buffer. */ -static void ZSTD_safecopy(BYTE* op, BYTE* const oend_w, BYTE const* ip, ptrdiff_t length, ZSTD_overlap_e ovtype) { +static void ZSTD_safecopy(BYTE* op, const BYTE* const oend_w, BYTE const* ip, ptrdiff_t length, ZSTD_overlap_e ovtype) { ptrdiff_t const diff = op - ip; BYTE* const oend = op + length; @@ -32321,6 +41883,7 @@ static void ZSTD_safecopy(BYTE* op, BYTE* const oend_w, BYTE const* ip, ptrdiff_ /* Copy 8 bytes and ensure the offset >= 8 when there can be overlap. */ assert(length >= 8); ZSTD_overlapCopy8(&op, &ip, diff); + length -= 8; assert(op - ip >= 8); assert(op <= oend); } @@ -32335,12 +41898,35 @@ static void ZSTD_safecopy(BYTE* op, BYTE* const oend_w, BYTE const* ip, ptrdiff_ assert(oend > oend_w); ZSTD_wildcopy(op, ip, oend_w - op, ovtype); ip += oend_w - op; - op = oend_w; + op += oend_w - op; } /* Handle the leftovers. */ while (op < oend) *op++ = *ip++; } +/* ZSTD_safecopyDstBeforeSrc(): + * This version allows overlap with dst before src, or handles the non-overlap case with dst after src + * Kept separate from more common ZSTD_safecopy case to avoid performance impact to the safecopy common case */ +static void ZSTD_safecopyDstBeforeSrc(BYTE* op, const BYTE* ip, ptrdiff_t length) { + ptrdiff_t const diff = op - ip; + BYTE* const oend = op + length; + + if (length < 8 || diff > -8) { + /* Handle short lengths, close overlaps, and dst not before src. */ + while (op < oend) *op++ = *ip++; + return; + } + + if (op <= oend - WILDCOPY_OVERLENGTH && diff < -WILDCOPY_VECLEN) { + ZSTD_wildcopy(op, ip, oend - WILDCOPY_OVERLENGTH - op, ZSTD_no_overlap); + ip += oend - WILDCOPY_OVERLENGTH - op; + op += oend - WILDCOPY_OVERLENGTH - op; + } + + /* Handle the leftovers. */ + while (op < oend) *op++ = *ip++; +} + /* ZSTD_execSequenceEnd(): * This version handles cases that are near the end of the output buffer. It requires * more careful checks to make sure there is no overflow. By separating out these hard @@ -32350,10 +41936,11 @@ static void ZSTD_safecopy(BYTE* op, BYTE* const oend_w, BYTE const* ip, ptrdiff_ * to be optimized for many small sequences, since those fall into ZSTD_execSequence(). */ FORCE_NOINLINE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR size_t ZSTD_execSequenceEnd(BYTE* op, - BYTE* const oend, seq_t sequence, - const BYTE** litPtr, const BYTE* const litLimit, - const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) { BYTE* const oLitEnd = op + sequence.litLength; size_t const sequenceLength = sequence.litLength + sequence.matchLength; @@ -32368,7 +41955,56 @@ size_t ZSTD_execSequenceEnd(BYTE* op, assert(oLitEnd < op + sequenceLength); /* copy literals */ - ZSTD_safecopy(op, oend_w, *litPtr, sequence.litLength, ZSTD_no_overlap); + ZSTD_safecopy(op, oend_w, *litPtr, sequence.litLength, ZSTD_no_overlap); + op = oLitEnd; + *litPtr = iLitEnd; + + /* copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix */ + RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); + match = dictEnd - (prefixStart - match); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } + } + ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst); + return sequenceLength; +} + +/* ZSTD_execSequenceEndSplitLitBuffer(): + * This version is intended to be used during instances where the litBuffer is still split. It is kept separate to avoid performance impact for the good case. + */ +FORCE_NOINLINE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_execSequenceEndSplitLitBuffer(BYTE* op, + BYTE* const oend, const BYTE* const oend_w, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + + /* bounds checks : careful of address space overflow in 32-bit mode */ + RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer"); + RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer"); + assert(op < op + sequenceLength); + assert(oLitEnd < op + sequenceLength); + + /* copy literals */ + RETURN_ERROR_IF(op > *litPtr && op < *litPtr + sequence.litLength, dstSize_tooSmall, "output should not catch up to and overwrite literal buffer"); + ZSTD_safecopyDstBeforeSrc(op, *litPtr, sequence.litLength); op = oLitEnd; *litPtr = iLitEnd; @@ -32376,27 +42012,29 @@ size_t ZSTD_execSequenceEnd(BYTE* op, if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { /* offset beyond prefix */ RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); - match = dictEnd - (prefixStart-match); + match = dictEnd - (prefixStart - match); if (match + sequence.matchLength <= dictEnd) { ZSTD_memmove(oLitEnd, match, sequence.matchLength); return sequenceLength; } /* span extDict & currentPrefixSegment */ { size_t const length1 = dictEnd - match; - ZSTD_memmove(oLitEnd, match, length1); - op = oLitEnd + length1; - sequence.matchLength -= length1; - match = prefixStart; - } } + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } + } ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst); return sequenceLength; } HINT_INLINE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR size_t ZSTD_execSequence(BYTE* op, - BYTE* const oend, seq_t sequence, - const BYTE** litPtr, const BYTE* const litLimit, - const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) { BYTE* const oLitEnd = op + sequence.litLength; size_t const sequenceLength = sequence.litLength + sequence.matchLength; @@ -32407,6 +42045,104 @@ size_t ZSTD_execSequence(BYTE* op, assert(op != NULL /* Precondition */); assert(oend_w < oend /* No underflow */); + +#if defined(__aarch64__) + /* prefetch sequence starting from match that will be used for copy later */ + PREFETCH_L1(match); +#endif + /* Handle edge cases in a slow path: + * - Read beyond end of literals + * - Match end is within WILDCOPY_OVERLIMIT of oend + * - 32-bit mode and the match length overflows + */ + if (UNLIKELY( + iLitEnd > litLimit || + oMatchEnd > oend_w || + (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH))) + return ZSTD_execSequenceEnd(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); + + /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */ + assert(op <= oLitEnd /* No overflow */); + assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */); + assert(oMatchEnd <= oend /* No underflow */); + assert(iLitEnd <= litLimit /* Literal length is in bounds */); + assert(oLitEnd <= oend_w /* Can wildcopy literals */); + assert(oMatchEnd <= oend_w /* Can wildcopy matches */); + + /* Copy Literals: + * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9. + * We likely don't need the full 32-byte wildcopy. + */ + assert(WILDCOPY_OVERLENGTH >= 16); + ZSTD_copy16(op, (*litPtr)); + if (UNLIKELY(sequence.litLength > 16)) { + ZSTD_wildcopy(op + 16, (*litPtr) + 16, sequence.litLength - 16, ZSTD_no_overlap); + } + op = oLitEnd; + *litPtr = iLitEnd; /* update for next sequence */ + + /* Copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix -> go into extDict */ + RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, ""); + match = dictEnd + (match - prefixStart); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } + } + /* Match within prefix of 1 or more bytes */ + assert(op <= oMatchEnd); + assert(oMatchEnd <= oend_w); + assert(match >= prefixStart); + assert(sequence.matchLength >= 1); + + /* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy + * without overlap checking. + */ + if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) { + /* We bet on a full wildcopy for matches, since we expect matches to be + * longer than literals (in general). In silesia, ~10% of matches are longer + * than 16 bytes. + */ + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap); + return sequenceLength; + } + assert(sequence.offset < WILDCOPY_VECLEN); + + /* Copy 8 bytes and spread the offset to be >= 8. */ + ZSTD_overlapCopy8(&op, &match, sequence.offset); + + /* If the match length is > 8 bytes, then continue with the wildcopy. */ + if (sequence.matchLength > 8) { + assert(op < oMatchEnd); + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8, ZSTD_overlap_src_before_dst); + } + return sequenceLength; +} + +HINT_INLINE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_execSequenceSplitLitBuffer(BYTE* op, + BYTE* const oend, const BYTE* const oend_w, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + assert(op != NULL /* Precondition */); + assert(oend_w < oend /* No underflow */); /* Handle edge cases in a slow path: * - Read beyond end of literals * - Match end is within WILDCOPY_OVERLIMIT of oend @@ -32416,7 +42152,7 @@ size_t ZSTD_execSequence(BYTE* op, iLitEnd > litLimit || oMatchEnd > oend_w || (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH))) - return ZSTD_execSequenceEnd(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); + return ZSTD_execSequenceEndSplitLitBuffer(op, oend, oend_w, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */ assert(op <= oLitEnd /* No overflow */); @@ -32484,6 +42220,7 @@ size_t ZSTD_execSequence(BYTE* op, return sequenceLength; } + static void ZSTD_initFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, const ZSTD_seqSymbol* dt) { @@ -32497,24 +42234,14 @@ ZSTD_initFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, const ZSTD_seqS } FORCE_INLINE_TEMPLATE void -ZSTD_updateFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD) -{ - ZSTD_seqSymbol const DInfo = DStatePtr->table[DStatePtr->state]; - U32 const nbBits = DInfo.nbBits; - size_t const lowBits = BIT_readBits(bitD, nbBits); - DStatePtr->state = DInfo.nextState + lowBits; -} - -FORCE_INLINE_TEMPLATE void -ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, ZSTD_seqSymbol const DInfo) +ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, U16 nextState, U32 nbBits) { - U32 const nbBits = DInfo.nbBits; size_t const lowBits = BIT_readBits(bitD, nbBits); - DStatePtr->state = DInfo.nextState + lowBits; + DStatePtr->state = nextState + lowBits; } /* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum - * offset bits. But we can only read at most (STREAM_ACCUMULATOR_MIN_32 - 1) + * offset bits. But we can only read at most STREAM_ACCUMULATOR_MIN_32 * bits before reloading. This value is the maximum number of bytes we read * after reloading when we are decoding long offsets. */ @@ -32524,123 +42251,136 @@ ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, ZSTD : 0) typedef enum { ZSTD_lo_isRegularOffset, ZSTD_lo_isLongOffset=1 } ZSTD_longOffset_e; -typedef enum { ZSTD_p_noPrefetch=0, ZSTD_p_prefetch=1 } ZSTD_prefetch_e; +/** + * ZSTD_decodeSequence(): + * @p longOffsets : tells the decoder to reload more bit while decoding large offsets + * only used in 32-bit mode + * @return : Sequence (litL + matchL + offset) + */ FORCE_INLINE_TEMPLATE seq_t -ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets, const ZSTD_prefetch_e prefetch) +ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets, const int isLastSeq) { seq_t seq; - ZSTD_seqSymbol const llDInfo = seqState->stateLL.table[seqState->stateLL.state]; - ZSTD_seqSymbol const mlDInfo = seqState->stateML.table[seqState->stateML.state]; - ZSTD_seqSymbol const ofDInfo = seqState->stateOffb.table[seqState->stateOffb.state]; - U32 const llBase = llDInfo.baseValue; - U32 const mlBase = mlDInfo.baseValue; - U32 const ofBase = ofDInfo.baseValue; - BYTE const llBits = llDInfo.nbAdditionalBits; - BYTE const mlBits = mlDInfo.nbAdditionalBits; - BYTE const ofBits = ofDInfo.nbAdditionalBits; - BYTE const totalBits = llBits+mlBits+ofBits; - - /* sequence */ - { size_t offset; - if (ofBits > 1) { - ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1); - ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5); - assert(ofBits <= MaxOff); - if (MEM_32bits() && longOffsets && (ofBits >= STREAM_ACCUMULATOR_MIN_32)) { - U32 const extraBits = ofBits - MIN(ofBits, 32 - seqState->DStream.bitsConsumed); - offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); - BIT_reloadDStream(&seqState->DStream); - if (extraBits) offset += BIT_readBitsFast(&seqState->DStream, extraBits); - assert(extraBits <= LONG_OFFSETS_MAX_EXTRA_BITS_32); /* to avoid another reload */ - } else { - offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ - if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); - } - seqState->prevOffset[2] = seqState->prevOffset[1]; - seqState->prevOffset[1] = seqState->prevOffset[0]; - seqState->prevOffset[0] = offset; - } else { - U32 const ll0 = (llBase == 0); - if (LIKELY((ofBits == 0))) { - if (LIKELY(!ll0)) - offset = seqState->prevOffset[0]; - else { - offset = seqState->prevOffset[1]; - seqState->prevOffset[1] = seqState->prevOffset[0]; - seqState->prevOffset[0] = offset; + /* + * ZSTD_seqSymbol is a 64 bits wide structure. + * It can be loaded in one operation + * and its fields extracted by simply shifting or bit-extracting on aarch64. + * GCC doesn't recognize this and generates more unnecessary ldr/ldrb/ldrh + * operations that cause performance drop. This can be avoided by using this + * ZSTD_memcpy hack. + */ +#if defined(__aarch64__) && (defined(__GNUC__) && !defined(__clang__)) + ZSTD_seqSymbol llDInfoS, mlDInfoS, ofDInfoS; + ZSTD_seqSymbol* const llDInfo = &llDInfoS; + ZSTD_seqSymbol* const mlDInfo = &mlDInfoS; + ZSTD_seqSymbol* const ofDInfo = &ofDInfoS; + ZSTD_memcpy(llDInfo, seqState->stateLL.table + seqState->stateLL.state, sizeof(ZSTD_seqSymbol)); + ZSTD_memcpy(mlDInfo, seqState->stateML.table + seqState->stateML.state, sizeof(ZSTD_seqSymbol)); + ZSTD_memcpy(ofDInfo, seqState->stateOffb.table + seqState->stateOffb.state, sizeof(ZSTD_seqSymbol)); +#else + const ZSTD_seqSymbol* const llDInfo = seqState->stateLL.table + seqState->stateLL.state; + const ZSTD_seqSymbol* const mlDInfo = seqState->stateML.table + seqState->stateML.state; + const ZSTD_seqSymbol* const ofDInfo = seqState->stateOffb.table + seqState->stateOffb.state; +#endif + seq.matchLength = mlDInfo->baseValue; + seq.litLength = llDInfo->baseValue; + { U32 const ofBase = ofDInfo->baseValue; + BYTE const llBits = llDInfo->nbAdditionalBits; + BYTE const mlBits = mlDInfo->nbAdditionalBits; + BYTE const ofBits = ofDInfo->nbAdditionalBits; + BYTE const totalBits = llBits+mlBits+ofBits; + + U16 const llNext = llDInfo->nextState; + U16 const mlNext = mlDInfo->nextState; + U16 const ofNext = ofDInfo->nextState; + U32 const llnbBits = llDInfo->nbBits; + U32 const mlnbBits = mlDInfo->nbBits; + U32 const ofnbBits = ofDInfo->nbBits; + + assert(llBits <= MaxLLBits); + assert(mlBits <= MaxMLBits); + assert(ofBits <= MaxOff); + /* + * As gcc has better branch and block analyzers, sometimes it is only + * valuable to mark likeliness for clang, it gives around 3-4% of + * performance. + */ + + /* sequence */ + { size_t offset; + if (ofBits > 1) { + ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1); + ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5); + ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 > LONG_OFFSETS_MAX_EXTRA_BITS_32); + ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 - LONG_OFFSETS_MAX_EXTRA_BITS_32 >= MaxMLBits); + if (MEM_32bits() && longOffsets && (ofBits >= STREAM_ACCUMULATOR_MIN_32)) { + /* Always read extra bits, this keeps the logic simple, + * avoids branches, and avoids accidentally reading 0 bits. + */ + U32 const extraBits = LONG_OFFSETS_MAX_EXTRA_BITS_32; + offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); + BIT_reloadDStream(&seqState->DStream); + offset += BIT_readBitsFast(&seqState->DStream, extraBits); + } else { + offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); } + seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset; } else { - offset = ofBase + ll0 + BIT_readBitsFast(&seqState->DStream, 1); - { size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; - temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ - if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; - seqState->prevOffset[1] = seqState->prevOffset[0]; - seqState->prevOffset[0] = offset = temp; - } } } - seq.offset = offset; - } - - seq.matchLength = mlBase; - if (mlBits > 0) - seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/); + U32 const ll0 = (llDInfo->baseValue == 0); + if (LIKELY((ofBits == 0))) { + offset = seqState->prevOffset[ll0]; + seqState->prevOffset[1] = seqState->prevOffset[!ll0]; + seqState->prevOffset[0] = offset; + } else { + offset = ofBase + ll0 + BIT_readBitsFast(&seqState->DStream, 1); + { size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; + temp -= !temp; /* 0 is not valid: input corrupted => force offset to -1 => corruption detected at execSequence */ + if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset = temp; + } } } + seq.offset = offset; + } - if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32)) - BIT_reloadDStream(&seqState->DStream); - if (MEM_64bits() && UNLIKELY(totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) - BIT_reloadDStream(&seqState->DStream); - /* Ensure there are enough bits to read the rest of data in 64-bit mode. */ - ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64); + if (mlBits > 0) + seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/); - seq.litLength = llBase; - if (llBits > 0) - seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/*>0*/); + if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32)) + BIT_reloadDStream(&seqState->DStream); + if (MEM_64bits() && UNLIKELY(totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) + BIT_reloadDStream(&seqState->DStream); + /* Ensure there are enough bits to read the rest of data in 64-bit mode. */ + ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64); - if (MEM_32bits()) - BIT_reloadDStream(&seqState->DStream); + if (llBits > 0) + seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/*>0*/); - DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u", - (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); + if (MEM_32bits()) + BIT_reloadDStream(&seqState->DStream); - if (prefetch == ZSTD_p_prefetch) { - size_t const pos = seqState->pos + seq.litLength; - const BYTE* const matchBase = (seq.offset > pos) ? seqState->dictEnd : seqState->prefixStart; - seq.match = matchBase + pos - seq.offset; /* note : this operation can overflow when seq.offset is really too large, which can only happen when input is corrupted. - * No consequence though : no memory access will occur, offset is only used for prefetching */ - seqState->pos = pos + seq.matchLength; - } + DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u", + (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); - /* ANS state update - * gcc-9.0.0 does 2.5% worse with ZSTD_updateFseStateWithDInfo(). - * clang-9.2.0 does 7% worse with ZSTD_updateFseState(). - * Naturally it seems like ZSTD_updateFseStateWithDInfo() should be the - * better option, so it is the default for other compilers. But, if you - * measure that it is worse, please put up a pull request. - */ - { -#if defined(__GNUC__) && !defined(__clang__) - const int kUseUpdateFseState = 1; -#else - const int kUseUpdateFseState = 0; -#endif - if (kUseUpdateFseState) { - ZSTD_updateFseState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */ - ZSTD_updateFseState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */ - if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ - ZSTD_updateFseState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */ - } else { - ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llDInfo); /* <= 9 bits */ - ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlDInfo); /* <= 9 bits */ + if (!isLastSeq) { + /* don't update FSE state for last Sequence */ + ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llNext, llnbBits); /* <= 9 bits */ + ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlNext, mlnbBits); /* <= 9 bits */ if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ - ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, ofDInfo); /* <= 8 bits */ + ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, ofNext, ofnbBits); /* <= 8 bits */ + BIT_reloadDStream(&seqState->DStream); } } return seq; } -#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -MEM_STATIC int ZSTD_dictionaryIsActive(ZSTD_DCtx const* dctx, BYTE const* prefixStart, BYTE const* oLitEnd) +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) +#if DEBUGLEVEL >= 1 +static int ZSTD_dictionaryIsActive(ZSTD_DCtx const* dctx, BYTE const* prefixStart, BYTE const* oLitEnd) { size_t const windowSize = dctx->fParams.windowSize; /* No dictionary used. */ @@ -32654,30 +42394,33 @@ MEM_STATIC int ZSTD_dictionaryIsActive(ZSTD_DCtx const* dctx, BYTE const* prefix /* Dictionary is active. */ return 1; } +#endif -MEM_STATIC void ZSTD_assertValidSequence( +static void ZSTD_assertValidSequence( ZSTD_DCtx const* dctx, BYTE const* op, BYTE const* oend, seq_t const seq, BYTE const* prefixStart, BYTE const* virtualStart) { #if DEBUGLEVEL >= 1 - size_t const windowSize = dctx->fParams.windowSize; - size_t const sequenceSize = seq.litLength + seq.matchLength; - BYTE const* const oLitEnd = op + seq.litLength; - DEBUGLOG(6, "Checking sequence: litL=%u matchL=%u offset=%u", - (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); - assert(op <= oend); - assert((size_t)(oend - op) >= sequenceSize); - assert(sequenceSize <= ZSTD_BLOCKSIZE_MAX); - if (ZSTD_dictionaryIsActive(dctx, prefixStart, oLitEnd)) { - size_t const dictSize = (size_t)((char const*)dctx->dictContentEndForFuzzing - (char const*)dctx->dictContentBeginForFuzzing); - /* Offset must be within the dictionary. */ - assert(seq.offset <= (size_t)(oLitEnd - virtualStart)); - assert(seq.offset <= windowSize + dictSize); - } else { - /* Offset must be within our window. */ - assert(seq.offset <= windowSize); + if (dctx->isFrameDecompression) { + size_t const windowSize = dctx->fParams.windowSize; + size_t const sequenceSize = seq.litLength + seq.matchLength; + BYTE const* const oLitEnd = op + seq.litLength; + DEBUGLOG(6, "Checking sequence: litL=%u matchL=%u offset=%u", + (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); + assert(op <= oend); + assert((size_t)(oend - op) >= sequenceSize); + assert(sequenceSize <= ZSTD_blockSizeMax(dctx)); + if (ZSTD_dictionaryIsActive(dctx, prefixStart, oLitEnd)) { + size_t const dictSize = (size_t)((char const*)dctx->dictContentEndForFuzzing - (char const*)dctx->dictContentBeginForFuzzing); + /* Offset must be within the dictionary. */ + assert(seq.offset <= (size_t)(oLitEnd - virtualStart)); + assert(seq.offset <= windowSize + dictSize); + } else { + /* Offset must be within our window. */ + assert(seq.offset <= windowSize); + } } #else (void)dctx, (void)op, (void)oend, (void)seq, (void)prefixStart, (void)virtualStart; @@ -32686,31 +42429,30 @@ MEM_STATIC void ZSTD_assertValidSequence( #endif #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG + + FORCE_INLINE_TEMPLATE size_t DONT_VECTORIZE -ZSTD_decompressSequences_body( ZSTD_DCtx* dctx, +ZSTD_decompressSequences_bodySplitLitBuffer( ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { const BYTE* ip = (const BYTE*)seqStart; const BYTE* const iend = ip + seqSize; BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = ostart + maxDstSize; + BYTE* const oend = ZSTD_maybeNullPtrAdd(ostart, maxDstSize); BYTE* op = ostart; const BYTE* litPtr = dctx->litPtr; - const BYTE* const litEnd = litPtr + dctx->litSize; + const BYTE* litBufferEnd = dctx->litBufferEnd; const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); const BYTE* const vBase = (const BYTE*) (dctx->virtualStart); const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); - DEBUGLOG(5, "ZSTD_decompressSequences_body"); - (void)frame; + DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer (%i seqs)", nbSeq); - /* Regen sequences */ + /* Literals are split between internal buffer & output buffer */ if (nbSeq) { seqState_t seqState; - size_t error = 0; dctx->fseEntropy = 1; { U32 i; for (i=0; ientropy.rep[i]; } RETURN_ERROR_IF( @@ -32726,134 +42468,331 @@ ZSTD_decompressSequences_body( ZSTD_DCtx* dctx, BIT_DStream_endOfBuffer < BIT_DStream_completed && BIT_DStream_completed < BIT_DStream_overflow); + /* decompress without overrunning litPtr begins */ + { seq_t sequence = {0,0,0}; /* some static analyzer believe that @sequence is not initialized (it necessarily is, since for(;;) loop as at least one iteration) */ + /* Align the decompression loop to 32 + 16 bytes. + * + * zstd compiled with gcc-9 on an Intel i9-9900k shows 10% decompression + * speed swings based on the alignment of the decompression loop. This + * performance swing is caused by parts of the decompression loop falling + * out of the DSB. The entire decompression loop should fit in the DSB, + * when it can't we get much worse performance. You can measure if you've + * hit the good case or the bad case with this perf command for some + * compressed file test.zst: + * + * perf stat -e cycles -e instructions -e idq.all_dsb_cycles_any_uops \ + * -e idq.all_mite_cycles_any_uops -- ./zstd -tq test.zst + * + * If you see most cycles served out of the MITE you've hit the bad case. + * If you see most cycles served out of the DSB you've hit the good case. + * If it is pretty even then you may be in an okay case. + * + * This issue has been reproduced on the following CPUs: + * - Kabylake: Macbook Pro (15-inch, 2019) 2.4 GHz Intel Core i9 + * Use Instruments->Counters to get DSB/MITE cycles. + * I never got performance swings, but I was able to + * go from the good case of mostly DSB to half of the + * cycles served from MITE. + * - Coffeelake: Intel i9-9900k + * - Coffeelake: Intel i7-9700k + * + * I haven't been able to reproduce the instability or DSB misses on any + * of the following CPUS: + * - Haswell + * - Broadwell: Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GH + * - Skylake + * + * Alignment is done for each of the three major decompression loops: + * - ZSTD_decompressSequences_bodySplitLitBuffer - presplit section of the literal buffer + * - ZSTD_decompressSequences_bodySplitLitBuffer - postsplit section of the literal buffer + * - ZSTD_decompressSequences_body + * Alignment choices are made to minimize large swings on bad cases and influence on performance + * from changes external to this code, rather than to overoptimize on the current commit. + * + * If you are seeing performance stability this script can help test. + * It tests on 4 commits in zstd where I saw performance change. + * + * https://gist.github.com/terrelln/9889fc06a423fd5ca6e99351564473f4 + */ #if defined(__GNUC__) && defined(__x86_64__) - /* Align the decompression loop to 32 + 16 bytes. - * - * zstd compiled with gcc-9 on an Intel i9-9900k shows 10% decompression - * speed swings based on the alignment of the decompression loop. This - * performance swing is caused by parts of the decompression loop falling - * out of the DSB. The entire decompression loop should fit in the DSB, - * when it can't we get much worse performance. You can measure if you've - * hit the good case or the bad case with this perf command for some - * compressed file test.zst: - * - * perf stat -e cycles -e instructions -e idq.all_dsb_cycles_any_uops \ - * -e idq.all_mite_cycles_any_uops -- ./zstd -tq test.zst - * - * If you see most cycles served out of the MITE you've hit the bad case. - * If you see most cycles served out of the DSB you've hit the good case. - * If it is pretty even then you may be in an okay case. - * - * I've been able to reproduce this issue on the following CPUs: - * - Kabylake: Macbook Pro (15-inch, 2019) 2.4 GHz Intel Core i9 - * Use Instruments->Counters to get DSB/MITE cycles. - * I never got performance swings, but I was able to - * go from the good case of mostly DSB to half of the - * cycles served from MITE. - * - Coffeelake: Intel i9-9900k - * - * I haven't been able to reproduce the instability or DSB misses on any - * of the following CPUS: - * - Haswell - * - Broadwell: Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GH - * - Skylake - * - * If you are seeing performance stability this script can help test. - * It tests on 4 commits in zstd where I saw performance change. - * - * https://gist.github.com/terrelln/9889fc06a423fd5ca6e99351564473f4 - */ - __asm__(".p2align 5"); - __asm__("nop"); - __asm__(".p2align 4"); + __asm__(".p2align 6"); +# if __GNUC__ >= 7 + /* good for gcc-7, gcc-9, and gcc-11 */ + __asm__("nop"); + __asm__(".p2align 5"); + __asm__("nop"); + __asm__(".p2align 4"); +# if __GNUC__ == 8 || __GNUC__ == 10 + /* good for gcc-8 and gcc-10 */ + __asm__("nop"); + __asm__(".p2align 3"); +# endif +# endif #endif - for ( ; ; ) { - seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, ZSTD_p_noPrefetch); - size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, prefixStart, vBase, dictEnd); + + /* Handle the initial state where litBuffer is currently split between dst and litExtraBuffer */ + for ( ; nbSeq; nbSeq--) { + sequence = ZSTD_decodeSequence(&seqState, isLongOffset, nbSeq==1); + if (litPtr + sequence.litLength > dctx->litBufferEnd) break; + { size_t const oneSeqSize = ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence.litLength - WILDCOPY_OVERLENGTH, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); #if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) - assert(!ZSTD_isError(oneSeqSize)); - if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); #endif - DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); - BIT_reloadDStream(&(seqState.DStream)); - op += oneSeqSize; - /* gcc and clang both don't like early returns in this loop. - * Instead break and check for an error at the end of the loop. - */ - if (UNLIKELY(ZSTD_isError(oneSeqSize))) { - error = oneSeqSize; - break; + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; + } } + DEBUGLOG(6, "reached: (litPtr + sequence.litLength > dctx->litBufferEnd)"); + + /* If there are more sequences, they will need to read literals from litExtraBuffer; copy over the remainder from dst and update litPtr and litEnd */ + if (nbSeq > 0) { + const size_t leftoverLit = dctx->litBufferEnd - litPtr; + DEBUGLOG(6, "There are %i sequences left, and %zu/%zu literals left in buffer", nbSeq, leftoverLit, sequence.litLength); + if (leftoverLit) { + RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer"); + ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit); + sequence.litLength -= leftoverLit; + op += leftoverLit; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + { size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; + } + nbSeq--; + } + } + + if (nbSeq > 0) { + /* there is remaining lit from extra buffer */ + +#if defined(__GNUC__) && defined(__x86_64__) + __asm__(".p2align 6"); + __asm__("nop"); +# if __GNUC__ != 7 + /* worse for gcc-7 better for gcc-8, gcc-9, and gcc-10 and clang */ + __asm__(".p2align 4"); + __asm__("nop"); + __asm__(".p2align 3"); +# elif __GNUC__ >= 11 + __asm__(".p2align 3"); +# else + __asm__(".p2align 5"); + __asm__("nop"); + __asm__(".p2align 3"); +# endif +#endif + + for ( ; nbSeq ; nbSeq--) { + seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, nbSeq==1); + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; } - if (UNLIKELY(!--nbSeq)) break; } /* check if reached exact end */ - DEBUGLOG(5, "ZSTD_decompressSequences_body: after decode loop, remaining nbSeq : %i", nbSeq); - if (ZSTD_isError(error)) return error; + DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer: after decode loop, remaining nbSeq : %i", nbSeq); RETURN_ERROR_IF(nbSeq, corruption_detected, ""); - RETURN_ERROR_IF(BIT_reloadDStream(&seqState.DStream) < BIT_DStream_completed, corruption_detected, ""); + DEBUGLOG(5, "bitStream : start=%p, ptr=%p, bitsConsumed=%u", seqState.DStream.start, seqState.DStream.ptr, seqState.DStream.bitsConsumed); + RETURN_ERROR_IF(!BIT_endOfDStream(&seqState.DStream), corruption_detected, ""); /* save reps for next block */ { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); } } /* last literal segment */ - { size_t const lastLLSize = litEnd - litPtr; + if (dctx->litBufferLocation == ZSTD_split) { + /* split hasn't been reached yet, first get dst then copy litExtraBuffer */ + size_t const lastLLSize = (size_t)(litBufferEnd - litPtr); + DEBUGLOG(6, "copy last literals from segment : %u", (U32)lastLLSize); + RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memmove(op, litPtr, lastLLSize); + op += lastLLSize; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + } + /* copy last literals from internal buffer */ + { size_t const lastLLSize = (size_t)(litBufferEnd - litPtr); + DEBUGLOG(6, "copy last literals from internal buffer : %u", (U32)lastLLSize); RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); if (op != NULL) { ZSTD_memcpy(op, litPtr, lastLLSize); op += lastLLSize; + } } + + DEBUGLOG(6, "decoded block of size %u bytes", (U32)(op - ostart)); + return (size_t)(op - ostart); +} + +FORCE_INLINE_TEMPLATE size_t +DONT_VECTORIZE +ZSTD_decompressSequences_body(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = dctx->litBufferLocation == ZSTD_not_in_dst ? ZSTD_maybeNullPtrAdd(ostart, maxDstSize) : dctx->litBuffer; + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* const litEnd = litPtr + dctx->litSize; + const BYTE* const prefixStart = (const BYTE*)(dctx->prefixStart); + const BYTE* const vBase = (const BYTE*)(dctx->virtualStart); + const BYTE* const dictEnd = (const BYTE*)(dctx->dictEnd); + DEBUGLOG(5, "ZSTD_decompressSequences_body: nbSeq = %d", nbSeq); + + /* Regen sequences */ + if (nbSeq) { + seqState_t seqState; + dctx->fseEntropy = 1; + { U32 i; for (i = 0; i < ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; } + RETURN_ERROR_IF( + ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend - ip)), + corruption_detected, ""); + ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + assert(dst != NULL); + +#if defined(__GNUC__) && defined(__x86_64__) + __asm__(".p2align 6"); + __asm__("nop"); +# if __GNUC__ >= 7 + __asm__(".p2align 5"); + __asm__("nop"); + __asm__(".p2align 3"); +# else + __asm__(".p2align 4"); + __asm__("nop"); + __asm__(".p2align 3"); +# endif +#endif + + for ( ; nbSeq ; nbSeq--) { + seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, nbSeq==1); + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; } + + /* check if reached exact end */ + assert(nbSeq == 0); + RETURN_ERROR_IF(!BIT_endOfDStream(&seqState.DStream), corruption_detected, ""); + /* save reps for next block */ + { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); } } - return op-ostart; + /* last literal segment */ + { size_t const lastLLSize = (size_t)(litEnd - litPtr); + DEBUGLOG(6, "copy last literals : %u", (U32)lastLLSize); + RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memcpy(op, litPtr, lastLLSize); + op += lastLLSize; + } } + + DEBUGLOG(6, "decoded block of size %u bytes", (U32)(op - ostart)); + return (size_t)(op - ostart); } static size_t ZSTD_decompressSequences_default(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) +{ + return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} + +static size_t +ZSTD_decompressSequencesSplitLitBuffer_default(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) { - return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT + +FORCE_INLINE_TEMPLATE + +size_t ZSTD_prefetchMatch(size_t prefetchPos, seq_t const sequence, + const BYTE* const prefixStart, const BYTE* const dictEnd) +{ + prefetchPos += sequence.litLength; + { const BYTE* const matchBase = (sequence.offset > prefetchPos) ? dictEnd : prefixStart; + /* note : this operation can overflow when seq.offset is really too large, which can only happen when input is corrupted. + * No consequence though : memory address is only used for prefetching, not for dereferencing */ + const BYTE* const match = ZSTD_wrappedPtrSub(ZSTD_wrappedPtrAdd(matchBase, prefetchPos), sequence.offset); + PREFETCH_L1(match); PREFETCH_L1(match+CACHELINE_SIZE); /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */ + } + return prefetchPos + sequence.matchLength; +} + +/* This decoding function employs prefetching + * to reduce latency impact of cache misses. + * It's generally employed when block contains a significant portion of long-distance matches + * or when coupled with a "cold" dictionary */ FORCE_INLINE_TEMPLATE size_t ZSTD_decompressSequencesLong_body( ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { const BYTE* ip = (const BYTE*)seqStart; const BYTE* const iend = ip + seqSize; BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = ostart + maxDstSize; + BYTE* const oend = dctx->litBufferLocation == ZSTD_in_dst ? dctx->litBuffer : ZSTD_maybeNullPtrAdd(ostart, maxDstSize); BYTE* op = ostart; const BYTE* litPtr = dctx->litPtr; - const BYTE* const litEnd = litPtr + dctx->litSize; + const BYTE* litBufferEnd = dctx->litBufferEnd; const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); const BYTE* const dictStart = (const BYTE*) (dctx->virtualStart); const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); - (void)frame; /* Regen sequences */ if (nbSeq) { -#define STORED_SEQS 4 +#define STORED_SEQS 8 #define STORED_SEQS_MASK (STORED_SEQS-1) -#define ADVANCED_SEQS 4 +#define ADVANCED_SEQS STORED_SEQS seq_t sequences[STORED_SEQS]; int const seqAdvance = MIN(nbSeq, ADVANCED_SEQS); seqState_t seqState; int seqNb; + size_t prefetchPos = (size_t)(op-prefixStart); /* track position relative to prefixStart */ + dctx->fseEntropy = 1; { int i; for (i=0; ientropy.rep[i]; } - seqState.prefixStart = prefixStart; - seqState.pos = (size_t)(op-prefixStart); - seqState.dictEnd = dictEnd; assert(dst != NULL); assert(iend >= ip); RETURN_ERROR_IF( @@ -32864,37 +42803,95 @@ ZSTD_decompressSequencesLong_body( ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); /* prepare in advance */ - for (seqNb=0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && (seqNblitBufferLocation == ZSTD_split && litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength > dctx->litBufferEnd) { + /* lit buffer is reaching split point, empty out the first buffer and transition to litExtraBuffer */ + const size_t leftoverLit = dctx->litBufferEnd - litPtr; + if (leftoverLit) + { + RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer"); + ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit); + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength -= leftoverLit; + op += leftoverLit; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + { size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); #if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) - assert(!ZSTD_isError(oneSeqSize)); - if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb-ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart); + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart); #endif - if (ZSTD_isError(oneSeqSize)) return oneSeqSize; - PREFETCH_L1(sequence.match); PREFETCH_L1(sequence.match + sequence.matchLength - 1); /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */ - sequences[seqNb & STORED_SEQS_MASK] = sequence; - op += oneSeqSize; + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + + prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd); + sequences[seqNb & STORED_SEQS_MASK] = sequence; + op += oneSeqSize; + } } + else + { + /* lit buffer is either wholly contained in first or second split, or not split at all*/ + size_t const oneSeqSize = dctx->litBufferLocation == ZSTD_split ? + ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength - WILDCOPY_OVERLENGTH, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) : + ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart); +#endif + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + + prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd); + sequences[seqNb & STORED_SEQS_MASK] = sequence; + op += oneSeqSize; + } } - RETURN_ERROR_IF(seqNblitBufferLocation == ZSTD_split && litPtr + sequence->litLength > dctx->litBufferEnd) { + const size_t leftoverLit = dctx->litBufferEnd - litPtr; + if (leftoverLit) { + RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer"); + ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit); + sequence->litLength -= leftoverLit; + op += leftoverLit; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + { size_t const oneSeqSize = ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); #if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) - assert(!ZSTD_isError(oneSeqSize)); - if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart); + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart); #endif - if (ZSTD_isError(oneSeqSize)) return oneSeqSize; - op += oneSeqSize; + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + op += oneSeqSize; + } + } + else + { + size_t const oneSeqSize = dctx->litBufferLocation == ZSTD_split ? + ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence->litLength - WILDCOPY_OVERLENGTH, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) : + ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart); +#endif + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + op += oneSeqSize; + } } /* save reps for next block */ @@ -32902,25 +42899,34 @@ ZSTD_decompressSequencesLong_body( } /* last literal segment */ - { size_t const lastLLSize = litEnd - litPtr; + if (dctx->litBufferLocation == ZSTD_split) { /* first deplete literal buffer in dst, then copy litExtraBuffer */ + size_t const lastLLSize = litBufferEnd - litPtr; + RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memmove(op, litPtr, lastLLSize); + op += lastLLSize; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + } + { size_t const lastLLSize = litBufferEnd - litPtr; RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); if (op != NULL) { - ZSTD_memcpy(op, litPtr, lastLLSize); + ZSTD_memmove(op, litPtr, lastLLSize); op += lastLLSize; } } - return op-ostart; + return (size_t)(op - ostart); } static size_t ZSTD_decompressSequencesLong_default(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { - return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ @@ -32929,53 +42935,65 @@ ZSTD_decompressSequencesLong_default(ZSTD_DCtx* dctx, #if DYNAMIC_BMI2 #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG -static TARGET_ATTRIBUTE("bmi2") size_t +static BMI2_TARGET_ATTRIBUTE size_t DONT_VECTORIZE ZSTD_decompressSequences_bmi2(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { - return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +static BMI2_TARGET_ATTRIBUTE size_t +DONT_VECTORIZE +ZSTD_decompressSequencesSplitLitBuffer_bmi2(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT -static TARGET_ATTRIBUTE("bmi2") size_t +static BMI2_TARGET_ATTRIBUTE size_t ZSTD_decompressSequencesLong_bmi2(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { - return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ #endif /* DYNAMIC_BMI2 */ -typedef size_t (*ZSTD_decompressSequences_t)( - ZSTD_DCtx* dctx, - void* dst, size_t maxDstSize, - const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame); - #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG static size_t ZSTD_decompressSequences(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { DEBUGLOG(5, "ZSTD_decompressSequences"); #if DYNAMIC_BMI2 - if (dctx->bmi2) { - return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + if (ZSTD_DCtx_get_bmi2(dctx)) { + return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); + } +#endif + return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +static size_t +ZSTD_decompressSequencesSplitLitBuffer(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + DEBUGLOG(5, "ZSTD_decompressSequencesSplitLitBuffer"); +#if DYNAMIC_BMI2 + if (ZSTD_DCtx_get_bmi2(dctx)) { + return ZSTD_decompressSequencesSplitLitBuffer_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif - return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequencesSplitLitBuffer_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ @@ -32990,69 +43008,114 @@ static size_t ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { DEBUGLOG(5, "ZSTD_decompressSequencesLong"); #if DYNAMIC_BMI2 - if (dctx->bmi2) { - return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + if (ZSTD_DCtx_get_bmi2(dctx)) { + return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif - return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ +/** + * @returns The total size of the history referenceable by zstd, including + * both the prefix and the extDict. At @p op any offset larger than this + * is invalid. + */ +static size_t ZSTD_totalHistorySize(BYTE* op, BYTE const* virtualStart) +{ + return (size_t)(op - virtualStart); +} + +typedef struct { + unsigned longOffsetShare; + unsigned maxNbAdditionalBits; +} ZSTD_OffsetInfo; -#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ - !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) -/* ZSTD_getLongOffsetsShare() : +/* ZSTD_getOffsetInfo() : * condition : offTable must be valid * @return : "share" of long offsets (arbitrarily defined as > (1<<23)) - * compared to maximum possible of (1< 22) total += 1; + assert(max <= (1 << OffFSELog)); /* max not too large */ + for (u=0; u 22) info.longOffsetShare += 1; + } + + assert(tableLog <= OffFSELog); + info.longOffsetShare <<= (OffFSELog - tableLog); /* scale to OffFSELog */ } - assert(tableLog <= OffFSELog); - total <<= (OffFSELog - tableLog); /* scale to OffFSELog */ + return info; +} - return total; +/** + * @returns The maximum offset we can decode in one read of our bitstream, without + * reloading more bits in the middle of the offset bits read. Any offsets larger + * than this must use the long offset decoder. + */ +static size_t ZSTD_maxShortOffset(void) +{ + if (MEM_64bits()) { + /* We can decode any offset without reloading bits. + * This might change if the max window size grows. + */ + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31); + return (size_t)-1; + } else { + /* The maximum offBase is (1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1. + * This offBase would require STREAM_ACCUMULATOR_MIN extra bits. + * Then we have to subtract ZSTD_REP_NUM to get the maximum possible offset. + */ + size_t const maxOffbase = ((size_t)1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1; + size_t const maxOffset = maxOffbase - ZSTD_REP_NUM; + assert(ZSTD_highbit32((U32)maxOffbase) == STREAM_ACCUMULATOR_MIN); + return maxOffset; + } } -#endif size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, - const void* src, size_t srcSize, const int frame) + const void* src, size_t srcSize, const streaming_operation streaming) { /* blockType == blockCompressed */ const BYTE* ip = (const BYTE*)src; - /* isLongOffset must be true if there are long offsets. - * Offsets are long if they are larger than 2^STREAM_ACCUMULATOR_MIN. - * We don't expect that to be the case in 64-bit mode. - * In block mode, window size is not known, so we have to be conservative. - * (note: but it could be evaluated from current-lowLimit) - */ - ZSTD_longOffset_e const isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (!frame || (dctx->fParams.windowSize > (1ULL << STREAM_ACCUMULATOR_MIN)))); - DEBUGLOG(5, "ZSTD_decompressBlock_internal (size : %u)", (U32)srcSize); - - RETURN_ERROR_IF(srcSize >= ZSTD_BLOCKSIZE_MAX, srcSize_wrong, ""); + DEBUGLOG(5, "ZSTD_decompressBlock_internal (cSize : %u)", (unsigned)srcSize); + + /* Note : the wording of the specification + * allows compressed block to be sized exactly ZSTD_blockSizeMax(dctx). + * This generally does not happen, as it makes little sense, + * since an uncompressed block would feature same size and have no decompression cost. + * Also, note that decoder from reference libzstd before < v1.5.4 + * would consider this edge case as an error. + * As a consequence, avoid generating compressed blocks of size ZSTD_blockSizeMax(dctx) + * for broader compatibility with the deployed ecosystem of zstd decoders */ + RETURN_ERROR_IF(srcSize > ZSTD_blockSizeMax(dctx), srcSize_wrong, ""); /* Decode literals section */ - { size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize); - DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : %u", (U32)litCSize); + { size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, streaming); + DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : cSize=%u, nbLiterals=%zu", (U32)litCSize, dctx->litSize); if (ZSTD_isError(litCSize)) return litCSize; ip += litCSize; srcSize -= litCSize; @@ -33060,6 +43123,23 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, /* Build Decoding Tables */ { + /* Compute the maximum block size, which must also work when !frame and fParams are unset. + * Additionally, take the min with dstCapacity to ensure that the totalHistorySize fits in a size_t. + */ + size_t const blockSizeMax = MIN(dstCapacity, ZSTD_blockSizeMax(dctx)); + size_t const totalHistorySize = ZSTD_totalHistorySize(ZSTD_maybeNullPtrAdd((BYTE*)dst, blockSizeMax), (BYTE const*)dctx->virtualStart); + /* isLongOffset must be true if there are long offsets. + * Offsets are long if they are larger than ZSTD_maxShortOffset(). + * We don't expect that to be the case in 64-bit mode. + * + * We check here to see if our history is large enough to allow long offsets. + * If it isn't, then we can't possible have (valid) long offsets. If the offset + * is invalid, then it is okay to read it incorrectly. + * + * If isLongOffsets is true, then we will later check our decoding table to see + * if it is even possible to generate long offsets. + */ + ZSTD_longOffset_e isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (totalHistorySize > ZSTD_maxShortOffset())); /* These macros control at build-time which decompressor implementation * we use. If neither is defined, we do some inspection and dispatch at * runtime. @@ -33067,6 +43147,11 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, #if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) int usePrefetchDecoder = dctx->ddictIsCold; +#else + /* Set to 1 to avoid computing offset info if we don't need to. + * Otherwise this value is ignored. + */ + int usePrefetchDecoder = 1; #endif int nbSeq; size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, srcSize); @@ -33074,37 +43159,55 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, ip += seqHSize; srcSize -= seqHSize; - RETURN_ERROR_IF(dst == NULL && nbSeq > 0, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF((dst == NULL || dstCapacity == 0) && nbSeq > 0, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(MEM_64bits() && sizeof(size_t) == sizeof(void*) && (size_t)(-1) - (size_t)dst < (size_t)(1 << 20), dstSize_tooSmall, + "invalid dst"); -#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ - !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) - if ( !usePrefetchDecoder - && (!frame || (dctx->fParams.windowSize > (1<<24))) - && (nbSeq>ADVANCED_SEQS) ) { /* could probably use a larger nbSeq limit */ - U32 const shareLongOffsets = ZSTD_getLongOffsetsShare(dctx->OFTptr); - U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */ - usePrefetchDecoder = (shareLongOffsets >= minShare); + /* If we could potentially have long offsets, or we might want to use the prefetch decoder, + * compute information about the share of long offsets, and the maximum nbAdditionalBits. + * NOTE: could probably use a larger nbSeq limit + */ + if (isLongOffset || (!usePrefetchDecoder && (totalHistorySize > (1u << 24)) && (nbSeq > 8))) { + ZSTD_OffsetInfo const info = ZSTD_getOffsetInfo(dctx->OFTptr, nbSeq); + if (isLongOffset && info.maxNbAdditionalBits <= STREAM_ACCUMULATOR_MIN) { + /* If isLongOffset, but the maximum number of additional bits that we see in our table is small + * enough, then we know it is impossible to have too long an offset in this block, so we can + * use the regular offset decoder. + */ + isLongOffset = ZSTD_lo_isRegularOffset; + } + if (!usePrefetchDecoder) { + U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */ + usePrefetchDecoder = (info.longOffsetShare >= minShare); + } } -#endif dctx->ddictIsCold = 0; #if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) - if (usePrefetchDecoder) + if (usePrefetchDecoder) { +#else + (void)usePrefetchDecoder; + { #endif #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT - return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); #endif + } #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG /* else */ - return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame); + if (dctx->litBufferLocation == ZSTD_split) + return ZSTD_decompressSequencesSplitLitBuffer(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); + else + return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); #endif } } +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize) { if (dst != dctx->previousDstEnd && dstSize > 0) { /* not contiguous */ @@ -33116,21 +43219,32 @@ void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize) } -size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize) +size_t ZSTD_decompressBlock_deprecated(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) { size_t dSize; + dctx->isFrameDecompression = 0; ZSTD_checkContinuity(dctx, dst, dstCapacity); - dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 0); + dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, not_streaming); + FORWARD_IF_ERROR(dSize, ""); dctx->previousDstEnd = (char*)dst + dSize; return dSize; } + + +/* NOTE: Must just wrap ZSTD_decompressBlock_deprecated() */ +size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + return ZSTD_decompressBlock_deprecated(dctx, dst, dstCapacity, src, srcSize); +} /**** ended inlining decompress/zstd_decompress_block.c ****/ /**** start inlining dictBuilder/cover.c ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -33152,39 +43266,32 @@ size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, /*-************************************* * Dependencies ***************************************/ +/* qsort_r is an extension. */ +#if defined(__linux) || defined(__linux__) || defined(linux) || defined(__gnu_linux__) || \ + defined(__CYGWIN__) || defined(__MSYS__) +#if !defined(_GNU_SOURCE) && !defined(__ANDROID__) /* NDK doesn't ship qsort_r(). */ +#define _GNU_SOURCE +#endif +#endif + #include /* fprintf */ -#include /* malloc, free, qsort */ +#include /* malloc, free, qsort_r */ + #include /* memset */ #include /* clock */ -/**** skipping file: ../common/mem.h ****/ -/**** skipping file: ../common/pool.h ****/ -/**** skipping file: ../common/threading.h ****/ -/**** start inlining cover.h ****/ -/* - * Copyright (c) 2017-2021, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ +#ifndef ZDICT_STATIC_LINKING_ONLY +# define ZDICT_STATIC_LINKING_ONLY +#endif -#include /* fprintf */ -#include /* malloc, free, qsort */ -#include /* memset */ -#include /* clock */ /**** skipping file: ../common/mem.h ****/ /**** skipping file: ../common/pool.h ****/ /**** skipping file: ../common/threading.h ****/ /**** skipping file: ../common/zstd_internal.h ****/ -#ifndef ZDICT_STATIC_LINKING_ONLY -#define ZDICT_STATIC_LINKING_ONLY -#endif -/**** start inlining zdict.h ****/ +/**** skipping file: ../common/bits.h ****/ +/**** start inlining ../zdict.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -33193,34 +43300,184 @@ size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, * You may select, at your option, one of the above-listed licenses. */ -#ifndef DICTBUILDER_H_001 -#define DICTBUILDER_H_001 - -#if defined (__cplusplus) -extern "C" { -#endif +#ifndef ZSTD_ZDICT_H +#define ZSTD_ZDICT_H /*====== Dependencies ======*/ #include /* size_t */ +#if defined (__cplusplus) +extern "C" { +#endif /* ===== ZDICTLIB_API : control library symbols visibility ===== */ -#ifndef ZDICTLIB_VISIBILITY -# if defined(__GNUC__) && (__GNUC__ >= 4) -# define ZDICTLIB_VISIBILITY __attribute__ ((visibility ("default"))) +#ifndef ZDICTLIB_VISIBLE + /* Backwards compatibility with old macro name */ +# ifdef ZDICTLIB_VISIBILITY +# define ZDICTLIB_VISIBLE ZDICTLIB_VISIBILITY +# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZDICTLIB_VISIBLE __attribute__ ((visibility ("default"))) +# else +# define ZDICTLIB_VISIBLE +# endif +#endif + +#ifndef ZDICTLIB_HIDDEN +# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZDICTLIB_HIDDEN __attribute__ ((visibility ("hidden"))) # else -# define ZDICTLIB_VISIBILITY +# define ZDICTLIB_HIDDEN # endif #endif + #if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) -# define ZDICTLIB_API __declspec(dllexport) ZDICTLIB_VISIBILITY +# define ZDICTLIB_API __declspec(dllexport) ZDICTLIB_VISIBLE #elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) -# define ZDICTLIB_API __declspec(dllimport) ZDICTLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +# define ZDICTLIB_API __declspec(dllimport) ZDICTLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ #else -# define ZDICTLIB_API ZDICTLIB_VISIBILITY +# define ZDICTLIB_API ZDICTLIB_VISIBLE #endif +/******************************************************************************* + * Zstd dictionary builder + * + * FAQ + * === + * Why should I use a dictionary? + * ------------------------------ + * + * Zstd can use dictionaries to improve compression ratio of small data. + * Traditionally small files don't compress well because there is very little + * repetition in a single sample, since it is small. But, if you are compressing + * many similar files, like a bunch of JSON records that share the same + * structure, you can train a dictionary on ahead of time on some samples of + * these files. Then, zstd can use the dictionary to find repetitions that are + * present across samples. This can vastly improve compression ratio. + * + * When is a dictionary useful? + * ---------------------------- + * + * Dictionaries are useful when compressing many small files that are similar. + * The larger a file is, the less benefit a dictionary will have. Generally, + * we don't expect dictionary compression to be effective past 100KB. And the + * smaller a file is, the more we would expect the dictionary to help. + * + * How do I use a dictionary? + * -------------------------- + * + * Simply pass the dictionary to the zstd compressor with + * `ZSTD_CCtx_loadDictionary()`. The same dictionary must then be passed to + * the decompressor, using `ZSTD_DCtx_loadDictionary()`. There are other + * more advanced functions that allow selecting some options, see zstd.h for + * complete documentation. + * + * What is a zstd dictionary? + * -------------------------- + * + * A zstd dictionary has two pieces: Its header, and its content. The header + * contains a magic number, the dictionary ID, and entropy tables. These + * entropy tables allow zstd to save on header costs in the compressed file, + * which really matters for small data. The content is just bytes, which are + * repeated content that is common across many samples. + * + * What is a raw content dictionary? + * --------------------------------- + * + * A raw content dictionary is just bytes. It doesn't have a zstd dictionary + * header, a dictionary ID, or entropy tables. Any buffer is a valid raw + * content dictionary. + * + * How do I train a dictionary? + * ---------------------------- + * + * Gather samples from your use case. These samples should be similar to each + * other. If you have several use cases, you could try to train one dictionary + * per use case. + * + * Pass those samples to `ZDICT_trainFromBuffer()` and that will train your + * dictionary. There are a few advanced versions of this function, but this + * is a great starting point. If you want to further tune your dictionary + * you could try `ZDICT_optimizeTrainFromBuffer_cover()`. If that is too slow + * you can try `ZDICT_optimizeTrainFromBuffer_fastCover()`. + * + * If the dictionary training function fails, that is likely because you + * either passed too few samples, or a dictionary would not be effective + * for your data. Look at the messages that the dictionary trainer printed, + * if it doesn't say too few samples, then a dictionary would not be effective. + * + * How large should my dictionary be? + * ---------------------------------- + * + * A reasonable dictionary size, the `dictBufferCapacity`, is about 100KB. + * The zstd CLI defaults to a 110KB dictionary. You likely don't need a + * dictionary larger than that. But, most use cases can get away with a + * smaller dictionary. The advanced dictionary builders can automatically + * shrink the dictionary for you, and select the smallest size that doesn't + * hurt compression ratio too much. See the `shrinkDict` parameter. + * A smaller dictionary can save memory, and potentially speed up + * compression. + * + * How many samples should I provide to the dictionary builder? + * ------------------------------------------------------------ + * + * We generally recommend passing ~100x the size of the dictionary + * in samples. A few thousand should suffice. Having too few samples + * can hurt the dictionaries effectiveness. Having more samples will + * only improve the dictionaries effectiveness. But having too many + * samples can slow down the dictionary builder. + * + * How do I determine if a dictionary will be effective? + * ----------------------------------------------------- + * + * Simply train a dictionary and try it out. You can use zstd's built in + * benchmarking tool to test the dictionary effectiveness. + * + * # Benchmark levels 1-3 without a dictionary + * zstd -b1e3 -r /path/to/my/files + * # Benchmark levels 1-3 with a dictionary + * zstd -b1e3 -r /path/to/my/files -D /path/to/my/dictionary + * + * When should I retrain a dictionary? + * ----------------------------------- + * + * You should retrain a dictionary when its effectiveness drops. Dictionary + * effectiveness drops as the data you are compressing changes. Generally, we do + * expect dictionaries to "decay" over time, as your data changes, but the rate + * at which they decay depends on your use case. Internally, we regularly + * retrain dictionaries, and if the new dictionary performs significantly + * better than the old dictionary, we will ship the new dictionary. + * + * I have a raw content dictionary, how do I turn it into a zstd dictionary? + * ------------------------------------------------------------------------- + * + * If you have a raw content dictionary, e.g. by manually constructing it, or + * using a third-party dictionary builder, you can turn it into a zstd + * dictionary by using `ZDICT_finalizeDictionary()`. You'll also have to + * provide some samples of the data. It will add the zstd header to the + * raw content, which contains a dictionary ID and entropy tables, which + * will improve compression ratio, and allow zstd to write the dictionary ID + * into the frame, if you so choose. + * + * Do I have to use zstd's dictionary builder? + * ------------------------------------------- + * + * No! You can construct dictionary content however you please, it is just + * bytes. It will always be valid as a raw content dictionary. If you want + * a zstd dictionary, which can improve compression ratio, use + * `ZDICT_finalizeDictionary()`. + * + * What is the attack surface of a zstd dictionary? + * ------------------------------------------------ + * + * Zstd is heavily fuzz tested, including loading fuzzed dictionaries, so + * zstd should never crash, or access out-of-bounds memory no matter what + * the dictionary is. However, if an attacker can control the dictionary + * during decompression, they can cause zstd to generate arbitrary bytes, + * just like if they controlled the compressed data. + * + ******************************************************************************/ + /*! ZDICT_trainFromBuffer(): * Train a dictionary from an array of samples. @@ -33247,9 +43504,16 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCap const size_t* samplesSizes, unsigned nbSamples); typedef struct { - int compressionLevel; /*< optimize for a specific zstd compression level; 0 means default */ - unsigned notificationLevel; /*< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */ - unsigned dictID; /*< force dictID value; 0 means auto mode (32-bits random value) */ + int compressionLevel; /**< optimize for a specific zstd compression level; 0 means default */ + unsigned notificationLevel; /**< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */ + unsigned dictID; /**< force dictID value; 0 means auto mode (32-bits random value) + * NOTE: The zstd format reserves some dictionary IDs for future use. + * You may use them in private settings, but be warned that they + * may be used by zstd in a public dictionary registry in the future. + * These dictionary IDs are: + * - low range : <= 32767 + * - high range : >= (2^31) + */ } ZDICT_params_t; /*! ZDICT_finalizeDictionary(): @@ -33276,8 +43540,7 @@ typedef struct { * is presumed that the most profitable content is at the end of the dictionary, * since that is the cheapest to reference. * - * `dictContentSize` must be >= ZDICT_CONTENTSIZE_MIN bytes. - * `maxDictSize` must be >= max(dictContentSize, ZSTD_DICTSIZE_MIN). + * `maxDictSize` must be >= max(dictContentSize, ZDICT_DICTSIZE_MIN). * * @return: size of dictionary stored into `dstDictBuffer` (<= `maxDictSize`), * or an error code, which can be tested by ZDICT_isError(). @@ -33300,9 +43563,29 @@ ZDICTLIB_API size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictS ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode); ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode); +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_ZDICT_H */ + +#if defined(ZDICT_STATIC_LINKING_ONLY) && !defined(ZSTD_ZDICT_H_STATIC) +#define ZSTD_ZDICT_H_STATIC +#if defined (__cplusplus) +extern "C" { +#endif -#ifdef ZDICT_STATIC_LINKING_ONLY +/* This can be overridden externally to hide static symbols. */ +#ifndef ZDICTLIB_STATIC_API +# if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZDICTLIB_STATIC_API __declspec(dllexport) ZDICTLIB_VISIBLE +# elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZDICTLIB_STATIC_API __declspec(dllimport) ZDICTLIB_VISIBLE +# else +# define ZDICTLIB_STATIC_API ZDICTLIB_VISIBLE +# endif +#endif /* ==================================================================================== * The definitions in this section are considered experimental. @@ -33311,8 +43594,9 @@ ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode); * Use them only in association with static linking. * ==================================================================================== */ -#define ZDICT_CONTENTSIZE_MIN 128 #define ZDICT_DICTSIZE_MIN 256 +/* Deprecated: Remove in v1.6.0 */ +#define ZDICT_CONTENTSIZE_MIN 128 /*! ZDICT_cover_params_t: * k and d are the only required parameters. @@ -33357,7 +43641,7 @@ typedef struct { * In general, it's recommended to provide a few thousands samples, though this can vary a lot. * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. */ -ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover( +ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_cover( void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples, ZDICT_cover_params_t parameters); @@ -33379,7 +43663,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover( * See ZDICT_trainFromBuffer() for details on failure modes. * Note: ZDICT_optimizeTrainFromBuffer_cover() requires about 8 bytes of memory for each input byte and additionally another 5 bytes of memory for each byte of memory for each thread. */ -ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover( +ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_cover( void* dictBuffer, size_t dictBufferCapacity, const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, ZDICT_cover_params_t* parameters); @@ -33400,7 +43684,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover( * In general, it's recommended to provide a few thousands samples, though this can vary a lot. * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. */ -ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer, +ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples, ZDICT_fastCover_params_t parameters); @@ -33423,7 +43707,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer, * See ZDICT_trainFromBuffer() for details on failure modes. * Note: ZDICT_optimizeTrainFromBuffer_fastCover() requires about 6 * 2^f bytes of memory for each thread. */ -ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer, +ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer, size_t dictBufferCapacity, const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, ZDICT_fastCover_params_t* parameters); @@ -33448,7 +43732,7 @@ typedef struct { * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. * Note: ZDICT_trainFromBuffer_legacy() will send notifications into stderr if instructed to, using notificationLevel>0. */ -ZDICTLIB_API size_t ZDICT_trainFromBuffer_legacy( +ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_legacy( void* dictBuffer, size_t dictBufferCapacity, const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, ZDICT_legacy_params_t parameters); @@ -33460,36 +43744,52 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_legacy( or _CRT_SECURE_NO_WARNINGS in Visual. Otherwise, it's also possible to manually define ZDICT_DISABLE_DEPRECATE_WARNINGS */ #ifdef ZDICT_DISABLE_DEPRECATE_WARNINGS -# define ZDICT_DEPRECATED(message) ZDICTLIB_API /* disable deprecation warnings */ +# define ZDICT_DEPRECATED(message) /* disable deprecation warnings */ #else # define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) # if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ -# define ZDICT_DEPRECATED(message) [[deprecated(message)]] ZDICTLIB_API +# define ZDICT_DEPRECATED(message) [[deprecated(message)]] # elif defined(__clang__) || (ZDICT_GCC_VERSION >= 405) -# define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated(message))) +# define ZDICT_DEPRECATED(message) __attribute__((deprecated(message))) # elif (ZDICT_GCC_VERSION >= 301) -# define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated)) +# define ZDICT_DEPRECATED(message) __attribute__((deprecated)) # elif defined(_MSC_VER) -# define ZDICT_DEPRECATED(message) ZDICTLIB_API __declspec(deprecated(message)) +# define ZDICT_DEPRECATED(message) __declspec(deprecated(message)) # else # pragma message("WARNING: You need to implement ZDICT_DEPRECATED for this compiler") -# define ZDICT_DEPRECATED(message) ZDICTLIB_API +# define ZDICT_DEPRECATED(message) # endif #endif /* ZDICT_DISABLE_DEPRECATE_WARNINGS */ ZDICT_DEPRECATED("use ZDICT_finalizeDictionary() instead") +ZDICTLIB_STATIC_API size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity, const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples); - -#endif /* ZDICT_STATIC_LINKING_ONLY */ - #if defined (__cplusplus) } #endif -#endif /* DICTBUILDER_H_001 */ -/**** ended inlining zdict.h ****/ +#endif /* ZSTD_ZDICT_H_STATIC */ +/**** ended inlining ../zdict.h ****/ +/**** start inlining cover.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZDICT_STATIC_LINKING_ONLY +# define ZDICT_STATIC_LINKING_ONLY +#endif + +/**** skipping file: ../common/threading.h ****/ +/**** skipping file: ../common/mem.h ****/ +/**** skipping file: ../zdict.h ****/ /** * COVER_best_t is used for two purposes: @@ -33626,15 +43926,17 @@ void COVER_dictSelectionFree(COVER_dictSelection_t selection); size_t dictContentSize, const BYTE* samplesBuffer, const size_t* samplesSizes, unsigned nbFinalizeSamples, size_t nbCheckSamples, size_t nbSamples, ZDICT_cover_params_t params, size_t* offsets, size_t totalCompressedSize); /**** ended inlining cover.h ****/ -/**** skipping file: ../common/zstd_internal.h ****/ -#ifndef ZDICT_STATIC_LINKING_ONLY -#define ZDICT_STATIC_LINKING_ONLY -#endif -/**** skipping file: zdict.h ****/ /*-************************************* * Constants ***************************************/ +/** +* There are 32bit indexes used to ref samples, so limit samples size to 4GB +* on 64bit builds. +* For 32bit builds we choose 1 GB. +* Most 32bit platforms have 2GB user-mode addressable space and we allocate a large +* contiguous buffer, so 1GB is already a high limit. +*/ #define COVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((unsigned)-1) : ((unsigned)1 GB)) #define COVER_DEFAULT_SPLITPOINT 1.0 @@ -33642,7 +43944,7 @@ void COVER_dictSelectionFree(COVER_dictSelection_t selection); * Console display ***************************************/ #ifndef LOCALDISPLAYLEVEL -static int g_displayLevel = 2; +static int g_displayLevel = 0; #endif #undef DISPLAY #define DISPLAY(...) \ @@ -33665,7 +43967,7 @@ static clock_t g_time = 0; #undef LOCALDISPLAYUPDATE #define LOCALDISPLAYUPDATE(displayLevel, l, ...) \ if (displayLevel >= l) { \ - if ((clock() - g_time > g_refreshRate) || (displayLevel >= 4)) { \ + if ((clock() - g_time > g_refreshRate) || (displayLevel >= 4)) { \ g_time = clock(); \ DISPLAY(__VA_ARGS__); \ } \ @@ -33819,8 +44121,10 @@ typedef struct { unsigned d; } COVER_ctx_t; -/* We need a global context for qsort... */ +#if !defined(_GNU_SOURCE) && !defined(__APPLE__) && !defined(_MSC_VER) +/* C90 only offers qsort() that needs a global context. */ static COVER_ctx_t *g_coverCtx = NULL; +#endif /*-************************************* * Helper functions @@ -33863,11 +44167,15 @@ static int COVER_cmp8(COVER_ctx_t *ctx, const void *lp, const void *rp) { /** * Same as COVER_cmp() except ties are broken by pointer value - * NOTE: g_coverCtx must be set to call this function. A global is required because - * qsort doesn't take an opaque pointer. */ -static int WIN_CDECL COVER_strict_cmp(const void *lp, const void *rp) { - int result = COVER_cmp(g_coverCtx, lp, rp); +#if (defined(_WIN32) && defined(_MSC_VER)) || defined(__APPLE__) +static int WIN_CDECL COVER_strict_cmp(void* g_coverCtx, const void* lp, const void* rp) { +#elif defined(_GNU_SOURCE) +static int COVER_strict_cmp(const void *lp, const void *rp, void *g_coverCtx) { +#else /* C90 fallback.*/ +static int COVER_strict_cmp(const void *lp, const void *rp) { +#endif + int result = COVER_cmp((COVER_ctx_t*)g_coverCtx, lp, rp); if (result == 0) { result = lp < rp ? -1 : 1; } @@ -33876,21 +44184,58 @@ static int WIN_CDECL COVER_strict_cmp(const void *lp, const void *rp) { /** * Faster version for d <= 8. */ -static int WIN_CDECL COVER_strict_cmp8(const void *lp, const void *rp) { - int result = COVER_cmp8(g_coverCtx, lp, rp); +#if (defined(_WIN32) && defined(_MSC_VER)) || defined(__APPLE__) +static int WIN_CDECL COVER_strict_cmp8(void* g_coverCtx, const void* lp, const void* rp) { +#elif defined(_GNU_SOURCE) +static int COVER_strict_cmp8(const void *lp, const void *rp, void *g_coverCtx) { +#else /* C90 fallback.*/ +static int COVER_strict_cmp8(const void *lp, const void *rp) { +#endif + int result = COVER_cmp8((COVER_ctx_t*)g_coverCtx, lp, rp); if (result == 0) { result = lp < rp ? -1 : 1; } return result; } +/** + * Abstract away divergence of qsort_r() parameters. + * Hopefully when C11 become the norm, we will be able + * to clean it up. + */ +static void stableSort(COVER_ctx_t *ctx) { +#if defined(__APPLE__) + qsort_r(ctx->suffix, ctx->suffixSize, sizeof(U32), + ctx, + (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp)); +#elif defined(_GNU_SOURCE) + qsort_r(ctx->suffix, ctx->suffixSize, sizeof(U32), + (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp), + ctx); +#elif defined(_WIN32) && defined(_MSC_VER) + qsort_s(ctx->suffix, ctx->suffixSize, sizeof(U32), + (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp), + ctx); +#elif defined(__OpenBSD__) + g_coverCtx = ctx; + mergesort(ctx->suffix, ctx->suffixSize, sizeof(U32), + (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp)); +#else /* C90 fallback.*/ + g_coverCtx = ctx; + /* TODO(cavalcanti): implement a reentrant qsort() when is not available. */ + qsort(ctx->suffix, ctx->suffixSize, sizeof(U32), + (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp)); +#endif +} + /** * Returns the first pointer in [first, last) whose element does not compare * less than value. If no such element exists it returns last. */ -static const size_t *COVER_lower_bound(const size_t *first, const size_t *last, +static const size_t *COVER_lower_bound(const size_t* first, const size_t* last, size_t value) { - size_t count = last - first; + size_t count = (size_t)(last - first); + assert(last >= first); while (count != 0) { size_t step = count / 2; const size_t *ptr = first; @@ -34129,14 +44474,15 @@ static void COVER_ctx_destroy(COVER_ctx_t *ctx) { /** * Prepare a context for dictionary building. - * The context is only dependent on the parameter `d` and can used multiple + * The context is only dependent on the parameter `d` and can be used multiple * times. * Returns 0 on success or error code on error. * The context must be destroyed with `COVER_ctx_destroy()`. */ static size_t COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples, - unsigned d, double splitPoint) { + unsigned d, double splitPoint) +{ const BYTE *const samples = (const BYTE *)samplesBuffer; const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples); /* Split samples into testing and training sets */ @@ -34205,17 +44551,7 @@ static size_t COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer, for (i = 0; i < ctx->suffixSize; ++i) { ctx->suffix[i] = i; } - /* qsort doesn't take an opaque pointer, so pass as a global. - * On OpenBSD qsort() is not guaranteed to be stable, their mergesort() is. - */ - g_coverCtx = ctx; -#if defined(__OpenBSD__) - mergesort(ctx->suffix, ctx->suffixSize, sizeof(U32), - (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp)); -#else - qsort(ctx->suffix, ctx->suffixSize, sizeof(U32), - (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp)); -#endif + stableSort(ctx); } DISPLAYLEVEL(2, "Computing frequencies\n"); /* For each dmer group (group of positions with the same first d bytes): @@ -34234,7 +44570,7 @@ static size_t COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer, void COVER_warnOnSmallCorpus(size_t maxDictSize, size_t nbDmers, int displayLevel) { - const double ratio = (double)nbDmers / maxDictSize; + const double ratio = (double)nbDmers / (double)maxDictSize; if (ratio >= 10) { return; } @@ -34320,7 +44656,7 @@ static size_t COVER_buildDictionary(const COVER_ctx_t *ctx, U32 *freqs, return tail; } -ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover( +ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_cover( void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples, ZDICT_cover_params_t parameters) @@ -34330,7 +44666,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover( COVER_map_t activeDmers; parameters.splitPoint = 1.0; /* Initialize global data */ - g_displayLevel = parameters.zParams.notificationLevel; + g_displayLevel = (int)parameters.zParams.notificationLevel; /* Checks */ if (!COVER_checkParameters(parameters, dictBufferCapacity)) { DISPLAYLEVEL(1, "Cover parameters incorrect\n"); @@ -34494,8 +44830,10 @@ void COVER_best_start(COVER_best_t *best) { * Decrements liveJobs and signals any waiting threads if liveJobs == 0. * If this dictionary is the best so far save it and its parameters. */ -void COVER_best_finish(COVER_best_t *best, ZDICT_cover_params_t parameters, - COVER_dictSelection_t selection) { +void COVER_best_finish(COVER_best_t* best, + ZDICT_cover_params_t parameters, + COVER_dictSelection_t selection) +{ void* dict = selection.dictContent; size_t compressedSize = selection.totalCompressedSize; size_t dictSize = selection.dictSize; @@ -34538,9 +44876,17 @@ void COVER_best_finish(COVER_best_t *best, ZDICT_cover_params_t parameters, } } +static COVER_dictSelection_t setDictSelection(BYTE* buf, size_t s, size_t csz) +{ + COVER_dictSelection_t ds; + ds.dictContent = buf; + ds.dictSize = s; + ds.totalCompressedSize = csz; + return ds; +} + COVER_dictSelection_t COVER_dictSelectionError(size_t error) { - COVER_dictSelection_t selection = { NULL, 0, error }; - return selection; + return setDictSelection(NULL, 0, error); } unsigned COVER_dictSelectionIsError(COVER_dictSelection_t selection) { @@ -34559,8 +44905,8 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBuffe size_t largestCompressed = 0; BYTE* customDictContentEnd = customDictContent + dictContentSize; - BYTE * largestDictbuffer = (BYTE *)malloc(dictBufferCapacity); - BYTE * candidateDictBuffer = (BYTE *)malloc(dictBufferCapacity); + BYTE* largestDictbuffer = (BYTE*)malloc(dictBufferCapacity); + BYTE* candidateDictBuffer = (BYTE*)malloc(dictBufferCapacity); double regressionTolerance = ((double)params.shrinkDictMaxRegression / 100.0) + 1.00; if (!largestDictbuffer || !candidateDictBuffer) { @@ -34593,9 +44939,8 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBuffe } if (params.shrinkDict == 0) { - COVER_dictSelection_t selection = { largestDictbuffer, dictContentSize, totalCompressedSize }; free(candidateDictBuffer); - return selection; + return setDictSelection(largestDictbuffer, dictContentSize, totalCompressedSize); } largestDict = dictContentSize; @@ -34627,20 +44972,16 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBuffe return COVER_dictSelectionError(totalCompressedSize); } - if (totalCompressedSize <= largestCompressed * regressionTolerance) { - COVER_dictSelection_t selection = { candidateDictBuffer, dictContentSize, totalCompressedSize }; + if ((double)totalCompressedSize <= (double)largestCompressed * regressionTolerance) { free(largestDictbuffer); - return selection; + return setDictSelection( candidateDictBuffer, dictContentSize, totalCompressedSize ); } dictContentSize *= 2; } dictContentSize = largestDict; totalCompressedSize = largestCompressed; - { - COVER_dictSelection_t selection = { largestDictbuffer, dictContentSize, totalCompressedSize }; - free(candidateDictBuffer); - return selection; - } + free(candidateDictBuffer); + return setDictSelection( largestDictbuffer, dictContentSize, totalCompressedSize ); } /** @@ -34703,7 +45044,7 @@ static void COVER_tryParameters(void *opaque) free(freqs); } -ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover( +ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_cover( void* dictBuffer, size_t dictBufferCapacity, const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, ZDICT_cover_params_t* parameters) @@ -34913,11 +45254,6 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover( #ifndef _DIVSUFSORT_H #define _DIVSUFSORT_H 1 -#ifdef __cplusplus -extern "C" { -#endif /* __cplusplus */ - - /*- Prototypes -*/ /** @@ -34945,11 +45281,6 @@ divsufsort(const unsigned char *T, int *SA, int n, int openMP); int divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char * num_indexes, int * indexes, int openMP); - -#ifdef __cplusplus -} /* extern "C" */ -#endif /* __cplusplus */ - #endif /* _DIVSUFSORT_H */ /**** ended inlining divsufsort.h ****/ @@ -36825,7 +47156,7 @@ divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char * /**** ended inlining dictBuilder/divsufsort.c ****/ /**** start inlining dictBuilder/fastcover.c ****/ /* - * Copyright (c) 2018-2021, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -36842,21 +47173,29 @@ divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char * #include /* memset */ #include /* clock */ +#ifndef ZDICT_STATIC_LINKING_ONLY +# define ZDICT_STATIC_LINKING_ONLY +#endif + /**** skipping file: ../common/mem.h ****/ /**** skipping file: ../common/pool.h ****/ /**** skipping file: ../common/threading.h ****/ -/**** skipping file: cover.h ****/ /**** skipping file: ../common/zstd_internal.h ****/ /**** skipping file: ../compress/zstd_compress_internal.h ****/ -#ifndef ZDICT_STATIC_LINKING_ONLY -#define ZDICT_STATIC_LINKING_ONLY -#endif -/**** skipping file: zdict.h ****/ +/**** skipping file: ../zdict.h ****/ +/**** skipping file: cover.h ****/ /*-************************************* * Constants ***************************************/ +/** +* There are 32bit indexes used to ref samples, so limit samples size to 4GB +* on 64bit builds. +* For 32bit builds we choose 1 GB. +* Most 32bit platforms have 2GB user-mode addressable space and we allocate a large +* contiguous buffer, so 1GB is already a high limit. +*/ #define FASTCOVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((unsigned)-1) : ((unsigned)1 GB)) #define FASTCOVER_MAX_F 31 #define FASTCOVER_MAX_ACCEL 10 @@ -36869,7 +47208,7 @@ divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char * * Console display ***************************************/ #ifndef LOCALDISPLAYLEVEL -static int g_displayLevel = 2; +static int g_displayLevel = 0; #endif #undef DISPLAY #define DISPLAY(...) \ @@ -37122,7 +47461,7 @@ FASTCOVER_computeFrequency(U32* freqs, const FASTCOVER_ctx_t* ctx) /** * Prepare a context for dictionary building. - * The context is only dependent on the parameter `d` and can used multiple + * The context is only dependent on the parameter `d` and can be used multiple * times. * Returns 0 on success or error code on error. * The context must be destroyed with `FASTCOVER_ctx_destroy()`. @@ -37363,7 +47702,7 @@ FASTCOVER_convertToFastCoverParams(ZDICT_cover_params_t coverParams, } -ZDICTLIB_API size_t +ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_fastCover(void* dictBuffer, size_t dictBufferCapacity, const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, @@ -37374,7 +47713,7 @@ ZDICT_trainFromBuffer_fastCover(void* dictBuffer, size_t dictBufferCapacity, ZDICT_cover_params_t coverParams; FASTCOVER_accel_t accelParams; /* Initialize global data */ - g_displayLevel = parameters.zParams.notificationLevel; + g_displayLevel = (int)parameters.zParams.notificationLevel; /* Assign splitPoint and f if not provided */ parameters.splitPoint = 1.0; parameters.f = parameters.f == 0 ? DEFAULT_F : parameters.f; @@ -37432,7 +47771,7 @@ ZDICT_trainFromBuffer_fastCover(void* dictBuffer, size_t dictBufferCapacity, } -ZDICTLIB_API size_t +ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_fastCover( void* dictBuffer, size_t dictBufferCapacity, const void* samplesBuffer, @@ -37457,7 +47796,7 @@ ZDICT_optimizeTrainFromBuffer_fastCover( const unsigned accel = parameters->accel == 0 ? DEFAULT_ACCEL : parameters->accel; const unsigned shrinkDict = 0; /* Local variables */ - const int displayLevel = parameters->zParams.notificationLevel; + const int displayLevel = (int)parameters->zParams.notificationLevel; unsigned iteration = 1; unsigned d; unsigned k; @@ -37541,7 +47880,7 @@ ZDICT_optimizeTrainFromBuffer_fastCover( data->parameters.splitPoint = splitPoint; data->parameters.steps = kSteps; data->parameters.shrinkDict = shrinkDict; - data->parameters.zParams.notificationLevel = g_displayLevel; + data->parameters.zParams.notificationLevel = (unsigned)g_displayLevel; /* Check the parameters */ if (!FASTCOVER_checkParameters(data->parameters, dictBufferCapacity, data->ctx->f, accel)) { @@ -37585,7 +47924,7 @@ ZDICT_optimizeTrainFromBuffer_fastCover( /**** ended inlining dictBuilder/fastcover.c ****/ /**** start inlining dictBuilder/zdict.c ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -37627,18 +47966,19 @@ ZDICT_optimizeTrainFromBuffer_fastCover( #include /* fprintf, fopen, ftello64 */ #include /* clock */ +#ifndef ZDICT_STATIC_LINKING_ONLY +# define ZDICT_STATIC_LINKING_ONLY +#endif + /**** skipping file: ../common/mem.h ****/ /**** skipping file: ../common/fse.h ****/ -#define HUF_STATIC_LINKING_ONLY /**** skipping file: ../common/huf.h ****/ /**** skipping file: ../common/zstd_internal.h ****/ /**** skipping file: ../common/xxhash.h ****/ -/**** skipping file: divsufsort.h ****/ -#ifndef ZDICT_STATIC_LINKING_ONLY -# define ZDICT_STATIC_LINKING_ONLY -#endif -/**** skipping file: zdict.h ****/ /**** skipping file: ../compress/zstd_compress_internal.h ****/ +/**** skipping file: ../zdict.h ****/ +/**** skipping file: divsufsort.h ****/ +/**** skipping file: ../common/bits.h ****/ /*-************************************* @@ -37659,9 +47999,9 @@ static const U32 g_selectivity_default = 9; * Console display ***************************************/ #undef DISPLAY -#define DISPLAY(...) { fprintf(stderr, __VA_ARGS__); fflush( stderr ); } +#define DISPLAY(...) do { fprintf(stderr, __VA_ARGS__); fflush( stderr ); } while (0) #undef DISPLAYLEVEL -#define DISPLAYLEVEL(l, ...) if (notificationLevel>=l) { DISPLAY(__VA_ARGS__); } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */ +#define DISPLAYLEVEL(l, ...) do { if (notificationLevel>=l) { DISPLAY(__VA_ARGS__); } } while (0) /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */ static clock_t ZDICT_clockSpan(clock_t nPrevious) { return clock() - nPrevious; } @@ -37715,65 +48055,6 @@ size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictSize) /*-******************************************************** * Dictionary training functions **********************************************************/ -static unsigned ZDICT_NbCommonBytes (size_t val) -{ - if (MEM_isLittleEndian()) { - if (MEM_64bits()) { -# if defined(_MSC_VER) && defined(_WIN64) - unsigned long r = 0; - _BitScanForward64( &r, (U64)val ); - return (unsigned)(r>>3); -# elif defined(__GNUC__) && (__GNUC__ >= 3) - return (__builtin_ctzll((U64)val) >> 3); -# else - static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; - return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; -# endif - } else { /* 32 bits */ -# if defined(_MSC_VER) - unsigned long r=0; - _BitScanForward( &r, (U32)val ); - return (unsigned)(r>>3); -# elif defined(__GNUC__) && (__GNUC__ >= 3) - return (__builtin_ctz((U32)val) >> 3); -# else - static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; - return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; -# endif - } - } else { /* Big Endian CPU */ - if (MEM_64bits()) { -# if defined(_MSC_VER) && defined(_WIN64) - unsigned long r = 0; - _BitScanReverse64( &r, val ); - return (unsigned)(r>>3); -# elif defined(__GNUC__) && (__GNUC__ >= 3) - return (__builtin_clzll(val) >> 3); -# else - unsigned r; - const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */ - if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; } - if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } - r += (!val); - return r; -# endif - } else { /* 32 bits */ -# if defined(_MSC_VER) - unsigned long r = 0; - _BitScanReverse( &r, (unsigned long)val ); - return (unsigned)(r>>3); -# elif defined(__GNUC__) && (__GNUC__ >= 3) - return (__builtin_clz((U32)val) >> 3); -# else - unsigned r; - if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } - r += (!val); - return r; -# endif - } } -} - - /*! ZDICT_count() : Count the nb of common bytes between 2 pointers. Note : this function presumes end of buffer followed by noisy guard band. @@ -37788,7 +48069,7 @@ static size_t ZDICT_count(const void* pIn, const void* pMatch) pMatch = (const char*)pMatch+sizeof(size_t); continue; } - pIn = (const char*)pIn+ZDICT_NbCommonBytes(diff); + pIn = (const char*)pIn+ZSTD_NbCommonBytes(diff); return (size_t)((const char*)pIn - pStart); } } @@ -37820,7 +48101,7 @@ static dictItem ZDICT_analyzePos( U32 savings[LLIMIT] = {0}; const BYTE* b = (const BYTE*)buffer; size_t maxLength = LLIMIT; - size_t pos = suffix[start]; + size_t pos = (size_t)suffix[start]; U32 end = start; dictItem solution; @@ -37954,7 +48235,7 @@ static dictItem ZDICT_analyzePos( savings[i] = savings[i-1] + (lengthList[i] * (i-3)); DISPLAYLEVEL(4, "Selected dict at position %u, of length %u : saves %u (ratio: %.2f) \n", - (unsigned)pos, (unsigned)maxLength, (unsigned)savings[maxLength], (double)savings[maxLength] / maxLength); + (unsigned)pos, (unsigned)maxLength, (unsigned)savings[maxLength], (double)savings[maxLength] / (double)maxLength); solution.pos = (U32)pos; solution.length = (U32)maxLength; @@ -37964,7 +48245,7 @@ static dictItem ZDICT_analyzePos( { U32 id; for (id=start; id1) && (table[u-1].savings < elt.savings)) - table[u] = table[u-1], u--; + table[u] = table[u-1], u--; table[u] = elt; return u; } } @@ -38027,7 +48308,7 @@ static U32 ZDICT_tryMerge(dictItem* table, dictItem elt, U32 eltNbToSkip, const if ((table[u].pos + table[u].length >= elt.pos) && (table[u].pos < elt.pos)) { /* overlap, existing < new */ /* append */ - int const addedLength = (int)eltEnd - (table[u].pos + table[u].length); + int const addedLength = (int)eltEnd - (int)(table[u].pos + table[u].length); table[u].savings += elt.length / 8; /* rough approx bonus */ if (addedLength > 0) { /* otherwise, elt fully included into existing */ table[u].length += addedLength; @@ -38121,10 +48402,16 @@ static size_t ZDICT_trainBuffer_legacy(dictItem* dictList, U32 dictListSize, clock_t const refreshRate = CLOCKS_PER_SEC * 3 / 10; # undef DISPLAYUPDATE -# define DISPLAYUPDATE(l, ...) if (notificationLevel>=l) { \ - if (ZDICT_clockSpan(displayClock) > refreshRate) \ - { displayClock = clock(); DISPLAY(__VA_ARGS__); \ - if (notificationLevel>=4) fflush(stderr); } } +# define DISPLAYUPDATE(l, ...) \ + do { \ + if (notificationLevel>=l) { \ + if (ZDICT_clockSpan(displayClock) > refreshRate) { \ + displayClock = clock(); \ + DISPLAY(__VA_ARGS__); \ + } \ + if (notificationLevel>=4) fflush(stderr); \ + } \ + } while (0) /* init */ DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */ @@ -38167,7 +48454,7 @@ static size_t ZDICT_trainBuffer_legacy(dictItem* dictList, U32 dictListSize, if (solution.length==0) { cursor++; continue; } ZDICT_insertDictItem(dictList, dictListSize, solution, buffer); cursor += solution.length; - DISPLAYUPDATE(2, "\r%4.2f %% \r", (double)cursor / bufferSize * 100); + DISPLAYUPDATE(2, "\r%4.2f %% \r", (double)cursor / (double)bufferSize * 100.0); } } _cleanup: @@ -38210,15 +48497,15 @@ static void ZDICT_countEStats(EStats_ress_t esr, const ZSTD_parameters* params, size_t cSize; if (srcSize > blockSizeMax) srcSize = blockSizeMax; /* protection vs large samples */ - { size_t const errorCode = ZSTD_compressBegin_usingCDict(esr.zc, esr.dict); + { size_t const errorCode = ZSTD_compressBegin_usingCDict_deprecated(esr.zc, esr.dict); if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_compressBegin_usingCDict failed \n"); return; } } - cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize); + cSize = ZSTD_compressBlock_deprecated(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize); if (ZSTD_isError(cSize)) { DISPLAYLEVEL(3, "warning : could not compress sample size %u \n", (unsigned)srcSize); return; } if (cSize) { /* if == 0; block is not compressible */ - const seqStore_t* const seqStorePtr = ZSTD_getSeqStore(esr.zc); + const SeqStore_t* const seqStorePtr = ZSTD_getSeqStore(esr.zc); /* literals stats */ { const BYTE* bytePtr; @@ -38246,9 +48533,9 @@ static void ZDICT_countEStats(EStats_ress_t esr, const ZSTD_parameters* params, } if (nbSeq >= 2) { /* rep offsets */ - const seqDef* const seq = seqStorePtr->sequencesStart; - U32 offset1 = seq[0].offset - 3; - U32 offset2 = seq[1].offset - 3; + const SeqDef* const seq = seqStorePtr->sequencesStart; + U32 offset1 = seq[0].offBase - ZSTD_REP_NUM; + U32 offset2 = seq[1].offBase - ZSTD_REP_NUM; if (offset1 >= MAXREPOFFSET) offset1 = 0; if (offset2 >= MAXREPOFFSET) offset2 = 0; repOffsets[offset1] += 3; @@ -38319,6 +48606,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, size_t const totalSrcSize = ZDICT_totalSampleSize(fileSizes, nbFiles); size_t const averageSampleSize = totalSrcSize / (nbFiles + !nbFiles); BYTE* dstPtr = (BYTE*)dstBuffer; + U32 wksp[HUF_CTABLE_WORKSPACE_SIZE_U32]; /* init */ DEBUGLOG(4, "ZDICT_analyzeEntropy"); @@ -38351,8 +48639,15 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, pos += fileSizes[u]; } + if (notificationLevel >= 4) { + /* writeStats */ + DISPLAYLEVEL(4, "Offset Code Frequencies : \n"); + for (u=0; u<=offcodeMax; u++) { + DISPLAYLEVEL(4, "%2u :%7u \n", u, offcodeCount[u]); + } } + /* analyze, build stats, starting with literals */ - { size_t maxNbBits = HUF_buildCTable (hufTable, countLit, 255, huffLog); + { size_t maxNbBits = HUF_buildCTable_wksp(hufTable, countLit, 255, huffLog, wksp, sizeof(wksp)); if (HUF_isError(maxNbBits)) { eSize = maxNbBits; DISPLAYLEVEL(1, " HUF_buildCTable error \n"); @@ -38361,7 +48656,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, if (maxNbBits==8) { /* not compressible : will fail on HUF_writeCTable() */ DISPLAYLEVEL(2, "warning : pathological dataset : literals are not compressible : samples are noisy or too regular \n"); ZDICT_flatLit(countLit); /* replace distribution by a fake "mostly flat but still compressible" distribution, that HUF_writeCTable() can encode */ - maxNbBits = HUF_buildCTable (hufTable, countLit, 255, huffLog); + maxNbBits = HUF_buildCTable_wksp(hufTable, countLit, 255, huffLog, wksp, sizeof(wksp)); assert(maxNbBits==9); } huffLog = (U32)maxNbBits; @@ -38402,7 +48697,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, llLog = (U32)errorCode; /* write result to buffer */ - { size_t const hhSize = HUF_writeCTable(dstPtr, maxDstSize, hufTable, 255, huffLog); + { size_t const hhSize = HUF_writeCTable_wksp(dstPtr, maxDstSize, hufTable, 255, huffLog, wksp, sizeof(wksp)); if (HUF_isError(hhSize)) { eSize = hhSize; DISPLAYLEVEL(1, "HUF_writeCTable error \n"); @@ -38457,7 +48752,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, MEM_writeLE32(dstPtr+8, bestRepOffset[2].offset); #else /* at this stage, we don't use the result of "most common first offset", - as the impact of statistics is not properly evaluated */ + * as the impact of statistics is not properly evaluated */ MEM_writeLE32(dstPtr+0, repStartValue[0]); MEM_writeLE32(dstPtr+4, repStartValue[1]); MEM_writeLE32(dstPtr+8, repStartValue[2]); @@ -38473,6 +48768,17 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, } +/** + * @returns the maximum repcode value + */ +static U32 ZDICT_maxRep(U32 const reps[ZSTD_REP_NUM]) +{ + U32 maxRep = reps[0]; + int r; + for (r = 1; r < ZSTD_REP_NUM; ++r) + maxRep = MAX(maxRep, reps[r]); + return maxRep; +} size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity, const void* customDictContent, size_t dictContentSize, @@ -38484,11 +48790,13 @@ size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity, BYTE header[HBUFFSIZE]; int const compressionLevel = (params.compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : params.compressionLevel; U32 const notificationLevel = params.notificationLevel; + /* The final dictionary content must be at least as large as the largest repcode */ + size_t const minContentSize = (size_t)ZDICT_maxRep(repStartValue); + size_t paddingSize; /* check conditions */ DEBUGLOG(4, "ZDICT_finalizeDictionary"); if (dictBufferCapacity < dictContentSize) return ERROR(dstSize_tooSmall); - if (dictContentSize < ZDICT_CONTENTSIZE_MIN) return ERROR(srcSize_wrong); if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) return ERROR(dstSize_tooSmall); /* dictionary header */ @@ -38512,12 +48820,43 @@ size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity, hSize += eSize; } - /* copy elements in final buffer ; note : src and dst buffer can overlap */ - if (hSize + dictContentSize > dictBufferCapacity) dictContentSize = dictBufferCapacity - hSize; - { size_t const dictSize = hSize + dictContentSize; - char* dictEnd = (char*)dictBuffer + dictSize; - memmove(dictEnd - dictContentSize, customDictContent, dictContentSize); - memcpy(dictBuffer, header, hSize); + /* Shrink the content size if it doesn't fit in the buffer */ + if (hSize + dictContentSize > dictBufferCapacity) { + dictContentSize = dictBufferCapacity - hSize; + } + + /* Pad the dictionary content with zeros if it is too small */ + if (dictContentSize < minContentSize) { + RETURN_ERROR_IF(hSize + minContentSize > dictBufferCapacity, dstSize_tooSmall, + "dictBufferCapacity too small to fit max repcode"); + paddingSize = minContentSize - dictContentSize; + } else { + paddingSize = 0; + } + + { + size_t const dictSize = hSize + paddingSize + dictContentSize; + + /* The dictionary consists of the header, optional padding, and the content. + * The padding comes before the content because the "best" position in the + * dictionary is the last byte. + */ + BYTE* const outDictHeader = (BYTE*)dictBuffer; + BYTE* const outDictPadding = outDictHeader + hSize; + BYTE* const outDictContent = outDictPadding + paddingSize; + + assert(dictSize <= dictBufferCapacity); + assert(outDictContent + dictContentSize == (BYTE*)dictBuffer + dictSize); + + /* First copy the customDictContent into its final location. + * `customDictContent` and `dictBuffer` may overlap, so we must + * do this before any other writes into the output buffer. + * Then copy the header & padding into the output buffer. + */ + memmove(outDictContent, customDictContent, dictContentSize); + memcpy(outDictHeader, header, hSize); + memset(outDictPadding, 0, paddingSize); + return dictSize; } } diff --git a/external/basis_universal/zstd/zstd.h b/external/basis_universal/zstd/zstd.h index 222339d71a..b8c0644a7e 100644 --- a/external/basis_universal/zstd/zstd.h +++ b/external/basis_universal/zstd/zstd.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -7,34 +7,73 @@ * in the COPYING file in the root directory of this source tree). * You may select, at your option, one of the above-listed licenses. */ -#if defined (__cplusplus) -extern "C" { -#endif #ifndef ZSTD_H_235446 #define ZSTD_H_235446 -/* ====== Dependency ======*/ -#include /* INT_MAX */ + +/* ====== Dependencies ======*/ #include /* size_t */ +#include "zstd_errors.h" /* list of errors */ +#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY) +#include /* INT_MAX */ +#endif /* ZSTD_STATIC_LINKING_ONLY */ + +#if defined (__cplusplus) +extern "C" { +#endif /* ===== ZSTDLIB_API : control library symbols visibility ===== */ -#ifndef ZSTDLIB_VISIBILITY -# if defined(__GNUC__) && (__GNUC__ >= 4) -# define ZSTDLIB_VISIBILITY __attribute__ ((visibility ("default"))) +#ifndef ZSTDLIB_VISIBLE + /* Backwards compatibility with old macro name */ +# ifdef ZSTDLIB_VISIBILITY +# define ZSTDLIB_VISIBLE ZSTDLIB_VISIBILITY +# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDLIB_VISIBLE __attribute__ ((visibility ("default"))) # else -# define ZSTDLIB_VISIBILITY +# define ZSTDLIB_VISIBLE # endif #endif + +#ifndef ZSTDLIB_HIDDEN +# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDLIB_HIDDEN __attribute__ ((visibility ("hidden"))) +# else +# define ZSTDLIB_HIDDEN +# endif +#endif + #if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) -# define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBILITY +# define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBLE #elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) -# define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +# define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ #else -# define ZSTDLIB_API ZSTDLIB_VISIBILITY +# define ZSTDLIB_API ZSTDLIB_VISIBLE #endif +/* Deprecation warnings : + * Should these warnings be a problem, it is generally possible to disable them, + * typically with -Wno-deprecated-declarations for gcc or _CRT_SECURE_NO_WARNINGS in Visual. + * Otherwise, it's also possible to define ZSTD_DISABLE_DEPRECATE_WARNINGS. + */ +#ifdef ZSTD_DISABLE_DEPRECATE_WARNINGS +# define ZSTD_DEPRECATED(message) /* disable deprecation warnings */ +#else +# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ +# define ZSTD_DEPRECATED(message) [[deprecated(message)]] +# elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__) || defined(__IAR_SYSTEMS_ICC__) +# define ZSTD_DEPRECATED(message) __attribute__((deprecated(message))) +# elif defined(__GNUC__) && (__GNUC__ >= 3) +# define ZSTD_DEPRECATED(message) __attribute__((deprecated)) +# elif defined(_MSC_VER) +# define ZSTD_DEPRECATED(message) __declspec(deprecated(message)) +# else +# pragma message("WARNING: You need to implement ZSTD_DEPRECATED for this compiler") +# define ZSTD_DEPRECATED(message) +# endif +#endif /* ZSTD_DISABLE_DEPRECATE_WARNINGS */ + /******************************************************************************* Introduction @@ -71,8 +110,8 @@ extern "C" { /*------ Version ------*/ #define ZSTD_VERSION_MAJOR 1 -#define ZSTD_VERSION_MINOR 4 -#define ZSTD_VERSION_RELEASE 9 +#define ZSTD_VERSION_MINOR 5 +#define ZSTD_VERSION_RELEASE 7 #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) /*! ZSTD_versionNumber() : @@ -109,13 +148,13 @@ ZSTDLIB_API const char* ZSTD_versionString(void); #define ZSTD_BLOCKSIZE_MAX (1<= `ZSTD_compressBound(srcSize)`. + * NOTE: Providing `dstCapacity >= ZSTD_compressBound(srcSize)` guarantees that zstd will have + * enough space to successfully compress the data. * @return : compressed size written into `dst` (<= `dstCapacity), * or an error code if it fails (which can be tested using ZSTD_isError()). */ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, @@ -123,65 +162,106 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, int compressionLevel); /*! ZSTD_decompress() : - * `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames. - * `dstCapacity` is an upper bound of originalSize to regenerate. - * If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data. - * @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), - * or an errorCode if it fails (which can be tested using ZSTD_isError()). */ + * `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames. + * Multiple compressed frames can be decompressed at once with this method. + * The result will be the concatenation of all decompressed frames, back to back. + * `dstCapacity` is an upper bound of originalSize to regenerate. + * First frame's decompressed size can be extracted using ZSTD_getFrameContentSize(). + * If maximum upper bound isn't known, prefer using streaming mode to decompress data. + * @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), + * or an errorCode if it fails (which can be tested using ZSTD_isError()). */ ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, const void* src, size_t compressedSize); + +/*====== Decompression helper functions ======*/ + /*! ZSTD_getFrameContentSize() : requires v1.3.0+ - * `src` should point to the start of a ZSTD encoded frame. - * `srcSize` must be at least as large as the frame header. - * hint : any size >= `ZSTD_frameHeaderSize_max` is large enough. - * @return : - decompressed size of `src` frame content, if known - * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined - * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) - * note 1 : a 0 return value means the frame is valid but "empty". - * note 2 : decompressed size is an optional field, it may not be present, typically in streaming mode. - * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. - * In which case, it's necessary to use streaming mode to decompress data. - * Optionally, application can rely on some implicit limit, - * as ZSTD_decompress() only needs an upper bound of decompressed size. - * (For example, data could be necessarily cut into blocks <= 16 KB). - * note 3 : decompressed size is always present when compression is completed using single-pass functions, - * such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict(). - * note 4 : decompressed size can be very large (64-bits value), - * potentially larger than what local system can handle as a single memory segment. - * In which case, it's necessary to use streaming mode to decompress data. - * note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified. - * Always ensure return value fits within application's authorized limits. - * Each application can set its own limits. - * note 6 : This function replaces ZSTD_getDecompressedSize() */ + * `src` should point to the start of a ZSTD encoded frame. + * `srcSize` must be at least as large as the frame header. + * hint : any size >= `ZSTD_frameHeaderSize_max` is large enough. + * @return : - decompressed size of `src` frame content, if known + * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined + * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) + * note 1 : a 0 return value means the frame is valid but "empty". + * When invoking this method on a skippable frame, it will return 0. + * note 2 : decompressed size is an optional field, it may not be present (typically in streaming mode). + * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. + * In which case, it's necessary to use streaming mode to decompress data. + * Optionally, application can rely on some implicit limit, + * as ZSTD_decompress() only needs an upper bound of decompressed size. + * (For example, data could be necessarily cut into blocks <= 16 KB). + * note 3 : decompressed size is always present when compression is completed using single-pass functions, + * such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict(). + * note 4 : decompressed size can be very large (64-bits value), + * potentially larger than what local system can handle as a single memory segment. + * In which case, it's necessary to use streaming mode to decompress data. + * note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified. + * Always ensure return value fits within application's authorized limits. + * Each application can set its own limits. + * note 6 : This function replaces ZSTD_getDecompressedSize() */ #define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1) #define ZSTD_CONTENTSIZE_ERROR (0ULL - 2) ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); -/*! ZSTD_getDecompressedSize() : - * NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize(). +/*! ZSTD_getDecompressedSize() (obsolete): + * This function is now obsolete, in favor of ZSTD_getFrameContentSize(). * Both functions work the same way, but ZSTD_getDecompressedSize() blends * "empty", "unknown" and "error" results to the same return value (0), * while ZSTD_getFrameContentSize() gives them separate return values. * @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */ +ZSTD_DEPRECATED("Replaced by ZSTD_getFrameContentSize") ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); -/*! ZSTD_findFrameCompressedSize() : +/*! ZSTD_findFrameCompressedSize() : Requires v1.4.0+ * `src` should point to the start of a ZSTD frame or skippable frame. * `srcSize` must be >= first frame size * @return : the compressed size of the first frame starting at `src`, * suitable to pass as `srcSize` to `ZSTD_decompress` or similar, - * or an error code if input is invalid */ + * or an error code if input is invalid + * Note 1: this method is called _find*() because it's not enough to read the header, + * it may have to scan through the frame's content, to reach its end. + * Note 2: this method also works with Skippable Frames. In which case, + * it returns the size of the complete skippable frame, + * which is always equal to its content size + 8 bytes for headers. */ ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize); -/*====== Helper functions ======*/ -#define ZSTD_COMPRESSBOUND(srcSize) ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */ -ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */ -ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ -ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */ -ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed */ -ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */ +/*====== Compression helper functions ======*/ + +/*! ZSTD_compressBound() : + * maximum compressed size in worst case single-pass scenario. + * When invoking `ZSTD_compress()`, or any other one-pass compression function, + * it's recommended to provide @dstCapacity >= ZSTD_compressBound(srcSize) + * as it eliminates one potential failure scenario, + * aka not enough room in dst buffer to write the compressed frame. + * Note : ZSTD_compressBound() itself can fail, if @srcSize >= ZSTD_MAX_INPUT_SIZE . + * In which case, ZSTD_compressBound() will return an error code + * which can be tested using ZSTD_isError(). + * + * ZSTD_COMPRESSBOUND() : + * same as ZSTD_compressBound(), but as a macro. + * It can be used to produce constants, which can be useful for static allocation, + * for example to size a static array on stack. + * Will produce constant value 0 if srcSize is too large. + */ +#define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00ULL : 0xFF00FF00U) +#define ZSTD_COMPRESSBOUND(srcSize) (((size_t)(srcSize) >= ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */ +ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */ + + +/*====== Error helper functions ======*/ +/* ZSTD_isError() : + * Most ZSTD_* functions returning a size_t value can be tested for error, + * using ZSTD_isError(). + * @return 1 if error, 0 otherwise + */ +ZSTDLIB_API unsigned ZSTD_isError(size_t result); /*!< tells if a `size_t` function result is an error code */ +ZSTDLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); /* convert a result into an error code, which can be compared to error enum list */ +ZSTDLIB_API const char* ZSTD_getErrorName(size_t result); /*!< provides readable string from a function result */ +ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed, requires v1.4.0+ */ +ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */ +ZSTDLIB_API int ZSTD_defaultCLevel(void); /*!< default compression level, specified by ZSTD_CLEVEL_DEFAULT, requires v1.5.0+ */ /*************************************** @@ -189,25 +269,25 @@ ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compres ***************************************/ /*= Compression context * When compressing many times, - * it is recommended to allocate a context just once, - * and re-use it for each successive compression operation. - * This will make workload friendlier for system's memory. + * it is recommended to allocate a compression context just once, + * and reuse it for each successive compression operation. + * This will make the workload easier for system's memory. * Note : re-using context is just a speed / resource optimization. * It doesn't change the compression ratio, which remains identical. - * Note 2 : In multi-threaded environments, - * use one different context per thread for parallel execution. + * Note 2: For parallel execution in multi-threaded environments, + * use one different context per thread . */ typedef struct ZSTD_CCtx_s ZSTD_CCtx; ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); -ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); +ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); /* compatible with NULL pointer */ /*! ZSTD_compressCCtx() : * Same as ZSTD_compress(), using an explicit ZSTD_CCtx. - * Important : in order to behave similarly to `ZSTD_compress()`, - * this function compresses at requested compression level, - * __ignoring any other parameter__ . + * Important : in order to mirror `ZSTD_compress()` behavior, + * this function compresses at the requested compression level, + * __ignoring any other advanced parameter__ . * If any advanced parameter was set using the advanced API, - * they will all be reset. Only `compressionLevel` remains. + * they will all be reset. Only @compressionLevel remains. */ ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, @@ -217,38 +297,38 @@ ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, /*= Decompression context * When decompressing many times, * it is recommended to allocate a context only once, - * and re-use it for each successive compression operation. + * and reuse it for each successive compression operation. * This will make workload friendlier for system's memory. * Use one context per thread for parallel execution. */ typedef struct ZSTD_DCtx_s ZSTD_DCtx; ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void); -ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); /* accept NULL pointer */ /*! ZSTD_decompressDCtx() : * Same as ZSTD_decompress(), * requires an allocated ZSTD_DCtx. - * Compatible with sticky parameters. + * Compatible with sticky parameters (see below). */ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -/*************************************** -* Advanced compression API -***************************************/ +/********************************************* +* Advanced compression API (Requires v1.4.0+) +**********************************************/ /* API design : * Parameters are pushed one by one into an existing context, * using ZSTD_CCtx_set*() functions. * Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame. * "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` ! - * __They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx()__ . + * __They do not apply to one-shot variants such as ZSTD_compressCCtx()__ . * * It's possible to reset all parameters to "default" using ZSTD_CCtx_reset(). * - * This API supercedes all other "advanced" API entry points in the experimental section. - * In the future, we expect to remove from experimental API entry points which are redundant with this API. + * This API supersedes all other "advanced" API entry points in the experimental section. + * In the future, we expect to remove API entry points from experimental which are redundant with this API. */ @@ -266,7 +346,6 @@ typedef enum { ZSTD_fast=1, Only the order (from fast to strong) is guaranteed */ } ZSTD_strategy; - typedef enum { /* compression parameters @@ -333,6 +412,18 @@ typedef enum { * resulting in stronger and slower compression. * Special: value 0 means "use default strategy". */ + ZSTD_c_targetCBlockSize=130, /* v1.5.6+ + * Attempts to fit compressed block size into approximately targetCBlockSize. + * Bound by ZSTD_TARGETCBLOCKSIZE_MIN and ZSTD_TARGETCBLOCKSIZE_MAX. + * Note that it's not a guarantee, just a convergence target (default:0). + * No target when targetCBlockSize == 0. + * This is helpful in low bandwidth streaming environments to improve end-to-end latency, + * when a client can make use of partial documents (a prominent example being Chrome). + * Note: this parameter is stable since v1.5.6. + * It was present as an experimental parameter in earlier versions, + * but it's not recommended using it with earlier library versions + * due to massive performance regressions. + */ /* LDM mode parameters */ ZSTD_c_enableLongDistanceMatching=160, /* Enable long distance matching. * This parameter is designed to improve compression ratio @@ -389,7 +480,7 @@ typedef enum { ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1. * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads. * 0 means default, which is dynamically determined based on compression parameters. - * Job size must be a minimum of overlap size, or 1 MB, whichever is largest. + * Job size must be a minimum of overlap size, or ZSTDMT_JOBSIZE_MIN (= 512 KB), whichever is largest. * The minimum size is automatically and transparently enforced. */ ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size. * The overlap size is an amount of data reloaded from previous job at the beginning of a new job. @@ -412,13 +503,18 @@ typedef enum { * ZSTD_c_forceMaxWindow * ZSTD_c_forceAttachDict * ZSTD_c_literalCompressionMode - * ZSTD_c_targetCBlockSize * ZSTD_c_srcSizeHint * ZSTD_c_enableDedicatedDictSearch * ZSTD_c_stableInBuffer * ZSTD_c_stableOutBuffer * ZSTD_c_blockDelimiters * ZSTD_c_validateSequences + * ZSTD_c_blockSplitterLevel + * ZSTD_c_splitAfterSequences + * ZSTD_c_useRowMatchFinder + * ZSTD_c_prefetchCDictTables + * ZSTD_c_enableSeqProducerFallback + * ZSTD_c_maxBlockSize * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. * note : never ever use experimentalParam? names directly; * also, the enums values themselves are unstable and can still change. @@ -428,13 +524,21 @@ typedef enum { ZSTD_c_experimentalParam3=1000, ZSTD_c_experimentalParam4=1001, ZSTD_c_experimentalParam5=1002, - ZSTD_c_experimentalParam6=1003, + /* was ZSTD_c_experimentalParam6=1003; is now ZSTD_c_targetCBlockSize */ ZSTD_c_experimentalParam7=1004, ZSTD_c_experimentalParam8=1005, ZSTD_c_experimentalParam9=1006, ZSTD_c_experimentalParam10=1007, ZSTD_c_experimentalParam11=1008, - ZSTD_c_experimentalParam12=1009 + ZSTD_c_experimentalParam12=1009, + ZSTD_c_experimentalParam13=1010, + ZSTD_c_experimentalParam14=1011, + ZSTD_c_experimentalParam15=1012, + ZSTD_c_experimentalParam16=1013, + ZSTD_c_experimentalParam17=1014, + ZSTD_c_experimentalParam18=1015, + ZSTD_c_experimentalParam19=1016, + ZSTD_c_experimentalParam20=1017 } ZSTD_cParameter; typedef struct { @@ -497,7 +601,7 @@ typedef enum { * They will be used to compress next frame. * Resetting session never fails. * - The parameters : changes all parameters back to "default". - * This removes any reference to any dictionary too. + * This also removes any reference to any dictionary or external sequence producer. * Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing) * otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError()) * - Both : similar to resetting the session, followed by resetting parameters. @@ -506,11 +610,13 @@ ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset); /*! ZSTD_compress2() : * Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API. + * (note that this entry point doesn't even expose a compression level parameter). * ZSTD_compress2() always starts a new frame. * Should cctx hold data from a previously unfinished frame, everything about it is forgotten. * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() * - The function is always blocking, returns when compression is completed. - * Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`. + * NOTE: Providing `dstCapacity >= ZSTD_compressBound(srcSize)` guarantees that zstd will have + * enough space to successfully compress the data, though it is possible it fails for other reasons. * @return : compressed size written into `dst` (<= `dstCapacity), * or an error code if it fails (which can be tested using ZSTD_isError()). */ @@ -519,9 +625,9 @@ ZSTDLIB_API size_t ZSTD_compress2( ZSTD_CCtx* cctx, const void* src, size_t srcSize); -/*************************************** -* Advanced decompression API -***************************************/ +/*********************************************** +* Advanced decompression API (Requires v1.4.0+) +************************************************/ /* The advanced API pushes parameters one by one into an existing DCtx context. * Parameters are sticky, and remain valid for all following frames @@ -547,13 +653,17 @@ typedef enum { * ZSTD_d_stableOutBuffer * ZSTD_d_forceIgnoreChecksum * ZSTD_d_refMultipleDDicts + * ZSTD_d_disableHuffmanAssembly + * ZSTD_d_maxBlockSize * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. * note : never ever use experimentalParam? names directly */ ZSTD_d_experimentalParam1=1000, ZSTD_d_experimentalParam2=1001, ZSTD_d_experimentalParam3=1002, - ZSTD_d_experimentalParam4=1003 + ZSTD_d_experimentalParam4=1003, + ZSTD_d_experimentalParam5=1004, + ZSTD_d_experimentalParam6=1005 } ZSTD_dParameter; @@ -608,14 +718,14 @@ typedef struct ZSTD_outBuffer_s { * A ZSTD_CStream object is required to track streaming operation. * Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. * ZSTD_CStream objects can be reused multiple times on consecutive compression operations. -* It is recommended to re-use ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory. +* It is recommended to reuse ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory. * * For parallel execution, use one separate ZSTD_CStream per thread. * * note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing. * * Parameters are sticky : when starting a new compression on the same context, -* it will re-use the same sticky parameters as previous compression session. +* it will reuse the same sticky parameters as previous compression session. * When in doubt, it's recommended to fully initialize the context before usage. * Use ZSTD_CCtx_reset() to reset the context and ZSTD_CCtx_setParameter(), * ZSTD_CCtx_setPledgedSrcSize(), or ZSTD_CCtx_loadDictionary() and friends to @@ -667,7 +777,7 @@ typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are now effectively same /* Continue to distinguish them for compatibility with older versions <= v1.2.0 */ /*===== ZSTD_CStream management functions =====*/ ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void); -ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); +ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); /* accept NULL pointer */ /*===== Streaming compression functions =====*/ typedef enum { @@ -683,7 +793,7 @@ typedef enum { : note : multithreaded compression will block to flush as much output as possible. */ } ZSTD_EndDirective; -/*! ZSTD_compressStream2() : +/*! ZSTD_compressStream2() : Requires v1.4.0+ * Behaves about the same as ZSTD_compressStream, with additional control on end directive. * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() * - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode) @@ -704,6 +814,11 @@ typedef enum { * only ZSTD_e_end or ZSTD_e_flush operations are allowed. * Before starting a new compression job, or changing compression parameters, * it is required to fully flush internal buffers. + * - note: if an operation ends with an error, it may leave @cctx in an undefined state. + * Therefore, it's UB to invoke ZSTD_compressStream2() of ZSTD_compressStream() on such a state. + * In order to be re-employed after an error, a state must be reset, + * which can be done explicitly (ZSTD_CCtx_reset()), + * or is sometimes implied by methods starting a new compression job (ZSTD_initCStream(), ZSTD_compressCCtx()) */ ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, ZSTD_outBuffer* output, @@ -729,11 +844,9 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output /* ***************************************************************************** - * This following is a legacy streaming API. + * This following is a legacy streaming API, available since v1.0+ . * It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2(). * It is redundant, but remains fully supported. - * Advanced parameters and dictionary compression can only be used through the - * new API. ******************************************************************************/ /*! @@ -742,6 +855,9 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * + * Note that ZSTD_initCStream() clears any previously set dictionary. Use the new API + * to compress with a dictionary. */ ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); /*! @@ -762,7 +878,7 @@ ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); * * A ZSTD_DStream object is required to track streaming operations. * Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. -* ZSTD_DStream objects can be re-used multiple times. +* ZSTD_DStream objects can be re-employed multiple times. * * Use ZSTD_initDStream() to start a new decompression operation. * @return : recommended first input size @@ -772,33 +888,63 @@ ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); * The function will update both `pos` fields. * If `input.pos < input.size`, some input has not been consumed. * It's up to the caller to present again remaining data. +* * The function tries to flush all data decoded immediately, respecting output buffer size. * If `output.pos < output.size`, decoder has flushed everything it could. -* But if `output.pos == output.size`, there might be some data left within internal buffers., +* +* However, when `output.pos == output.size`, it's more difficult to know. +* If @return > 0, the frame is not complete, meaning +* either there is still some data left to flush within internal buffers, +* or there is more input to read to complete the frame (or both). * In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer. * Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX. * @return : 0 when a frame is completely decoded and fully flushed, * or an error code, which can be tested using ZSTD_isError(), * or any other value > 0, which means there is still some decoding or flushing to do to complete current frame : * the return value is a suggested next input size (just a hint for better latency) -* that will never request more than the remaining frame size. +* that will never request more than the remaining content of the compressed frame. * *******************************************************************************/ typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */ /* For compatibility with versions <= v1.2.0, prefer differentiating them. */ /*===== ZSTD_DStream management functions =====*/ ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void); -ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); +ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); /* accept NULL pointer */ /*===== Streaming decompression functions =====*/ -/* This function is redundant with the advanced API and equivalent to: +/*! ZSTD_initDStream() : + * Initialize/reset DStream state for new decompression operation. + * Call before new decompression operation using same DStream. * + * Note : This function is redundant with the advanced API and equivalent to: * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); * ZSTD_DCtx_refDDict(zds, NULL); */ ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); +/*! ZSTD_decompressStream() : + * Streaming decompression function. + * Call repetitively to consume full input updating it as necessary. + * Function will update both input and output `pos` fields exposing current state via these fields: + * - `input.pos < input.size`, some input remaining and caller should provide remaining input + * on the next call. + * - `output.pos < output.size`, decoder flushed internal output buffer. + * - `output.pos == output.size`, unflushed data potentially present in the internal buffers, + * check ZSTD_decompressStream() @return value, + * if > 0, invoke it again to flush remaining data to output. + * Note : with no additional input, amount of data flushed <= ZSTD_BLOCKSIZE_MAX. + * + * @return : 0 when a frame is completely decoded and fully flushed, + * or an error code, which can be tested using ZSTD_isError(), + * or any other value > 0, which means there is some decoding or flushing to do to complete current frame. + * + * Note: when an operation returns with an error code, the @zds state may be left in undefined state. + * It's UB to invoke `ZSTD_decompressStream()` on such a state. + * In order to re-use such a state, it must be first reset, + * which can be done explicitly (`ZSTD_DCtx_reset()`), + * or is implied for operations starting some new decompression job (`ZSTD_initDStream`, `ZSTD_decompressDCtx()`, `ZSTD_decompress_usingDict()`) + */ ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ @@ -811,7 +957,7 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output /*! ZSTD_compress_usingDict() : * Compression at an explicit compression level using a Dictionary. * A dictionary can be any arbitrary data segment (also called a prefix), - * or a buffer with specified information (see dictBuilder/zdict.h). + * or a buffer with specified information (see zdict.h). * Note : This function loads the dictionary, resulting in significant startup delay. * It's intended for a dictionary used only once. * Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */ @@ -854,7 +1000,8 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize int compressionLevel); /*! ZSTD_freeCDict() : - * Function frees memory allocated by ZSTD_createCDict(). */ + * Function frees memory allocated by ZSTD_createCDict(). + * If a NULL pointer is passed, no operation is performed. */ ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); /*! ZSTD_compress_usingCDict() : @@ -876,7 +1023,8 @@ typedef struct ZSTD_DDict_s ZSTD_DDict; ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize); /*! ZSTD_freeDDict() : - * Function frees memory allocated with ZSTD_createDDict() */ + * Function frees memory allocated with ZSTD_createDDict() + * If a NULL pointer is passed, no operation is performed. */ ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict); /*! ZSTD_decompress_usingDDict() : @@ -892,24 +1040,30 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, * Dictionary helper functions *******************************/ -/*! ZSTD_getDictID_fromDict() : +/*! ZSTD_getDictID_fromDict() : Requires v1.4.0+ * Provides the dictID stored within dictionary. * if @return == 0, the dictionary is not conformant with Zstandard specification. * It can still be loaded, but as a content-only dictionary. */ ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize); -/*! ZSTD_getDictID_fromDDict() : +/*! ZSTD_getDictID_fromCDict() : Requires v1.5.0+ + * Provides the dictID of the dictionary loaded into `cdict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict); + +/*! ZSTD_getDictID_fromDDict() : Requires v1.4.0+ * Provides the dictID of the dictionary loaded into `ddict`. * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict); -/*! ZSTD_getDictID_fromFrame() : +/*! ZSTD_getDictID_fromFrame() : Requires v1.4.0+ * Provides the dictID required to decompressed the frame stored within `src`. * If @return == 0, the dictID could not be decoded. * This could for one of the following reasons : * - The frame does not require a dictionary to be decoded (most common case). - * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information. + * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden piece of information. * Note : this use case also happens when using a non-conformant dictionary. * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). * - This is not a Zstandard frame. @@ -918,23 +1072,26 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); /******************************************************************************* - * Advanced dictionary and prefix API + * Advanced dictionary and prefix API (Requires v1.4.0+) * * This API allows dictionaries to be used with ZSTD_compress2(), - * ZSTD_compressStream2(), and ZSTD_decompress(). Dictionaries are sticky, and - * only reset with the context is reset with ZSTD_reset_parameters or - * ZSTD_reset_session_and_parameters. Prefixes are single-use. + * ZSTD_compressStream2(), and ZSTD_decompressDCtx(). + * Dictionaries are sticky, they remain valid when same context is reused, + * they only reset when the context is reset + * with ZSTD_reset_parameters or ZSTD_reset_session_and_parameters. + * In contrast, Prefixes are single-use. ******************************************************************************/ -/*! ZSTD_CCtx_loadDictionary() : +/*! ZSTD_CCtx_loadDictionary() : Requires v1.4.0+ * Create an internal CDict from `dict` buffer. * Decompression will have to use same dictionary. * @result : 0, or an error code (which can be tested with ZSTD_isError()). * Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary, * meaning "return to no-dictionary mode". - * Note 1 : Dictionary is sticky, it will be used for all future compressed frames. - * To return to "no-dictionary" situation, load a NULL dictionary (or reset parameters). + * Note 1 : Dictionary is sticky, it will be used for all future compressed frames, + * until parameters are reset, a new dictionary is loaded, or the dictionary + * is explicitly invalidated by loading a NULL dictionary. * Note 2 : Loading a dictionary involves building tables. * It's also a CPU consuming operation, with non-negligible impact on latency. * Tables are dependent on compression parameters, and for this reason, @@ -943,11 +1100,15 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); * Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead. * In such a case, dictionary buffer must outlive its users. * Note 4 : Use ZSTD_CCtx_loadDictionary_advanced() - * to precisely select how dictionary content must be interpreted. */ + * to precisely select how dictionary content must be interpreted. + * Note 5 : This method does not benefit from LDM (long distance mode). + * If you want to employ LDM on some large dictionary content, + * prefer employing ZSTD_CCtx_refPrefix() described below. + */ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); -/*! ZSTD_CCtx_refCDict() : - * Reference a prepared dictionary, to be used for all next compressed frames. +/*! ZSTD_CCtx_refCDict() : Requires v1.4.0+ + * Reference a prepared dictionary, to be used for all future compressed frames. * Note that compression parameters are enforced from within CDict, * and supersede any compression parameter previously set within CCtx. * The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs. @@ -960,12 +1121,13 @@ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, s * Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); -/*! ZSTD_CCtx_refPrefix() : +/*! ZSTD_CCtx_refPrefix() : Requires v1.4.0+ * Reference a prefix (single-usage dictionary) for next compressed frame. * A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end). * Decompression will need same prefix to properly regenerate data. * Compressing with a prefix is similar in outcome as performing a diff and compressing it, * but performs much faster, especially during decompression (compression speed is tunable with compression level). + * This method is compatible with LDM (long distance mode). * @result : 0, or an error code (which can be tested with ZSTD_isError()). * Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary * Note 1 : Prefix buffer is referenced. It **must** outlive compression. @@ -981,10 +1143,10 @@ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize); -/*! ZSTD_DCtx_loadDictionary() : - * Create an internal DDict from dict buffer, - * to be used to decompress next frames. - * The dictionary remains valid for all future frames, until explicitly invalidated. +/*! ZSTD_DCtx_loadDictionary() : Requires v1.4.0+ + * Create an internal DDict from dict buffer, to be used to decompress all future frames. + * The dictionary remains valid for all future frames, until explicitly invalidated, or + * a new dictionary is loaded. * @result : 0, or an error code (which can be tested with ZSTD_isError()). * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary, * meaning "return to no-dictionary mode". @@ -998,7 +1160,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, */ ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); -/*! ZSTD_DCtx_refDDict() : +/*! ZSTD_DCtx_refDDict() : Requires v1.4.0+ * Reference a prepared dictionary, to be used to decompress next frames. * The dictionary remains active for decompression of future frames using same DCtx. * @@ -1008,15 +1170,16 @@ ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, s * The memory for the table is allocated on the first call to refDDict, and can be * freed with ZSTD_freeDCtx(). * + * If called with ZSTD_d_refMultipleDDicts disabled (the default), only one dictionary + * will be managed, and referencing a dictionary effectively "discards" any previous one. + * * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Note 1 : Currently, only one dictionary can be managed. - * Referencing a new dictionary effectively "discards" any previous one. * Special: referencing a NULL DDict means "return to no-dictionary mode". * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx. */ ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); -/*! ZSTD_DCtx_refPrefix() : +/*! ZSTD_DCtx_refPrefix() : Requires v1.4.0+ * Reference a prefix (single-usage dictionary) to decompress next frame. * This is the reverse operation of ZSTD_CCtx_refPrefix(), * and must use the same prefix as the one used during compression. @@ -1037,7 +1200,7 @@ ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, /* === Memory management === */ -/*! ZSTD_sizeof_*() : +/*! ZSTD_sizeof_*() : Requires v1.4.0+ * These functions give the _current_ memory usage of selected object. * Note that object memory usage can evolve (increase or decrease) over time. */ ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); @@ -1047,6 +1210,10 @@ ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict); ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); +#if defined (__cplusplus) +} +#endif + #endif /* ZSTD_H_235446 */ @@ -1062,6 +1229,21 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); #if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY) #define ZSTD_H_ZSTD_STATIC_LINKING_ONLY +#if defined (__cplusplus) +extern "C" { +#endif + +/* This can be overridden externally to hide static symbols. */ +#ifndef ZSTDLIB_STATIC_API +# if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDLIB_STATIC_API __declspec(dllexport) ZSTDLIB_VISIBLE +# elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDLIB_STATIC_API __declspec(dllimport) ZSTDLIB_VISIBLE +# else +# define ZSTDLIB_STATIC_API ZSTDLIB_VISIBLE +# endif +#endif + /**************************************************************************************** * experimental API (static linking only) **************************************************************************************** @@ -1096,6 +1278,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); #define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */ #define ZSTD_STRATEGY_MIN ZSTD_fast #define ZSTD_STRATEGY_MAX ZSTD_btultra2 +#define ZSTD_BLOCKSIZE_MAX_MIN (1 << 10) /* The minimum valid max blocksize. Maximum blocksizes smaller than this make compressBound() inaccurate. */ #define ZSTD_OVERLAPLOG_MIN 0 @@ -1119,14 +1302,11 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); #define ZSTD_LDM_HASHRATELOG_MAX (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN) /* Advanced parameter bounds */ -#define ZSTD_TARGETCBLOCKSIZE_MIN 64 +#define ZSTD_TARGETCBLOCKSIZE_MIN 1340 /* suitable to fit into an ethernet / wifi / 4G transport frame */ #define ZSTD_TARGETCBLOCKSIZE_MAX ZSTD_BLOCKSIZE_MAX #define ZSTD_SRCSIZEHINT_MIN 0 #define ZSTD_SRCSIZEHINT_MAX INT_MAX -/* internal */ -#define ZSTD_HASHLOG3_MAX 17 - /* --- Advanced types --- */ @@ -1164,7 +1344,7 @@ typedef struct { * * Note: This field is optional. ZSTD_generateSequences() will calculate the value of * 'rep', but repeat offsets do not necessarily need to be calculated from an external - * sequence provider's perspective. For example, ZSTD_compressSequences() does not + * sequence provider perspective. For example, ZSTD_compressSequences() does not * use this 'rep' field at all (as of now). */ } ZSTD_Sequence; @@ -1268,9 +1448,19 @@ typedef enum { ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */ } ZSTD_literalCompressionMode_e; +typedef enum { + /* Note: This enum controls features which are conditionally beneficial. + * Zstd can take a decision on whether or not to enable the feature (ZSTD_ps_auto), + * but setting the switch to ZSTD_ps_enable or ZSTD_ps_disable force enable/disable the feature. + */ + ZSTD_ps_auto = 0, /* Let the library automatically determine whether the feature shall be enabled */ + ZSTD_ps_enable = 1, /* Force-enable the feature */ + ZSTD_ps_disable = 2 /* Do not use the feature */ +} ZSTD_ParamSwitch_e; +#define ZSTD_paramSwitch_e ZSTD_ParamSwitch_e /* old name */ /*************************************** -* Frame size functions +* Frame header and size functions ***************************************/ /*! ZSTD_findDecompressedSize() : @@ -1294,7 +1484,7 @@ typedef enum { * note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to * read each contained frame header. This is fast as most of the data is skipped, * however it does mean that all frame data must be present and valid. */ -ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); +ZSTDLIB_STATIC_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); /*! ZSTD_decompressBound() : * `src` should point to the start of a series of ZSTD encoded and/or skippable frames @@ -1309,41 +1499,137 @@ ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t * note 3 : when the decompressed size field isn't available, the upper-bound for that frame is calculated by: * upper-bound = # blocks * min(128 KB, Window_Size) */ -ZSTDLIB_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize); +ZSTDLIB_STATIC_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize); /*! ZSTD_frameHeaderSize() : - * srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX. + * srcSize must be large enough, aka >= ZSTD_FRAMEHEADERSIZE_PREFIX. * @return : size of the Frame Header, * or an error code (if srcSize is too small) */ -ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); +ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); + +typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_FrameType_e; +#define ZSTD_frameType_e ZSTD_FrameType_e /* old name */ +typedef struct { + unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */ + unsigned long long windowSize; /* can be very large, up to <= frameContentSize */ + unsigned blockSizeMax; + ZSTD_FrameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */ + unsigned headerSize; + unsigned dictID; /* for ZSTD_skippableFrame, contains the skippable magic variant [0-15] */ + unsigned checksumFlag; + unsigned _reserved1; + unsigned _reserved2; +} ZSTD_FrameHeader; +#define ZSTD_frameHeader ZSTD_FrameHeader /* old name */ + +/*! ZSTD_getFrameHeader() : + * decode Frame Header into `zfhPtr`, or requires larger `srcSize`. + * @return : 0 => header is complete, `zfhPtr` is correctly filled, + * >0 => `srcSize` is too small, @return value is the wanted `srcSize` amount, `zfhPtr` is not filled, + * or an error code, which can be tested using ZSTD_isError() */ +ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_FrameHeader* zfhPtr, const void* src, size_t srcSize); +/*! ZSTD_getFrameHeader_advanced() : + * same as ZSTD_getFrameHeader(), + * with added capability to select a format (like ZSTD_f_zstd1_magicless) */ +ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_FrameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format); + +/*! ZSTD_decompressionMargin() : + * Zstd supports in-place decompression, where the input and output buffers overlap. + * In this case, the output buffer must be at least (Margin + Output_Size) bytes large, + * and the input buffer must be at the end of the output buffer. + * + * _______________________ Output Buffer ________________________ + * | | + * | ____ Input Buffer ____| + * | | | + * v v v + * |---------------------------------------|-----------|----------| + * ^ ^ ^ + * |___________________ Output_Size ___________________|_ Margin _| + * + * NOTE: See also ZSTD_DECOMPRESSION_MARGIN(). + * NOTE: This applies only to single-pass decompression through ZSTD_decompress() or + * ZSTD_decompressDCtx(). + * NOTE: This function supports multi-frame input. + * + * @param src The compressed frame(s) + * @param srcSize The size of the compressed frame(s) + * @returns The decompression margin or an error that can be checked with ZSTD_isError(). + */ +ZSTDLIB_STATIC_API size_t ZSTD_decompressionMargin(const void* src, size_t srcSize); + +/*! ZSTD_DECOMPRESS_MARGIN() : + * Similar to ZSTD_decompressionMargin(), but instead of computing the margin from + * the compressed frame, compute it from the original size and the blockSizeLog. + * See ZSTD_decompressionMargin() for details. + * + * WARNING: This macro does not support multi-frame input, the input must be a single + * zstd frame. If you need that support use the function, or implement it yourself. + * + * @param originalSize The original uncompressed size of the data. + * @param blockSize The block size == MIN(windowSize, ZSTD_BLOCKSIZE_MAX). + * Unless you explicitly set the windowLog smaller than + * ZSTD_BLOCKSIZELOG_MAX you can just use ZSTD_BLOCKSIZE_MAX. + */ +#define ZSTD_DECOMPRESSION_MARGIN(originalSize, blockSize) ((size_t)( \ + ZSTD_FRAMEHEADERSIZE_MAX /* Frame header */ + \ + 4 /* checksum */ + \ + ((originalSize) == 0 ? 0 : 3 * (((originalSize) + (blockSize) - 1) / blockSize)) /* 3 bytes per block */ + \ + (blockSize) /* One block of margin */ \ + )) typedef enum { - ZSTD_sf_noBlockDelimiters = 0, /* Representation of ZSTD_Sequence has no block delimiters, sequences only */ - ZSTD_sf_explicitBlockDelimiters = 1 /* Representation of ZSTD_Sequence contains explicit block delimiters */ -} ZSTD_sequenceFormat_e; + ZSTD_sf_noBlockDelimiters = 0, /* ZSTD_Sequence[] has no block delimiters, just sequences */ + ZSTD_sf_explicitBlockDelimiters = 1 /* ZSTD_Sequence[] contains explicit block delimiters */ +} ZSTD_SequenceFormat_e; +#define ZSTD_sequenceFormat_e ZSTD_SequenceFormat_e /* old name */ + +/*! ZSTD_sequenceBound() : + * `srcSize` : size of the input buffer + * @return : upper-bound for the number of sequences that can be generated + * from a buffer of srcSize bytes + * + * note : returns number of sequences - to get bytes, multiply by sizeof(ZSTD_Sequence). + */ +ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize); /*! ZSTD_generateSequences() : - * Generate sequences using ZSTD_compress2, given a source buffer. + * WARNING: This function is meant for debugging and informational purposes ONLY! + * Its implementation is flawed, and it will be deleted in a future version. + * It is not guaranteed to succeed, as there are several cases where it will give + * up and fail. You should NOT use this function in production code. + * + * This function is deprecated, and will be removed in a future version. + * + * Generate sequences using ZSTD_compress2(), given a source buffer. + * + * @param zc The compression context to be used for ZSTD_compress2(). Set any + * compression parameters you need on this context. + * @param outSeqs The output sequences buffer of size @p outSeqsSize + * @param outSeqsCapacity The size of the output sequences buffer. + * ZSTD_sequenceBound(srcSize) is an upper bound on the number + * of sequences that can be generated. + * @param src The source buffer to generate sequences from of size @p srcSize. + * @param srcSize The size of the source buffer. * * Each block will end with a dummy sequence * with offset == 0, matchLength == 0, and litLength == length of last literals. * litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0) * simply acts as a block delimiter. * - * zc can be used to insert custom compression params. - * This function invokes ZSTD_compress2 - * - * The output of this function can be fed into ZSTD_compressSequences() with CCtx - * setting of ZSTD_c_blockDelimiters as ZSTD_sf_explicitBlockDelimiters - * @return : number of sequences generated + * @returns The number of sequences generated, necessarily less than + * ZSTD_sequenceBound(srcSize), or an error code that can be checked + * with ZSTD_isError(). */ - -ZSTDLIB_API size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, - size_t outSeqsSize, const void* src, size_t srcSize); +ZSTD_DEPRECATED("For debugging only, will be replaced by ZSTD_extractSequences()") +ZSTDLIB_STATIC_API size_t +ZSTD_generateSequences(ZSTD_CCtx* zc, + ZSTD_Sequence* outSeqs, size_t outSeqsCapacity, + const void* src, size_t srcSize); /*! ZSTD_mergeBlockDelimiters() : * Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals - * by merging them into into the literals of the next sequence. + * by merging them into the literals of the next sequence. * * As such, the final generated result has no explicit representation of block boundaries, * and the final last literals segment is not represented in the sequences. @@ -1352,11 +1638,13 @@ ZSTDLIB_API size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, * setting of ZSTD_c_blockDelimiters as ZSTD_sf_noBlockDelimiters * @return : number of sequences left after merging */ -ZSTDLIB_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize); +ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize); /*! ZSTD_compressSequences() : - * Compress an array of ZSTD_Sequence, generated from the original source buffer, into dst. - * If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.) + * Compress an array of ZSTD_Sequence, associated with @src buffer, into dst. + * @src contains the entire input (not just the literals). + * If @srcSize > sum(sequence.length), the remaining bytes are considered all literals + * If a dictionary is included, then the cctx should reference the dict (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.). * The entire source is compressed into a single frame. * * The compression behavior changes based on cctx params. In particular: @@ -1365,11 +1653,17 @@ ZSTDLIB_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t se * the block size derived from the cctx, and sequences may be split. This is the default setting. * * If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain - * block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided. + * valid block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided. + * + * When ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, it's possible to decide generating repcodes + * using the advanced parameter ZSTD_c_repcodeResolution. Repcodes will improve compression ratio, though the benefit + * can vary greatly depending on Sequences. On the other hand, repcode resolution is an expensive operation. + * By default, it's disabled at low (<10) compression levels, and enabled above the threshold (>=10). + * ZSTD_c_repcodeResolution makes it possible to directly manage this processing in either direction. * - * If ZSTD_c_validateSequences == 0, this function will blindly accept the sequences provided. Invalid sequences cause undefined - * behavior. If ZSTD_c_validateSequences == 1, then if sequence is invalid (see doc/zstd_compression_format.md for - * specifics regarding offset/matchlength requirements) then the function will bail out and return an error. + * If ZSTD_c_validateSequences == 0, this function blindly accepts the Sequences provided. Invalid Sequences cause undefined + * behavior. If ZSTD_c_validateSequences == 1, then the function will detect invalid Sequences (see doc/zstd_compression_format.md for + * specifics regarding offset/matchlength requirements) and then bail out and return an error. * * In addition to the two adjustable experimental params, there are other important cctx params. * - ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN. @@ -1377,31 +1671,81 @@ ZSTDLIB_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t se * - ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset * is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md * - * Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused. - * Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly, - * and cannot emit an RLE block that disagrees with the repcode history - * @return : final compressed size or a ZSTD error. - */ -ZSTDLIB_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstSize, - const ZSTD_Sequence* inSeqs, size_t inSeqsSize, - const void* src, size_t srcSize); + * Note: Repcodes are, as of now, always re-calculated within this function, ZSTD_Sequence.rep is effectively unused. + * Dev Note: Once ability to ingest repcodes become available, the explicit block delims mode must respect those repcodes exactly, + * and cannot emit an RLE block that disagrees with the repcode history. + * @return : final compressed size, or a ZSTD error code. + */ +ZSTDLIB_STATIC_API size_t +ZSTD_compressSequences(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + const void* src, size_t srcSize); + + +/*! ZSTD_compressSequencesAndLiterals() : + * This is a variant of ZSTD_compressSequences() which, + * instead of receiving (src,srcSize) as input parameter, receives (literals,litSize), + * aka all the literals, already extracted and laid out into a single continuous buffer. + * This can be useful if the process generating the sequences also happens to generate the buffer of literals, + * thus skipping an extraction + caching stage. + * It's a speed optimization, useful when the right conditions are met, + * but it also features the following limitations: + * - Only supports explicit delimiter mode + * - Currently does not support Sequences validation (so input Sequences are trusted) + * - Not compatible with frame checksum, which must be disabled + * - If any block is incompressible, will fail and return an error + * - @litSize must be == sum of all @.litLength fields in @inSeqs. Any discrepancy will generate an error. + * - @litBufCapacity is the size of the underlying buffer into which literals are written, starting at address @literals. + * @litBufCapacity must be at least 8 bytes larger than @litSize. + * - @decompressedSize must be correct, and correspond to the sum of all Sequences. Any discrepancy will generate an error. + * @return : final compressed size, or a ZSTD error code. + */ +ZSTDLIB_STATIC_API size_t +ZSTD_compressSequencesAndLiterals(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const ZSTD_Sequence* inSeqs, size_t nbSequences, + const void* literals, size_t litSize, size_t litBufCapacity, + size_t decompressedSize); /*! ZSTD_writeSkippableFrame() : * Generates a zstd skippable frame containing data given by src, and writes it to dst buffer. * - * Skippable frames begin with a a 4-byte magic number. There are 16 possible choices of magic number, + * Skippable frames begin with a 4-byte magic number. There are 16 possible choices of magic number, * ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15. - * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so - * the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant. + * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, + * so the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant. * * Returns an error if destination buffer is not large enough, if the source size is not representable * with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid). * * @return : number of bytes written or a ZSTD error. */ -ZSTDLIB_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity, - const void* src, size_t srcSize, unsigned magicVariant); +ZSTDLIB_STATIC_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + unsigned magicVariant); + +/*! ZSTD_readSkippableFrame() : + * Retrieves the content of a zstd skippable frame starting at @src, and writes it to @dst buffer. + * + * The parameter @magicVariant will receive the magicVariant that was supplied when the frame was written, + * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. + * This can be NULL if the caller is not interested in the magicVariant. + * + * Returns an error if destination buffer is not large enough, or if the frame is not skippable. + * + * @return : number of bytes written or a ZSTD error. + */ +ZSTDLIB_STATIC_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, + unsigned* magicVariant, + const void* src, size_t srcSize); + +/*! ZSTD_isSkippableFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame. + */ +ZSTDLIB_STATIC_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size); + /*************************************** @@ -1411,58 +1755,69 @@ ZSTDLIB_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity, /*! ZSTD_estimate*() : * These functions make it possible to estimate memory usage * of a future {D,C}Ctx, before its creation. + * This is useful in combination with ZSTD_initStatic(), + * which makes it possible to employ a static buffer for ZSTD_CCtx* state. * * ZSTD_estimateCCtxSize() will provide a memory budget large enough - * for any compression level up to selected one. - * Note : Unlike ZSTD_estimateCStreamSize*(), this estimate - * does not include space for a window buffer. - * Therefore, the estimation is only guaranteed for single-shot compressions, not streaming. + * to compress data of any size using one-shot compression ZSTD_compressCCtx() or ZSTD_compress2() + * associated with any compression level up to max specified one. * The estimate will assume the input may be arbitrarily large, * which is the worst case. * + * Note that the size estimation is specific for one-shot compression, + * it is not valid for streaming (see ZSTD_estimateCStreamSize*()) + * nor other potential ways of using a ZSTD_CCtx* state. + * * When srcSize can be bound by a known and rather "small" value, - * this fact can be used to provide a tighter estimation - * because the CCtx compression context will need less memory. - * This tighter estimation can be provided by more advanced functions + * this knowledge can be used to provide a tighter budget estimation + * because the ZSTD_CCtx* state will need less memory for small inputs. + * This tighter estimation can be provided by employing more advanced functions * ZSTD_estimateCCtxSize_usingCParams(), which can be used in tandem with ZSTD_getCParams(), * and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter(). * Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits. * - * Note 2 : only single-threaded compression is supported. + * Note : only single-threaded compression is supported. * ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1. */ -ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel); -ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); -ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params); -ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int maxCompressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void); /*! ZSTD_estimateCStreamSize() : - * ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one. - * It will also consider src size to be arbitrarily "large", which is worst case. + * ZSTD_estimateCStreamSize() will provide a memory budget large enough for streaming compression + * using any compression level up to the max specified one. + * It will also consider src size to be arbitrarily "large", which is a worst case scenario. * If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation. * ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. * ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1. * Note : CStream size estimation is only correct for single-threaded compression. - * ZSTD_DStream memory budget depends on window Size. + * ZSTD_estimateCStreamSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1. + * Note 2 : ZSTD_estimateCStreamSize* functions are not compatible with the Block-Level Sequence Producer API at this time. + * Size estimates assume that no external sequence producer is registered. + * + * ZSTD_DStream memory budget depends on frame's window Size. * This information can be passed manually, using ZSTD_estimateDStreamSize, * or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame(); + * Any frame requesting a window size larger than max specified one will be rejected. * Note : if streaming is init with function ZSTD_init?Stream_usingDict(), * an internal ?Dict will be created, which additional size is not estimated here. - * In this case, get total size by adding ZSTD_estimate?DictSize */ -ZSTDLIB_API size_t ZSTD_estimateCStreamSize(int compressionLevel); -ZSTDLIB_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams); -ZSTDLIB_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params); -ZSTDLIB_API size_t ZSTD_estimateDStreamSize(size_t windowSize); -ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize); + * In this case, get total size by adding ZSTD_estimate?DictSize + */ +ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int maxCompressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize(size_t maxWindowSize); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize); /*! ZSTD_estimate?DictSize() : * ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict(). * ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, like ZSTD_createCDict_advanced(). * Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller. */ -ZSTDLIB_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel); -ZSTDLIB_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod); -ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod); /*! ZSTD_initStatic*() : * Initialize an object using a pre-allocated fixed-size buffer. @@ -1485,20 +1840,20 @@ ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e * Limitation 2 : static cctx currently not compatible with multi-threading. * Limitation 3 : static dctx is incompatible with legacy support. */ -ZSTDLIB_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize); -ZSTDLIB_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */ +ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize); +ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */ -ZSTDLIB_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize); -ZSTDLIB_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */ +ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize); +ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */ -ZSTDLIB_API const ZSTD_CDict* ZSTD_initStaticCDict( +ZSTDLIB_STATIC_API const ZSTD_CDict* ZSTD_initStaticCDict( void* workspace, size_t workspaceSize, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType, ZSTD_compressionParameters cParams); -ZSTDLIB_API const ZSTD_DDict* ZSTD_initStaticDDict( +ZSTDLIB_STATIC_API const ZSTD_DDict* ZSTD_initStaticDDict( void* workspace, size_t workspaceSize, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, @@ -1517,46 +1872,54 @@ static #ifdef __GNUC__ __attribute__((__unused__)) #endif + +#if defined(__clang__) && __clang_major__ >= 5 +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant" +#endif ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */ +#if defined(__clang__) && __clang_major__ >= 5 +#pragma clang diagnostic pop +#endif -ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); -ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); -ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); -ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); -ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, +ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType, ZSTD_compressionParameters cParams, ZSTD_customMem customMem); -/* ! Thread pool : - * These prototypes make it possible to share a thread pool among multiple compression contexts. - * This can limit resources for applications with multiple threads where each one uses - * a threaded compression mode (via ZSTD_c_nbWorkers parameter). - * ZSTD_createThreadPool creates a new thread pool with a given number of threads. - * Note that the lifetime of such pool must exist while being used. - * ZSTD_CCtx_refThreadPool assigns a thread pool to a context (use NULL argument value - * to use an internal thread pool). - * ZSTD_freeThreadPool frees a thread pool. +/*! Thread pool : + * These prototypes make it possible to share a thread pool among multiple compression contexts. + * This can limit resources for applications with multiple threads where each one uses + * a threaded compression mode (via ZSTD_c_nbWorkers parameter). + * ZSTD_createThreadPool creates a new thread pool with a given number of threads. + * Note that the lifetime of such pool must exist while being used. + * ZSTD_CCtx_refThreadPool assigns a thread pool to a context (use NULL argument value + * to use an internal thread pool). + * ZSTD_freeThreadPool frees a thread pool, accepts NULL pointer. */ typedef struct POOL_ctx_s ZSTD_threadPool; -ZSTDLIB_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads); -ZSTDLIB_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool); -ZSTDLIB_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool); +ZSTDLIB_STATIC_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads); +ZSTDLIB_STATIC_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool); /* accept NULL pointer */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool); /* * This API is temporary and is expected to change or disappear in the future! */ -ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced2( +ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced2( const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType, const ZSTD_CCtx_params* cctxParams, ZSTD_customMem customMem); -ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced( +ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_advanced( const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType, @@ -1573,28 +1936,22 @@ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced( * As a consequence, `dictBuffer` **must** outlive CDict, * and its content must remain unmodified throughout the lifetime of CDict. * note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */ -ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); - -/*! ZSTD_getDictID_fromCDict() : - * Provides the dictID of the dictionary loaded into `cdict`. - * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. - * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ -ZSTDLIB_API unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict); +ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); /*! ZSTD_getCParams() : * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize. * `estimatedSrcSize` value is optional, select 0 if not known */ -ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); +ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); /*! ZSTD_getParams() : * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`. * All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */ -ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); +ZSTDLIB_STATIC_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); /*! ZSTD_checkCParams() : * Ensure param values remain within authorized range. * @return 0 on success, or an error code (can be checked with ZSTD_isError()) */ -ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); +ZSTDLIB_STATIC_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); /*! ZSTD_adjustCParams() : * optimize params for a given `srcSize` and `dictSize`. @@ -1602,23 +1959,48 @@ ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); * `dictSize` must be `0` when there is no dictionary. * cPar can be invalid : all parameters will be clamped within valid range in the @return struct. * This function never fails (wide contract) */ -ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); +ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); + +/*! ZSTD_CCtx_setCParams() : + * Set all parameters provided within @p cparams into the working @p cctx. + * Note : if modifying parameters during compression (MT mode only), + * note that changes to the .windowLog parameter will be ignored. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()). + * On failure, no parameters are updated. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams); + +/*! ZSTD_CCtx_setFParams() : + * Set all parameters provided within @p fparams into the working @p cctx. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setFParams(ZSTD_CCtx* cctx, ZSTD_frameParameters fparams); + +/*! ZSTD_CCtx_setParams() : + * Set all parameters provided within @p params into the working @p cctx. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParams(ZSTD_CCtx* cctx, ZSTD_parameters params); /*! ZSTD_compress_advanced() : * Note : this function is now DEPRECATED. * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters. - * This prototype will be marked as deprecated and generate compilation warning on reaching v1.5.x */ -ZSTDLIB_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize, - ZSTD_parameters params); + * This prototype will generate compilation warnings. */ +ZSTD_DEPRECATED("use ZSTD_compress2") +ZSTDLIB_STATIC_API +size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + ZSTD_parameters params); /*! ZSTD_compress_usingCDict_advanced() : - * Note : this function is now REDUNDANT. + * Note : this function is now DEPRECATED. * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters. - * This prototype will be marked as deprecated and generate compilation warning in some future version */ -ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, + * This prototype will generate compilation warnings. */ +ZSTD_DEPRECATED("use ZSTD_compress2 with ZSTD_CCtx_loadDictionary") +ZSTDLIB_STATIC_API +size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const ZSTD_CDict* cdict, @@ -1628,18 +2010,18 @@ ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, /*! ZSTD_CCtx_loadDictionary_byReference() : * Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx. * It saves some memory, but also requires that `dict` outlives its usage within `cctx` */ -ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); /*! ZSTD_CCtx_loadDictionary_advanced() : * Same as ZSTD_CCtx_loadDictionary(), but gives finer control over * how to load the dictionary (by copy ? by reference ?) * and how to interpret it (automatic ? force raw mode ? full mode only ?) */ -ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); /*! ZSTD_CCtx_refPrefix_advanced() : * Same as ZSTD_CCtx_refPrefix(), but gives finer control over * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ -ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); /* === experimental parameters === */ /* these parameters can be used with ZSTD_setParameter() @@ -1678,17 +2060,18 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre * See the comments on that enum for an explanation of the feature. */ #define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4 -/* Controls how the literals are compressed (default is auto). - * The value must be of type ZSTD_literalCompressionMode_e. - * See ZSTD_literalCompressionMode_t enum definition for details. +/* Controlled with ZSTD_ParamSwitch_e enum. + * Default is ZSTD_ps_auto. + * Set to ZSTD_ps_disable to never compress literals. + * Set to ZSTD_ps_enable to always compress literals. (Note: uncompressed literals + * may still be emitted if huffman is not beneficial to use.) + * + * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use + * literals compression based on the compression parameters - specifically, + * negative compression levels do not use literal compression. */ #define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5 -/* Tries to fit compressed block size to be around targetCBlockSize. - * No target when targetCBlockSize == 0. - * There is no guarantee on compressed block size (default:0) */ -#define ZSTD_c_targetCBlockSize ZSTD_c_experimentalParam6 - /* User's best guess of source size. * Hint is not valid when srcSizeHint == 0. * There is no guarantee that hint is close to actual source size, @@ -1743,7 +2126,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre * * Note that this means that the CDict tables can no longer be copied into the * CCtx, so the dict attachment mode ZSTD_dictForceCopy will no longer be - * useable. The dictionary can only be attached or reloaded. + * usable. The dictionary can only be attached or reloaded. * * In general, you should expect compression to be faster--sometimes very much * so--and CDict creation to be slightly slower. Eventually, we will probably @@ -1755,13 +2138,16 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre * Experimental parameter. * Default is 0 == disabled. Set to 1 to enable. * - * Tells the compressor that the ZSTD_inBuffer will ALWAYS be the same - * between calls, except for the modifications that zstd makes to pos (the - * caller must not modify pos). This is checked by the compressor, and - * compression will fail if it ever changes. This means the only flush - * mode that makes sense is ZSTD_e_end, so zstd will error if ZSTD_e_end - * is not used. The data in the ZSTD_inBuffer in the range [src, src + pos) - * MUST not be modified during compression or you will get data corruption. + * Tells the compressor that input data presented with ZSTD_inBuffer + * will ALWAYS be the same between calls. + * Technically, the @src pointer must never be changed, + * and the @pos field can only be updated by zstd. + * However, it's possible to increase the @size field, + * allowing scenarios where more data can be appended after compressions starts. + * These conditions are checked by the compressor, + * and compression will fail if they are not respected. + * Also, data in the ZSTD_inBuffer within the range [src, src + pos) + * MUST not be modified during compression or it will result in data corruption. * * When this flag is enabled zstd won't allocate an input window buffer, * because the user guarantees it can reference the ZSTD_inBuffer until @@ -1769,18 +2155,15 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre * large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also * avoid the memcpy() from the input buffer to the input window buffer. * - * NOTE: ZSTD_compressStream2() will error if ZSTD_e_end is not used. - * That means this flag cannot be used with ZSTD_compressStream(). - * * NOTE: So long as the ZSTD_inBuffer always points to valid memory, using * this flag is ALWAYS memory safe, and will never access out-of-bounds - * memory. However, compression WILL fail if you violate the preconditions. + * memory. However, compression WILL fail if conditions are not respected. * - * WARNING: The data in the ZSTD_inBuffer in the range [dst, dst + pos) MUST - * not be modified during compression or you will get data corruption. This - * is because zstd needs to reference data in the ZSTD_inBuffer to find + * WARNING: The data in the ZSTD_inBuffer in the range [src, src + pos) MUST + * not be modified during compression or it will result in data corruption. + * This is because zstd needs to reference data in the ZSTD_inBuffer to find * matches. Normally zstd maintains its own window buffer for this purpose, - * but passing this flag tells zstd to use the user provided buffer. + * but passing this flag tells zstd to rely on user provided buffer instead. */ #define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9 @@ -1818,26 +2201,167 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre /* ZSTD_c_validateSequences * Default is 0 == disabled. Set to 1 to enable sequence validation. * - * For use with sequence compression API: ZSTD_compressSequences(). - * Designates whether or not we validate sequences provided to ZSTD_compressSequences() + * For use with sequence compression API: ZSTD_compressSequences*(). + * Designates whether or not provided sequences are validated within ZSTD_compressSequences*() * during function execution. * - * Without validation, providing a sequence that does not conform to the zstd spec will cause - * undefined behavior, and may produce a corrupted block. + * When Sequence validation is disabled (default), Sequences are compressed as-is, + * so they must correct, otherwise it would result in a corruption error. * - * With validation enabled, a if sequence is invalid (see doc/zstd_compression_format.md for + * Sequence validation adds some protection, by ensuring that all values respect boundary conditions. + * If a Sequence is detected invalid (see doc/zstd_compression_format.md for * specifics regarding offset/matchlength requirements) then the function will bail out and * return an error. - * */ #define ZSTD_c_validateSequences ZSTD_c_experimentalParam12 +/* ZSTD_c_blockSplitterLevel + * note: this parameter only influences the first splitter stage, + * which is active before producing the sequences. + * ZSTD_c_splitAfterSequences controls the next splitter stage, + * which is active after sequence production. + * Note that both can be combined. + * Allowed values are between 0 and ZSTD_BLOCKSPLITTER_LEVEL_MAX included. + * 0 means "auto", which will select a value depending on current ZSTD_c_strategy. + * 1 means no splitting. + * Then, values from 2 to 6 are sorted in increasing cpu load order. + * + * Note that currently the first block is never split, + * to ensure expansion guarantees in presence of incompressible data. + */ +#define ZSTD_BLOCKSPLITTER_LEVEL_MAX 6 +#define ZSTD_c_blockSplitterLevel ZSTD_c_experimentalParam20 + +/* ZSTD_c_splitAfterSequences + * This is a stronger splitter algorithm, + * based on actual sequences previously produced by the selected parser. + * It's also slower, and as a consequence, mostly used for high compression levels. + * While the post-splitter does overlap with the pre-splitter, + * both can nonetheless be combined, + * notably with ZSTD_c_blockSplitterLevel at ZSTD_BLOCKSPLITTER_LEVEL_MAX, + * resulting in higher compression ratio than just one of them. + * + * Default is ZSTD_ps_auto. + * Set to ZSTD_ps_disable to never use block splitter. + * Set to ZSTD_ps_enable to always use block splitter. + * + * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use + * block splitting based on the compression parameters. + */ +#define ZSTD_c_splitAfterSequences ZSTD_c_experimentalParam13 + +/* ZSTD_c_useRowMatchFinder + * Controlled with ZSTD_ParamSwitch_e enum. + * Default is ZSTD_ps_auto. + * Set to ZSTD_ps_disable to never use row-based matchfinder. + * Set to ZSTD_ps_enable to force usage of row-based matchfinder. + * + * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use + * the row-based matchfinder based on support for SIMD instructions and the window log. + * Note that this only pertains to compression strategies: greedy, lazy, and lazy2 + */ +#define ZSTD_c_useRowMatchFinder ZSTD_c_experimentalParam14 + +/* ZSTD_c_deterministicRefPrefix + * Default is 0 == disabled. Set to 1 to enable. + * + * Zstd produces different results for prefix compression when the prefix is + * directly adjacent to the data about to be compressed vs. when it isn't. + * This is because zstd detects that the two buffers are contiguous and it can + * use a more efficient match finding algorithm. However, this produces different + * results than when the two buffers are non-contiguous. This flag forces zstd + * to always load the prefix in non-contiguous mode, even if it happens to be + * adjacent to the data, to guarantee determinism. + * + * If you really care about determinism when using a dictionary or prefix, + * like when doing delta compression, you should select this option. It comes + * at a speed penalty of about ~2.5% if the dictionary and data happened to be + * contiguous, and is free if they weren't contiguous. We don't expect that + * intentionally making the dictionary and data contiguous will be worth the + * cost to memcpy() the data. + */ +#define ZSTD_c_deterministicRefPrefix ZSTD_c_experimentalParam15 + +/* ZSTD_c_prefetchCDictTables + * Controlled with ZSTD_ParamSwitch_e enum. Default is ZSTD_ps_auto. + * + * In some situations, zstd uses CDict tables in-place rather than copying them + * into the working context. (See docs on ZSTD_dictAttachPref_e above for details). + * In such situations, compression speed is seriously impacted when CDict tables are + * "cold" (outside CPU cache). This parameter instructs zstd to prefetch CDict tables + * when they are used in-place. + * + * For sufficiently small inputs, the cost of the prefetch will outweigh the benefit. + * For sufficiently large inputs, zstd will by default memcpy() CDict tables + * into the working context, so there is no need to prefetch. This parameter is + * targeted at a middle range of input sizes, where a prefetch is cheap enough to be + * useful but memcpy() is too expensive. The exact range of input sizes where this + * makes sense is best determined by careful experimentation. + * + * Note: for this parameter, ZSTD_ps_auto is currently equivalent to ZSTD_ps_disable, + * but in the future zstd may conditionally enable this feature via an auto-detection + * heuristic for cold CDicts. + * Use ZSTD_ps_disable to opt out of prefetching under any circumstances. + */ +#define ZSTD_c_prefetchCDictTables ZSTD_c_experimentalParam16 + +/* ZSTD_c_enableSeqProducerFallback + * Allowed values are 0 (disable) and 1 (enable). The default setting is 0. + * + * Controls whether zstd will fall back to an internal sequence producer if an + * external sequence producer is registered and returns an error code. This fallback + * is block-by-block: the internal sequence producer will only be called for blocks + * where the external sequence producer returns an error code. Fallback parsing will + * follow any other cParam settings, such as compression level, the same as in a + * normal (fully-internal) compression operation. + * + * The user is strongly encouraged to read the full Block-Level Sequence Producer API + * documentation (below) before setting this parameter. */ +#define ZSTD_c_enableSeqProducerFallback ZSTD_c_experimentalParam17 + +/* ZSTD_c_maxBlockSize + * Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB). + * The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default. + * + * This parameter can be used to set an upper bound on the blocksize + * that overrides the default ZSTD_BLOCKSIZE_MAX. It cannot be used to set upper + * bounds greater than ZSTD_BLOCKSIZE_MAX or bounds lower than 1KB (will make + * compressBound() inaccurate). Only currently meant to be used for testing. + */ +#define ZSTD_c_maxBlockSize ZSTD_c_experimentalParam18 + +/* ZSTD_c_repcodeResolution + * This parameter only has an effect if ZSTD_c_blockDelimiters is + * set to ZSTD_sf_explicitBlockDelimiters (may change in the future). + * + * This parameter affects how zstd parses external sequences, + * provided via the ZSTD_compressSequences*() API + * or from an external block-level sequence producer. + * + * If set to ZSTD_ps_enable, the library will check for repeated offsets within + * external sequences, even if those repcodes are not explicitly indicated in + * the "rep" field. Note that this is the only way to exploit repcode matches + * while using compressSequences*() or an external sequence producer, since zstd + * currently ignores the "rep" field of external sequences. + * + * If set to ZSTD_ps_disable, the library will not exploit repeated offsets in + * external sequences, regardless of whether the "rep" field has been set. This + * reduces sequence compression overhead by about 25% while sacrificing some + * compression ratio. + * + * The default value is ZSTD_ps_auto, for which the library will enable/disable + * based on compression level (currently: level<10 disables, level>=10 enables). + */ +#define ZSTD_c_repcodeResolution ZSTD_c_experimentalParam19 +#define ZSTD_c_searchForExternalRepcodes ZSTD_c_experimentalParam19 /* older name */ + + /*! ZSTD_CCtx_getParameter() : * Get the requested compression parameter value, selected by enum ZSTD_cParameter, * and store it into int* value. * @return : 0, or an error code (which can be tested with ZSTD_isError()). */ -ZSTDLIB_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value); +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value); /*! ZSTD_CCtx_params : @@ -1852,32 +2376,32 @@ ZSTDLIB_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter * These parameters will be applied to * all subsequent frames. * - ZSTD_compressStream2() : Do compression using the CCtx. - * - ZSTD_freeCCtxParams() : Free the memory. + * - ZSTD_freeCCtxParams() : Free the memory, accept NULL pointer. * * This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams() * for static allocation of CCtx for single-threaded compression. */ -ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void); -ZSTDLIB_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); +ZSTDLIB_STATIC_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void); +ZSTDLIB_STATIC_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); /* accept NULL pointer */ /*! ZSTD_CCtxParams_reset() : * Reset params to default values. */ -ZSTDLIB_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params); +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params); /*! ZSTD_CCtxParams_init() : * Initializes the compression parameters of cctxParams according to * compression level. All other parameters are reset to their default values. */ -ZSTDLIB_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel); /*! ZSTD_CCtxParams_init_advanced() : * Initializes the compression and frame parameters of cctxParams according to * params. All other parameters are reset to their default values. */ -ZSTDLIB_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params); +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params); -/*! ZSTD_CCtxParams_setParameter() : +/*! ZSTD_CCtxParams_setParameter() : Requires v1.4.0+ * Similar to ZSTD_CCtx_setParameter. * Set one compression parameter, selected by enum ZSTD_cParameter. * Parameters must be applied to a ZSTD_CCtx using @@ -1885,14 +2409,14 @@ ZSTDLIB_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, Z * @result : a code representing success or failure (which can be tested with * ZSTD_isError()). */ -ZSTDLIB_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value); +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value); /*! ZSTD_CCtxParams_getParameter() : * Similar to ZSTD_CCtx_getParameter. * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter. * @result : 0, or an error code (which can be tested with ZSTD_isError()). */ -ZSTDLIB_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value); +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value); /*! ZSTD_CCtx_setParametersUsingCCtxParams() : * Apply a set of ZSTD_CCtx_params to the compression context. @@ -1901,7 +2425,7 @@ ZSTDLIB_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, * if nbWorkers>=1, new parameters will be picked up at next job, * with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated). */ -ZSTDLIB_API size_t ZSTD_CCtx_setParametersUsingCCtxParams( +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParametersUsingCCtxParams( ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params); /*! ZSTD_compressStream2_simpleArgs() : @@ -1910,7 +2434,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_setParametersUsingCCtxParams( * This variant might be helpful for binders from dynamic languages * which have troubles handling structures containing memory pointers. */ -ZSTDLIB_API size_t ZSTD_compressStream2_simpleArgs ( +ZSTDLIB_STATIC_API size_t ZSTD_compressStream2_simpleArgs ( ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, size_t* dstPos, const void* src, size_t srcSize, size_t* srcPos, @@ -1926,33 +2450,33 @@ ZSTDLIB_API size_t ZSTD_compressStream2_simpleArgs ( * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. * Note 3 : Skippable Frame Identifiers are considered valid. */ -ZSTDLIB_API unsigned ZSTD_isFrame(const void* buffer, size_t size); +ZSTDLIB_STATIC_API unsigned ZSTD_isFrame(const void* buffer, size_t size); /*! ZSTD_createDDict_byReference() : * Create a digested dictionary, ready to start decompression operation without startup delay. * Dictionary content is referenced, and therefore stays in dictBuffer. * It is important that dictBuffer outlives DDict, * it must remain read accessible throughout the lifetime of DDict */ -ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize); +ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize); /*! ZSTD_DCtx_loadDictionary_byReference() : * Same as ZSTD_DCtx_loadDictionary(), * but references `dict` content instead of copying it into `dctx`. * This saves memory if `dict` remains around., * However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression. */ -ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); /*! ZSTD_DCtx_loadDictionary_advanced() : * Same as ZSTD_DCtx_loadDictionary(), * but gives direct control over * how to load the dictionary (by copy ? by reference ?) * and how to interpret it (automatic ? force raw mode ? full mode only ?). */ -ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); /*! ZSTD_DCtx_refPrefix_advanced() : * Same as ZSTD_DCtx_refPrefix(), but gives finer control over * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ -ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); /*! ZSTD_DCtx_setMaxWindowSize() : * Refuses allocating internal buffers for frames requiring a window size larger than provided limit. @@ -1961,14 +2485,14 @@ ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* pre * By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) * @return : 0, or an error code (which can be tested using ZSTD_isError()). */ -ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize); +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize); /*! ZSTD_DCtx_getParameter() : * Get the requested decompression parameter value, selected by enum ZSTD_dParameter, * and store it into int* value. * @return : 0, or an error code (which can be tested with ZSTD_isError()). */ -ZSTDLIB_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value); +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value); /* ZSTD_d_format * experimental parameter, @@ -1988,7 +2512,7 @@ ZSTDLIB_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param * in the range [dst, dst + pos) MUST not be modified during decompression * or you will get data corruption. * - * When this flags is enabled zstd won't allocate an output buffer, because + * When this flag is enabled zstd won't allocate an output buffer, because * it can write directly to the ZSTD_outBuffer, but it will still allocate * an input buffer large enough to fit any compressed block. This will also * avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer. @@ -2041,13 +2565,43 @@ ZSTDLIB_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param */ #define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4 +/* ZSTD_d_disableHuffmanAssembly + * Set to 1 to disable the Huffman assembly implementation. + * The default value is 0, which allows zstd to use the Huffman assembly + * implementation if available. + * + * This parameter can be used to disable Huffman assembly at runtime. + * If you want to disable it at compile time you can define the macro + * ZSTD_DISABLE_ASM. + */ +#define ZSTD_d_disableHuffmanAssembly ZSTD_d_experimentalParam5 + +/* ZSTD_d_maxBlockSize + * Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB). + * The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default. + * + * Forces the decompressor to reject blocks whose content size is + * larger than the configured maxBlockSize. When maxBlockSize is + * larger than the windowSize, the windowSize is used instead. + * This saves memory on the decoder when you know all blocks are small. + * + * This option is typically used in conjunction with ZSTD_c_maxBlockSize. + * + * WARNING: This causes the decoder to reject otherwise valid frames + * that have block sizes larger than the configured maxBlockSize. + */ +#define ZSTD_d_maxBlockSize ZSTD_d_experimentalParam6 + /*! ZSTD_DCtx_setFormat() : + * This function is REDUNDANT. Prefer ZSTD_DCtx_setParameter(). * Instruct the decoder context about what kind of data to decode next. * This instruction is mandatory to decode data without a fully-formed header, * such ZSTD_f_zstd1_magicless for example. * @return : 0, or an error code (which can be tested using ZSTD_isError()). */ -ZSTDLIB_API size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); +ZSTD_DEPRECATED("use ZSTD_DCtx_setParameter() instead") +ZSTDLIB_STATIC_API +size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); /*! ZSTD_decompressStream_simpleArgs() : * Same as ZSTD_decompressStream(), @@ -2055,7 +2609,7 @@ ZSTDLIB_API size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); * This can be helpful for binders from dynamic languages * which have troubles handling structures containing memory pointers. */ -ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs ( +ZSTDLIB_STATIC_API size_t ZSTD_decompressStream_simpleArgs ( ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, size_t* dstPos, const void* src, size_t srcSize, size_t* srcPos); @@ -2071,7 +2625,7 @@ ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs ( /*===== Advanced Streaming compression functions =====*/ /*! ZSTD_initCStream_srcSize() : - * This function is deprecated, and equivalent to: + * This function is DEPRECATED, and equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); @@ -2080,15 +2634,16 @@ ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs ( * pledgedSrcSize must be correct. If it is not known at init time, use * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, * "0" also disables frame content size field. It may be enabled in the future. - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + * This prototype will generate compilation warnings. */ -ZSTDLIB_API size_t -ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /*! ZSTD_initCStream_usingDict() : - * This function is deprecated, and is equivalent to: + * This function is DEPRECATED, and is equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); @@ -2097,81 +2652,85 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, * dict == NULL or dictSize < 8, in which case no dict is used. * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy. - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + * This prototype will generate compilation warnings. */ -ZSTDLIB_API size_t -ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /*! ZSTD_initCStream_advanced() : - * This function is deprecated, and is approximately equivalent to: + * This function is DEPRECATED, and is equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * // Pseudocode: Set each zstd parameter and leave the rest as-is. - * for ((param, value) : params) { - * ZSTD_CCtx_setParameter(zcs, param, value); - * } + * ZSTD_CCtx_setParams(zcs, params); * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); * * dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy. * pledgedSrcSize must be correct. * If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + * This prototype will generate compilation warnings. */ -ZSTDLIB_API size_t -ZSTD_initCStream_advanced(ZSTD_CStream* zcs, +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /*! ZSTD_initCStream_usingCDict() : - * This function is deprecated, and equivalent to: + * This function is DEPRECATED, and equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); * ZSTD_CCtx_refCDict(zcs, cdict); * * note : cdict will just be referenced, and must outlive compression session - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + * This prototype will generate compilation warnings. */ -ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); +ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /*! ZSTD_initCStream_usingCDict_advanced() : - * This function is DEPRECATED, and is approximately equivalent to: + * This function is DEPRECATED, and is equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * // Pseudocode: Set each zstd frame parameter and leave the rest as-is. - * for ((fParam, value) : fParams) { - * ZSTD_CCtx_setParameter(zcs, fParam, value); - * } + * ZSTD_CCtx_setFParams(zcs, fParams); * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); * ZSTD_CCtx_refCDict(zcs, cdict); * * same as ZSTD_initCStream_usingCDict(), with control over frame parameters. * pledgedSrcSize must be correct. If srcSize is not known at init time, use * value ZSTD_CONTENTSIZE_UNKNOWN. - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + * This prototype will generate compilation warnings. */ -ZSTDLIB_API size_t -ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, +ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); /*! ZSTD_resetCStream() : - * This function is deprecated, and is equivalent to: + * This function is DEPRECATED, and is equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * Note: ZSTD_resetCStream() interprets pledgedSrcSize == 0 as ZSTD_CONTENTSIZE_UNKNOWN, but + * ZSTD_CCtx_setPledgedSrcSize() does not do the same, so ZSTD_CONTENTSIZE_UNKNOWN must be + * explicitly specified. * * start a new frame, using same parameters from previous frame. - * This is typically useful to skip dictionary loading stage, since it will re-use it in-place. + * This is typically useful to skip dictionary loading stage, since it will reuse it in-place. * Note that zcs must be init at least once before using ZSTD_resetCStream(). * If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN. * If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end. * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs, * but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead. * @return : 0, or an error code (which can be tested using ZSTD_isError()) - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + * This prototype will generate compilation warnings. */ -ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); typedef struct { @@ -2189,7 +2748,7 @@ typedef struct { * Note : (ingested - consumed) is amount of input data buffered internally, not yet compressed. * Aggregates progression inside active worker threads. */ -ZSTDLIB_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx); +ZSTDLIB_STATIC_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx); /*! ZSTD_toFlushNow() : * Tell how many bytes are ready to be flushed immediately. @@ -2204,7 +2763,7 @@ ZSTDLIB_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx * therefore flush speed is limited by production speed of oldest job * irrespective of the speed of concurrent (and newer) jobs. */ -ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx); +ZSTDLIB_STATIC_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx); /*===== Advanced Streaming decompression functions =====*/ @@ -2216,9 +2775,9 @@ ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx); * ZSTD_DCtx_loadDictionary(zds, dict, dictSize); * * note: no dictionary will be used if dict == NULL or dictSize < 8 - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x */ -ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); +ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_loadDictionary, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /*! * This function is deprecated, and is equivalent to: @@ -2227,27 +2786,211 @@ ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dic * ZSTD_DCtx_refDDict(zds, ddict); * * note : ddict is referenced, it must outlive decompression session - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x */ -ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); +ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_refDDict, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /*! * This function is deprecated, and is equivalent to: * * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); * - * re-use decompression parameters from previous init; saves dictionary loading - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x - */ -ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); + * reuse decompression parameters from previous init; saves dictionary loading + */ +ZSTD_DEPRECATED("use ZSTD_DCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); + + +/* ********************* BLOCK-LEVEL SEQUENCE PRODUCER API ********************* + * + * *** OVERVIEW *** + * The Block-Level Sequence Producer API allows users to provide their own custom + * sequence producer which libzstd invokes to process each block. The produced list + * of sequences (literals and matches) is then post-processed by libzstd to produce + * valid compressed blocks. + * + * This block-level offload API is a more granular complement of the existing + * frame-level offload API compressSequences() (introduced in v1.5.1). It offers + * an easier migration story for applications already integrated with libzstd: the + * user application continues to invoke the same compression functions + * ZSTD_compress2() or ZSTD_compressStream2() as usual, and transparently benefits + * from the specific advantages of the external sequence producer. For example, + * the sequence producer could be tuned to take advantage of known characteristics + * of the input, to offer better speed / ratio, or could leverage hardware + * acceleration not available within libzstd itself. + * + * See contrib/externalSequenceProducer for an example program employing the + * Block-Level Sequence Producer API. + * + * *** USAGE *** + * The user is responsible for implementing a function of type + * ZSTD_sequenceProducer_F. For each block, zstd will pass the following + * arguments to the user-provided function: + * + * - sequenceProducerState: a pointer to a user-managed state for the sequence + * producer. + * + * - outSeqs, outSeqsCapacity: an output buffer for the sequence producer. + * outSeqsCapacity is guaranteed >= ZSTD_sequenceBound(srcSize). The memory + * backing outSeqs is managed by the CCtx. + * + * - src, srcSize: an input buffer for the sequence producer to parse. + * srcSize is guaranteed to be <= ZSTD_BLOCKSIZE_MAX. + * + * - dict, dictSize: a history buffer, which may be empty, which the sequence + * producer may reference as it parses the src buffer. Currently, zstd will + * always pass dictSize == 0 into external sequence producers, but this will + * change in the future. + * + * - compressionLevel: a signed integer representing the zstd compression level + * set by the user for the current operation. The sequence producer may choose + * to use this information to change its compression strategy and speed/ratio + * tradeoff. Note: the compression level does not reflect zstd parameters set + * through the advanced API. + * + * - windowSize: a size_t representing the maximum allowed offset for external + * sequences. Note that sequence offsets are sometimes allowed to exceed the + * windowSize if a dictionary is present, see doc/zstd_compression_format.md + * for details. + * + * The user-provided function shall return a size_t representing the number of + * sequences written to outSeqs. This return value will be treated as an error + * code if it is greater than outSeqsCapacity. The return value must be non-zero + * if srcSize is non-zero. The ZSTD_SEQUENCE_PRODUCER_ERROR macro is provided + * for convenience, but any value greater than outSeqsCapacity will be treated as + * an error code. + * + * If the user-provided function does not return an error code, the sequences + * written to outSeqs must be a valid parse of the src buffer. Data corruption may + * occur if the parse is not valid. A parse is defined to be valid if the + * following conditions hold: + * - The sum of matchLengths and literalLengths must equal srcSize. + * - All sequences in the parse, except for the final sequence, must have + * matchLength >= ZSTD_MINMATCH_MIN. The final sequence must have + * matchLength >= ZSTD_MINMATCH_MIN or matchLength == 0. + * - All offsets must respect the windowSize parameter as specified in + * doc/zstd_compression_format.md. + * - If the final sequence has matchLength == 0, it must also have offset == 0. + * + * zstd will only validate these conditions (and fail compression if they do not + * hold) if the ZSTD_c_validateSequences cParam is enabled. Note that sequence + * validation has a performance cost. + * + * If the user-provided function returns an error, zstd will either fall back + * to an internal sequence producer or fail the compression operation. The user can + * choose between the two behaviors by setting the ZSTD_c_enableSeqProducerFallback + * cParam. Fallback compression will follow any other cParam settings, such as + * compression level, the same as in a normal compression operation. + * + * The user shall instruct zstd to use a particular ZSTD_sequenceProducer_F + * function by calling + * ZSTD_registerSequenceProducer(cctx, + * sequenceProducerState, + * sequenceProducer) + * This setting will persist until the next parameter reset of the CCtx. + * + * The sequenceProducerState must be initialized by the user before calling + * ZSTD_registerSequenceProducer(). The user is responsible for destroying the + * sequenceProducerState. + * + * *** LIMITATIONS *** + * This API is compatible with all zstd compression APIs which respect advanced parameters. + * However, there are three limitations: + * + * First, the ZSTD_c_enableLongDistanceMatching cParam is not currently supported. + * COMPRESSION WILL FAIL if it is enabled and the user tries to compress with a block-level + * external sequence producer. + * - Note that ZSTD_c_enableLongDistanceMatching is auto-enabled by default in some + * cases (see its documentation for details). Users must explicitly set + * ZSTD_c_enableLongDistanceMatching to ZSTD_ps_disable in such cases if an external + * sequence producer is registered. + * - As of this writing, ZSTD_c_enableLongDistanceMatching is disabled by default + * whenever ZSTD_c_windowLog < 128MB, but that's subject to change. Users should + * check the docs on ZSTD_c_enableLongDistanceMatching whenever the Block-Level Sequence + * Producer API is used in conjunction with advanced settings (like ZSTD_c_windowLog). + * + * Second, history buffers are not currently supported. Concretely, zstd will always pass + * dictSize == 0 to the external sequence producer (for now). This has two implications: + * - Dictionaries are not currently supported. Compression will *not* fail if the user + * references a dictionary, but the dictionary won't have any effect. + * - Stream history is not currently supported. All advanced compression APIs, including + * streaming APIs, work with external sequence producers, but each block is treated as + * an independent chunk without history from previous blocks. + * + * Third, multi-threading within a single compression is not currently supported. In other words, + * COMPRESSION WILL FAIL if ZSTD_c_nbWorkers > 0 and an external sequence producer is registered. + * Multi-threading across compressions is fine: simply create one CCtx per thread. + * + * Long-term, we plan to overcome all three limitations. There is no technical blocker to + * overcoming them. It is purely a question of engineering effort. + */ + +#define ZSTD_SEQUENCE_PRODUCER_ERROR ((size_t)(-1)) + +typedef size_t (*ZSTD_sequenceProducer_F) ( + void* sequenceProducerState, + ZSTD_Sequence* outSeqs, size_t outSeqsCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize, + int compressionLevel, + size_t windowSize +); + +/*! ZSTD_registerSequenceProducer() : + * Instruct zstd to use a block-level external sequence producer function. + * + * The sequenceProducerState must be initialized by the caller, and the caller is + * responsible for managing its lifetime. This parameter is sticky across + * compressions. It will remain set until the user explicitly resets compression + * parameters. + * + * Sequence producer registration is considered to be an "advanced parameter", + * part of the "advanced API". This means it will only have an effect on compression + * APIs which respect advanced parameters, such as compress2() and compressStream2(). + * Older compression APIs such as compressCCtx(), which predate the introduction of + * "advanced parameters", will ignore any external sequence producer setting. + * + * The sequence producer can be "cleared" by registering a NULL function pointer. This + * removes all limitations described above in the "LIMITATIONS" section of the API docs. + * + * The user is strongly encouraged to read the full API documentation (above) before + * calling this function. */ +ZSTDLIB_STATIC_API void +ZSTD_registerSequenceProducer( + ZSTD_CCtx* cctx, + void* sequenceProducerState, + ZSTD_sequenceProducer_F sequenceProducer +); + +/*! ZSTD_CCtxParams_registerSequenceProducer() : + * Same as ZSTD_registerSequenceProducer(), but operates on ZSTD_CCtx_params. + * This is used for accurate size estimation with ZSTD_estimateCCtxSize_usingCCtxParams(), + * which is needed when creating a ZSTD_CCtx with ZSTD_initStaticCCtx(). + * + * If you are using the external sequence producer API in a scenario where ZSTD_initStaticCCtx() + * is required, then this function is for you. Otherwise, you probably don't need it. + * + * See tests/zstreamtest.c for example usage. */ +ZSTDLIB_STATIC_API void +ZSTD_CCtxParams_registerSequenceProducer( + ZSTD_CCtx_params* params, + void* sequenceProducerState, + ZSTD_sequenceProducer_F sequenceProducer +); /********************************************************************* -* Buffer-less and synchronous inner streaming functions +* Buffer-less and synchronous inner streaming functions (DEPRECATED) +* +* This API is deprecated, and will be removed in a future version. +* It allows streaming (de)compression with user allocated buffers. +* However, it is hard to use, and not as well tested as the rest of +* our API. * -* This is an advanced API, giving full control over buffer management, for users which need direct control over memory. -* But it's also a complex one, with several restrictions, documented below. -* Prefer normal streaming API for an easier experience. +* Please use the normal streaming API instead: ZSTD_compressStream2, +* and ZSTD_decompressStream. +* If there is functionality that you need, but it doesn't provide, +* please open an issue on our GitHub. ********************************************************************* */ /** @@ -2255,12 +2998,10 @@ ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); A ZSTD_CCtx object is required to track streaming operations. Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource. - ZSTD_CCtx object can be re-used multiple times within successive compression operations. + ZSTD_CCtx object can be reused multiple times within successive compression operations. Start by initializing a context. - Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression, - or ZSTD_compressBegin_advanced(), for finer parameter control. - It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx() + Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression. Then, consume your input using ZSTD_compressContinue(). There are some important considerations to keep in mind when using this advanced function : @@ -2278,37 +3019,49 @@ ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame. Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders. - `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress again. + `ZSTD_CCtx` object can be reused (ZSTD_compressBegin()) to compress again. */ /*===== Buffer-less streaming compression functions =====*/ -ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); -ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); -ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */ -ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */ -ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */ -ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */ - -ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); - - +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */ + +ZSTD_DEPRECATED("This function will likely be removed in a future release. It is misleading and has very limited utility.") +ZSTDLIB_STATIC_API +size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */ + +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +/* The ZSTD_compressBegin_advanced() and ZSTD_compressBegin_usingCDict_advanced() are now DEPRECATED and will generate a compiler warning */ +ZSTD_DEPRECATED("use advanced API to access custom parameters") +ZSTDLIB_STATIC_API +size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */ +ZSTD_DEPRECATED("use advanced API to access custom parameters") +ZSTDLIB_STATIC_API +size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */ /** Buffer-less streaming decompression (synchronous mode) A ZSTD_DCtx object is required to track streaming operations. Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. - A ZSTD_DCtx object can be re-used multiple times. + A ZSTD_DCtx object can be reused multiple times. First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader(). Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough. Data fragment must be large enough to ensure successful decoding. `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough. - @result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled. - >0 : `srcSize` is too small, please provide at least @result bytes on next attempt. + result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled. + >0 : `srcSize` is too small, please provide at least result bytes on next attempt. errorCode, which can be tested using ZSTD_isError(). - It fills a ZSTD_frameHeader structure with important information to correctly decode the frame, + It fills a ZSTD_FrameHeader structure with important information to correctly decode the frame, such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`). Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information. As a consequence, check that values remain within valid application range. @@ -2324,7 +3077,7 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci The most memory efficient way is to use a round buffer of sufficient size. Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(), - which can @return an error code if required value is too large for current system (in 32-bits mode). + which can return an error code if required value is too large for current system (in 32-bits mode). In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one, up to the moment there is not enough room left in the buffer to guarantee decoding another full block, which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`. @@ -2344,7 +3097,7 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue(). ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail. - @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). + result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item. It can also be an error code, which can be tested with ZSTD_isError(). @@ -2367,49 +3120,42 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci */ /*===== Buffer-less streaming decompression functions =====*/ -typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e; -typedef struct { - unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */ - unsigned long long windowSize; /* can be very large, up to <= frameContentSize */ - unsigned blockSizeMax; - ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */ - unsigned headerSize; - unsigned dictID; - unsigned checksumFlag; -} ZSTD_frameHeader; -/*! ZSTD_getFrameHeader() : - * decode Frame Header, or requires larger `srcSize`. - * @return : 0, `zfhPtr` is correctly filled, - * >0, `srcSize` is too small, value is wanted `srcSize` amount, - * or an error code, which can be tested using ZSTD_isError() */ -ZSTDLIB_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */ -/*! ZSTD_getFrameHeader_advanced() : - * same as ZSTD_getFrameHeader(), - * with added capability to select a format (like ZSTD_f_zstd1_magicless) */ -ZSTDLIB_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format); -ZSTDLIB_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */ +ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */ -ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); -ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); -ZSTDLIB_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); +ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); +ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); +ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); -ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); -ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTDLIB_STATIC_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); +ZSTDLIB_STATIC_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); /* misc */ -ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); +ZSTD_DEPRECATED("This function will likely be removed in the next minor release. It is misleading and has very limited utility.") +ZSTDLIB_STATIC_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; -ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); +ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); -/* ============================ */ -/** Block level API */ -/* ============================ */ +/* ========================================= */ +/** Block level API (DEPRECATED) */ +/* ========================================= */ /*! + + This API is deprecated in favor of the regular compression API. + You can get the frame header down to 2 bytes by setting: + - ZSTD_c_format = ZSTD_f_zstd1_magicless + - ZSTD_c_contentSizeFlag = 0 + - ZSTD_c_checksumFlag = 0 + - ZSTD_c_dictIDFlag = 0 + + This API is not as well tested as our normal API, so we recommend not using it. + We will be removing it in a future version. If the normal API doesn't provide + the functionality you need, please open a GitHub issue. + Block functions produce and decode raw zstd blocks, without frame metadata. Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes). But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes. @@ -2420,7 +3166,6 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); - It is necessary to init context before starting + compression : any ZSTD_compressBegin*() variant, including with dictionary + decompression : any ZSTD_decompressBegin*() variant, including with dictionary - + copyCCtx() and copyDCtx() can be used too - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB + If input is larger than a block size, it's necessary to split input data into multiple blocks + For inputs larger than a single block, consider using regular ZSTD_compress() instead. @@ -2437,14 +3182,17 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); */ /*===== Raw zstd block functions =====*/ -ZSTDLIB_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx); -ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -ZSTDLIB_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */ - - -#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */ +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx); +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */ #if defined (__cplusplus) } #endif + +#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */ diff --git a/external/basis_universal/zstd/zstd_errors.h b/external/basis_universal/zstd/zstd_errors.h new file mode 100644 index 0000000000..8ebc95cbb2 --- /dev/null +++ b/external/basis_universal/zstd/zstd_errors.h @@ -0,0 +1,107 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_ERRORS_H_398273423 +#define ZSTD_ERRORS_H_398273423 + +#if defined (__cplusplus) +extern "C" { +#endif + +/* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */ +#ifndef ZSTDERRORLIB_VISIBLE + /* Backwards compatibility with old macro name */ +# ifdef ZSTDERRORLIB_VISIBILITY +# define ZSTDERRORLIB_VISIBLE ZSTDERRORLIB_VISIBILITY +# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDERRORLIB_VISIBLE __attribute__ ((visibility ("default"))) +# else +# define ZSTDERRORLIB_VISIBLE +# endif +#endif + +#ifndef ZSTDERRORLIB_HIDDEN +# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDERRORLIB_HIDDEN __attribute__ ((visibility ("hidden"))) +# else +# define ZSTDERRORLIB_HIDDEN +# endif +#endif + +#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBLE +#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBLE +#endif + +/*-********************************************* + * Error codes list + *-********************************************* + * Error codes _values_ are pinned down since v1.3.1 only. + * Therefore, don't rely on values if you may link to any version < v1.3.1. + * + * Only values < 100 are considered stable. + * + * note 1 : this API shall be used with static linking only. + * dynamic linking is not yet officially supported. + * note 2 : Prefer relying on the enum than on its value whenever possible + * This is the only supported way to use the error list < v1.3.1 + * note 3 : ZSTD_isError() is always correct, whatever the library version. + **********************************************/ +typedef enum { + ZSTD_error_no_error = 0, + ZSTD_error_GENERIC = 1, + ZSTD_error_prefix_unknown = 10, + ZSTD_error_version_unsupported = 12, + ZSTD_error_frameParameter_unsupported = 14, + ZSTD_error_frameParameter_windowTooLarge = 16, + ZSTD_error_corruption_detected = 20, + ZSTD_error_checksum_wrong = 22, + ZSTD_error_literals_headerWrong = 24, + ZSTD_error_dictionary_corrupted = 30, + ZSTD_error_dictionary_wrong = 32, + ZSTD_error_dictionaryCreation_failed = 34, + ZSTD_error_parameter_unsupported = 40, + ZSTD_error_parameter_combination_unsupported = 41, + ZSTD_error_parameter_outOfBound = 42, + ZSTD_error_tableLog_tooLarge = 44, + ZSTD_error_maxSymbolValue_tooLarge = 46, + ZSTD_error_maxSymbolValue_tooSmall = 48, + ZSTD_error_cannotProduce_uncompressedBlock = 49, + ZSTD_error_stabilityCondition_notRespected = 50, + ZSTD_error_stage_wrong = 60, + ZSTD_error_init_missing = 62, + ZSTD_error_memory_allocation = 64, + ZSTD_error_workSpace_tooSmall= 66, + ZSTD_error_dstSize_tooSmall = 70, + ZSTD_error_srcSize_wrong = 72, + ZSTD_error_dstBuffer_null = 74, + ZSTD_error_noForwardProgress_destFull = 80, + ZSTD_error_noForwardProgress_inputEmpty = 82, + /* following error codes are __NOT STABLE__, they can be removed or changed in future versions */ + ZSTD_error_frameIndex_tooLarge = 100, + ZSTD_error_seekableIO = 102, + ZSTD_error_dstBuffer_wrong = 104, + ZSTD_error_srcBuffer_wrong = 105, + ZSTD_error_sequenceProducer_failed = 106, + ZSTD_error_externalSequences_invalid = 107, + ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */ +} ZSTD_ErrorCode; + +ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); /**< Same as ZSTD_getErrorName, but using a `ZSTD_ErrorCode` enum argument */ + + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_ERRORS_H_398273423 */ diff --git a/external/basis_universal/zstd/zstddeclib.c b/external/basis_universal/zstd/zstddeclib.c index de206f64bf..875d3b735a 100644 --- a/external/basis_universal/zstd/zstddeclib.c +++ b/external/basis_universal/zstd/zstddeclib.c @@ -4,11 +4,11 @@ * * Generate using: * \code - * combine.sh -r ../../lib -o zstddeclib.c zstddeclib-in.c + * python combine.py -r ../../lib -x legacy/zstd_legacy.h -o zstddeclib.c zstddeclib-in.c * \endcode */ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -25,8 +25,12 @@ * Note: MEM_MODULE stops xxhash redefining BYTE, U16, etc., which are also * defined in mem.h (breaking C99 compatibility). * - * Note: the undefs for xxHash allow Zstd's implementation to coinside with with + * Note: the undefs for xxHash allow Zstd's implementation to coincide with * standalone xxHash usage (with global defines). + * + * Note: if you enable ZSTD_LEGACY_SUPPORT the combine.py script will need + * re-running without the "-x legacy/zstd_legacy.h" option (it excludes the + * legacy support at the source level). */ #define DEBUGLEVEL 0 #define MEM_MODULE @@ -39,12 +43,14 @@ #define ZSTD_LEGACY_SUPPORT 0 #define ZSTD_STRIP_ERROR_STRINGS #define ZSTD_TRACE 0 +/* TODO: Can't amalgamate ASM function */ +#define ZSTD_DISABLE_ASM 1 /* Include zstd_deps.h first with all the options we need enabled. */ #define ZSTD_DEPS_NEED_MALLOC /**** start inlining common/zstd_deps.h ****/ /* - * Copyright (c) 2016-2021, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -69,6 +75,18 @@ #ifndef ZSTD_DEPS_COMMON #define ZSTD_DEPS_COMMON +/* Even though we use qsort_r only for the dictionary builder, the macro + * _GNU_SOURCE has to be declared *before* the inclusion of any standard + * header and the script 'combine.sh' combines the whole zstd source code + * in a single file. + */ +#if defined(__linux) || defined(__linux__) || defined(linux) || defined(__gnu_linux__) || \ + defined(__CYGWIN__) || defined(__MSYS__) +#if !defined(_GNU_SOURCE) && !defined(__ANDROID__) /* NDK doesn't ship qsort_r(). */ +#define _GNU_SOURCE +#endif +#endif + #include #include #include @@ -160,7 +178,7 @@ /* ****************************************************************** * debug * Part of FSE library - * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -182,7 +200,7 @@ /* ****************************************************************** * debug * Part of FSE library - * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -213,10 +231,6 @@ #ifndef DEBUG_H_12987983217 #define DEBUG_H_12987983217 -#if defined (__cplusplus) -extern "C" { -#endif - /* static assert is triggered at compile time, leaving no runtime artefact. * static assert only works with compile-time constants. @@ -266,34 +280,44 @@ extern int g_debuglevel; /* the variable is only declared, It's useful when enabling very verbose levels on selective conditions (such as position in src) */ -# define RAWLOG(l, ...) { \ - if (l<=g_debuglevel) { \ - ZSTD_DEBUG_PRINT(__VA_ARGS__); \ - } } -# define DEBUGLOG(l, ...) { \ - if (l<=g_debuglevel) { \ - ZSTD_DEBUG_PRINT(__FILE__ ": " __VA_ARGS__); \ - ZSTD_DEBUG_PRINT(" \n"); \ - } } +# define RAWLOG(l, ...) \ + do { \ + if (l<=g_debuglevel) { \ + ZSTD_DEBUG_PRINT(__VA_ARGS__); \ + } \ + } while (0) + +#define STRINGIFY(x) #x +#define TOSTRING(x) STRINGIFY(x) +#define LINE_AS_STRING TOSTRING(__LINE__) + +# define DEBUGLOG(l, ...) \ + do { \ + if (l<=g_debuglevel) { \ + ZSTD_DEBUG_PRINT(__FILE__ ":" LINE_AS_STRING ": " __VA_ARGS__); \ + ZSTD_DEBUG_PRINT(" \n"); \ + } \ + } while (0) #else -# define RAWLOG(l, ...) {} /* disabled */ -# define DEBUGLOG(l, ...) {} /* disabled */ -#endif - - -#if defined (__cplusplus) -} +# define RAWLOG(l, ...) do { } while (0) /* disabled */ +# define DEBUGLOG(l, ...) do { } while (0) /* disabled */ #endif #endif /* DEBUG_H_12987983217 */ /**** ended inlining debug.h ****/ +#if !defined(ZSTD_LINUX_KERNEL) || (DEBUGLEVEL>=2) +/* We only use this when DEBUGLEVEL>=2, but we get -Werror=pedantic errors if a + * translation unit is empty. So remove this from Linux kernel builds, but + * otherwise just leave it in. + */ int g_debuglevel = DEBUGLEVEL; +#endif /**** ended inlining common/debug.c ****/ /**** start inlining common/entropy_common.c ****/ /* ****************************************************************** * Common functions of New Generation Entropy library - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -310,7 +334,7 @@ int g_debuglevel = DEBUGLEVEL; ***************************************/ /**** start inlining mem.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -322,17 +346,13 @@ int g_debuglevel = DEBUGLEVEL; #ifndef MEM_H_MODULE #define MEM_H_MODULE -#if defined (__cplusplus) -extern "C" { -#endif - /*-**************************************** * Dependencies ******************************************/ #include /* size_t, ptrdiff_t */ /**** start inlining compiler.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -344,6 +364,182 @@ extern "C" { #ifndef ZSTD_COMPILER_H #define ZSTD_COMPILER_H +#include + +/**** start inlining portability_macros.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_PORTABILITY_MACROS_H +#define ZSTD_PORTABILITY_MACROS_H + +/** + * This header file contains macro definitions to support portability. + * This header is shared between C and ASM code, so it MUST only + * contain macro definitions. It MUST not contain any C code. + * + * This header ONLY defines macros to detect platforms/feature support. + * + */ + + +/* compat. with non-clang compilers */ +#ifndef __has_attribute + #define __has_attribute(x) 0 +#endif + +/* compat. with non-clang compilers */ +#ifndef __has_builtin +# define __has_builtin(x) 0 +#endif + +/* compat. with non-clang compilers */ +#ifndef __has_feature +# define __has_feature(x) 0 +#endif + +/* detects whether we are being compiled under msan */ +#ifndef ZSTD_MEMORY_SANITIZER +# if __has_feature(memory_sanitizer) +# define ZSTD_MEMORY_SANITIZER 1 +# else +# define ZSTD_MEMORY_SANITIZER 0 +# endif +#endif + +/* detects whether we are being compiled under asan */ +#ifndef ZSTD_ADDRESS_SANITIZER +# if __has_feature(address_sanitizer) +# define ZSTD_ADDRESS_SANITIZER 1 +# elif defined(__SANITIZE_ADDRESS__) +# define ZSTD_ADDRESS_SANITIZER 1 +# else +# define ZSTD_ADDRESS_SANITIZER 0 +# endif +#endif + +/* detects whether we are being compiled under dfsan */ +#ifndef ZSTD_DATAFLOW_SANITIZER +# if __has_feature(dataflow_sanitizer) +# define ZSTD_DATAFLOW_SANITIZER 1 +# else +# define ZSTD_DATAFLOW_SANITIZER 0 +# endif +#endif + +/* Mark the internal assembly functions as hidden */ +#ifdef __ELF__ +# define ZSTD_HIDE_ASM_FUNCTION(func) .hidden func +#elif defined(__APPLE__) +# define ZSTD_HIDE_ASM_FUNCTION(func) .private_extern func +#else +# define ZSTD_HIDE_ASM_FUNCTION(func) +#endif + +/* Compile time determination of BMI2 support */ +#ifndef STATIC_BMI2 +# if defined(__BMI2__) +# define STATIC_BMI2 1 +# elif defined(_MSC_VER) && defined(__AVX2__) +# define STATIC_BMI2 1 /* MSVC does not have a BMI2 specific flag, but every CPU that supports AVX2 also supports BMI2 */ +# endif +#endif + +#ifndef STATIC_BMI2 +# define STATIC_BMI2 0 +#endif + +/* Enable runtime BMI2 dispatch based on the CPU. + * Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default. + */ +#ifndef DYNAMIC_BMI2 +# if ((defined(__clang__) && __has_attribute(__target__)) \ + || (defined(__GNUC__) \ + && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \ + && (defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) || defined(_M_X64)) \ + && !defined(__BMI2__) +# define DYNAMIC_BMI2 1 +# else +# define DYNAMIC_BMI2 0 +# endif +#endif + +/** + * Only enable assembly for GNU C compatible compilers, + * because other platforms may not support GAS assembly syntax. + * + * Only enable assembly for Linux / MacOS / Win32, other platforms may + * work, but they haven't been tested. This could likely be + * extended to BSD systems. + * + * Disable assembly when MSAN is enabled, because MSAN requires + * 100% of code to be instrumented to work. + */ +#if defined(__GNUC__) +# if defined(__linux__) || defined(__linux) || defined(__APPLE__) || defined(_WIN32) +# if ZSTD_MEMORY_SANITIZER +# define ZSTD_ASM_SUPPORTED 0 +# elif ZSTD_DATAFLOW_SANITIZER +# define ZSTD_ASM_SUPPORTED 0 +# else +# define ZSTD_ASM_SUPPORTED 1 +# endif +# else +# define ZSTD_ASM_SUPPORTED 0 +# endif +#else +# define ZSTD_ASM_SUPPORTED 0 +#endif + +/** + * Determines whether we should enable assembly for x86-64 + * with BMI2. + * + * Enable if all of the following conditions hold: + * - ASM hasn't been explicitly disabled by defining ZSTD_DISABLE_ASM + * - Assembly is supported + * - We are compiling for x86-64 and either: + * - DYNAMIC_BMI2 is enabled + * - BMI2 is supported at compile time + */ +#if !defined(ZSTD_DISABLE_ASM) && \ + ZSTD_ASM_SUPPORTED && \ + defined(__x86_64__) && \ + (DYNAMIC_BMI2 || defined(__BMI2__)) +# define ZSTD_ENABLE_ASM_X86_64_BMI2 1 +#else +# define ZSTD_ENABLE_ASM_X86_64_BMI2 0 +#endif + +/* + * For x86 ELF targets, add .note.gnu.property section for Intel CET in + * assembly sources when CET is enabled. + * + * Additionally, any function that may be called indirectly must begin + * with ZSTD_CET_ENDBRANCH. + */ +#if defined(__ELF__) && (defined(__x86_64__) || defined(__i386__)) \ + && defined(__has_include) +# if __has_include() +# include +# define ZSTD_CET_ENDBRANCH _CET_ENDBR +# endif +#endif + +#ifndef ZSTD_CET_ENDBRANCH +# define ZSTD_CET_ENDBRANCH +#endif + +#endif /* ZSTD_PORTABILITY_MACROS_H */ +/**** ended inlining portability_macros.h ****/ + /*-******************************************************* * Compiler specifics *********************************************************/ @@ -356,7 +552,7 @@ extern "C" { # define INLINE_KEYWORD #endif -#if defined(__GNUC__) || defined(__ICCARM__) +#if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__) # define FORCE_INLINE_ATTR __attribute__((always_inline)) #elif defined(_MSC_VER) # define FORCE_INLINE_ATTR __forceinline @@ -373,7 +569,7 @@ extern "C" { /** On MSVC qsort requires that functions passed into it use the __cdecl calling conversion(CC). - This explictly marks such functions as __cdecl so that the code will still compile + This explicitly marks such functions as __cdecl so that the code will still compile if a CC other than __cdecl has been made the default. */ #if defined(_MSC_VER) @@ -382,12 +578,19 @@ extern "C" { # define WIN_CDECL #endif +/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */ +#if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__) +# define UNUSED_ATTR __attribute__((unused)) +#else +# define UNUSED_ATTR +#endif + /** * FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant * parameters. They must be inlined for the compiler to eliminate the constant * branches. */ -#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR +#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR UNUSED_ATTR /** * HINT_INLINE is used to help the compiler generate better code. It is *not* * used for "templates", so it can be tweaked based on the compilers @@ -402,21 +605,37 @@ extern "C" { #if !defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 8 && __GNUC__ < 5 # define HINT_INLINE static INLINE_KEYWORD #else -# define HINT_INLINE static INLINE_KEYWORD FORCE_INLINE_ATTR +# define HINT_INLINE FORCE_INLINE_TEMPLATE #endif -/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */ +/* "soft" inline : + * The compiler is free to select if it's a good idea to inline or not. + * The main objective is to silence compiler warnings + * when a defined function in included but not used. + * + * Note : this macro is prefixed `MEM_` because it used to be provided by `mem.h` unit. + * Updating the prefix is probably preferable, but requires a fairly large codemod, + * since this name is used everywhere. + */ +#ifndef MEM_STATIC /* already defined in Linux Kernel mem.h */ #if defined(__GNUC__) -# define UNUSED_ATTR __attribute__((unused)) +# define MEM_STATIC static __inline UNUSED_ATTR +#elif defined(__IAR_SYSTEMS_ICC__) +# define MEM_STATIC static inline UNUSED_ATTR +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define MEM_STATIC static inline +#elif defined(_MSC_VER) +# define MEM_STATIC static __inline #else -# define UNUSED_ATTR +# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif #endif /* force no inlining */ #ifdef _MSC_VER # define FORCE_NOINLINE static __declspec(noinline) #else -# if defined(__GNUC__) || defined(__ICCARM__) +# if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__) # define FORCE_NOINLINE static __attribute__((__noinline__)) # else # define FORCE_NOINLINE static @@ -425,37 +644,25 @@ extern "C" { /* target attribute */ -#ifndef __has_attribute - #define __has_attribute(x) 0 /* Compatibility with non-clang compilers. */ -#endif -#if defined(__GNUC__) || defined(__ICCARM__) +#if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__) # define TARGET_ATTRIBUTE(target) __attribute__((__target__(target))) #else # define TARGET_ATTRIBUTE(target) #endif -/* Enable runtime BMI2 dispatch based on the CPU. - * Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default. +/* Target attribute for BMI2 dynamic dispatch. + * Enable lzcnt, bmi, and bmi2. + * We test for bmi1 & bmi2. lzcnt is included in bmi1. */ -#ifndef DYNAMIC_BMI2 - #if ((defined(__clang__) && __has_attribute(__target__)) \ - || (defined(__GNUC__) \ - && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \ - && (defined(__x86_64__) || defined(_M_X86)) \ - && !defined(__BMI2__) - # define DYNAMIC_BMI2 1 - #else - # define DYNAMIC_BMI2 0 - #endif -#endif +#define BMI2_TARGET_ATTRIBUTE TARGET_ATTRIBUTE("lzcnt,bmi,bmi2") /* prefetch * can be disabled, by declaring NO_PREFETCH build macro */ #if defined(NO_PREFETCH) -# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */ -# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */ +# define PREFETCH_L1(ptr) do { (void)(ptr); } while (0) /* disabled */ +# define PREFETCH_L2(ptr) do { (void)(ptr); } while (0) /* disabled */ #else -# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */ +# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) && !defined(_M_ARM64EC) /* _mm_prefetch() is not defined outside of x86/x64 */ # include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ # define PREFETCH_L1(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) # define PREFETCH_L2(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T1) @@ -463,28 +670,30 @@ extern "C" { # define PREFETCH_L1(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) # define PREFETCH_L2(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */) # elif defined(__aarch64__) -# define PREFETCH_L1(ptr) __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(ptr))) -# define PREFETCH_L2(ptr) __asm__ __volatile__("prfm pldl2keep, %0" ::"Q"(*(ptr))) +# define PREFETCH_L1(ptr) do { __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(ptr))); } while (0) +# define PREFETCH_L2(ptr) do { __asm__ __volatile__("prfm pldl2keep, %0" ::"Q"(*(ptr))); } while (0) # else -# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */ -# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */ +# define PREFETCH_L1(ptr) do { (void)(ptr); } while (0) /* disabled */ +# define PREFETCH_L2(ptr) do { (void)(ptr); } while (0) /* disabled */ # endif #endif /* NO_PREFETCH */ #define CACHELINE_SIZE 64 -#define PREFETCH_AREA(p, s) { \ - const char* const _ptr = (const char*)(p); \ - size_t const _size = (size_t)(s); \ - size_t _pos; \ - for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) { \ - PREFETCH_L2(_ptr + _pos); \ - } \ -} +#define PREFETCH_AREA(p, s) \ + do { \ + const char* const _ptr = (const char*)(p); \ + size_t const _size = (size_t)(s); \ + size_t _pos; \ + for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) { \ + PREFETCH_L2(_ptr + _pos); \ + } \ + } while (0) /* vectorization - * older GCC (pre gcc-4.3 picked as the cutoff) uses a different syntax */ -#if !defined(__INTEL_COMPILER) && !defined(__clang__) && defined(__GNUC__) + * older GCC (pre gcc-4.3 picked as the cutoff) uses a different syntax, + * and some compilers, like Intel ICC and MCST LCC, do not support it at all. */ +#if !defined(__INTEL_COMPILER) && !defined(__clang__) && defined(__GNUC__) && !defined(__LCC__) # if (__GNUC__ == 4 && __GNUC_MINOR__ > 3) || (__GNUC__ >= 5) # define DONT_VECTORIZE __attribute__((optimize("no-tree-vectorize"))) # else @@ -507,6 +716,12 @@ extern "C" { #define UNLIKELY(x) (x) #endif +#if __has_builtin(__builtin_unreachable) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5))) +# define ZSTD_UNREACHABLE do { assert(0), __builtin_unreachable(); } while (0) +#else +# define ZSTD_UNREACHABLE do { assert(0); } while (0) +#endif + /* disable warnings */ #ifdef _MSC_VER /* Visual Studio */ # include /* For Visual 2005 */ @@ -517,39 +732,198 @@ extern "C" { # pragma warning(disable : 4324) /* disable: C4324: padded structure */ #endif -/*Like DYNAMIC_BMI2 but for compile time determination of BMI2 support*/ -#ifndef STATIC_BMI2 -# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) -# ifdef __AVX2__ //MSVC does not have a BMI2 specific flag, but every CPU that supports AVX2 also supports BMI2 -# define STATIC_BMI2 1 -# endif +/* compile time determination of SIMD support */ +#if !defined(ZSTD_NO_INTRINSICS) +# if defined(__AVX2__) +# define ZSTD_ARCH_X86_AVX2 +# endif +# if defined(__SSE2__) || defined(_M_X64) || (defined (_M_IX86) && defined(_M_IX86_FP) && (_M_IX86_FP >= 2)) +# define ZSTD_ARCH_X86_SSE2 +# endif +# if defined(__ARM_NEON) || defined(_M_ARM64) +# define ZSTD_ARCH_ARM_NEON +# endif +# +# if defined(ZSTD_ARCH_X86_AVX2) +# include +# endif +# if defined(ZSTD_ARCH_X86_SSE2) +# include +# elif defined(ZSTD_ARCH_ARM_NEON) +# include # endif #endif -#ifndef STATIC_BMI2 - #define STATIC_BMI2 0 +/* C-language Attributes are added in C23. */ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201710L) && defined(__has_c_attribute) +# define ZSTD_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) +#else +# define ZSTD_HAS_C_ATTRIBUTE(x) 0 #endif -/* compat. with non-clang compilers */ -#ifndef __has_builtin -# define __has_builtin(x) 0 +/* Only use C++ attributes in C++. Some compilers report support for C++ + * attributes when compiling with C. + */ +#if defined(__cplusplus) && defined(__has_cpp_attribute) +# define ZSTD_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) +#else +# define ZSTD_HAS_CPP_ATTRIBUTE(x) 0 #endif -/* compat. with non-clang compilers */ -#ifndef __has_feature -# define __has_feature(x) 0 +/* Define ZSTD_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute. + * - C23: https://en.cppreference.com/w/c/language/attributes/fallthrough + * - CPP17: https://en.cppreference.com/w/cpp/language/attributes/fallthrough + * - Else: __attribute__((__fallthrough__)) + */ +#ifndef ZSTD_FALLTHROUGH +# if ZSTD_HAS_C_ATTRIBUTE(fallthrough) +# define ZSTD_FALLTHROUGH [[fallthrough]] +# elif ZSTD_HAS_CPP_ATTRIBUTE(fallthrough) +# define ZSTD_FALLTHROUGH [[fallthrough]] +# elif __has_attribute(__fallthrough__) +/* Leading semicolon is to satisfy gcc-11 with -pedantic. Without the semicolon + * gcc complains about: a label can only be part of a statement and a declaration is not a statement. + */ +# define ZSTD_FALLTHROUGH ; __attribute__((__fallthrough__)) +# else +# define ZSTD_FALLTHROUGH +# endif #endif -/* detects whether we are being compiled under msan */ -#ifndef ZSTD_MEMORY_SANITIZER -# if __has_feature(memory_sanitizer) -# define ZSTD_MEMORY_SANITIZER 1 +/*-************************************************************** +* Alignment +*****************************************************************/ + +/* @return 1 if @u is a 2^n value, 0 otherwise + * useful to check a value is valid for alignment restrictions */ +MEM_STATIC int ZSTD_isPower2(size_t u) { + return (u & (u-1)) == 0; +} + +/* this test was initially positioned in mem.h, + * but this file is removed (or replaced) for linux kernel + * so it's now hosted in compiler.h, + * which remains valid for both user & kernel spaces. + */ + +#ifndef ZSTD_ALIGNOF +# if defined(__GNUC__) || defined(_MSC_VER) +/* covers gcc, clang & MSVC */ +/* note : this section must come first, before C11, + * due to a limitation in the kernel source generator */ +# define ZSTD_ALIGNOF(T) __alignof(T) + +# elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) +/* C11 support */ +# include +# define ZSTD_ALIGNOF(T) alignof(T) + +# else +/* No known support for alignof() - imperfect backup */ +# define ZSTD_ALIGNOF(T) (sizeof(void*) < sizeof(T) ? sizeof(void*) : sizeof(T)) + +# endif +#endif /* ZSTD_ALIGNOF */ + +#ifndef ZSTD_ALIGNED +/* C90-compatible alignment macro (GCC/Clang). Adjust for other compilers if needed. */ +# if defined(__GNUC__) || defined(__clang__) +# define ZSTD_ALIGNED(a) __attribute__((aligned(a))) +# elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */ +# define ZSTD_ALIGNED(a) _Alignas(a) +#elif defined(_MSC_VER) +# define ZSTD_ALIGNED(n) __declspec(align(n)) +# else + /* this compiler will require its own alignment instruction */ +# define ZSTD_ALIGNED(...) +# endif +#endif /* ZSTD_ALIGNED */ + + +/*-************************************************************** +* Sanitizer +*****************************************************************/ + +/** + * Zstd relies on pointer overflow in its decompressor. + * We add this attribute to functions that rely on pointer overflow. + */ +#ifndef ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +# if __has_attribute(no_sanitize) +# if !defined(__clang__) && defined(__GNUC__) && __GNUC__ < 8 + /* gcc < 8 only has signed-integer-overlow which triggers on pointer overflow */ +# define ZSTD_ALLOW_POINTER_OVERFLOW_ATTR __attribute__((no_sanitize("signed-integer-overflow"))) +# else + /* older versions of clang [3.7, 5.0) will warn that pointer-overflow is ignored. */ +# define ZSTD_ALLOW_POINTER_OVERFLOW_ATTR __attribute__((no_sanitize("pointer-overflow"))) +# endif # else -# define ZSTD_MEMORY_SANITIZER 0 +# define ZSTD_ALLOW_POINTER_OVERFLOW_ATTR # endif #endif -#if ZSTD_MEMORY_SANITIZER +/** + * Helper function to perform a wrapped pointer difference without triggering + * UBSAN. + * + * @returns lhs - rhs with wrapping + */ +MEM_STATIC +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +ptrdiff_t ZSTD_wrappedPtrDiff(unsigned char const* lhs, unsigned char const* rhs) +{ + return lhs - rhs; +} + +/** + * Helper function to perform a wrapped pointer add without triggering UBSAN. + * + * @return ptr + add with wrapping + */ +MEM_STATIC +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +unsigned char const* ZSTD_wrappedPtrAdd(unsigned char const* ptr, ptrdiff_t add) +{ + return ptr + add; +} + +/** + * Helper function to perform a wrapped pointer subtraction without triggering + * UBSAN. + * + * @return ptr - sub with wrapping + */ +MEM_STATIC +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +unsigned char const* ZSTD_wrappedPtrSub(unsigned char const* ptr, ptrdiff_t sub) +{ + return ptr - sub; +} + +/** + * Helper function to add to a pointer that works around C's undefined behavior + * of adding 0 to NULL. + * + * @returns `ptr + add` except it defines `NULL + 0 == NULL`. + */ +MEM_STATIC +unsigned char* ZSTD_maybeNullPtrAdd(unsigned char* ptr, ptrdiff_t add) +{ + return add > 0 ? ptr + add : ptr; +} + +/* Issue #3240 reports an ASAN failure on an llvm-mingw build. Out of an + * abundance of caution, disable our custom poisoning on mingw. */ +#ifdef __MINGW32__ +#ifndef ZSTD_ASAN_DONT_POISON_WORKSPACE +#define ZSTD_ASAN_DONT_POISON_WORKSPACE 1 +#endif +#ifndef ZSTD_MSAN_DONT_POISON_WORKSPACE +#define ZSTD_MSAN_DONT_POISON_WORKSPACE 1 +#endif +#endif + +#if ZSTD_MEMORY_SANITIZER && !defined(ZSTD_MSAN_DONT_POISON_WORKSPACE) /* Not all platforms that support msan provide sanitizers/msan_interface.h. * We therefore declare the functions we need ourselves, rather than trying to * include the header file... */ @@ -568,20 +942,13 @@ void __msan_poison(const volatile void *a, size_t size); /* Returns the offset of the first (at least partially) poisoned byte in the memory range, or -1 if the whole range is good. */ intptr_t __msan_test_shadow(const volatile void *x, size_t size); -#endif -/* detects whether we are being compiled under asan */ -#ifndef ZSTD_ADDRESS_SANITIZER -# if __has_feature(address_sanitizer) -# define ZSTD_ADDRESS_SANITIZER 1 -# elif defined(__SANITIZE_ADDRESS__) -# define ZSTD_ADDRESS_SANITIZER 1 -# else -# define ZSTD_ADDRESS_SANITIZER 0 -# endif +/* Print shadow and origin for the memory range to stderr in a human-readable + format. */ +void __msan_print_shadow(const volatile void *x, size_t size); #endif -#if ZSTD_ADDRESS_SANITIZER +#if ZSTD_ADDRESS_SANITIZER && !defined(ZSTD_ASAN_DONT_POISON_WORKSPACE) /* Not all platforms that support asan provide sanitizers/asan_interface.h. * We therefore declare the functions we need ourselves, rather than trying to * include the header file... */ @@ -631,15 +998,8 @@ void __asan_unpoison_memory_region(void const volatile *addr, size_t size); #if defined(_MSC_VER) /* Visual Studio */ # include /* _byteswap_ulong */ # include /* _byteswap_* */ -#endif -#if defined(__GNUC__) -# define MEM_STATIC static __inline __attribute__((unused)) -#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define MEM_STATIC static inline -#elif defined(_MSC_VER) -# define MEM_STATIC static __inline -#else -# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#elif defined(__ICCARM__) +# include #endif /*-************************************************************** @@ -652,6 +1012,8 @@ void __asan_unpoison_memory_region(void const volatile *addr, size_t size); # include /* intptr_t */ # endif typedef uint8_t BYTE; + typedef uint8_t U8; + typedef int8_t S8; typedef uint16_t U16; typedef int16_t S16; typedef uint32_t U32; @@ -664,6 +1026,8 @@ void __asan_unpoison_memory_region(void const volatile *addr, size_t size); # error "this implementation requires char to be exactly 8-bit type" #endif typedef unsigned char BYTE; + typedef unsigned char U8; + typedef signed char S8; #if USHRT_MAX != 65535 # error "this implementation requires short to be exactly 16-bit type" #endif @@ -680,7 +1044,6 @@ void __asan_unpoison_memory_region(void const volatile *addr, size_t size); typedef signed long long S64; #endif - /*-************************************************************** * Memory I/O API *****************************************************************/ @@ -730,23 +1093,15 @@ MEM_STATIC size_t MEM_swapST(size_t in); /*-************************************************************** * Memory I/O Implementation *****************************************************************/ -/* MEM_FORCE_MEMORY_ACCESS : - * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. - * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. - * The below switch allow to select different access method for improved performance. - * Method 0 (default) : use `memcpy()`. Safe and portable. - * Method 1 : `__packed` statement. It depends on compiler extension (i.e., not portable). - * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. +/* MEM_FORCE_MEMORY_ACCESS : For accessing unaligned memory: + * Method 0 : always use `memcpy()`. Safe and portable. + * Method 1 : Use compiler extension to set unaligned access. * Method 2 : direct access. This method is portable but violate C standard. * It can generate buggy code on targets depending on alignment. - * In some circumstances, it's the only known way to get the most performance (i.e. GCC + ARMv6) - * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. - * Prefer these methods in priority order (0 > 1 > 2) + * Default : method 1 if supported, else method 0 */ #ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ -# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) -# define MEM_FORCE_MEMORY_ACCESS 2 -# elif defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__) +# ifdef __GNUC__ # define MEM_FORCE_MEMORY_ACCESS 1 # endif #endif @@ -756,8 +1111,24 @@ MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; } MEM_STATIC unsigned MEM_isLittleEndian(void) { +#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + return 1; +#elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + return 0; +#elif defined(__clang__) && __LITTLE_ENDIAN__ + return 1; +#elif defined(__clang__) && __BIG_ENDIAN__ + return 0; +#elif defined(_MSC_VER) && (_M_X64 || _M_IX86) + return 1; +#elif defined(__DMC__) && defined(_M_IX86) + return 1; +#elif defined(__IAR_SYSTEMS_ICC__) && __LITTLE_ENDIAN__ + return 1; +#else const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ return one.c[0]; +#endif } #if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2) @@ -775,30 +1146,19 @@ MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; } #elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1) -/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ -/* currently only defined for gcc and icc */ -#if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32)) - __pragma( pack(push, 1) ) - typedef struct { U16 v; } unalign16; - typedef struct { U32 v; } unalign32; - typedef struct { U64 v; } unalign64; - typedef struct { size_t v; } unalignArch; - __pragma( pack(pop) ) -#else - typedef struct { U16 v; } __attribute__((packed)) unalign16; - typedef struct { U32 v; } __attribute__((packed)) unalign32; - typedef struct { U64 v; } __attribute__((packed)) unalign64; - typedef struct { size_t v; } __attribute__((packed)) unalignArch; -#endif +typedef __attribute__((aligned(1))) U16 unalign16; +typedef __attribute__((aligned(1))) U32 unalign32; +typedef __attribute__((aligned(1))) U64 unalign64; +typedef __attribute__((aligned(1))) size_t unalignArch; -MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign16*)ptr)->v; } -MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign32*)ptr)->v; } -MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign64*)ptr)->v; } -MEM_STATIC size_t MEM_readST(const void* ptr) { return ((const unalignArch*)ptr)->v; } +MEM_STATIC U16 MEM_read16(const void* ptr) { return *(const unalign16*)ptr; } +MEM_STATIC U32 MEM_read32(const void* ptr) { return *(const unalign32*)ptr; } +MEM_STATIC U64 MEM_read64(const void* ptr) { return *(const unalign64*)ptr; } +MEM_STATIC size_t MEM_readST(const void* ptr) { return *(const unalignArch*)ptr; } -MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign16*)memPtr)->v = value; } -MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign32*)memPtr)->v = value; } -MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign64*)memPtr)->v = value; } +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(unalign16*)memPtr = value; } +MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(unalign32*)memPtr = value; } +MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(unalign64*)memPtr = value; } #else @@ -842,6 +1202,14 @@ MEM_STATIC void MEM_write64(void* memPtr, U64 value) #endif /* MEM_FORCE_MEMORY_ACCESS */ +MEM_STATIC U32 MEM_swap32_fallback(U32 in) +{ + return ((in << 24) & 0xff000000 ) | + ((in << 8) & 0x00ff0000 ) | + ((in >> 8) & 0x0000ff00 ) | + ((in >> 24) & 0x000000ff ); +} + MEM_STATIC U32 MEM_swap32(U32 in) { #if defined(_MSC_VER) /* Visual Studio */ @@ -849,23 +1217,16 @@ MEM_STATIC U32 MEM_swap32(U32 in) #elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \ || (defined(__clang__) && __has_builtin(__builtin_bswap32)) return __builtin_bswap32(in); +#elif defined(__ICCARM__) + return __REV(in); #else - return ((in << 24) & 0xff000000 ) | - ((in << 8) & 0x00ff0000 ) | - ((in >> 8) & 0x0000ff00 ) | - ((in >> 24) & 0x000000ff ); + return MEM_swap32_fallback(in); #endif } -MEM_STATIC U64 MEM_swap64(U64 in) +MEM_STATIC U64 MEM_swap64_fallback(U64 in) { -#if defined(_MSC_VER) /* Visual Studio */ - return _byteswap_uint64(in); -#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \ - || (defined(__clang__) && __has_builtin(__builtin_bswap64)) - return __builtin_bswap64(in); -#else - return ((in << 56) & 0xff00000000000000ULL) | + return ((in << 56) & 0xff00000000000000ULL) | ((in << 40) & 0x00ff000000000000ULL) | ((in << 24) & 0x0000ff0000000000ULL) | ((in << 8) & 0x000000ff00000000ULL) | @@ -873,6 +1234,17 @@ MEM_STATIC U64 MEM_swap64(U64 in) ((in >> 24) & 0x0000000000ff0000ULL) | ((in >> 40) & 0x000000000000ff00ULL) | ((in >> 56) & 0x00000000000000ffULL); +} + +MEM_STATIC U64 MEM_swap64(U64 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_uint64(in); +#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \ + || (defined(__clang__) && __has_builtin(__builtin_bswap64)) + return __builtin_bswap64(in); +#else + return MEM_swap64_fallback(in); #endif } @@ -909,7 +1281,7 @@ MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val) MEM_STATIC U32 MEM_readLE24(const void* memPtr) { - return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16); + return (U32)MEM_readLE16(memPtr) + ((U32)(((const BYTE*)memPtr)[2]) << 16); } MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val) @@ -1019,16 +1391,11 @@ MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val) /* code only tested on 32 and 64 bits systems */ MEM_STATIC void MEM_check(void) { DEBUG_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); } - -#if defined (__cplusplus) -} -#endif - #endif /* MEM_H_MODULE */ /**** ended inlining mem.h ****/ /**** start inlining error_private.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -1042,18 +1409,12 @@ MEM_STATIC void MEM_check(void) { DEBUG_STATIC_ASSERT((sizeof(size_t)==4) || (si #ifndef ERROR_H_MODULE #define ERROR_H_MODULE -#if defined (__cplusplus) -extern "C" { -#endif - - /* **************************************** * Dependencies ******************************************/ -/**** skipping file: zstd_deps.h ****/ -/**** start inlining zstd_errors.h ****/ +/**** start inlining ../zstd_errors.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -1069,24 +1430,32 @@ extern "C" { extern "C" { #endif -/*===== dependency =====*/ -#include /* size_t */ - - /* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */ -#ifndef ZSTDERRORLIB_VISIBILITY -# if defined(__GNUC__) && (__GNUC__ >= 4) -# define ZSTDERRORLIB_VISIBILITY __attribute__ ((visibility ("default"))) +#ifndef ZSTDERRORLIB_VISIBLE + /* Backwards compatibility with old macro name */ +# ifdef ZSTDERRORLIB_VISIBILITY +# define ZSTDERRORLIB_VISIBLE ZSTDERRORLIB_VISIBILITY +# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDERRORLIB_VISIBLE __attribute__ ((visibility ("default"))) +# else +# define ZSTDERRORLIB_VISIBLE +# endif +#endif + +#ifndef ZSTDERRORLIB_HIDDEN +# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDERRORLIB_HIDDEN __attribute__ ((visibility ("hidden"))) # else -# define ZSTDERRORLIB_VISIBILITY +# define ZSTDERRORLIB_HIDDEN # endif #endif + #if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) -# define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBILITY +# define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBLE #elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) -# define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +# define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ #else -# define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBILITY +# define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBLE #endif /*-********************************************* @@ -1112,14 +1481,18 @@ typedef enum { ZSTD_error_frameParameter_windowTooLarge = 16, ZSTD_error_corruption_detected = 20, ZSTD_error_checksum_wrong = 22, + ZSTD_error_literals_headerWrong = 24, ZSTD_error_dictionary_corrupted = 30, ZSTD_error_dictionary_wrong = 32, ZSTD_error_dictionaryCreation_failed = 34, ZSTD_error_parameter_unsupported = 40, + ZSTD_error_parameter_combination_unsupported = 41, ZSTD_error_parameter_outOfBound = 42, ZSTD_error_tableLog_tooLarge = 44, ZSTD_error_maxSymbolValue_tooLarge = 46, ZSTD_error_maxSymbolValue_tooSmall = 48, + ZSTD_error_cannotProduce_uncompressedBlock = 49, + ZSTD_error_stabilityCondition_notRespected = 50, ZSTD_error_stage_wrong = 60, ZSTD_error_init_missing = 62, ZSTD_error_memory_allocation = 64, @@ -1127,18 +1500,18 @@ typedef enum { ZSTD_error_dstSize_tooSmall = 70, ZSTD_error_srcSize_wrong = 72, ZSTD_error_dstBuffer_null = 74, + ZSTD_error_noForwardProgress_destFull = 80, + ZSTD_error_noForwardProgress_inputEmpty = 82, /* following error codes are __NOT STABLE__, they can be removed or changed in future versions */ ZSTD_error_frameIndex_tooLarge = 100, ZSTD_error_seekableIO = 102, ZSTD_error_dstBuffer_wrong = 104, ZSTD_error_srcBuffer_wrong = 105, + ZSTD_error_sequenceProducer_failed = 106, + ZSTD_error_externalSequences_invalid = 107, ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */ } ZSTD_ErrorCode; -/*! ZSTD_getErrorCode() : - convert a `size_t` function result into a `ZSTD_ErrorCode` enum type, - which can be used to compare with enum list published above */ -ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); /**< Same as ZSTD_getErrorName, but using a `ZSTD_ErrorCode` enum argument */ @@ -1147,8 +1520,10 @@ ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); /**< Sa #endif #endif /* ZSTD_ERRORS_H_398273423 */ -/**** ended inlining zstd_errors.h ****/ - +/**** ended inlining ../zstd_errors.h ****/ +/**** skipping file: compiler.h ****/ +/**** skipping file: debug.h ****/ +/**** skipping file: zstd_deps.h ****/ /* **************************************** * Compiler-specific @@ -1183,8 +1558,13 @@ ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } /* check and forward error code */ -#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return e -#define CHECK_F(f) { CHECK_V_F(_var_err__, f); } +#define CHECK_V_F(e, f) \ + size_t const e = f; \ + do { \ + if (ERR_isError(e)) \ + return e; \ + } while (0) +#define CHECK_F(f) do { CHECK_V_F(_var_err__, f); } while (0) /*-**************************************** @@ -1198,9 +1578,86 @@ ERR_STATIC const char* ERR_getErrorName(size_t code) return ERR_getErrorString(ERR_getErrorCode(code)); } -#if defined (__cplusplus) +/** + * Ignore: this is an internal helper. + * + * This is a helper function to help force C99-correctness during compilation. + * Under strict compilation modes, variadic macro arguments can't be empty. + * However, variadic function arguments can be. Using a function therefore lets + * us statically check that at least one (string) argument was passed, + * independent of the compilation flags. + */ +static INLINE_KEYWORD UNUSED_ATTR +void _force_has_format_string(const char *format, ...) { + (void)format; } -#endif + +/** + * Ignore: this is an internal helper. + * + * We want to force this function invocation to be syntactically correct, but + * we don't want to force runtime evaluation of its arguments. + */ +#define _FORCE_HAS_FORMAT_STRING(...) \ + do { \ + if (0) { \ + _force_has_format_string(__VA_ARGS__); \ + } \ + } while (0) + +#define ERR_QUOTE(str) #str + +/** + * Return the specified error if the condition evaluates to true. + * + * In debug modes, prints additional information. + * In order to do that (particularly, printing the conditional that failed), + * this can't just wrap RETURN_ERROR(). + */ +#define RETURN_ERROR_IF(cond, err, ...) \ + do { \ + if (cond) { \ + RAWLOG(3, "%s:%d: ERROR!: check %s failed, returning %s", \ + __FILE__, __LINE__, ERR_QUOTE(cond), ERR_QUOTE(ERROR(err))); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return ERROR(err); \ + } \ + } while (0) + +/** + * Unconditionally return the specified error. + * + * In debug modes, prints additional information. + */ +#define RETURN_ERROR(err, ...) \ + do { \ + RAWLOG(3, "%s:%d: ERROR!: unconditional check failed, returning %s", \ + __FILE__, __LINE__, ERR_QUOTE(ERROR(err))); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return ERROR(err); \ + } while(0) + +/** + * If the provided expression evaluates to an error code, returns that error code. + * + * In debug modes, prints additional information. + */ +#define FORWARD_IF_ERROR(err, ...) \ + do { \ + size_t const err_code = (err); \ + if (ERR_isError(err_code)) { \ + RAWLOG(3, "%s:%d: ERROR!: forwarding error in %s: %s", \ + __FILE__, __LINE__, ERR_QUOTE(err), ERR_getErrorName(err_code)); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return err_code; \ + } \ + } while(0) #endif /* ERROR_H_MODULE */ /**** ended inlining error_private.h ****/ @@ -1209,7 +1666,7 @@ ERR_STATIC const char* ERR_getErrorName(size_t code) /* ****************************************************************** * FSE : Finite State Entropy codec * Public Prototypes declaration - * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -1219,11 +1676,6 @@ ERR_STATIC const char* ERR_getErrorName(size_t code) * in the COPYING file in the root directory of this source tree). * You may select, at your option, one of the above-listed licenses. ****************************************************************** */ - -#if defined (__cplusplus) -extern "C" { -#endif - #ifndef FSE_H #define FSE_H @@ -1233,7 +1685,6 @@ extern "C" { ******************************************/ /**** skipping file: zstd_deps.h ****/ - /*-***************************************** * FSE_PUBLIC_API : control library symbols visibility ******************************************/ @@ -1261,34 +1712,6 @@ extern "C" { FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */ -/*-**************************************** -* FSE simple functions -******************************************/ -/*! FSE_compress() : - Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'. - 'dst' buffer must be already allocated. Compression runs faster is dstCapacity >= FSE_compressBound(srcSize). - @return : size of compressed data (<= dstCapacity). - Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! - if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead. - if FSE_isError(return), compression failed (more details using FSE_getErrorName()) -*/ -FSE_PUBLIC_API size_t FSE_compress(void* dst, size_t dstCapacity, - const void* src, size_t srcSize); - -/*! FSE_decompress(): - Decompress FSE data from buffer 'cSrc', of size 'cSrcSize', - into already allocated destination buffer 'dst', of size 'dstCapacity'. - @return : size of regenerated data (<= maxDstSize), - or an error code, which can be tested using FSE_isError() . - - ** Important ** : FSE_decompress() does not decompress non-compressible nor RLE data !!! - Why ? : making this distinction requires a header. - Header management is intentionally delegated to the user layer, which can better manage special cases. -*/ -FSE_PUBLIC_API size_t FSE_decompress(void* dst, size_t dstCapacity, - const void* cSrc, size_t cSrcSize); - - /*-***************************************** * Tool functions ******************************************/ @@ -1299,20 +1722,6 @@ FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return FSE_PUBLIC_API const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */ -/*-***************************************** -* FSE advanced functions -******************************************/ -/*! FSE_compress2() : - Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog' - Both parameters can be defined as '0' to mean : use default value - @return : size of compressed data - Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!! - if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression. - if FSE_isError(return), it's an error code. -*/ -FSE_PUBLIC_API size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); - - /*-***************************************** * FSE detailed API ******************************************/ @@ -1372,8 +1781,6 @@ FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize, /*! Constructor and Destructor of FSE_CTable. Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */ typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */ -FSE_PUBLIC_API FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog); -FSE_PUBLIC_API void FSE_freeCTable (FSE_CTable* ct); /*! FSE_buildCTable(): Builds `ct`, which must be already allocated, using FSE_createCTable(). @@ -1449,23 +1856,7 @@ FSE_PUBLIC_API size_t FSE_readNCount_bmi2(short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize, int bmi2); -/*! Constructor and Destructor of FSE_DTable. - Note that its size depends on 'tableLog' */ typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */ -FSE_PUBLIC_API FSE_DTable* FSE_createDTable(unsigned tableLog); -FSE_PUBLIC_API void FSE_freeDTable(FSE_DTable* dt); - -/*! FSE_buildDTable(): - Builds 'dt', which must be already allocated, using FSE_createDTable(). - return : 0, or an errorCode, which can be tested using FSE_isError() */ -FSE_PUBLIC_API size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); - -/*! FSE_decompress_usingDTable(): - Decompress compressed source `cSrc` of size `cSrcSize` using `dt` - into `dst` which must be already allocated. - @return : size of regenerated data (necessarily <= `dstCapacity`), - or an errorCode, which can be tested using FSE_isError() */ -FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt); /*! Tutorial : @@ -1497,15 +1888,14 @@ If there is an error, the function will return an error code, which can be teste #endif /* FSE_H */ + #if defined(FSE_STATIC_LINKING_ONLY) && !defined(FSE_H_FSE_STATIC_LINKING_ONLY) #define FSE_H_FSE_STATIC_LINKING_ONLY - -/* *** Dependency *** */ /**** start inlining bitstream.h ****/ /* ****************************************************************** * bitstream * Part of FSE library - * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -1518,9 +1908,6 @@ If there is an error, the function will return an error code, which can be teste #ifndef BITSTREAM_H_MODULE #define BITSTREAM_H_MODULE -#if defined (__cplusplus) -extern "C" { -#endif /* * This API consists of small unitary functions, which must be inlined for best performance. * Since link-time-optimization is not available for all compilers, @@ -1534,14 +1921,220 @@ extern "C" { /**** skipping file: compiler.h ****/ /**** skipping file: debug.h ****/ /**** skipping file: error_private.h ****/ +/**** start inlining bits.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_BITS_H +#define ZSTD_BITS_H + +/**** skipping file: mem.h ****/ + +MEM_STATIC unsigned ZSTD_countTrailingZeros32_fallback(U32 val) +{ + assert(val != 0); + { + static const U32 DeBruijnBytePos[32] = {0, 1, 28, 2, 29, 14, 24, 3, + 30, 22, 20, 15, 25, 17, 4, 8, + 31, 27, 13, 23, 21, 19, 16, 7, + 26, 12, 18, 6, 11, 5, 10, 9}; + return DeBruijnBytePos[((U32) ((val & -(S32) val) * 0x077CB531U)) >> 27]; + } +} + +MEM_STATIC unsigned ZSTD_countTrailingZeros32(U32 val) +{ + assert(val != 0); +#if defined(_MSC_VER) +# if STATIC_BMI2 + return (unsigned)_tzcnt_u32(val); +# else + if (val != 0) { + unsigned long r; + _BitScanForward(&r, val); + return (unsigned)r; + } else { + __assume(0); /* Should not reach this code path */ + } +# endif +#elif defined(__GNUC__) && (__GNUC__ >= 4) + return (unsigned)__builtin_ctz(val); +#elif defined(__ICCARM__) + return (unsigned)__builtin_ctz(val); +#else + return ZSTD_countTrailingZeros32_fallback(val); +#endif +} + +MEM_STATIC unsigned ZSTD_countLeadingZeros32_fallback(U32 val) +{ + assert(val != 0); + { + static const U32 DeBruijnClz[32] = {0, 9, 1, 10, 13, 21, 2, 29, + 11, 14, 16, 18, 22, 25, 3, 30, + 8, 12, 20, 28, 15, 17, 24, 7, + 19, 27, 23, 6, 26, 5, 4, 31}; + val |= val >> 1; + val |= val >> 2; + val |= val >> 4; + val |= val >> 8; + val |= val >> 16; + return 31 - DeBruijnClz[(val * 0x07C4ACDDU) >> 27]; + } +} + +MEM_STATIC unsigned ZSTD_countLeadingZeros32(U32 val) +{ + assert(val != 0); +#if defined(_MSC_VER) +# if STATIC_BMI2 + return (unsigned)_lzcnt_u32(val); +# else + if (val != 0) { + unsigned long r; + _BitScanReverse(&r, val); + return (unsigned)(31 - r); + } else { + __assume(0); /* Should not reach this code path */ + } +# endif +#elif defined(__GNUC__) && (__GNUC__ >= 4) + return (unsigned)__builtin_clz(val); +#elif defined(__ICCARM__) + return (unsigned)__builtin_clz(val); +#else + return ZSTD_countLeadingZeros32_fallback(val); +#endif +} + +MEM_STATIC unsigned ZSTD_countTrailingZeros64(U64 val) +{ + assert(val != 0); +#if defined(_MSC_VER) && defined(_WIN64) +# if STATIC_BMI2 + return (unsigned)_tzcnt_u64(val); +# else + if (val != 0) { + unsigned long r; + _BitScanForward64(&r, val); + return (unsigned)r; + } else { + __assume(0); /* Should not reach this code path */ + } +# endif +#elif defined(__GNUC__) && (__GNUC__ >= 4) && defined(__LP64__) + return (unsigned)__builtin_ctzll(val); +#elif defined(__ICCARM__) + return (unsigned)__builtin_ctzll(val); +#else + { + U32 mostSignificantWord = (U32)(val >> 32); + U32 leastSignificantWord = (U32)val; + if (leastSignificantWord == 0) { + return 32 + ZSTD_countTrailingZeros32(mostSignificantWord); + } else { + return ZSTD_countTrailingZeros32(leastSignificantWord); + } + } +#endif +} + +MEM_STATIC unsigned ZSTD_countLeadingZeros64(U64 val) +{ + assert(val != 0); +#if defined(_MSC_VER) && defined(_WIN64) +# if STATIC_BMI2 + return (unsigned)_lzcnt_u64(val); +# else + if (val != 0) { + unsigned long r; + _BitScanReverse64(&r, val); + return (unsigned)(63 - r); + } else { + __assume(0); /* Should not reach this code path */ + } +# endif +#elif defined(__GNUC__) && (__GNUC__ >= 4) + return (unsigned)(__builtin_clzll(val)); +#elif defined(__ICCARM__) + return (unsigned)(__builtin_clzll(val)); +#else + { + U32 mostSignificantWord = (U32)(val >> 32); + U32 leastSignificantWord = (U32)val; + if (mostSignificantWord == 0) { + return 32 + ZSTD_countLeadingZeros32(leastSignificantWord); + } else { + return ZSTD_countLeadingZeros32(mostSignificantWord); + } + } +#endif +} + +MEM_STATIC unsigned ZSTD_NbCommonBytes(size_t val) +{ + if (MEM_isLittleEndian()) { + if (MEM_64bits()) { + return ZSTD_countTrailingZeros64((U64)val) >> 3; + } else { + return ZSTD_countTrailingZeros32((U32)val) >> 3; + } + } else { /* Big Endian CPU */ + if (MEM_64bits()) { + return ZSTD_countLeadingZeros64((U64)val) >> 3; + } else { + return ZSTD_countLeadingZeros32((U32)val) >> 3; + } + } +} + +MEM_STATIC unsigned ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus */ +{ + assert(val != 0); + return 31 - ZSTD_countLeadingZeros32(val); +} + +/* ZSTD_rotateRight_*(): + * Rotates a bitfield to the right by "count" bits. + * https://en.wikipedia.org/w/index.php?title=Circular_shift&oldid=991635599#Implementing_circular_shifts + */ +MEM_STATIC +U64 ZSTD_rotateRight_U64(U64 const value, U32 count) { + assert(count < 64); + count &= 0x3F; /* for fickle pattern recognition */ + return (value >> count) | (U64)(value << ((0U - count) & 0x3F)); +} + +MEM_STATIC +U32 ZSTD_rotateRight_U32(U32 const value, U32 count) { + assert(count < 32); + count &= 0x1F; /* for fickle pattern recognition */ + return (value >> count) | (U32)(value << ((0U - count) & 0x1F)); +} + +MEM_STATIC +U16 ZSTD_rotateRight_U16(U16 const value, U32 count) { + assert(count < 16); + count &= 0x0F; /* for fickle pattern recognition */ + return (value >> count) | (U16)(value << ((0U - count) & 0x0F)); +} +#endif /* ZSTD_BITS_H */ +/**** ended inlining bits.h ****/ /*========================================= * Target specific =========================================*/ #ifndef ZSTD_NO_INTRINSICS -# if defined(__BMI__) && defined(__GNUC__) -# include /* support for bextr (experimental) */ +# if (defined(__BMI__) || defined(__BMI2__)) && defined(__GNUC__) +# include /* support for bextr (experimental)/bzhi */ # elif defined(__ICCARM__) # include # endif @@ -1555,12 +2148,13 @@ extern "C" { /*-****************************************** * bitStream encoding API (write forward) ********************************************/ +typedef size_t BitContainerType; /* bitStream can mix input from multiple sources. * A critical property of these streams is that they encode and decode in **reverse** direction. * So the first bit sequence you add will be the last to be read, like a LIFO stack. */ typedef struct { - size_t bitContainer; + BitContainerType bitContainer; unsigned bitPos; char* startPtr; char* ptr; @@ -1568,7 +2162,7 @@ typedef struct { } BIT_CStream_t; MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity); -MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits); +MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, BitContainerType value, unsigned nbBits); MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC); MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC); @@ -1577,7 +2171,7 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC); * `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code. * * bits are first added to a local register. -* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems. +* Local register is BitContainerType, 64-bits on 64-bits systems, or 32-bits on 32-bits systems. * Writing data into memory is an explicit operation, performed by the flushBits function. * Hence keep track how many bits are potentially stored into local register to avoid register overflow. * After a flushBits, a maximum of 7 bits might still be stored into local register. @@ -1594,28 +2188,28 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC); * bitStream decoding API (read backward) **********************************************/ typedef struct { - size_t bitContainer; + BitContainerType bitContainer; unsigned bitsConsumed; const char* ptr; const char* start; const char* limitPtr; } BIT_DStream_t; -typedef enum { BIT_DStream_unfinished = 0, - BIT_DStream_endOfBuffer = 1, - BIT_DStream_completed = 2, - BIT_DStream_overflow = 3 } BIT_DStream_status; /* result of BIT_reloadDStream() */ - /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */ +typedef enum { BIT_DStream_unfinished = 0, /* fully refilled */ + BIT_DStream_endOfBuffer = 1, /* still some bits left in bitstream */ + BIT_DStream_completed = 2, /* bitstream entirely consumed, bit-exact */ + BIT_DStream_overflow = 3 /* user requested more bits than present in bitstream */ + } BIT_DStream_status; /* result of BIT_reloadDStream() */ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize); -MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits); +MEM_STATIC BitContainerType BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits); MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD); MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD); /* Start by invoking BIT_initDStream(). * A chunk of the bitStream is then stored into a local register. -* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). +* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (BitContainerType). * You can then retrieve bitFields stored into the local register, **in reverse order**. * Local register is explicitly reloaded from memory by the BIT_reloadDStream() method. * A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished. @@ -1627,7 +2221,7 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD); /*-**************************************** * unsafe API ******************************************/ -MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits); +MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, BitContainerType value, unsigned nbBits); /* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */ MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC); @@ -1636,42 +2230,6 @@ MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC); MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits); /* faster, but works only if nbBits >= 1 */ - - -/*-************************************************************** -* Internal functions -****************************************************************/ -MEM_STATIC unsigned BIT_highbit32 (U32 val) -{ - assert(val != 0); - { -# if defined(_MSC_VER) /* Visual */ -# if STATIC_BMI2 == 1 - return _lzcnt_u32(val) ^ 31; -# else - unsigned long r = 0; - return _BitScanReverse(&r, val) ? (unsigned)r : 0; -# endif -# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */ - return __builtin_clz (val) ^ 31; -# elif defined(__ICCARM__) /* IAR Intrinsic */ - return 31 - __CLZ(val); -# else /* Software version */ - static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, - 11, 14, 16, 18, 22, 25, 3, 30, - 8, 12, 20, 28, 15, 17, 24, 7, - 19, 27, 23, 6, 26, 5, 4, 31 }; - U32 v = val; - v |= v >> 1; - v |= v >> 2; - v |= v >> 4; - v |= v >> 8; - v |= v >> 16; - return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27]; -# endif - } -} - /*===== Local Constants =====*/ static const unsigned BIT_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, @@ -1701,16 +2259,31 @@ MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, return 0; } +FORCE_INLINE_TEMPLATE BitContainerType BIT_getLowerBits(BitContainerType bitContainer, U32 const nbBits) +{ +#if STATIC_BMI2 && !defined(ZSTD_NO_INTRINSICS) +# if (defined(__x86_64__) || defined(_M_X64)) && !defined(__ILP32__) + return _bzhi_u64(bitContainer, nbBits); +# else + DEBUG_STATIC_ASSERT(sizeof(bitContainer) == sizeof(U32)); + return _bzhi_u32(bitContainer, nbBits); +# endif +#else + assert(nbBits < BIT_MASK_SIZE); + return bitContainer & BIT_mask[nbBits]; +#endif +} + /*! BIT_addBits() : * can add up to 31 bits into `bitC`. * Note : does not check for register overflow ! */ MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, - size_t value, unsigned nbBits) + BitContainerType value, unsigned nbBits) { DEBUG_STATIC_ASSERT(BIT_MASK_SIZE == 32); assert(nbBits < BIT_MASK_SIZE); assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8); - bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos; + bitC->bitContainer |= BIT_getLowerBits(value, nbBits) << bitC->bitPos; bitC->bitPos += nbBits; } @@ -1718,7 +2291,7 @@ MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, * works only if `value` is _clean_, * meaning all high bits above nbBits are 0 */ MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, - size_t value, unsigned nbBits) + BitContainerType value, unsigned nbBits) { assert((value>>nbBits) == 0); assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8); @@ -1765,7 +2338,7 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC) BIT_addBitsFast(bitC, 1, 1); /* endMark */ BIT_flushBits(bitC); if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */ - return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0); + return (size_t)(bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0); } @@ -1789,35 +2362,35 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer); bitD->bitContainer = MEM_readLEST(bitD->ptr); { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; - bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */ + bitD->bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */ if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } } else { bitD->ptr = bitD->start; bitD->bitContainer = *(const BYTE*)(bitD->start); switch(srcSize) { - case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16); - /* fall-through */ + case 7: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16); + ZSTD_FALLTHROUGH; - case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24); - /* fall-through */ + case 6: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24); + ZSTD_FALLTHROUGH; - case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32); - /* fall-through */ + case 5: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32); + ZSTD_FALLTHROUGH; - case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24; - /* fall-through */ + case 4: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[3]) << 24; + ZSTD_FALLTHROUGH; - case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16; - /* fall-through */ + case 3: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[2]) << 16; + ZSTD_FALLTHROUGH; - case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8; - /* fall-through */ + case 2: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[1]) << 8; + ZSTD_FALLTHROUGH; default: break; } { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; - bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; + bitD->bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0; if (lastByte == 0) return ERROR(corruption_detected); /* endMark not present */ } bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8; @@ -1826,26 +2399,25 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si return srcSize; } -MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getUpperBits(size_t bitContainer, U32 const start) +FORCE_INLINE_TEMPLATE BitContainerType BIT_getUpperBits(BitContainerType bitContainer, U32 const start) { return bitContainer >> start; } -MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) +FORCE_INLINE_TEMPLATE BitContainerType BIT_getMiddleBits(BitContainerType bitContainer, U32 const start, U32 const nbBits) { U32 const regMask = sizeof(bitContainer)*8 - 1; /* if start > regMask, bitstream is corrupted, and result is undefined */ assert(nbBits < BIT_MASK_SIZE); - return (bitContainer >> (start & regMask)) & BIT_mask[nbBits]; -} - -MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) -{ -#if defined(STATIC_BMI2) && STATIC_BMI2 == 1 - return _bzhi_u64(bitContainer, nbBits); + /* x86 transform & ((1 << nbBits) - 1) to bzhi instruction, it is better + * than accessing memory. When bmi2 instruction is not present, we consider + * such cpus old (pre-Haswell, 2013) and their performance is not of that + * importance. + */ +#if defined(__x86_64__) || defined(_M_X64) + return (bitContainer >> (start & regMask)) & ((((U64)1) << nbBits) - 1); #else - assert(nbBits < BIT_MASK_SIZE); - return bitContainer & BIT_mask[nbBits]; + return (bitContainer >> (start & regMask)) & BIT_mask[nbBits]; #endif } @@ -1855,7 +2427,7 @@ MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getLowerBits(size_t bitContainer, U32 co * On 32-bits, maxNbBits==24. * On 64-bits, maxNbBits==56. * @return : value extracted */ -MEM_STATIC FORCE_INLINE_ATTR size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits) +FORCE_INLINE_TEMPLATE BitContainerType BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits) { /* arbitrate between double-shift and shift+mask */ #if 1 @@ -1871,14 +2443,14 @@ MEM_STATIC FORCE_INLINE_ATTR size_t BIT_lookBits(const BIT_DStream_t* bitD, U3 /*! BIT_lookBitsFast() : * unsafe version; only works if nbBits >= 1 */ -MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits) +MEM_STATIC BitContainerType BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits) { U32 const regMask = sizeof(bitD->bitContainer)*8 - 1; assert(nbBits >= 1); return (bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> (((regMask+1)-nbBits) & regMask); } -MEM_STATIC FORCE_INLINE_ATTR void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) +FORCE_INLINE_TEMPLATE void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) { bitD->bitsConsumed += nbBits; } @@ -1887,23 +2459,38 @@ MEM_STATIC FORCE_INLINE_ATTR void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) * Read (consume) next n bits from local register and update. * Pay attention to not read more than nbBits contained into local register. * @return : extracted value. */ -MEM_STATIC FORCE_INLINE_ATTR size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits) +FORCE_INLINE_TEMPLATE BitContainerType BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits) { - size_t const value = BIT_lookBits(bitD, nbBits); + BitContainerType const value = BIT_lookBits(bitD, nbBits); BIT_skipBits(bitD, nbBits); return value; } /*! BIT_readBitsFast() : - * unsafe version; only works only if nbBits >= 1 */ -MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits) + * unsafe version; only works if nbBits >= 1 */ +MEM_STATIC BitContainerType BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits) { - size_t const value = BIT_lookBitsFast(bitD, nbBits); + BitContainerType const value = BIT_lookBitsFast(bitD, nbBits); assert(nbBits >= 1); BIT_skipBits(bitD, nbBits); return value; } +/*! BIT_reloadDStream_internal() : + * Simple variant of BIT_reloadDStream(), with two conditions: + * 1. bitstream is valid : bitsConsumed <= sizeof(bitD->bitContainer)*8 + * 2. look window is valid after shifted down : bitD->ptr >= bitD->start + */ +MEM_STATIC BIT_DStream_status BIT_reloadDStream_internal(BIT_DStream_t* bitD) +{ + assert(bitD->bitsConsumed <= sizeof(bitD->bitContainer)*8); + bitD->ptr -= bitD->bitsConsumed >> 3; + assert(bitD->ptr >= bitD->start); + bitD->bitsConsumed &= 7; + bitD->bitContainer = MEM_readLEST(bitD->ptr); + return BIT_DStream_unfinished; +} + /*! BIT_reloadDStreamFast() : * Similar to BIT_reloadDStream(), but with two differences: * 1. bitsConsumed <= sizeof(bitD->bitContainer)*8 must hold! @@ -1914,31 +2501,35 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStreamFast(BIT_DStream_t* bitD) { if (UNLIKELY(bitD->ptr < bitD->limitPtr)) return BIT_DStream_overflow; - assert(bitD->bitsConsumed <= sizeof(bitD->bitContainer)*8); - bitD->ptr -= bitD->bitsConsumed >> 3; - bitD->bitsConsumed &= 7; - bitD->bitContainer = MEM_readLEST(bitD->ptr); - return BIT_DStream_unfinished; + return BIT_reloadDStream_internal(bitD); } /*! BIT_reloadDStream() : * Refill `bitD` from buffer previously set in BIT_initDStream() . - * This function is safe, it guarantees it will not read beyond src buffer. + * This function is safe, it guarantees it will not never beyond src buffer. * @return : status of `BIT_DStream_t` internal register. * when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */ -MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) +FORCE_INLINE_TEMPLATE BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) { - if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* overflow detected, like end of stream */ + /* note : once in overflow mode, a bitstream remains in this mode until it's reset */ + if (UNLIKELY(bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8))) { + static const BitContainerType zeroFilled = 0; + bitD->ptr = (const char*)&zeroFilled; /* aliasing is allowed for char */ + /* overflow detected, erroneous scenario or end of stream: no update */ return BIT_DStream_overflow; + } + + assert(bitD->ptr >= bitD->start); if (bitD->ptr >= bitD->limitPtr) { - return BIT_reloadDStreamFast(bitD); + return BIT_reloadDStream_internal(bitD); } if (bitD->ptr == bitD->start) { + /* reached end of bitStream => no update */ if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer; return BIT_DStream_completed; } - /* start < ptr < limitPtr */ + /* start < ptr < limitPtr => cautious update */ { U32 nbBytes = bitD->bitsConsumed >> 3; BIT_DStream_status result = BIT_DStream_unfinished; if (bitD->ptr - nbBytes < bitD->start) { @@ -1960,14 +2551,9 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream) return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8)); } -#if defined (__cplusplus) -} -#endif - #endif /* BITSTREAM_H_MODULE */ /**** ended inlining bitstream.h ****/ - /* ***************************************** * Static allocation *******************************************/ @@ -1992,24 +2578,15 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream) unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus); /**< same as FSE_optimalTableLog(), which used `minus==2` */ -/* FSE_compress_wksp() : - * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`). - * FSE_COMPRESS_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable. - */ -#define FSE_COMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + ((maxTableLog > 12) ? (1 << (maxTableLog - 2)) : 1024) ) -size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); - -size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits); -/**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */ - size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue); /**< build a fake FSE_CTable, designed to compress always the same symbolValue */ /* FSE_buildCTable_wksp() : * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). * `wkspSize` must be >= `FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)` of `unsigned`. + * See FSE_buildCTable_wksp() for breakdown of workspace usage. */ -#define FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog) (maxSymbolValue + 2 + (1ull << (tableLog - 2))) +#define FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog) (((maxSymbolValue + 2) + (1ull << (tableLog)))/2 + sizeof(U64)/sizeof(U32) /* additional 8 bytes for potential table overwrite */) #define FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) (sizeof(unsigned) * FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)) size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); @@ -2018,19 +2595,11 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsi FSE_PUBLIC_API size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< Same as FSE_buildDTable(), using an externally allocated `workspace` produced with `FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxSymbolValue)` */ -size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits); -/**< build a fake FSE_DTable, designed to read a flat distribution where each symbol uses nbBits */ - -size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue); -/**< build a fake FSE_DTable, designed to always generate the same symbolValue */ - -#define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue)) +#define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + 1 + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) + (FSE_MAX_SYMBOL_VALUE + 1) / 2 + 1) #define FSE_DECOMPRESS_WKSP_SIZE(maxTableLog, maxSymbolValue) (FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(unsigned)) -size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize); -/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)` */ - size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2); -/**< Same as FSE_decompress_wksp() but with dynamic BMI2 support. Pass 1 if your CPU supports BMI2 or 0 if it doesn't. */ +/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)`. + * Set bmi2 to 1 if your CPU supports BMI2 or 0 if it doesn't */ typedef enum { FSE_repeat_none, /**< Cannot use the previous table */ @@ -2213,20 +2782,20 @@ MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, un FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; const U16* const stateTable = (const U16*)(statePtr->stateTable); U32 const nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16); - BIT_addBits(bitC, statePtr->value, nbBitsOut); + BIT_addBits(bitC, (BitContainerType)statePtr->value, nbBitsOut); statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; } MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr) { - BIT_addBits(bitC, statePtr->value, statePtr->stateLog); + BIT_addBits(bitC, (BitContainerType)statePtr->value, statePtr->stateLog); BIT_flushBits(bitC); } /* FSE_getMaxNbBits() : * Approximate maximum cost of a symbol, in bits. - * Fractional get rounded up (i.e : a symbol with a normalized frequency of 3 gives the same result as a frequency of 2) + * Fractional get rounded up (i.e. a symbol with a normalized frequency of 3 gives the same result as a frequency of 2) * note 1 : assume symbolValue is valid (<= maxSymbolValue) * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */ MEM_STATIC U32 FSE_getMaxNbBits(const void* symbolTTPtr, U32 symbolValue) @@ -2379,20 +2948,13 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) #define FSE_TABLESTEP(tableSize) (((tableSize)>>1) + ((tableSize)>>3) + 3) - #endif /* FSE_STATIC_LINKING_ONLY */ - - -#if defined (__cplusplus) -} -#endif /**** ended inlining fse.h ****/ -#define HUF_STATIC_LINKING_ONLY /* HUF_TABLELOG_ABSOLUTEMAX */ /**** start inlining huf.h ****/ /* ****************************************************************** * huff0 huffman codec, * part of Finite State Entropy library - * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -2403,115 +2965,33 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) * You may select, at your option, one of the above-listed licenses. ****************************************************************** */ -#if defined (__cplusplus) -extern "C" { -#endif - #ifndef HUF_H_298734234 #define HUF_H_298734234 /* *** Dependencies *** */ /**** skipping file: zstd_deps.h ****/ - - -/* *** library symbols visibility *** */ -/* Note : when linking with -fvisibility=hidden on gcc, or by default on Visual, - * HUF symbols remain "private" (internal symbols for library only). - * Set macro FSE_DLL_EXPORT to 1 if you want HUF symbols visible on DLL interface */ -#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4) -# define HUF_PUBLIC_API __attribute__ ((visibility ("default"))) -#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */ -# define HUF_PUBLIC_API __declspec(dllexport) -#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1) -# define HUF_PUBLIC_API __declspec(dllimport) /* not required, just to generate faster code (saves a function pointer load from IAT and an indirect jump) */ -#else -# define HUF_PUBLIC_API -#endif - - -/* ========================== */ -/* *** simple functions *** */ -/* ========================== */ - -/** HUF_compress() : - * Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'. - * 'dst' buffer must be already allocated. - * Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize). - * `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB. - * @return : size of compressed data (<= `dstCapacity`). - * Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! - * if HUF_isError(return), compression failed (more details using HUF_getErrorName()) - */ -HUF_PUBLIC_API size_t HUF_compress(void* dst, size_t dstCapacity, - const void* src, size_t srcSize); - -/** HUF_decompress() : - * Decompress HUF data from buffer 'cSrc', of size 'cSrcSize', - * into already allocated buffer 'dst', of minimum size 'dstSize'. - * `originalSize` : **must** be the ***exact*** size of original (uncompressed) data. - * Note : in contrast with FSE, HUF_decompress can regenerate - * RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data, - * because it knows size to regenerate (originalSize). - * @return : size of regenerated data (== originalSize), - * or an error code, which can be tested using HUF_isError() - */ -HUF_PUBLIC_API size_t HUF_decompress(void* dst, size_t originalSize, - const void* cSrc, size_t cSrcSize); - +/**** skipping file: mem.h ****/ +#define FSE_STATIC_LINKING_ONLY +/**** skipping file: fse.h ****/ /* *** Tool functions *** */ -#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */ -HUF_PUBLIC_API size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ +#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */ +size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ /* Error Management */ -HUF_PUBLIC_API unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ -HUF_PUBLIC_API const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ - - -/* *** Advanced function *** */ - -/** HUF_compress2() : - * Same as HUF_compress(), but offers control over `maxSymbolValue` and `tableLog`. - * `maxSymbolValue` must be <= HUF_SYMBOLVALUE_MAX . - * `tableLog` must be `<= HUF_TABLELOG_MAX` . */ -HUF_PUBLIC_API size_t HUF_compress2 (void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned tableLog); - -/** HUF_compress4X_wksp() : - * Same as HUF_compress2(), but uses externally allocated `workSpace`. - * `workspace` must have minimum alignment of 4, and be at least as large as HUF_WORKSPACE_SIZE */ -#define HUF_WORKSPACE_SIZE ((6 << 10) + 256) -#define HUF_WORKSPACE_SIZE_U32 (HUF_WORKSPACE_SIZE / sizeof(U32)) -HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned tableLog, - void* workSpace, size_t wkspSize); +unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ +const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ -#endif /* HUF_H_298734234 */ - -/* ****************************************************************** - * WARNING !! - * The following section contains advanced and experimental definitions - * which shall never be used in the context of a dynamic library, - * because they are not guaranteed to remain stable in the future. - * Only consider them in association with static linking. - * *****************************************************************/ -#if defined(HUF_STATIC_LINKING_ONLY) && !defined(HUF_H_HUF_STATIC_LINKING_ONLY) -#define HUF_H_HUF_STATIC_LINKING_ONLY - -/* *** Dependencies *** */ -/**** skipping file: mem.h ****/ -#define FSE_STATIC_LINKING_ONLY -/**** skipping file: fse.h ****/ +#define HUF_WORKSPACE_SIZE ((8 << 10) + 512 /* sorting scratch space */) +#define HUF_WORKSPACE_SIZE_U64 (HUF_WORKSPACE_SIZE / sizeof(U64)) /* *** Constants *** */ -#define HUF_TABLELOG_MAX 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */ +#define HUF_TABLELOG_MAX 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_TABLELOG_ABSOLUTEMAX */ #define HUF_TABLELOG_DEFAULT 11 /* default tableLog value when none specified */ #define HUF_SYMBOLVALUE_MAX 255 -#define HUF_TABLELOG_ABSOLUTEMAX 15 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ +#define HUF_TABLELOG_ABSOLUTEMAX 12 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ #if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX) # error "HUF_TABLELOG_MAX is too large !" #endif @@ -2527,15 +3007,11 @@ HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity, /* static allocation of HUF's Compression Table */ /* this is a private definition, just exposed for allocation and strict aliasing purpose. never EVER access its members directly */ -struct HUF_CElt_s { - U16 val; - BYTE nbBits; -}; /* typedef'd to HUF_CElt */ -typedef struct HUF_CElt_s HUF_CElt; /* consider it an incomplete type */ -#define HUF_CTABLE_SIZE_U32(maxSymbolValue) ((maxSymbolValue)+1) /* Use tables of U32, for proper alignment */ -#define HUF_CTABLE_SIZE(maxSymbolValue) (HUF_CTABLE_SIZE_U32(maxSymbolValue) * sizeof(U32)) +typedef size_t HUF_CElt; /* consider it an incomplete type */ +#define HUF_CTABLE_SIZE_ST(maxSymbolValue) ((maxSymbolValue)+2) /* Use tables of size_t, for proper alignment */ +#define HUF_CTABLE_SIZE(maxSymbolValue) (HUF_CTABLE_SIZE_ST(maxSymbolValue) * sizeof(size_t)) #define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \ - HUF_CElt name[HUF_CTABLE_SIZE_U32(maxSymbolValue)] /* no final ; */ + HUF_CElt name[HUF_CTABLE_SIZE_ST(maxSymbolValue)] /* no final ; */ /* static allocation of HUF's DTable */ typedef U32 HUF_DTable; @@ -2549,25 +3025,49 @@ typedef U32 HUF_DTable; /* **************************************** * Advanced decompression functions ******************************************/ -size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ -#endif -size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */ -size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */ -size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< considers RLE and uncompressed as errors */ -size_t HUF_decompress4X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ -size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */ -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ -size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */ -#endif +/** + * Huffman flags bitset. + * For all flags, 0 is the default value. + */ +typedef enum { + /** + * If compiled with DYNAMIC_BMI2: Set flag only if the CPU supports BMI2 at runtime. + * Otherwise: Ignored. + */ + HUF_flags_bmi2 = (1 << 0), + /** + * If set: Test possible table depths to find the one that produces the smallest header + encoded size. + * If unset: Use heuristic to find the table depth. + */ + HUF_flags_optimalDepth = (1 << 1), + /** + * If set: If the previous table can encode the input, always reuse the previous table. + * If unset: If the previous table can encode the input, reuse the previous table if it results in a smaller output. + */ + HUF_flags_preferRepeat = (1 << 2), + /** + * If set: Sample the input and check if the sample is uncompressible, if it is then don't attempt to compress. + * If unset: Always histogram the entire input. + */ + HUF_flags_suspectUncompressible = (1 << 3), + /** + * If set: Don't use assembly implementations + * If unset: Allow using assembly implementations + */ + HUF_flags_disableAsm = (1 << 4), + /** + * If set: Don't use the fast decoding loop, always use the fallback decoding loop. + * If unset: Use the fast decoding loop when possible. + */ + HUF_flags_disableFast = (1 << 5) +} HUF_flags_e; /* **************************************** * HUF detailed API * ****************************************/ +#define HUF_OPTIMAL_DEPTH_THRESHOLD ZSTD_btultra /*! HUF_compress() does the following: * 1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h") @@ -2580,10 +3080,12 @@ size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, * For example, it's possible to compress several blocks using the same 'CTable', * or to save and regenerate 'CTable' using external methods. */ -unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); -size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); /* @return : maxNbBits; CTable and count can overlap. In which case, CTable will overwrite count content */ -size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog); -size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); +unsigned HUF_minTableLog(unsigned symbolCardinality); +unsigned HUF_cardinality(const unsigned* count, unsigned maxSymbolValue); +unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace, + size_t wkspSize, HUF_CElt* table, const unsigned* count, int flags); /* table is used as scratch space for building and testing tables, not a return value */ +size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, void* workspace, size_t workspaceSize); +size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags); size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue); int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue); @@ -2592,22 +3094,24 @@ typedef enum { HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */ HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */ } HUF_repeat; + /** HUF_compress4X_repeat() : * Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. * If it uses hufTable it does not modify hufTable or repeat. * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. - * If preferRepeat then the old table will always be used if valid. */ + * If preferRepeat then the old table will always be used if valid. + * If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */ size_t HUF_compress4X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */ - HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2); + HUF_CElt* hufTable, HUF_repeat* repeat, int flags); /** HUF_buildCTable_wksp() : * Same as HUF_buildCTable(), but using externally allocated scratch buffer. * `workSpace` must be aligned on 4-bytes boundaries, and its size must be >= HUF_CTABLE_WORKSPACE_SIZE. */ -#define HUF_CTABLE_WORKSPACE_SIZE_U32 (2*HUF_SYMBOLVALUE_MAX +1 +1) +#define HUF_CTABLE_WORKSPACE_SIZE_U32 ((4 * (HUF_SYMBOLVALUE_MAX + 1)) + 192) #define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned)) size_t HUF_buildCTable_wksp (HUF_CElt* tree, const unsigned* count, U32 maxSymbolValue, U32 maxNbBits, @@ -2633,17 +3137,29 @@ size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, const void* src, size_t srcSize, void* workspace, size_t wkspSize, - int bmi2); + int flags); /** HUF_readCTable() : * Loading a CTable saved with HUF_writeCTable() */ size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned *hasZeroWeights); -/** HUF_getNbBits() : +/** HUF_getNbBitsFromCTable() : * Read nbBits from CTable symbolTable, for symbol `symbolValue` presumed <= HUF_SYMBOLVALUE_MAX - * Note 1 : is not inlined, as HUF_CElt definition is private - * Note 2 : const void* used, so that it can provide a statically allocated table as argument (which uses type U32) */ -U32 HUF_getNbBits(const void* symbolTable, U32 symbolValue); + * Note 1 : If symbolValue > HUF_readCTableHeader(symbolTable).maxSymbolValue, returns 0 + * Note 2 : is not inlined, as HUF_CElt definition is private + */ +U32 HUF_getNbBitsFromCTable(const HUF_CElt* symbolTable, U32 symbolValue); + +typedef struct { + BYTE tableLog; + BYTE maxSymbolValue; + BYTE unused[sizeof(size_t) - 2]; +} HUF_CTableHeader; + +/** HUF_readCTableHeader() : + * @returns The header from the CTable specifying the tableLog and the maxSymbolValue. + */ +HUF_CTableHeader HUF_readCTableHeader(HUF_CElt const* ctable); /* * HUF_decompress() does the following: @@ -2669,88 +3185,51 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize); * a required workspace size greater than that specified in the following * macro. */ -#define HUF_DECOMPRESS_WORKSPACE_SIZE (2 << 10) +#define HUF_DECOMPRESS_WORKSPACE_SIZE ((2 << 10) + (1 << 9)) #define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32)) -#ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_readDTableX1 (HUF_DTable* DTable, const void* src, size_t srcSize); -size_t HUF_readDTableX1_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize); -#endif -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize); -size_t HUF_readDTableX2_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize); -#endif - -size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); -#ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_decompress4X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); -#endif -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); -#endif - /* ====================== */ /* single stream variants */ /* ====================== */ -size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); -size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */ -size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags); /** HUF_compress1X_repeat() : * Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. * If it uses hufTable it does not modify hufTable or repeat. * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. - * If preferRepeat then the old table will always be used if valid. */ + * If preferRepeat then the old table will always be used if valid. + * If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */ size_t HUF_compress1X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */ - HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2); - -size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */ -#endif - -size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); -size_t HUF_decompress1X_DCtx_wksp (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); -#ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_decompress1X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ -size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */ -#endif -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ -size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */ -#endif + HUF_CElt* hufTable, HUF_repeat* repeat, int flags); -size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */ -#ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_decompress1X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); -#endif +size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); #ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); /**< double-symbols decoder */ #endif /* BMI2 variants. * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0. */ -size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2); +size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags); #ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2); +size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); #endif -size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2); -size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2); +size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags); +size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); #ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2); +size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags); #endif - -#endif /* HUF_STATIC_LINKING_ONLY */ - -#if defined (__cplusplus) -} +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags); #endif + +#endif /* HUF_H_298734234 */ /**** ended inlining huf.h ****/ +/**** skipping file: bits.h ****/ /*=== Version ===*/ @@ -2768,28 +3247,6 @@ const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); } /*-************************************************************** * FSE NCount encoding-decoding ****************************************************************/ -static U32 FSE_ctz(U32 val) -{ - assert(val != 0); - { -# if defined(_MSC_VER) /* Visual */ - unsigned long r=0; - return _BitScanForward(&r, val) ? (unsigned)r : 0; -# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */ - return __builtin_ctz(val); -# elif defined(__ICCARM__) /* IAR Intrinsic */ - return __CTZ(val); -# else /* Software version */ - U32 count = 0; - while ((val & 1) == 0) { - val >>= 1; - ++count; - } - return count; -# endif - } -} - FORCE_INLINE_TEMPLATE size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, const void* headerBuffer, size_t hbSize) @@ -2837,7 +3294,7 @@ size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigne * repeat. * Avoid UB by setting the high bit to 1. */ - int repeats = FSE_ctz(~bitStream | 0x80000000) >> 1; + int repeats = ZSTD_countTrailingZeros32(~bitStream | 0x80000000) >> 1; while (repeats >= 12) { charnum += 3 * 12; if (LIKELY(ip <= iend-7)) { @@ -2848,7 +3305,7 @@ size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigne ip = iend - 4; } bitStream = MEM_readLE32(ip) >> bitCount; - repeats = FSE_ctz(~bitStream | 0x80000000) >> 1; + repeats = ZSTD_countTrailingZeros32(~bitStream | 0x80000000) >> 1; } charnum += 3 * repeats; bitStream >>= 2 * repeats; @@ -2913,7 +3370,7 @@ size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigne * know that threshold > 1. */ if (remaining <= 1) break; - nbBits = BIT_highbit32(remaining) + 1; + nbBits = ZSTD_highbit32(remaining) + 1; threshold = 1 << (nbBits - 1); } if (charnum >= maxSV1) break; @@ -2947,7 +3404,7 @@ static size_t FSE_readNCount_body_default( } #if DYNAMIC_BMI2 -TARGET_ATTRIBUTE("bmi2") static size_t FSE_readNCount_body_bmi2( +BMI2_TARGET_ATTRIBUTE static size_t FSE_readNCount_body_bmi2( short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, const void* headerBuffer, size_t hbSize) { @@ -2988,7 +3445,7 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, const void* src, size_t srcSize) { U32 wksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; - return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* bmi2 */ 0); + return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* flags */ 0); } FORCE_INLINE_TEMPLATE size_t @@ -3029,21 +3486,21 @@ HUF_readStats_body(BYTE* huffWeight, size_t hwSize, U32* rankStats, ZSTD_memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32)); weightTotal = 0; { U32 n; for (n=0; n= HUF_TABLELOG_MAX) return ERROR(corruption_detected); + if (huffWeight[n] > HUF_TABLELOG_MAX) return ERROR(corruption_detected); rankStats[huffWeight[n]]++; weightTotal += (1 << huffWeight[n]) >> 1; } } if (weightTotal == 0) return ERROR(corruption_detected); /* get last non-null symbol weight (implied, total must be 2^n) */ - { U32 const tableLog = BIT_highbit32(weightTotal) + 1; + { U32 const tableLog = ZSTD_highbit32(weightTotal) + 1; if (tableLog > HUF_TABLELOG_MAX) return ERROR(corruption_detected); *tableLogPtr = tableLog; /* determine last weight */ { U32 const total = 1 << tableLog; U32 const rest = total - weightTotal; - U32 const verif = 1 << BIT_highbit32(rest); - U32 const lastWeight = BIT_highbit32(rest) + 1; + U32 const verif = 1 << ZSTD_highbit32(rest); + U32 const lastWeight = ZSTD_highbit32(rest) + 1; if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ huffWeight[oSize] = (BYTE)lastWeight; rankStats[lastWeight]++; @@ -3067,7 +3524,7 @@ static size_t HUF_readStats_body_default(BYTE* huffWeight, size_t hwSize, U32* r } #if DYNAMIC_BMI2 -static TARGET_ATTRIBUTE("bmi2") size_t HUF_readStats_body_bmi2(BYTE* huffWeight, size_t hwSize, U32* rankStats, +static BMI2_TARGET_ATTRIBUTE size_t HUF_readStats_body_bmi2(BYTE* huffWeight, size_t hwSize, U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, const void* src, size_t srcSize, void* workSpace, size_t wkspSize) @@ -3080,20 +3537,20 @@ size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, - int bmi2) + int flags) { #if DYNAMIC_BMI2 - if (bmi2) { + if (flags & HUF_flags_bmi2) { return HUF_readStats_body_bmi2(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize); } #endif - (void)bmi2; + (void)flags; return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize); } /**** ended inlining common/entropy_common.c ****/ /**** start inlining common/error_private.c ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -3121,9 +3578,11 @@ const char* ERR_getErrorString(ERR_enum code) case PREFIX(version_unsupported): return "Version not supported"; case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding"; - case PREFIX(corruption_detected): return "Corrupted block detected"; + case PREFIX(corruption_detected): return "Data corruption detected"; case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; + case PREFIX(literals_headerWrong): return "Header of Literals' block doesn't respect format specification"; case PREFIX(parameter_unsupported): return "Unsupported parameter"; + case PREFIX(parameter_combination_unsupported): return "Unsupported combination of parameters"; case PREFIX(parameter_outOfBound): return "Parameter is out of bound"; case PREFIX(init_missing): return "Context should be init first"; case PREFIX(memory_allocation): return "Allocation error : not enough memory"; @@ -3132,17 +3591,23 @@ const char* ERR_getErrorString(ERR_enum code) case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; + case PREFIX(cannotProduce_uncompressedBlock): return "This mode cannot generate an uncompressed block"; + case PREFIX(stabilityCondition_notRespected): return "pledged buffer stability condition is not respected"; case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; case PREFIX(dictionary_wrong): return "Dictionary mismatch"; case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples"; case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; case PREFIX(srcSize_wrong): return "Src size is incorrect"; case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer"; + case PREFIX(noForwardProgress_destFull): return "Operation made no progress over multiple calls, due to output buffer being full"; + case PREFIX(noForwardProgress_inputEmpty): return "Operation made no progress over multiple calls, due to input being empty"; /* following error codes are not stable and may be removed or changed in a future version */ case PREFIX(frameIndex_tooLarge): return "Frame index is too large"; case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking"; case PREFIX(dstBuffer_wrong): return "Destination buffer is wrong"; case PREFIX(srcBuffer_wrong): return "Source buffer is wrong"; + case PREFIX(sequenceProducer_failed): return "Block-level external sequence producer returned an error code"; + case PREFIX(externalSequences_invalid): return "External sequences are not valid"; case PREFIX(maxCode): default: return notErrorCode; } @@ -3152,7 +3617,7 @@ const char* ERR_getErrorString(ERR_enum code) /**** start inlining common/fse_decompress.c ****/ /* ****************************************************************** * FSE : Finite State Entropy decoder - * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -3174,8 +3639,8 @@ const char* ERR_getErrorString(ERR_enum code) #define FSE_STATIC_LINKING_ONLY /**** skipping file: fse.h ****/ /**** skipping file: error_private.h ****/ -#define ZSTD_DEPS_NEED_MALLOC /**** skipping file: zstd_deps.h ****/ +/**** skipping file: bits.h ****/ /* ************************************************************** @@ -3207,19 +3672,6 @@ const char* ERR_getErrorString(ERR_enum code) #define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) #define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) - -/* Function templates */ -FSE_DTable* FSE_createDTable (unsigned tableLog) -{ - if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; - return (FSE_DTable*)ZSTD_malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) ); -} - -void FSE_freeDTable (FSE_DTable* dt) -{ - ZSTD_free(dt); -} - static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize) { void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */ @@ -3248,7 +3700,7 @@ static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCo symbolNext[s] = 1; } else { if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0; - symbolNext[s] = normalizedCounter[s]; + symbolNext[s] = (U16)normalizedCounter[s]; } } } ZSTD_memcpy(dt, &DTableH, sizeof(DTableH)); } @@ -3263,8 +3715,7 @@ static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCo * all symbols have counts <= 8. We ensure we have 8 bytes at the end of * our buffer to handle the over-write. */ - { - U64 const add = 0x0101010101010101ull; + { U64 const add = 0x0101010101010101ull; size_t pos = 0; U64 sv = 0; U32 s; @@ -3275,14 +3726,13 @@ static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCo for (i = 8; i < n; i += 8) { MEM_write64(spread + pos + i, sv); } - pos += n; - } - } + pos += (size_t)n; + } } /* Now we spread those positions across the table. - * The benefit of doing it in two stages is that we avoid the the + * The benefit of doing it in two stages is that we avoid the * variable size inner loop, which caused lots of branch misses. * Now we can run through all the positions without any branch misses. - * We unroll the loop twice, since that is what emperically worked best. + * We unroll the loop twice, since that is what empirically worked best. */ { size_t position = 0; @@ -3318,7 +3768,7 @@ static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCo for (u=0; utableLog = 0; - DTableH->fastMode = 0; - - cell->newState = 0; - cell->symbol = symbolValue; - cell->nbBits = 0; - - return 0; -} - - -size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits) -{ - void* ptr = dt; - FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; - void* dPtr = dt + 1; - FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr; - const unsigned tableSize = 1 << nbBits; - const unsigned tableMask = tableSize - 1; - const unsigned maxSV1 = tableMask+1; - unsigned s; - - /* Sanity checks */ - if (nbBits < 1) return ERROR(GENERIC); /* min size */ - - /* Build Decoding Table */ - DTableH->tableLog = (U16)nbBits; - DTableH->fastMode = 1; - for (s=0; sfastMode; - - /* select fast mode (static) */ - if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); - return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); + assert(op >= ostart); + return (size_t)(op-ostart); } +typedef struct { + short ncount[FSE_MAX_SYMBOL_VALUE + 1]; +} FSE_DecompressWksp; -size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) -{ - return FSE_decompress_wksp_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, /* bmi2 */ 0); -} FORCE_INLINE_TEMPLATE size_t FSE_decompress_wksp_body( void* dst, size_t dstCapacity, @@ -3470,24 +3865,34 @@ FORCE_INLINE_TEMPLATE size_t FSE_decompress_wksp_body( { const BYTE* const istart = (const BYTE*)cSrc; const BYTE* ip = istart; - short counting[FSE_MAX_SYMBOL_VALUE+1]; unsigned tableLog; unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; - FSE_DTable* const dtable = (FSE_DTable*)workSpace; + FSE_DecompressWksp* const wksp = (FSE_DecompressWksp*)workSpace; + size_t const dtablePos = sizeof(FSE_DecompressWksp) / sizeof(FSE_DTable); + FSE_DTable* const dtable = (FSE_DTable*)workSpace + dtablePos; + + FSE_STATIC_ASSERT((FSE_MAX_SYMBOL_VALUE + 1) % 2 == 0); + if (wkspSize < sizeof(*wksp)) return ERROR(GENERIC); + + /* correct offset to dtable depends on this property */ + FSE_STATIC_ASSERT(sizeof(FSE_DecompressWksp) % sizeof(FSE_DTable) == 0); /* normal FSE decoding mode */ - size_t const NCountLength = FSE_readNCount_bmi2(counting, &maxSymbolValue, &tableLog, istart, cSrcSize, bmi2); - if (FSE_isError(NCountLength)) return NCountLength; - if (tableLog > maxLog) return ERROR(tableLog_tooLarge); - assert(NCountLength <= cSrcSize); - ip += NCountLength; - cSrcSize -= NCountLength; + { size_t const NCountLength = + FSE_readNCount_bmi2(wksp->ncount, &maxSymbolValue, &tableLog, istart, cSrcSize, bmi2); + if (FSE_isError(NCountLength)) return NCountLength; + if (tableLog > maxLog) return ERROR(tableLog_tooLarge); + assert(NCountLength <= cSrcSize); + ip += NCountLength; + cSrcSize -= NCountLength; + } if (FSE_DECOMPRESS_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(tableLog_tooLarge); - workSpace = dtable + FSE_DTABLE_SIZE_U32(tableLog); - wkspSize -= FSE_DTABLE_SIZE(tableLog); + assert(sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog) <= wkspSize); + workSpace = (BYTE*)workSpace + sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog); + wkspSize -= sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog); - CHECK_F( FSE_buildDTable_internal(dtable, counting, maxSymbolValue, tableLog, workSpace, wkspSize) ); + CHECK_F( FSE_buildDTable_internal(dtable, wksp->ncount, maxSymbolValue, tableLog, workSpace, wkspSize) ); { const void* ptr = dtable; @@ -3507,7 +3912,7 @@ static size_t FSE_decompress_wksp_body_default(void* dst, size_t dstCapacity, co } #if DYNAMIC_BMI2 -TARGET_ATTRIBUTE("bmi2") static size_t FSE_decompress_wksp_body_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) +BMI2_TARGET_ATTRIBUTE static size_t FSE_decompress_wksp_body_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) { return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 1); } @@ -3524,29 +3929,11 @@ size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, return FSE_decompress_wksp_body_default(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize); } - -typedef FSE_DTable DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)]; - -#ifndef ZSTD_NO_UNUSED_FUNCTIONS -size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) { - U32 wksp[FSE_BUILD_DTABLE_WKSP_SIZE_U32(FSE_TABLELOG_ABSOLUTE_MAX, FSE_MAX_SYMBOL_VALUE)]; - return FSE_buildDTable_wksp(dt, normalizedCounter, maxSymbolValue, tableLog, wksp, sizeof(wksp)); -} - -size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize) -{ - /* Static analyzer seems unable to understand this table will be properly initialized later */ - U32 wksp[FSE_DECOMPRESS_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)]; - return FSE_decompress_wksp(dst, dstCapacity, cSrc, cSrcSize, FSE_MAX_TABLELOG, wksp, sizeof(wksp)); -} -#endif - - #endif /* FSE_COMMONDEFS_ONLY */ /**** ended inlining common/fse_decompress.c ****/ /**** start inlining common/zstd_common.c ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -3561,11 +3948,10 @@ size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cS * Dependencies ***************************************/ #define ZSTD_DEPS_NEED_MALLOC -/**** skipping file: zstd_deps.h ****/ /**** skipping file: error_private.h ****/ /**** start inlining zstd_internal.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -3585,17 +3971,10 @@ size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cS /*-************************************* * Dependencies ***************************************/ -#if !defined(ZSTD_NO_INTRINSICS) && defined(__ARM_NEON) -#include -#endif /**** skipping file: compiler.h ****/ -/**** skipping file: mem.h ****/ -/**** skipping file: debug.h ****/ -/**** skipping file: error_private.h ****/ -#define ZSTD_STATIC_LINKING_ONLY -/**** start inlining ../zstd.h ****/ +/**** start inlining cpu.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -3603,2908 +3982,6744 @@ size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cS * in the COPYING file in the root directory of this source tree). * You may select, at your option, one of the above-listed licenses. */ -#if defined (__cplusplus) -extern "C" { -#endif -#ifndef ZSTD_H_235446 -#define ZSTD_H_235446 +#ifndef ZSTD_COMMON_CPU_H +#define ZSTD_COMMON_CPU_H -/* ====== Dependency ======*/ -#include /* INT_MAX */ -#include /* size_t */ +/** + * Implementation taken from folly/CpuId.h + * https://github.com/facebook/folly/blob/master/folly/CpuId.h + */ +/**** skipping file: mem.h ****/ -/* ===== ZSTDLIB_API : control library symbols visibility ===== */ -#ifndef ZSTDLIB_VISIBILITY -# if defined(__GNUC__) && (__GNUC__ >= 4) -# define ZSTDLIB_VISIBILITY __attribute__ ((visibility ("default"))) -# else -# define ZSTDLIB_VISIBILITY -# endif -#endif -#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) -# define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBILITY -#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) -# define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ -#else -# define ZSTDLIB_API ZSTDLIB_VISIBILITY +#ifdef _MSC_VER +#include #endif +typedef struct { + U32 f1c; + U32 f1d; + U32 f7b; + U32 f7c; +} ZSTD_cpuid_t; -/******************************************************************************* - Introduction - - zstd, short for Zstandard, is a fast lossless compression algorithm, targeting - real-time compression scenarios at zlib-level and better compression ratios. - The zstd compression library provides in-memory compression and decompression - functions. - - The library supports regular compression levels from 1 up to ZSTD_maxCLevel(), - which is currently 22. Levels >= 20, labeled `--ultra`, should be used with - caution, as they require more memory. The library also offers negative - compression levels, which extend the range of speed vs. ratio preferences. - The lower the level, the faster the speed (at the cost of compression). - - Compression can be done in: - - a single step (described as Simple API) - - a single step, reusing a context (described as Explicit context) - - unbounded multiple steps (described as Streaming compression) - - The compression ratio achievable on small data can be highly improved using - a dictionary. Dictionary compression can be performed in: - - a single step (described as Simple dictionary API) - - a single step, reusing a dictionary (described as Bulk-processing - dictionary API) - - Advanced experimental functions can be accessed using - `#define ZSTD_STATIC_LINKING_ONLY` before including zstd.h. - - Advanced experimental APIs should never be used with a dynamically-linked - library. They are not "stable"; their definitions or signatures may change in - the future. Only static linking is allowed. -*******************************************************************************/ - -/*------ Version ------*/ -#define ZSTD_VERSION_MAJOR 1 -#define ZSTD_VERSION_MINOR 4 -#define ZSTD_VERSION_RELEASE 9 -#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) - -/*! ZSTD_versionNumber() : - * Return runtime library version, the value is (MAJOR*100*100 + MINOR*100 + RELEASE). */ -ZSTDLIB_API unsigned ZSTD_versionNumber(void); - -#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE -#define ZSTD_QUOTE(str) #str -#define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str) -#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION) - -/*! ZSTD_versionString() : - * Return runtime library version, like "1.4.5". Requires v1.3.0+. */ -ZSTDLIB_API const char* ZSTD_versionString(void); - -/* ************************************* - * Default constant - ***************************************/ -#ifndef ZSTD_CLEVEL_DEFAULT -# define ZSTD_CLEVEL_DEFAULT 3 -#endif +MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) { + U32 f1c = 0; + U32 f1d = 0; + U32 f7b = 0; + U32 f7c = 0; +#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) +#if !defined(_M_X64) || !defined(__clang__) || __clang_major__ >= 16 + int reg[4]; + __cpuid((int*)reg, 0); + { + int const n = reg[0]; + if (n >= 1) { + __cpuid((int*)reg, 1); + f1c = (U32)reg[2]; + f1d = (U32)reg[3]; + } + if (n >= 7) { + __cpuidex((int*)reg, 7, 0); + f7b = (U32)reg[1]; + f7c = (U32)reg[2]; + } + } +#else + /* Clang compiler has a bug (fixed in https://reviews.llvm.org/D101338) in + * which the `__cpuid` intrinsic does not save and restore `rbx` as it needs + * to due to being a reserved register. So in that case, do the `cpuid` + * ourselves. Clang supports inline assembly anyway. + */ + U32 n; + __asm__( + "pushq %%rbx\n\t" + "cpuid\n\t" + "popq %%rbx\n\t" + : "=a"(n) + : "a"(0) + : "rcx", "rdx"); + if (n >= 1) { + U32 f1a; + __asm__( + "pushq %%rbx\n\t" + "cpuid\n\t" + "popq %%rbx\n\t" + : "=a"(f1a), "=c"(f1c), "=d"(f1d) + : "a"(1) + :); + } + if (n >= 7) { + __asm__( + "pushq %%rbx\n\t" + "cpuid\n\t" + "movq %%rbx, %%rax\n\t" + "popq %%rbx" + : "=a"(f7b), "=c"(f7c) + : "a"(7), "c"(0) + : "rdx"); + } +#endif +#elif defined(__i386__) && defined(__PIC__) && !defined(__clang__) && defined(__GNUC__) + /* The following block like the normal cpuid branch below, but gcc + * reserves ebx for use of its pic register so we must specially + * handle the save and restore to avoid clobbering the register + */ + U32 n; + __asm__( + "pushl %%ebx\n\t" + "cpuid\n\t" + "popl %%ebx\n\t" + : "=a"(n) + : "a"(0) + : "ecx", "edx"); + if (n >= 1) { + U32 f1a; + __asm__( + "pushl %%ebx\n\t" + "cpuid\n\t" + "popl %%ebx\n\t" + : "=a"(f1a), "=c"(f1c), "=d"(f1d) + : "a"(1)); + } + if (n >= 7) { + __asm__( + "pushl %%ebx\n\t" + "cpuid\n\t" + "movl %%ebx, %%eax\n\t" + "popl %%ebx" + : "=a"(f7b), "=c"(f7c) + : "a"(7), "c"(0) + : "edx"); + } +#elif defined(__x86_64__) || defined(_M_X64) || defined(__i386__) + U32 n; + __asm__("cpuid" : "=a"(n) : "a"(0) : "ebx", "ecx", "edx"); + if (n >= 1) { + U32 f1a; + __asm__("cpuid" : "=a"(f1a), "=c"(f1c), "=d"(f1d) : "a"(1) : "ebx"); + } + if (n >= 7) { + U32 f7a; + __asm__("cpuid" + : "=a"(f7a), "=b"(f7b), "=c"(f7c) + : "a"(7), "c"(0) + : "edx"); + } +#endif + { + ZSTD_cpuid_t cpuid; + cpuid.f1c = f1c; + cpuid.f1d = f1d; + cpuid.f7b = f7b; + cpuid.f7c = f7c; + return cpuid; + } +} + +#define X(name, r, bit) \ + MEM_STATIC int ZSTD_cpuid_##name(ZSTD_cpuid_t const cpuid) { \ + return ((cpuid.r) & (1U << bit)) != 0; \ + } + +/* cpuid(1): Processor Info and Feature Bits. */ +#define C(name, bit) X(name, f1c, bit) + C(sse3, 0) + C(pclmuldq, 1) + C(dtes64, 2) + C(monitor, 3) + C(dscpl, 4) + C(vmx, 5) + C(smx, 6) + C(eist, 7) + C(tm2, 8) + C(ssse3, 9) + C(cnxtid, 10) + C(fma, 12) + C(cx16, 13) + C(xtpr, 14) + C(pdcm, 15) + C(pcid, 17) + C(dca, 18) + C(sse41, 19) + C(sse42, 20) + C(x2apic, 21) + C(movbe, 22) + C(popcnt, 23) + C(tscdeadline, 24) + C(aes, 25) + C(xsave, 26) + C(osxsave, 27) + C(avx, 28) + C(f16c, 29) + C(rdrand, 30) +#undef C +#define D(name, bit) X(name, f1d, bit) + D(fpu, 0) + D(vme, 1) + D(de, 2) + D(pse, 3) + D(tsc, 4) + D(msr, 5) + D(pae, 6) + D(mce, 7) + D(cx8, 8) + D(apic, 9) + D(sep, 11) + D(mtrr, 12) + D(pge, 13) + D(mca, 14) + D(cmov, 15) + D(pat, 16) + D(pse36, 17) + D(psn, 18) + D(clfsh, 19) + D(ds, 21) + D(acpi, 22) + D(mmx, 23) + D(fxsr, 24) + D(sse, 25) + D(sse2, 26) + D(ss, 27) + D(htt, 28) + D(tm, 29) + D(pbe, 31) +#undef D + +/* cpuid(7): Extended Features. */ +#define B(name, bit) X(name, f7b, bit) + B(bmi1, 3) + B(hle, 4) + B(avx2, 5) + B(smep, 7) + B(bmi2, 8) + B(erms, 9) + B(invpcid, 10) + B(rtm, 11) + B(mpx, 14) + B(avx512f, 16) + B(avx512dq, 17) + B(rdseed, 18) + B(adx, 19) + B(smap, 20) + B(avx512ifma, 21) + B(pcommit, 22) + B(clflushopt, 23) + B(clwb, 24) + B(avx512pf, 26) + B(avx512er, 27) + B(avx512cd, 28) + B(sha, 29) + B(avx512bw, 30) + B(avx512vl, 31) +#undef B +#define C(name, bit) X(name, f7c, bit) + C(prefetchwt1, 0) + C(avx512vbmi, 1) +#undef C + +#undef X + +#endif /* ZSTD_COMMON_CPU_H */ +/**** ended inlining cpu.h ****/ +/**** skipping file: mem.h ****/ +/**** skipping file: debug.h ****/ +/**** skipping file: error_private.h ****/ +#define ZSTD_STATIC_LINKING_ONLY +/**** start inlining ../zstd.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_H_235446 +#define ZSTD_H_235446 + + +/* ====== Dependencies ======*/ +#include /* size_t */ + +/**** skipping file: zstd_errors.h ****/ +#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY) +#include /* INT_MAX */ +#endif /* ZSTD_STATIC_LINKING_ONLY */ + +#if defined (__cplusplus) +extern "C" { +#endif + +/* ===== ZSTDLIB_API : control library symbols visibility ===== */ +#ifndef ZSTDLIB_VISIBLE + /* Backwards compatibility with old macro name */ +# ifdef ZSTDLIB_VISIBILITY +# define ZSTDLIB_VISIBLE ZSTDLIB_VISIBILITY +# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDLIB_VISIBLE __attribute__ ((visibility ("default"))) +# else +# define ZSTDLIB_VISIBLE +# endif +#endif + +#ifndef ZSTDLIB_HIDDEN +# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDLIB_HIDDEN __attribute__ ((visibility ("hidden"))) +# else +# define ZSTDLIB_HIDDEN +# endif +#endif + +#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBLE +#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define ZSTDLIB_API ZSTDLIB_VISIBLE +#endif + +/* Deprecation warnings : + * Should these warnings be a problem, it is generally possible to disable them, + * typically with -Wno-deprecated-declarations for gcc or _CRT_SECURE_NO_WARNINGS in Visual. + * Otherwise, it's also possible to define ZSTD_DISABLE_DEPRECATE_WARNINGS. + */ +#ifdef ZSTD_DISABLE_DEPRECATE_WARNINGS +# define ZSTD_DEPRECATED(message) /* disable deprecation warnings */ +#else +# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ +# define ZSTD_DEPRECATED(message) [[deprecated(message)]] +# elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__) || defined(__IAR_SYSTEMS_ICC__) +# define ZSTD_DEPRECATED(message) __attribute__((deprecated(message))) +# elif defined(__GNUC__) && (__GNUC__ >= 3) +# define ZSTD_DEPRECATED(message) __attribute__((deprecated)) +# elif defined(_MSC_VER) +# define ZSTD_DEPRECATED(message) __declspec(deprecated(message)) +# else +# pragma message("WARNING: You need to implement ZSTD_DEPRECATED for this compiler") +# define ZSTD_DEPRECATED(message) +# endif +#endif /* ZSTD_DISABLE_DEPRECATE_WARNINGS */ + + +/******************************************************************************* + Introduction + + zstd, short for Zstandard, is a fast lossless compression algorithm, targeting + real-time compression scenarios at zlib-level and better compression ratios. + The zstd compression library provides in-memory compression and decompression + functions. + + The library supports regular compression levels from 1 up to ZSTD_maxCLevel(), + which is currently 22. Levels >= 20, labeled `--ultra`, should be used with + caution, as they require more memory. The library also offers negative + compression levels, which extend the range of speed vs. ratio preferences. + The lower the level, the faster the speed (at the cost of compression). + + Compression can be done in: + - a single step (described as Simple API) + - a single step, reusing a context (described as Explicit context) + - unbounded multiple steps (described as Streaming compression) + + The compression ratio achievable on small data can be highly improved using + a dictionary. Dictionary compression can be performed in: + - a single step (described as Simple dictionary API) + - a single step, reusing a dictionary (described as Bulk-processing + dictionary API) + + Advanced experimental functions can be accessed using + `#define ZSTD_STATIC_LINKING_ONLY` before including zstd.h. + + Advanced experimental APIs should never be used with a dynamically-linked + library. They are not "stable"; their definitions or signatures may change in + the future. Only static linking is allowed. +*******************************************************************************/ + +/*------ Version ------*/ +#define ZSTD_VERSION_MAJOR 1 +#define ZSTD_VERSION_MINOR 5 +#define ZSTD_VERSION_RELEASE 7 +#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) + +/*! ZSTD_versionNumber() : + * Return runtime library version, the value is (MAJOR*100*100 + MINOR*100 + RELEASE). */ +ZSTDLIB_API unsigned ZSTD_versionNumber(void); + +#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE +#define ZSTD_QUOTE(str) #str +#define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str) +#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION) + +/*! ZSTD_versionString() : + * Return runtime library version, like "1.4.5". Requires v1.3.0+. */ +ZSTDLIB_API const char* ZSTD_versionString(void); + +/* ************************************* + * Default constant + ***************************************/ +#ifndef ZSTD_CLEVEL_DEFAULT +# define ZSTD_CLEVEL_DEFAULT 3 +#endif + +/* ************************************* + * Constants + ***************************************/ + +/* All magic numbers are supposed read/written to/from files/memory using little-endian convention */ +#define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */ +#define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* valid since v0.7.0 */ +#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50 /* all 16 values, from 0x184D2A50 to 0x184D2A5F, signal the beginning of a skippable frame */ +#define ZSTD_MAGIC_SKIPPABLE_MASK 0xFFFFFFF0 + +#define ZSTD_BLOCKSIZELOG_MAX 17 +#define ZSTD_BLOCKSIZE_MAX (1<= ZSTD_compressBound(srcSize)` guarantees that zstd will have + * enough space to successfully compress the data. + * @return : compressed size written into `dst` (<= `dstCapacity), + * or an error code if it fails (which can be tested using ZSTD_isError()). */ +ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel); + +/*! ZSTD_decompress() : + * `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames. + * Multiple compressed frames can be decompressed at once with this method. + * The result will be the concatenation of all decompressed frames, back to back. + * `dstCapacity` is an upper bound of originalSize to regenerate. + * First frame's decompressed size can be extracted using ZSTD_getFrameContentSize(). + * If maximum upper bound isn't known, prefer using streaming mode to decompress data. + * @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), + * or an errorCode if it fails (which can be tested using ZSTD_isError()). */ +ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, + const void* src, size_t compressedSize); + + +/*====== Decompression helper functions ======*/ + +/*! ZSTD_getFrameContentSize() : requires v1.3.0+ + * `src` should point to the start of a ZSTD encoded frame. + * `srcSize` must be at least as large as the frame header. + * hint : any size >= `ZSTD_frameHeaderSize_max` is large enough. + * @return : - decompressed size of `src` frame content, if known + * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined + * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) + * note 1 : a 0 return value means the frame is valid but "empty". + * When invoking this method on a skippable frame, it will return 0. + * note 2 : decompressed size is an optional field, it may not be present (typically in streaming mode). + * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. + * In which case, it's necessary to use streaming mode to decompress data. + * Optionally, application can rely on some implicit limit, + * as ZSTD_decompress() only needs an upper bound of decompressed size. + * (For example, data could be necessarily cut into blocks <= 16 KB). + * note 3 : decompressed size is always present when compression is completed using single-pass functions, + * such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict(). + * note 4 : decompressed size can be very large (64-bits value), + * potentially larger than what local system can handle as a single memory segment. + * In which case, it's necessary to use streaming mode to decompress data. + * note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified. + * Always ensure return value fits within application's authorized limits. + * Each application can set its own limits. + * note 6 : This function replaces ZSTD_getDecompressedSize() */ +#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1) +#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2) +ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); + +/*! ZSTD_getDecompressedSize() (obsolete): + * This function is now obsolete, in favor of ZSTD_getFrameContentSize(). + * Both functions work the same way, but ZSTD_getDecompressedSize() blends + * "empty", "unknown" and "error" results to the same return value (0), + * while ZSTD_getFrameContentSize() gives them separate return values. + * @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */ +ZSTD_DEPRECATED("Replaced by ZSTD_getFrameContentSize") +ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); + +/*! ZSTD_findFrameCompressedSize() : Requires v1.4.0+ + * `src` should point to the start of a ZSTD frame or skippable frame. + * `srcSize` must be >= first frame size + * @return : the compressed size of the first frame starting at `src`, + * suitable to pass as `srcSize` to `ZSTD_decompress` or similar, + * or an error code if input is invalid + * Note 1: this method is called _find*() because it's not enough to read the header, + * it may have to scan through the frame's content, to reach its end. + * Note 2: this method also works with Skippable Frames. In which case, + * it returns the size of the complete skippable frame, + * which is always equal to its content size + 8 bytes for headers. */ +ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize); + + +/*====== Compression helper functions ======*/ + +/*! ZSTD_compressBound() : + * maximum compressed size in worst case single-pass scenario. + * When invoking `ZSTD_compress()`, or any other one-pass compression function, + * it's recommended to provide @dstCapacity >= ZSTD_compressBound(srcSize) + * as it eliminates one potential failure scenario, + * aka not enough room in dst buffer to write the compressed frame. + * Note : ZSTD_compressBound() itself can fail, if @srcSize >= ZSTD_MAX_INPUT_SIZE . + * In which case, ZSTD_compressBound() will return an error code + * which can be tested using ZSTD_isError(). + * + * ZSTD_COMPRESSBOUND() : + * same as ZSTD_compressBound(), but as a macro. + * It can be used to produce constants, which can be useful for static allocation, + * for example to size a static array on stack. + * Will produce constant value 0 if srcSize is too large. + */ +#define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00ULL : 0xFF00FF00U) +#define ZSTD_COMPRESSBOUND(srcSize) (((size_t)(srcSize) >= ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */ +ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */ + + +/*====== Error helper functions ======*/ +/* ZSTD_isError() : + * Most ZSTD_* functions returning a size_t value can be tested for error, + * using ZSTD_isError(). + * @return 1 if error, 0 otherwise + */ +ZSTDLIB_API unsigned ZSTD_isError(size_t result); /*!< tells if a `size_t` function result is an error code */ +ZSTDLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); /* convert a result into an error code, which can be compared to error enum list */ +ZSTDLIB_API const char* ZSTD_getErrorName(size_t result); /*!< provides readable string from a function result */ +ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed, requires v1.4.0+ */ +ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */ +ZSTDLIB_API int ZSTD_defaultCLevel(void); /*!< default compression level, specified by ZSTD_CLEVEL_DEFAULT, requires v1.5.0+ */ + + +/*************************************** +* Explicit context +***************************************/ +/*= Compression context + * When compressing many times, + * it is recommended to allocate a compression context just once, + * and reuse it for each successive compression operation. + * This will make the workload easier for system's memory. + * Note : re-using context is just a speed / resource optimization. + * It doesn't change the compression ratio, which remains identical. + * Note 2: For parallel execution in multi-threaded environments, + * use one different context per thread . + */ +typedef struct ZSTD_CCtx_s ZSTD_CCtx; +ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); +ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); /* compatible with NULL pointer */ + +/*! ZSTD_compressCCtx() : + * Same as ZSTD_compress(), using an explicit ZSTD_CCtx. + * Important : in order to mirror `ZSTD_compress()` behavior, + * this function compresses at the requested compression level, + * __ignoring any other advanced parameter__ . + * If any advanced parameter was set using the advanced API, + * they will all be reset. Only @compressionLevel remains. + */ +ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel); + +/*= Decompression context + * When decompressing many times, + * it is recommended to allocate a context only once, + * and reuse it for each successive compression operation. + * This will make workload friendlier for system's memory. + * Use one context per thread for parallel execution. */ +typedef struct ZSTD_DCtx_s ZSTD_DCtx; +ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void); +ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); /* accept NULL pointer */ + +/*! ZSTD_decompressDCtx() : + * Same as ZSTD_decompress(), + * requires an allocated ZSTD_DCtx. + * Compatible with sticky parameters (see below). + */ +ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + + +/********************************************* +* Advanced compression API (Requires v1.4.0+) +**********************************************/ + +/* API design : + * Parameters are pushed one by one into an existing context, + * using ZSTD_CCtx_set*() functions. + * Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame. + * "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` ! + * __They do not apply to one-shot variants such as ZSTD_compressCCtx()__ . + * + * It's possible to reset all parameters to "default" using ZSTD_CCtx_reset(). + * + * This API supersedes all other "advanced" API entry points in the experimental section. + * In the future, we expect to remove API entry points from experimental which are redundant with this API. + */ + + +/* Compression strategies, listed from fastest to strongest */ +typedef enum { ZSTD_fast=1, + ZSTD_dfast=2, + ZSTD_greedy=3, + ZSTD_lazy=4, + ZSTD_lazy2=5, + ZSTD_btlazy2=6, + ZSTD_btopt=7, + ZSTD_btultra=8, + ZSTD_btultra2=9 + /* note : new strategies _might_ be added in the future. + Only the order (from fast to strong) is guaranteed */ +} ZSTD_strategy; + +typedef enum { + + /* compression parameters + * Note: When compressing with a ZSTD_CDict these parameters are superseded + * by the parameters used to construct the ZSTD_CDict. + * See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict). */ + ZSTD_c_compressionLevel=100, /* Set compression parameters according to pre-defined cLevel table. + * Note that exact compression parameters are dynamically determined, + * depending on both compression level and srcSize (when known). + * Default level is ZSTD_CLEVEL_DEFAULT==3. + * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT. + * Note 1 : it's possible to pass a negative compression level. + * Note 2 : setting a level does not automatically set all other compression parameters + * to default. Setting this will however eventually dynamically impact the compression + * parameters which have not been manually set. The manually set + * ones will 'stick'. */ + /* Advanced compression parameters : + * It's possible to pin down compression parameters to some specific values. + * In which case, these values are no longer dynamically selected by the compressor */ + ZSTD_c_windowLog=101, /* Maximum allowed back-reference distance, expressed as power of 2. + * This will set a memory budget for streaming decompression, + * with larger values requiring more memory + * and typically compressing more. + * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX. + * Special: value 0 means "use default windowLog". + * Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT + * requires explicitly allowing such size at streaming decompression stage. */ + ZSTD_c_hashLog=102, /* Size of the initial probe table, as a power of 2. + * Resulting memory usage is (1 << (hashLog+2)). + * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX. + * Larger tables improve compression ratio of strategies <= dFast, + * and improve speed of strategies > dFast. + * Special: value 0 means "use default hashLog". */ + ZSTD_c_chainLog=103, /* Size of the multi-probe search table, as a power of 2. + * Resulting memory usage is (1 << (chainLog+2)). + * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX. + * Larger tables result in better and slower compression. + * This parameter is useless for "fast" strategy. + * It's still useful when using "dfast" strategy, + * in which case it defines a secondary probe table. + * Special: value 0 means "use default chainLog". */ + ZSTD_c_searchLog=104, /* Number of search attempts, as a power of 2. + * More attempts result in better and slower compression. + * This parameter is useless for "fast" and "dFast" strategies. + * Special: value 0 means "use default searchLog". */ + ZSTD_c_minMatch=105, /* Minimum size of searched matches. + * Note that Zstandard can still find matches of smaller size, + * it just tweaks its search algorithm to look for this size and larger. + * Larger values increase compression and decompression speed, but decrease ratio. + * Must be clamped between ZSTD_MINMATCH_MIN and ZSTD_MINMATCH_MAX. + * Note that currently, for all strategies < btopt, effective minimum is 4. + * , for all strategies > fast, effective maximum is 6. + * Special: value 0 means "use default minMatchLength". */ + ZSTD_c_targetLength=106, /* Impact of this field depends on strategy. + * For strategies btopt, btultra & btultra2: + * Length of Match considered "good enough" to stop search. + * Larger values make compression stronger, and slower. + * For strategy fast: + * Distance between match sampling. + * Larger values make compression faster, and weaker. + * Special: value 0 means "use default targetLength". */ + ZSTD_c_strategy=107, /* See ZSTD_strategy enum definition. + * The higher the value of selected strategy, the more complex it is, + * resulting in stronger and slower compression. + * Special: value 0 means "use default strategy". */ + + ZSTD_c_targetCBlockSize=130, /* v1.5.6+ + * Attempts to fit compressed block size into approximately targetCBlockSize. + * Bound by ZSTD_TARGETCBLOCKSIZE_MIN and ZSTD_TARGETCBLOCKSIZE_MAX. + * Note that it's not a guarantee, just a convergence target (default:0). + * No target when targetCBlockSize == 0. + * This is helpful in low bandwidth streaming environments to improve end-to-end latency, + * when a client can make use of partial documents (a prominent example being Chrome). + * Note: this parameter is stable since v1.5.6. + * It was present as an experimental parameter in earlier versions, + * but it's not recommended using it with earlier library versions + * due to massive performance regressions. + */ + /* LDM mode parameters */ + ZSTD_c_enableLongDistanceMatching=160, /* Enable long distance matching. + * This parameter is designed to improve compression ratio + * for large inputs, by finding large matches at long distance. + * It increases memory usage and window size. + * Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB + * except when expressly set to a different value. + * Note: will be enabled by default if ZSTD_c_windowLog >= 128 MB and + * compression strategy >= ZSTD_btopt (== compression level 16+) */ + ZSTD_c_ldmHashLog=161, /* Size of the table for long distance matching, as a power of 2. + * Larger values increase memory usage and compression ratio, + * but decrease compression speed. + * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX + * default: windowlog - 7. + * Special: value 0 means "automatically determine hashlog". */ + ZSTD_c_ldmMinMatch=162, /* Minimum match size for long distance matcher. + * Larger/too small values usually decrease compression ratio. + * Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX. + * Special: value 0 means "use default value" (default: 64). */ + ZSTD_c_ldmBucketSizeLog=163, /* Log size of each bucket in the LDM hash table for collision resolution. + * Larger values improve collision resolution but decrease compression speed. + * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX. + * Special: value 0 means "use default value" (default: 3). */ + ZSTD_c_ldmHashRateLog=164, /* Frequency of inserting/looking up entries into the LDM hash table. + * Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN). + * Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage. + * Larger values improve compression speed. + * Deviating far from default value will likely result in a compression ratio decrease. + * Special: value 0 means "automatically determine hashRateLog". */ + + /* frame parameters */ + ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1) + * Content size must be known at the beginning of compression. + * This is automatically the case when using ZSTD_compress2(), + * For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */ + ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */ + ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */ + + /* multi-threading parameters */ + /* These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD). + * Otherwise, trying to set any other value than default (0) will be a no-op and return an error. + * In a situation where it's unknown if the linked library supports multi-threading or not, + * setting ZSTD_c_nbWorkers to any value >= 1 and consulting the return value provides a quick way to check this property. + */ + ZSTD_c_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel. + * When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() : + * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller, + * while compression is performed in parallel, within worker thread(s). + * (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end : + * in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call). + * More workers improve speed, but also increase memory usage. + * Default value is `0`, aka "single-threaded mode" : no worker is spawned, + * compression is performed inside Caller's thread, and all invocations are blocking */ + ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1. + * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads. + * 0 means default, which is dynamically determined based on compression parameters. + * Job size must be a minimum of overlap size, or ZSTDMT_JOBSIZE_MIN (= 512 KB), whichever is largest. + * The minimum size is automatically and transparently enforced. */ + ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size. + * The overlap size is an amount of data reloaded from previous job at the beginning of a new job. + * It helps preserve compression ratio, while each job is compressed in parallel. + * This value is enforced only when nbWorkers >= 1. + * Larger values increase compression ratio, but decrease speed. + * Possible values range from 0 to 9 : + * - 0 means "default" : value will be determined by the library, depending on strategy + * - 1 means "no overlap" + * - 9 means "full overlap", using a full window size. + * Each intermediate rank increases/decreases load size by a factor 2 : + * 9: full window; 8: w/2; 7: w/4; 6: w/8; 5:w/16; 4: w/32; 3:w/64; 2:w/128; 1:no overlap; 0:default + * default value varies between 6 and 9, depending on strategy */ + + /* note : additional experimental parameters are also available + * within the experimental section of the API. + * At the time of this writing, they include : + * ZSTD_c_rsyncable + * ZSTD_c_format + * ZSTD_c_forceMaxWindow + * ZSTD_c_forceAttachDict + * ZSTD_c_literalCompressionMode + * ZSTD_c_srcSizeHint + * ZSTD_c_enableDedicatedDictSearch + * ZSTD_c_stableInBuffer + * ZSTD_c_stableOutBuffer + * ZSTD_c_blockDelimiters + * ZSTD_c_validateSequences + * ZSTD_c_blockSplitterLevel + * ZSTD_c_splitAfterSequences + * ZSTD_c_useRowMatchFinder + * ZSTD_c_prefetchCDictTables + * ZSTD_c_enableSeqProducerFallback + * ZSTD_c_maxBlockSize + * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. + * note : never ever use experimentalParam? names directly; + * also, the enums values themselves are unstable and can still change. + */ + ZSTD_c_experimentalParam1=500, + ZSTD_c_experimentalParam2=10, + ZSTD_c_experimentalParam3=1000, + ZSTD_c_experimentalParam4=1001, + ZSTD_c_experimentalParam5=1002, + /* was ZSTD_c_experimentalParam6=1003; is now ZSTD_c_targetCBlockSize */ + ZSTD_c_experimentalParam7=1004, + ZSTD_c_experimentalParam8=1005, + ZSTD_c_experimentalParam9=1006, + ZSTD_c_experimentalParam10=1007, + ZSTD_c_experimentalParam11=1008, + ZSTD_c_experimentalParam12=1009, + ZSTD_c_experimentalParam13=1010, + ZSTD_c_experimentalParam14=1011, + ZSTD_c_experimentalParam15=1012, + ZSTD_c_experimentalParam16=1013, + ZSTD_c_experimentalParam17=1014, + ZSTD_c_experimentalParam18=1015, + ZSTD_c_experimentalParam19=1016, + ZSTD_c_experimentalParam20=1017 +} ZSTD_cParameter; + +typedef struct { + size_t error; + int lowerBound; + int upperBound; +} ZSTD_bounds; + +/*! ZSTD_cParam_getBounds() : + * All parameters must belong to an interval with lower and upper bounds, + * otherwise they will either trigger an error or be automatically clamped. + * @return : a structure, ZSTD_bounds, which contains + * - an error status field, which must be tested using ZSTD_isError() + * - lower and upper bounds, both inclusive + */ +ZSTDLIB_API ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter cParam); + +/*! ZSTD_CCtx_setParameter() : + * Set one compression parameter, selected by enum ZSTD_cParameter. + * All parameters have valid bounds. Bounds can be queried using ZSTD_cParam_getBounds(). + * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). + * Setting a parameter is generally only possible during frame initialization (before starting compression). + * Exception : when using multi-threading mode (nbWorkers >= 1), + * the following parameters can be updated _during_ compression (within same frame): + * => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy. + * new parameters will be active for next job only (after a flush()). + * @return : an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value); + +/*! ZSTD_CCtx_setPledgedSrcSize() : + * Total input data size to be compressed as a single frame. + * Value will be written in frame header, unless if explicitly forbidden using ZSTD_c_contentSizeFlag. + * This value will also be controlled at end of frame, and trigger an error if not respected. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame. + * In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN. + * ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame. + * Note 2 : pledgedSrcSize is only valid once, for the next frame. + * It's discarded at the end of the frame, and replaced by ZSTD_CONTENTSIZE_UNKNOWN. + * Note 3 : Whenever all input data is provided and consumed in a single round, + * for example with ZSTD_compress2(), + * or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end), + * this value is automatically overridden by srcSize instead. + */ +ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize); + +typedef enum { + ZSTD_reset_session_only = 1, + ZSTD_reset_parameters = 2, + ZSTD_reset_session_and_parameters = 3 +} ZSTD_ResetDirective; + +/*! ZSTD_CCtx_reset() : + * There are 2 different things that can be reset, independently or jointly : + * - The session : will stop compressing current frame, and make CCtx ready to start a new one. + * Useful after an error, or to interrupt any ongoing compression. + * Any internal data not yet flushed is cancelled. + * Compression parameters and dictionary remain unchanged. + * They will be used to compress next frame. + * Resetting session never fails. + * - The parameters : changes all parameters back to "default". + * This also removes any reference to any dictionary or external sequence producer. + * Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing) + * otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError()) + * - Both : similar to resetting the session, followed by resetting parameters. + */ +ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset); + +/*! ZSTD_compress2() : + * Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API. + * (note that this entry point doesn't even expose a compression level parameter). + * ZSTD_compress2() always starts a new frame. + * Should cctx hold data from a previously unfinished frame, everything about it is forgotten. + * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() + * - The function is always blocking, returns when compression is completed. + * NOTE: Providing `dstCapacity >= ZSTD_compressBound(srcSize)` guarantees that zstd will have + * enough space to successfully compress the data, though it is possible it fails for other reasons. + * @return : compressed size written into `dst` (<= `dstCapacity), + * or an error code if it fails (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_compress2( ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + + +/*********************************************** +* Advanced decompression API (Requires v1.4.0+) +************************************************/ + +/* The advanced API pushes parameters one by one into an existing DCtx context. + * Parameters are sticky, and remain valid for all following frames + * using the same DCtx context. + * It's possible to reset parameters to default values using ZSTD_DCtx_reset(). + * Note : This API is compatible with existing ZSTD_decompressDCtx() and ZSTD_decompressStream(). + * Therefore, no new decompression function is necessary. + */ + +typedef enum { + + ZSTD_d_windowLogMax=100, /* Select a size limit (in power of 2) beyond which + * the streaming API will refuse to allocate memory buffer + * in order to protect the host from unreasonable memory requirements. + * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. + * By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT). + * Special: value 0 means "use default maximum windowLog". */ + + /* note : additional experimental parameters are also available + * within the experimental section of the API. + * At the time of this writing, they include : + * ZSTD_d_format + * ZSTD_d_stableOutBuffer + * ZSTD_d_forceIgnoreChecksum + * ZSTD_d_refMultipleDDicts + * ZSTD_d_disableHuffmanAssembly + * ZSTD_d_maxBlockSize + * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. + * note : never ever use experimentalParam? names directly + */ + ZSTD_d_experimentalParam1=1000, + ZSTD_d_experimentalParam2=1001, + ZSTD_d_experimentalParam3=1002, + ZSTD_d_experimentalParam4=1003, + ZSTD_d_experimentalParam5=1004, + ZSTD_d_experimentalParam6=1005 + +} ZSTD_dParameter; + +/*! ZSTD_dParam_getBounds() : + * All parameters must belong to an interval with lower and upper bounds, + * otherwise they will either trigger an error or be automatically clamped. + * @return : a structure, ZSTD_bounds, which contains + * - an error status field, which must be tested using ZSTD_isError() + * - both lower and upper bounds, inclusive + */ +ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam); + +/*! ZSTD_DCtx_setParameter() : + * Set one compression parameter, selected by enum ZSTD_dParameter. + * All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds(). + * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). + * Setting a parameter is only possible during frame initialization (before starting decompression). + * @return : 0, or an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value); + +/*! ZSTD_DCtx_reset() : + * Return a DCtx to clean state. + * Session and parameters can be reset jointly or separately. + * Parameters can only be reset when no active frame is being decompressed. + * @return : 0, or an error code, which can be tested with ZSTD_isError() + */ +ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset); + + +/**************************** +* Streaming +****************************/ + +typedef struct ZSTD_inBuffer_s { + const void* src; /**< start of input buffer */ + size_t size; /**< size of input buffer */ + size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_inBuffer; + +typedef struct ZSTD_outBuffer_s { + void* dst; /**< start of output buffer */ + size_t size; /**< size of output buffer */ + size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_outBuffer; + + + +/*-*********************************************************************** +* Streaming compression - HowTo +* +* A ZSTD_CStream object is required to track streaming operation. +* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. +* ZSTD_CStream objects can be reused multiple times on consecutive compression operations. +* It is recommended to reuse ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory. +* +* For parallel execution, use one separate ZSTD_CStream per thread. +* +* note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing. +* +* Parameters are sticky : when starting a new compression on the same context, +* it will reuse the same sticky parameters as previous compression session. +* When in doubt, it's recommended to fully initialize the context before usage. +* Use ZSTD_CCtx_reset() to reset the context and ZSTD_CCtx_setParameter(), +* ZSTD_CCtx_setPledgedSrcSize(), or ZSTD_CCtx_loadDictionary() and friends to +* set more specific parameters, the pledged source size, or load a dictionary. +* +* Use ZSTD_compressStream2() with ZSTD_e_continue as many times as necessary to +* consume input stream. The function will automatically update both `pos` +* fields within `input` and `output`. +* Note that the function may not consume the entire input, for example, because +* the output buffer is already full, in which case `input.pos < input.size`. +* The caller must check if input has been entirely consumed. +* If not, the caller must make some room to receive more compressed data, +* and then present again remaining input data. +* note: ZSTD_e_continue is guaranteed to make some forward progress when called, +* but doesn't guarantee maximal forward progress. This is especially relevant +* when compressing with multiple threads. The call won't block if it can +* consume some input, but if it can't it will wait for some, but not all, +* output to be flushed. +* @return : provides a minimum amount of data remaining to be flushed from internal buffers +* or an error code, which can be tested using ZSTD_isError(). +* +* At any moment, it's possible to flush whatever data might remain stuck within internal buffer, +* using ZSTD_compressStream2() with ZSTD_e_flush. `output->pos` will be updated. +* Note that, if `output->size` is too small, a single invocation with ZSTD_e_flush might not be enough (return code > 0). +* In which case, make some room to receive more compressed data, and call again ZSTD_compressStream2() with ZSTD_e_flush. +* You must continue calling ZSTD_compressStream2() with ZSTD_e_flush until it returns 0, at which point you can change the +* operation. +* note: ZSTD_e_flush will flush as much output as possible, meaning when compressing with multiple threads, it will +* block until the flush is complete or the output buffer is full. +* @return : 0 if internal buffers are entirely flushed, +* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), +* or an error code, which can be tested using ZSTD_isError(). +* +* Calling ZSTD_compressStream2() with ZSTD_e_end instructs to finish a frame. +* It will perform a flush and write frame epilogue. +* The epilogue is required for decoders to consider a frame completed. +* flush operation is the same, and follows same rules as calling ZSTD_compressStream2() with ZSTD_e_flush. +* You must continue calling ZSTD_compressStream2() with ZSTD_e_end until it returns 0, at which point you are free to +* start a new frame. +* note: ZSTD_e_end will flush as much output as possible, meaning when compressing with multiple threads, it will +* block until the flush is complete or the output buffer is full. +* @return : 0 if frame fully completed and fully flushed, +* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), +* or an error code, which can be tested using ZSTD_isError(). +* +* *******************************************************************/ + +typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are now effectively same object (>= v1.3.0) */ + /* Continue to distinguish them for compatibility with older versions <= v1.2.0 */ +/*===== ZSTD_CStream management functions =====*/ +ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void); +ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); /* accept NULL pointer */ + +/*===== Streaming compression functions =====*/ +typedef enum { + ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */ + ZSTD_e_flush=1, /* flush any data provided so far, + * it creates (at least) one new block, that can be decoded immediately on reception; + * frame will continue: any future data can still reference previously compressed data, improving compression. + * note : multithreaded compression will block to flush as much output as possible. */ + ZSTD_e_end=2 /* flush any remaining data _and_ close current frame. + * note that frame is only closed after compressed data is fully flushed (return value == 0). + * After that point, any additional data starts a new frame. + * note : each frame is independent (does not reference any content from previous frame). + : note : multithreaded compression will block to flush as much output as possible. */ +} ZSTD_EndDirective; + +/*! ZSTD_compressStream2() : Requires v1.4.0+ + * Behaves about the same as ZSTD_compressStream, with additional control on end directive. + * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() + * - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode) + * - output->pos must be <= dstCapacity, input->pos must be <= srcSize + * - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit. + * - endOp must be a valid directive + * - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller. + * - When nbWorkers>=1, function is non-blocking : it copies a portion of input, distributes jobs to internal worker threads, flush to output whatever is available, + * and then immediately returns, just indicating that there is some data remaining to be flushed. + * The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte. + * - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking. + * - @return provides a minimum amount of data remaining to be flushed from internal buffers + * or an error code, which can be tested using ZSTD_isError(). + * if @return != 0, flush is not fully completed, there is still some data left within internal buffers. + * This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers. + * For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed. + * - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0), + * only ZSTD_e_end or ZSTD_e_flush operations are allowed. + * Before starting a new compression job, or changing compression parameters, + * it is required to fully flush internal buffers. + * - note: if an operation ends with an error, it may leave @cctx in an undefined state. + * Therefore, it's UB to invoke ZSTD_compressStream2() of ZSTD_compressStream() on such a state. + * In order to be re-employed after an error, a state must be reset, + * which can be done explicitly (ZSTD_CCtx_reset()), + * or is sometimes implied by methods starting a new compression job (ZSTD_initCStream(), ZSTD_compressCCtx()) + */ +ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective endOp); + + +/* These buffer sizes are softly recommended. + * They are not required : ZSTD_compressStream*() happily accepts any buffer size, for both input and output. + * Respecting the recommended size just makes it a bit easier for ZSTD_compressStream*(), + * reducing the amount of memory shuffling and buffering, resulting in minor performance savings. + * + * However, note that these recommendations are from the perspective of a C caller program. + * If the streaming interface is invoked from some other language, + * especially managed ones such as Java or Go, through a foreign function interface such as jni or cgo, + * a major performance rule is to reduce crossing such interface to an absolute minimum. + * It's not rare that performance ends being spent more into the interface, rather than compression itself. + * In which cases, prefer using large buffers, as large as practical, + * for both input and output, to reduce the nb of roundtrips. + */ +ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block. */ + + +/* ***************************************************************************** + * This following is a legacy streaming API, available since v1.0+ . + * It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2(). + * It is redundant, but remains fully supported. + ******************************************************************************/ + +/*! + * Equivalent to: + * + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * + * Note that ZSTD_initCStream() clears any previously set dictionary. Use the new API + * to compress with a dictionary. + */ +ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); +/*! + * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue). + * NOTE: The return value is different. ZSTD_compressStream() returns a hint for + * the next read size (if non-zero and not an error). ZSTD_compressStream2() + * returns the minimum nb of bytes left to flush (if non-zero and not an error). + */ +ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); +/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */ +ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); +/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */ +ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); + + +/*-*************************************************************************** +* Streaming decompression - HowTo +* +* A ZSTD_DStream object is required to track streaming operations. +* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. +* ZSTD_DStream objects can be re-employed multiple times. +* +* Use ZSTD_initDStream() to start a new decompression operation. +* @return : recommended first input size +* Alternatively, use advanced API to set specific properties. +* +* Use ZSTD_decompressStream() repetitively to consume your input. +* The function will update both `pos` fields. +* If `input.pos < input.size`, some input has not been consumed. +* It's up to the caller to present again remaining data. +* +* The function tries to flush all data decoded immediately, respecting output buffer size. +* If `output.pos < output.size`, decoder has flushed everything it could. +* +* However, when `output.pos == output.size`, it's more difficult to know. +* If @return > 0, the frame is not complete, meaning +* either there is still some data left to flush within internal buffers, +* or there is more input to read to complete the frame (or both). +* In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer. +* Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX. +* @return : 0 when a frame is completely decoded and fully flushed, +* or an error code, which can be tested using ZSTD_isError(), +* or any other value > 0, which means there is still some decoding or flushing to do to complete current frame : +* the return value is a suggested next input size (just a hint for better latency) +* that will never request more than the remaining content of the compressed frame. +* *******************************************************************************/ + +typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */ + /* For compatibility with versions <= v1.2.0, prefer differentiating them. */ +/*===== ZSTD_DStream management functions =====*/ +ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void); +ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); /* accept NULL pointer */ + +/*===== Streaming decompression functions =====*/ + +/*! ZSTD_initDStream() : + * Initialize/reset DStream state for new decompression operation. + * Call before new decompression operation using same DStream. + * + * Note : This function is redundant with the advanced API and equivalent to: + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_refDDict(zds, NULL); + */ +ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); + +/*! ZSTD_decompressStream() : + * Streaming decompression function. + * Call repetitively to consume full input updating it as necessary. + * Function will update both input and output `pos` fields exposing current state via these fields: + * - `input.pos < input.size`, some input remaining and caller should provide remaining input + * on the next call. + * - `output.pos < output.size`, decoder flushed internal output buffer. + * - `output.pos == output.size`, unflushed data potentially present in the internal buffers, + * check ZSTD_decompressStream() @return value, + * if > 0, invoke it again to flush remaining data to output. + * Note : with no additional input, amount of data flushed <= ZSTD_BLOCKSIZE_MAX. + * + * @return : 0 when a frame is completely decoded and fully flushed, + * or an error code, which can be tested using ZSTD_isError(), + * or any other value > 0, which means there is some decoding or flushing to do to complete current frame. + * + * Note: when an operation returns with an error code, the @zds state may be left in undefined state. + * It's UB to invoke `ZSTD_decompressStream()` on such a state. + * In order to re-use such a state, it must be first reset, + * which can be done explicitly (`ZSTD_DCtx_reset()`), + * or is implied for operations starting some new decompression job (`ZSTD_initDStream`, `ZSTD_decompressDCtx()`, `ZSTD_decompress_usingDict()`) + */ +ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); + +ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */ + + +/************************** +* Simple dictionary API +***************************/ +/*! ZSTD_compress_usingDict() : + * Compression at an explicit compression level using a Dictionary. + * A dictionary can be any arbitrary data segment (also called a prefix), + * or a buffer with specified information (see zdict.h). + * Note : This function loads the dictionary, resulting in significant startup delay. + * It's intended for a dictionary used only once. + * Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */ +ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + int compressionLevel); + +/*! ZSTD_decompress_usingDict() : + * Decompression using a known Dictionary. + * Dictionary must be identical to the one used during compression. + * Note : This function loads the dictionary, resulting in significant startup delay. + * It's intended for a dictionary used only once. + * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */ +ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize); + + +/*********************************** + * Bulk processing dictionary API + **********************************/ +typedef struct ZSTD_CDict_s ZSTD_CDict; + +/*! ZSTD_createCDict() : + * When compressing multiple messages or blocks using the same dictionary, + * it's recommended to digest the dictionary only once, since it's a costly operation. + * ZSTD_createCDict() will create a state from digesting a dictionary. + * The resulting state can be used for future compression operations with very limited startup cost. + * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. + * @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict. + * Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content. + * Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer, + * in which case the only thing that it transports is the @compressionLevel. + * This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively, + * expecting a ZSTD_CDict parameter with any data, including those without a known dictionary. */ +ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, + int compressionLevel); + +/*! ZSTD_freeCDict() : + * Function frees memory allocated by ZSTD_createCDict(). + * If a NULL pointer is passed, no operation is performed. */ +ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); + +/*! ZSTD_compress_usingCDict() : + * Compression using a digested Dictionary. + * Recommended when same dictionary is used multiple times. + * Note : compression level is _decided at dictionary creation time_, + * and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */ +ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict); + + +typedef struct ZSTD_DDict_s ZSTD_DDict; + +/*! ZSTD_createDDict() : + * Create a digested dictionary, ready to start decompression operation without startup delay. + * dictBuffer can be released after DDict creation, as its content is copied inside DDict. */ +ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize); + +/*! ZSTD_freeDDict() : + * Function frees memory allocated with ZSTD_createDDict() + * If a NULL pointer is passed, no operation is performed. */ +ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict); + +/*! ZSTD_decompress_usingDDict() : + * Decompression using a digested Dictionary. + * Recommended when same dictionary is used multiple times. */ +ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_DDict* ddict); + + +/******************************** + * Dictionary helper functions + *******************************/ + +/*! ZSTD_getDictID_fromDict() : Requires v1.4.0+ + * Provides the dictID stored within dictionary. + * if @return == 0, the dictionary is not conformant with Zstandard specification. + * It can still be loaded, but as a content-only dictionary. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize); + +/*! ZSTD_getDictID_fromCDict() : Requires v1.5.0+ + * Provides the dictID of the dictionary loaded into `cdict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict); + +/*! ZSTD_getDictID_fromDDict() : Requires v1.4.0+ + * Provides the dictID of the dictionary loaded into `ddict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict); + +/*! ZSTD_getDictID_fromFrame() : Requires v1.4.0+ + * Provides the dictID required to decompressed the frame stored within `src`. + * If @return == 0, the dictID could not be decoded. + * This could for one of the following reasons : + * - The frame does not require a dictionary to be decoded (most common case). + * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden piece of information. + * Note : this use case also happens when using a non-conformant dictionary. + * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). + * - This is not a Zstandard frame. + * When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); + + +/******************************************************************************* + * Advanced dictionary and prefix API (Requires v1.4.0+) + * + * This API allows dictionaries to be used with ZSTD_compress2(), + * ZSTD_compressStream2(), and ZSTD_decompressDCtx(). + * Dictionaries are sticky, they remain valid when same context is reused, + * they only reset when the context is reset + * with ZSTD_reset_parameters or ZSTD_reset_session_and_parameters. + * In contrast, Prefixes are single-use. + ******************************************************************************/ + + +/*! ZSTD_CCtx_loadDictionary() : Requires v1.4.0+ + * Create an internal CDict from `dict` buffer. + * Decompression will have to use same dictionary. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary, + * meaning "return to no-dictionary mode". + * Note 1 : Dictionary is sticky, it will be used for all future compressed frames, + * until parameters are reset, a new dictionary is loaded, or the dictionary + * is explicitly invalidated by loading a NULL dictionary. + * Note 2 : Loading a dictionary involves building tables. + * It's also a CPU consuming operation, with non-negligible impact on latency. + * Tables are dependent on compression parameters, and for this reason, + * compression parameters can no longer be changed after loading a dictionary. + * Note 3 :`dict` content will be copied internally. + * Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead. + * In such a case, dictionary buffer must outlive its users. + * Note 4 : Use ZSTD_CCtx_loadDictionary_advanced() + * to precisely select how dictionary content must be interpreted. + * Note 5 : This method does not benefit from LDM (long distance mode). + * If you want to employ LDM on some large dictionary content, + * prefer employing ZSTD_CCtx_refPrefix() described below. + */ +ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); + +/*! ZSTD_CCtx_refCDict() : Requires v1.4.0+ + * Reference a prepared dictionary, to be used for all future compressed frames. + * Note that compression parameters are enforced from within CDict, + * and supersede any compression parameter previously set within CCtx. + * The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs. + * The ignored parameters will be used again if the CCtx is returned to no-dictionary mode. + * The dictionary will remain valid for future compressed frames using same CCtx. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special : Referencing a NULL CDict means "return to no-dictionary mode". + * Note 1 : Currently, only one dictionary can be managed. + * Referencing a new dictionary effectively "discards" any previous one. + * Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */ +ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); + +/*! ZSTD_CCtx_refPrefix() : Requires v1.4.0+ + * Reference a prefix (single-usage dictionary) for next compressed frame. + * A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end). + * Decompression will need same prefix to properly regenerate data. + * Compressing with a prefix is similar in outcome as performing a diff and compressing it, + * but performs much faster, especially during decompression (compression speed is tunable with compression level). + * This method is compatible with LDM (long distance mode). + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary + * Note 1 : Prefix buffer is referenced. It **must** outlive compression. + * Its content must remain unmodified during compression. + * Note 2 : If the intention is to diff some large src data blob with some prior version of itself, + * ensure that the window size is large enough to contain the entire source. + * See ZSTD_c_windowLog. + * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters. + * It's a CPU consuming operation, with non-negligible impact on latency. + * If there is a need to use the same prefix multiple times, consider loadDictionary instead. + * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent). + * Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */ +ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, + const void* prefix, size_t prefixSize); + +/*! ZSTD_DCtx_loadDictionary() : Requires v1.4.0+ + * Create an internal DDict from dict buffer, to be used to decompress all future frames. + * The dictionary remains valid for all future frames, until explicitly invalidated, or + * a new dictionary is loaded. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary, + * meaning "return to no-dictionary mode". + * Note 1 : Loading a dictionary involves building tables, + * which has a non-negligible impact on CPU usage and latency. + * It's recommended to "load once, use many times", to amortize the cost + * Note 2 :`dict` content will be copied internally, so `dict` can be released after loading. + * Use ZSTD_DCtx_loadDictionary_byReference() to reference dictionary content instead. + * Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to take control of + * how dictionary content is loaded and interpreted. + */ +ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); + +/*! ZSTD_DCtx_refDDict() : Requires v1.4.0+ + * Reference a prepared dictionary, to be used to decompress next frames. + * The dictionary remains active for decompression of future frames using same DCtx. + * + * If called with ZSTD_d_refMultipleDDicts enabled, repeated calls of this function + * will store the DDict references in a table, and the DDict used for decompression + * will be determined at decompression time, as per the dict ID in the frame. + * The memory for the table is allocated on the first call to refDDict, and can be + * freed with ZSTD_freeDCtx(). + * + * If called with ZSTD_d_refMultipleDDicts disabled (the default), only one dictionary + * will be managed, and referencing a dictionary effectively "discards" any previous one. + * + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: referencing a NULL DDict means "return to no-dictionary mode". + * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx. + */ +ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + +/*! ZSTD_DCtx_refPrefix() : Requires v1.4.0+ + * Reference a prefix (single-usage dictionary) to decompress next frame. + * This is the reverse operation of ZSTD_CCtx_refPrefix(), + * and must use the same prefix as the one used during compression. + * Prefix is **only used once**. Reference is discarded at end of frame. + * End of frame is reached when ZSTD_decompressStream() returns 0. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary + * Note 2 : Prefix buffer is referenced. It **must** outlive decompression. + * Prefix buffer must remain unmodified up to the end of frame, + * reached when ZSTD_decompressStream() returns 0. + * Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent). + * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section) + * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost. + * A full dictionary is more costly, as it requires building tables. + */ +ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, + const void* prefix, size_t prefixSize); + +/* === Memory management === */ + +/*! ZSTD_sizeof_*() : Requires v1.4.0+ + * These functions give the _current_ memory usage of selected object. + * Note that object memory usage can evolve (increase or decrease) over time. */ +ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); +ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); +ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); +ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict); +ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_H_235446 */ + + +/* ************************************************************************************** + * ADVANCED AND EXPERIMENTAL FUNCTIONS + **************************************************************************************** + * The definitions in the following section are considered experimental. + * They are provided for advanced scenarios. + * They should never be used with a dynamic library, as prototypes may change in the future. + * Use them only in association with static linking. + * ***************************************************************************************/ + +#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY) +#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY + +#if defined (__cplusplus) +extern "C" { +#endif + +/* This can be overridden externally to hide static symbols. */ +#ifndef ZSTDLIB_STATIC_API +# if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDLIB_STATIC_API __declspec(dllexport) ZSTDLIB_VISIBLE +# elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDLIB_STATIC_API __declspec(dllimport) ZSTDLIB_VISIBLE +# else +# define ZSTDLIB_STATIC_API ZSTDLIB_VISIBLE +# endif +#endif + +/**************************************************************************************** + * experimental API (static linking only) + **************************************************************************************** + * The following symbols and constants + * are not planned to join "stable API" status in the near future. + * They can still change in future versions. + * Some of them are planned to remain in the static_only section indefinitely. + * Some of them might be removed in the future (especially when redundant with existing stable functions) + * ***************************************************************************************/ + +#define ZSTD_FRAMEHEADERSIZE_PREFIX(format) ((format) == ZSTD_f_zstd1 ? 5 : 1) /* minimum input size required to query frame header size */ +#define ZSTD_FRAMEHEADERSIZE_MIN(format) ((format) == ZSTD_f_zstd1 ? 6 : 2) +#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */ +#define ZSTD_SKIPPABLEHEADERSIZE 8 + +/* compression parameter bounds */ +#define ZSTD_WINDOWLOG_MAX_32 30 +#define ZSTD_WINDOWLOG_MAX_64 31 +#define ZSTD_WINDOWLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64)) +#define ZSTD_WINDOWLOG_MIN 10 +#define ZSTD_HASHLOG_MAX ((ZSTD_WINDOWLOG_MAX < 30) ? ZSTD_WINDOWLOG_MAX : 30) +#define ZSTD_HASHLOG_MIN 6 +#define ZSTD_CHAINLOG_MAX_32 29 +#define ZSTD_CHAINLOG_MAX_64 30 +#define ZSTD_CHAINLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_CHAINLOG_MAX_32 : ZSTD_CHAINLOG_MAX_64)) +#define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN +#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1) +#define ZSTD_SEARCHLOG_MIN 1 +#define ZSTD_MINMATCH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */ +#define ZSTD_MINMATCH_MIN 3 /* only for ZSTD_btopt+, faster strategies are limited to 4 */ +#define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX +#define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */ +#define ZSTD_STRATEGY_MIN ZSTD_fast +#define ZSTD_STRATEGY_MAX ZSTD_btultra2 +#define ZSTD_BLOCKSIZE_MAX_MIN (1 << 10) /* The minimum valid max blocksize. Maximum blocksizes smaller than this make compressBound() inaccurate. */ + + +#define ZSTD_OVERLAPLOG_MIN 0 +#define ZSTD_OVERLAPLOG_MAX 9 + +#define ZSTD_WINDOWLOG_LIMIT_DEFAULT 27 /* by default, the streaming decoder will refuse any frame + * requiring larger than (1< 0: + * If litLength != 0: + * rep == 1 --> offset == repeat_offset_1 + * rep == 2 --> offset == repeat_offset_2 + * rep == 3 --> offset == repeat_offset_3 + * If litLength == 0: + * rep == 1 --> offset == repeat_offset_2 + * rep == 2 --> offset == repeat_offset_3 + * rep == 3 --> offset == repeat_offset_1 - 1 + * + * Note: This field is optional. ZSTD_generateSequences() will calculate the value of + * 'rep', but repeat offsets do not necessarily need to be calculated from an external + * sequence provider perspective. For example, ZSTD_compressSequences() does not + * use this 'rep' field at all (as of now). + */ +} ZSTD_Sequence; + +typedef struct { + unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */ + unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */ + unsigned hashLog; /**< dispatch table : larger == faster, more memory */ + unsigned searchLog; /**< nb of searches : larger == more compression, slower */ + unsigned minMatch; /**< match length searched : larger == faster decompression, sometimes less compression */ + unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */ + ZSTD_strategy strategy; /**< see ZSTD_strategy definition above */ +} ZSTD_compressionParameters; + +typedef struct { + int contentSizeFlag; /**< 1: content size will be in frame header (when known) */ + int checksumFlag; /**< 1: generate a 32-bits checksum using XXH64 algorithm at end of frame, for error detection */ + int noDictIDFlag; /**< 1: no dictID will be saved into frame header (dictID is only useful for dictionary compression) */ +} ZSTD_frameParameters; + +typedef struct { + ZSTD_compressionParameters cParams; + ZSTD_frameParameters fParams; +} ZSTD_parameters; + +typedef enum { + ZSTD_dct_auto = 0, /* dictionary is "full" when starting with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */ + ZSTD_dct_rawContent = 1, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */ + ZSTD_dct_fullDict = 2 /* refuses to load a dictionary if it does not respect Zstandard's specification, starting with ZSTD_MAGIC_DICTIONARY */ +} ZSTD_dictContentType_e; + +typedef enum { + ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */ + ZSTD_dlm_byRef = 1 /**< Reference dictionary content -- the dictionary buffer must outlive its users. */ +} ZSTD_dictLoadMethod_e; + +typedef enum { + ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */ + ZSTD_f_zstd1_magicless = 1 /* Variant of zstd frame format, without initial 4-bytes magic number. + * Useful to save 4 bytes per generated frame. + * Decoder cannot recognise automatically this format, requiring this instruction. */ +} ZSTD_format_e; + +typedef enum { + /* Note: this enum controls ZSTD_d_forceIgnoreChecksum */ + ZSTD_d_validateChecksum = 0, + ZSTD_d_ignoreChecksum = 1 +} ZSTD_forceIgnoreChecksum_e; + +typedef enum { + /* Note: this enum controls ZSTD_d_refMultipleDDicts */ + ZSTD_rmd_refSingleDDict = 0, + ZSTD_rmd_refMultipleDDicts = 1 +} ZSTD_refMultipleDDicts_e; + +typedef enum { + /* Note: this enum and the behavior it controls are effectively internal + * implementation details of the compressor. They are expected to continue + * to evolve and should be considered only in the context of extremely + * advanced performance tuning. + * + * Zstd currently supports the use of a CDict in three ways: + * + * - The contents of the CDict can be copied into the working context. This + * means that the compression can search both the dictionary and input + * while operating on a single set of internal tables. This makes + * the compression faster per-byte of input. However, the initial copy of + * the CDict's tables incurs a fixed cost at the beginning of the + * compression. For small compressions (< 8 KB), that copy can dominate + * the cost of the compression. + * + * - The CDict's tables can be used in-place. In this model, compression is + * slower per input byte, because the compressor has to search two sets of + * tables. However, this model incurs no start-up cost (as long as the + * working context's tables can be reused). For small inputs, this can be + * faster than copying the CDict's tables. + * + * - The CDict's tables are not used at all, and instead we use the working + * context alone to reload the dictionary and use params based on the source + * size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict(). + * This method is effective when the dictionary sizes are very small relative + * to the input size, and the input size is fairly large to begin with. + * + * Zstd has a simple internal heuristic that selects which strategy to use + * at the beginning of a compression. However, if experimentation shows that + * Zstd is making poor choices, it is possible to override that choice with + * this enum. + */ + ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */ + ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */ + ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */ + ZSTD_dictForceLoad = 3 /* Always reload the dictionary */ +} ZSTD_dictAttachPref_e; + +typedef enum { + ZSTD_lcm_auto = 0, /**< Automatically determine the compression mode based on the compression level. + * Negative compression levels will be uncompressed, and positive compression + * levels will be compressed. */ + ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be + * emitted if Huffman compression is not profitable. */ + ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */ +} ZSTD_literalCompressionMode_e; + +typedef enum { + /* Note: This enum controls features which are conditionally beneficial. + * Zstd can take a decision on whether or not to enable the feature (ZSTD_ps_auto), + * but setting the switch to ZSTD_ps_enable or ZSTD_ps_disable force enable/disable the feature. + */ + ZSTD_ps_auto = 0, /* Let the library automatically determine whether the feature shall be enabled */ + ZSTD_ps_enable = 1, /* Force-enable the feature */ + ZSTD_ps_disable = 2 /* Do not use the feature */ +} ZSTD_ParamSwitch_e; +#define ZSTD_paramSwitch_e ZSTD_ParamSwitch_e /* old name */ + +/*************************************** +* Frame header and size functions +***************************************/ + +/*! ZSTD_findDecompressedSize() : + * `src` should point to the start of a series of ZSTD encoded and/or skippable frames + * `srcSize` must be the _exact_ size of this series + * (i.e. there should be a frame boundary at `src + srcSize`) + * @return : - decompressed size of all data in all successive frames + * - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN + * - if an error occurred: ZSTD_CONTENTSIZE_ERROR + * + * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. + * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. + * In which case, it's necessary to use streaming mode to decompress data. + * note 2 : decompressed size is always present when compression is done with ZSTD_compress() + * note 3 : decompressed size can be very large (64-bits value), + * potentially larger than what local system can handle as a single memory segment. + * In which case, it's necessary to use streaming mode to decompress data. + * note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified. + * Always ensure result fits within application's authorized limits. + * Each application can set its own limits. + * note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to + * read each contained frame header. This is fast as most of the data is skipped, + * however it does mean that all frame data must be present and valid. */ +ZSTDLIB_STATIC_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); + +/*! ZSTD_decompressBound() : + * `src` should point to the start of a series of ZSTD encoded and/or skippable frames + * `srcSize` must be the _exact_ size of this series + * (i.e. there should be a frame boundary at `src + srcSize`) + * @return : - upper-bound for the decompressed size of all data in all successive frames + * - if an error occurred: ZSTD_CONTENTSIZE_ERROR + * + * note 1 : an error can occur if `src` contains an invalid or incorrectly formatted frame. + * note 2 : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`. + * in this case, `ZSTD_findDecompressedSize` and `ZSTD_decompressBound` return the same value. + * note 3 : when the decompressed size field isn't available, the upper-bound for that frame is calculated by: + * upper-bound = # blocks * min(128 KB, Window_Size) + */ +ZSTDLIB_STATIC_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize); + +/*! ZSTD_frameHeaderSize() : + * srcSize must be large enough, aka >= ZSTD_FRAMEHEADERSIZE_PREFIX. + * @return : size of the Frame Header, + * or an error code (if srcSize is too small) */ +ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); + +typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_FrameType_e; +#define ZSTD_frameType_e ZSTD_FrameType_e /* old name */ +typedef struct { + unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */ + unsigned long long windowSize; /* can be very large, up to <= frameContentSize */ + unsigned blockSizeMax; + ZSTD_FrameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */ + unsigned headerSize; + unsigned dictID; /* for ZSTD_skippableFrame, contains the skippable magic variant [0-15] */ + unsigned checksumFlag; + unsigned _reserved1; + unsigned _reserved2; +} ZSTD_FrameHeader; +#define ZSTD_frameHeader ZSTD_FrameHeader /* old name */ + +/*! ZSTD_getFrameHeader() : + * decode Frame Header into `zfhPtr`, or requires larger `srcSize`. + * @return : 0 => header is complete, `zfhPtr` is correctly filled, + * >0 => `srcSize` is too small, @return value is the wanted `srcSize` amount, `zfhPtr` is not filled, + * or an error code, which can be tested using ZSTD_isError() */ +ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_FrameHeader* zfhPtr, const void* src, size_t srcSize); +/*! ZSTD_getFrameHeader_advanced() : + * same as ZSTD_getFrameHeader(), + * with added capability to select a format (like ZSTD_f_zstd1_magicless) */ +ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_FrameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format); + +/*! ZSTD_decompressionMargin() : + * Zstd supports in-place decompression, where the input and output buffers overlap. + * In this case, the output buffer must be at least (Margin + Output_Size) bytes large, + * and the input buffer must be at the end of the output buffer. + * + * _______________________ Output Buffer ________________________ + * | | + * | ____ Input Buffer ____| + * | | | + * v v v + * |---------------------------------------|-----------|----------| + * ^ ^ ^ + * |___________________ Output_Size ___________________|_ Margin _| + * + * NOTE: See also ZSTD_DECOMPRESSION_MARGIN(). + * NOTE: This applies only to single-pass decompression through ZSTD_decompress() or + * ZSTD_decompressDCtx(). + * NOTE: This function supports multi-frame input. + * + * @param src The compressed frame(s) + * @param srcSize The size of the compressed frame(s) + * @returns The decompression margin or an error that can be checked with ZSTD_isError(). + */ +ZSTDLIB_STATIC_API size_t ZSTD_decompressionMargin(const void* src, size_t srcSize); + +/*! ZSTD_DECOMPRESS_MARGIN() : + * Similar to ZSTD_decompressionMargin(), but instead of computing the margin from + * the compressed frame, compute it from the original size and the blockSizeLog. + * See ZSTD_decompressionMargin() for details. + * + * WARNING: This macro does not support multi-frame input, the input must be a single + * zstd frame. If you need that support use the function, or implement it yourself. + * + * @param originalSize The original uncompressed size of the data. + * @param blockSize The block size == MIN(windowSize, ZSTD_BLOCKSIZE_MAX). + * Unless you explicitly set the windowLog smaller than + * ZSTD_BLOCKSIZELOG_MAX you can just use ZSTD_BLOCKSIZE_MAX. + */ +#define ZSTD_DECOMPRESSION_MARGIN(originalSize, blockSize) ((size_t)( \ + ZSTD_FRAMEHEADERSIZE_MAX /* Frame header */ + \ + 4 /* checksum */ + \ + ((originalSize) == 0 ? 0 : 3 * (((originalSize) + (blockSize) - 1) / blockSize)) /* 3 bytes per block */ + \ + (blockSize) /* One block of margin */ \ + )) + +typedef enum { + ZSTD_sf_noBlockDelimiters = 0, /* ZSTD_Sequence[] has no block delimiters, just sequences */ + ZSTD_sf_explicitBlockDelimiters = 1 /* ZSTD_Sequence[] contains explicit block delimiters */ +} ZSTD_SequenceFormat_e; +#define ZSTD_sequenceFormat_e ZSTD_SequenceFormat_e /* old name */ + +/*! ZSTD_sequenceBound() : + * `srcSize` : size of the input buffer + * @return : upper-bound for the number of sequences that can be generated + * from a buffer of srcSize bytes + * + * note : returns number of sequences - to get bytes, multiply by sizeof(ZSTD_Sequence). + */ +ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize); + +/*! ZSTD_generateSequences() : + * WARNING: This function is meant for debugging and informational purposes ONLY! + * Its implementation is flawed, and it will be deleted in a future version. + * It is not guaranteed to succeed, as there are several cases where it will give + * up and fail. You should NOT use this function in production code. + * + * This function is deprecated, and will be removed in a future version. + * + * Generate sequences using ZSTD_compress2(), given a source buffer. + * + * @param zc The compression context to be used for ZSTD_compress2(). Set any + * compression parameters you need on this context. + * @param outSeqs The output sequences buffer of size @p outSeqsSize + * @param outSeqsCapacity The size of the output sequences buffer. + * ZSTD_sequenceBound(srcSize) is an upper bound on the number + * of sequences that can be generated. + * @param src The source buffer to generate sequences from of size @p srcSize. + * @param srcSize The size of the source buffer. + * + * Each block will end with a dummy sequence + * with offset == 0, matchLength == 0, and litLength == length of last literals. + * litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0) + * simply acts as a block delimiter. + * + * @returns The number of sequences generated, necessarily less than + * ZSTD_sequenceBound(srcSize), or an error code that can be checked + * with ZSTD_isError(). + */ +ZSTD_DEPRECATED("For debugging only, will be replaced by ZSTD_extractSequences()") +ZSTDLIB_STATIC_API size_t +ZSTD_generateSequences(ZSTD_CCtx* zc, + ZSTD_Sequence* outSeqs, size_t outSeqsCapacity, + const void* src, size_t srcSize); + +/*! ZSTD_mergeBlockDelimiters() : + * Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals + * by merging them into the literals of the next sequence. + * + * As such, the final generated result has no explicit representation of block boundaries, + * and the final last literals segment is not represented in the sequences. + * + * The output of this function can be fed into ZSTD_compressSequences() with CCtx + * setting of ZSTD_c_blockDelimiters as ZSTD_sf_noBlockDelimiters + * @return : number of sequences left after merging + */ +ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize); + +/*! ZSTD_compressSequences() : + * Compress an array of ZSTD_Sequence, associated with @src buffer, into dst. + * @src contains the entire input (not just the literals). + * If @srcSize > sum(sequence.length), the remaining bytes are considered all literals + * If a dictionary is included, then the cctx should reference the dict (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.). + * The entire source is compressed into a single frame. + * + * The compression behavior changes based on cctx params. In particular: + * If ZSTD_c_blockDelimiters == ZSTD_sf_noBlockDelimiters, the array of ZSTD_Sequence is expected to contain + * no block delimiters (defined in ZSTD_Sequence). Block boundaries are roughly determined based on + * the block size derived from the cctx, and sequences may be split. This is the default setting. + * + * If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain + * valid block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided. + * + * When ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, it's possible to decide generating repcodes + * using the advanced parameter ZSTD_c_repcodeResolution. Repcodes will improve compression ratio, though the benefit + * can vary greatly depending on Sequences. On the other hand, repcode resolution is an expensive operation. + * By default, it's disabled at low (<10) compression levels, and enabled above the threshold (>=10). + * ZSTD_c_repcodeResolution makes it possible to directly manage this processing in either direction. + * + * If ZSTD_c_validateSequences == 0, this function blindly accepts the Sequences provided. Invalid Sequences cause undefined + * behavior. If ZSTD_c_validateSequences == 1, then the function will detect invalid Sequences (see doc/zstd_compression_format.md for + * specifics regarding offset/matchlength requirements) and then bail out and return an error. + * + * In addition to the two adjustable experimental params, there are other important cctx params. + * - ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN. + * - ZSTD_c_compressionLevel accordingly adjusts the strength of the entropy coder, as it would in typical compression. + * - ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset + * is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md + * + * Note: Repcodes are, as of now, always re-calculated within this function, ZSTD_Sequence.rep is effectively unused. + * Dev Note: Once ability to ingest repcodes become available, the explicit block delims mode must respect those repcodes exactly, + * and cannot emit an RLE block that disagrees with the repcode history. + * @return : final compressed size, or a ZSTD error code. + */ +ZSTDLIB_STATIC_API size_t +ZSTD_compressSequences(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + const void* src, size_t srcSize); + + +/*! ZSTD_compressSequencesAndLiterals() : + * This is a variant of ZSTD_compressSequences() which, + * instead of receiving (src,srcSize) as input parameter, receives (literals,litSize), + * aka all the literals, already extracted and laid out into a single continuous buffer. + * This can be useful if the process generating the sequences also happens to generate the buffer of literals, + * thus skipping an extraction + caching stage. + * It's a speed optimization, useful when the right conditions are met, + * but it also features the following limitations: + * - Only supports explicit delimiter mode + * - Currently does not support Sequences validation (so input Sequences are trusted) + * - Not compatible with frame checksum, which must be disabled + * - If any block is incompressible, will fail and return an error + * - @litSize must be == sum of all @.litLength fields in @inSeqs. Any discrepancy will generate an error. + * - @litBufCapacity is the size of the underlying buffer into which literals are written, starting at address @literals. + * @litBufCapacity must be at least 8 bytes larger than @litSize. + * - @decompressedSize must be correct, and correspond to the sum of all Sequences. Any discrepancy will generate an error. + * @return : final compressed size, or a ZSTD error code. + */ +ZSTDLIB_STATIC_API size_t +ZSTD_compressSequencesAndLiterals(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const ZSTD_Sequence* inSeqs, size_t nbSequences, + const void* literals, size_t litSize, size_t litBufCapacity, + size_t decompressedSize); + + +/*! ZSTD_writeSkippableFrame() : + * Generates a zstd skippable frame containing data given by src, and writes it to dst buffer. + * + * Skippable frames begin with a 4-byte magic number. There are 16 possible choices of magic number, + * ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15. + * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, + * so the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant. + * + * Returns an error if destination buffer is not large enough, if the source size is not representable + * with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid). + * + * @return : number of bytes written or a ZSTD error. + */ +ZSTDLIB_STATIC_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + unsigned magicVariant); + +/*! ZSTD_readSkippableFrame() : + * Retrieves the content of a zstd skippable frame starting at @src, and writes it to @dst buffer. + * + * The parameter @magicVariant will receive the magicVariant that was supplied when the frame was written, + * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. + * This can be NULL if the caller is not interested in the magicVariant. + * + * Returns an error if destination buffer is not large enough, or if the frame is not skippable. + * + * @return : number of bytes written or a ZSTD error. + */ +ZSTDLIB_STATIC_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, + unsigned* magicVariant, + const void* src, size_t srcSize); + +/*! ZSTD_isSkippableFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame. + */ +ZSTDLIB_STATIC_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size); + + + +/*************************************** +* Memory management +***************************************/ + +/*! ZSTD_estimate*() : + * These functions make it possible to estimate memory usage + * of a future {D,C}Ctx, before its creation. + * This is useful in combination with ZSTD_initStatic(), + * which makes it possible to employ a static buffer for ZSTD_CCtx* state. + * + * ZSTD_estimateCCtxSize() will provide a memory budget large enough + * to compress data of any size using one-shot compression ZSTD_compressCCtx() or ZSTD_compress2() + * associated with any compression level up to max specified one. + * The estimate will assume the input may be arbitrarily large, + * which is the worst case. + * + * Note that the size estimation is specific for one-shot compression, + * it is not valid for streaming (see ZSTD_estimateCStreamSize*()) + * nor other potential ways of using a ZSTD_CCtx* state. + * + * When srcSize can be bound by a known and rather "small" value, + * this knowledge can be used to provide a tighter budget estimation + * because the ZSTD_CCtx* state will need less memory for small inputs. + * This tighter estimation can be provided by employing more advanced functions + * ZSTD_estimateCCtxSize_usingCParams(), which can be used in tandem with ZSTD_getCParams(), + * and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter(). + * Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits. + * + * Note : only single-threaded compression is supported. + * ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1. + */ +ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int maxCompressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void); + +/*! ZSTD_estimateCStreamSize() : + * ZSTD_estimateCStreamSize() will provide a memory budget large enough for streaming compression + * using any compression level up to the max specified one. + * It will also consider src size to be arbitrarily "large", which is a worst case scenario. + * If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation. + * ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. + * ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1. + * Note : CStream size estimation is only correct for single-threaded compression. + * ZSTD_estimateCStreamSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1. + * Note 2 : ZSTD_estimateCStreamSize* functions are not compatible with the Block-Level Sequence Producer API at this time. + * Size estimates assume that no external sequence producer is registered. + * + * ZSTD_DStream memory budget depends on frame's window Size. + * This information can be passed manually, using ZSTD_estimateDStreamSize, + * or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame(); + * Any frame requesting a window size larger than max specified one will be rejected. + * Note : if streaming is init with function ZSTD_init?Stream_usingDict(), + * an internal ?Dict will be created, which additional size is not estimated here. + * In this case, get total size by adding ZSTD_estimate?DictSize + */ +ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int maxCompressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize(size_t maxWindowSize); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize); + +/*! ZSTD_estimate?DictSize() : + * ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict(). + * ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, like ZSTD_createCDict_advanced(). + * Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller. + */ +ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod); + +/*! ZSTD_initStatic*() : + * Initialize an object using a pre-allocated fixed-size buffer. + * workspace: The memory area to emplace the object into. + * Provided pointer *must be 8-bytes aligned*. + * Buffer must outlive object. + * workspaceSize: Use ZSTD_estimate*Size() to determine + * how large workspace must be to support target scenario. + * @return : pointer to object (same address as workspace, just different type), + * or NULL if error (size too small, incorrect alignment, etc.) + * Note : zstd will never resize nor malloc() when using a static buffer. + * If the object requires more memory than available, + * zstd will just error out (typically ZSTD_error_memory_allocation). + * Note 2 : there is no corresponding "free" function. + * Since workspace is allocated externally, it must be freed externally too. + * Note 3 : cParams : use ZSTD_getCParams() to convert a compression level + * into its associated cParams. + * Limitation 1 : currently not compatible with internal dictionary creation, triggered by + * ZSTD_CCtx_loadDictionary(), ZSTD_initCStream_usingDict() or ZSTD_initDStream_usingDict(). + * Limitation 2 : static cctx currently not compatible with multi-threading. + * Limitation 3 : static dctx is incompatible with legacy support. + */ +ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize); +ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */ + +ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize); +ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */ + +ZSTDLIB_STATIC_API const ZSTD_CDict* ZSTD_initStaticCDict( + void* workspace, size_t workspaceSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_compressionParameters cParams); + +ZSTDLIB_STATIC_API const ZSTD_DDict* ZSTD_initStaticDDict( + void* workspace, size_t workspaceSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType); + + +/*! Custom memory allocation : + * These prototypes make it possible to pass your own allocation/free functions. + * ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below. + * All allocation/free operations will be completed using these custom variants instead of regular ones. + */ +typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size); +typedef void (*ZSTD_freeFunction) (void* opaque, void* address); +typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; +static +#ifdef __GNUC__ +__attribute__((__unused__)) +#endif + +#if defined(__clang__) && __clang_major__ >= 5 +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant" +#endif +ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */ +#if defined(__clang__) && __clang_major__ >= 5 +#pragma clang diagnostic pop +#endif + +ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); + +ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_compressionParameters cParams, + ZSTD_customMem customMem); + +/*! Thread pool : + * These prototypes make it possible to share a thread pool among multiple compression contexts. + * This can limit resources for applications with multiple threads where each one uses + * a threaded compression mode (via ZSTD_c_nbWorkers parameter). + * ZSTD_createThreadPool creates a new thread pool with a given number of threads. + * Note that the lifetime of such pool must exist while being used. + * ZSTD_CCtx_refThreadPool assigns a thread pool to a context (use NULL argument value + * to use an internal thread pool). + * ZSTD_freeThreadPool frees a thread pool, accepts NULL pointer. + */ +typedef struct POOL_ctx_s ZSTD_threadPool; +ZSTDLIB_STATIC_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads); +ZSTDLIB_STATIC_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool); /* accept NULL pointer */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool); + + +/* + * This API is temporary and is expected to change or disappear in the future! + */ +ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced2( + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + const ZSTD_CCtx_params* cctxParams, + ZSTD_customMem customMem); + +ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_advanced( + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_customMem customMem); + + +/*************************************** +* Advanced compression functions +***************************************/ + +/*! ZSTD_createCDict_byReference() : + * Create a digested dictionary for compression + * Dictionary content is just referenced, not duplicated. + * As a consequence, `dictBuffer` **must** outlive CDict, + * and its content must remain unmodified throughout the lifetime of CDict. + * note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */ +ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); + +/*! ZSTD_getCParams() : + * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize. + * `estimatedSrcSize` value is optional, select 0 if not known */ +ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); + +/*! ZSTD_getParams() : + * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`. + * All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */ +ZSTDLIB_STATIC_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); + +/*! ZSTD_checkCParams() : + * Ensure param values remain within authorized range. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()) */ +ZSTDLIB_STATIC_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); + +/*! ZSTD_adjustCParams() : + * optimize params for a given `srcSize` and `dictSize`. + * `srcSize` can be unknown, in which case use ZSTD_CONTENTSIZE_UNKNOWN. + * `dictSize` must be `0` when there is no dictionary. + * cPar can be invalid : all parameters will be clamped within valid range in the @return struct. + * This function never fails (wide contract) */ +ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); + +/*! ZSTD_CCtx_setCParams() : + * Set all parameters provided within @p cparams into the working @p cctx. + * Note : if modifying parameters during compression (MT mode only), + * note that changes to the .windowLog parameter will be ignored. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()). + * On failure, no parameters are updated. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams); + +/*! ZSTD_CCtx_setFParams() : + * Set all parameters provided within @p fparams into the working @p cctx. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setFParams(ZSTD_CCtx* cctx, ZSTD_frameParameters fparams); + +/*! ZSTD_CCtx_setParams() : + * Set all parameters provided within @p params into the working @p cctx. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParams(ZSTD_CCtx* cctx, ZSTD_parameters params); + +/*! ZSTD_compress_advanced() : + * Note : this function is now DEPRECATED. + * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters. + * This prototype will generate compilation warnings. */ +ZSTD_DEPRECATED("use ZSTD_compress2") +ZSTDLIB_STATIC_API +size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + ZSTD_parameters params); + +/*! ZSTD_compress_usingCDict_advanced() : + * Note : this function is now DEPRECATED. + * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters. + * This prototype will generate compilation warnings. */ +ZSTD_DEPRECATED("use ZSTD_compress2 with ZSTD_CCtx_loadDictionary") +ZSTDLIB_STATIC_API +size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fParams); + + +/*! ZSTD_CCtx_loadDictionary_byReference() : + * Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx. + * It saves some memory, but also requires that `dict` outlives its usage within `cctx` */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); + +/*! ZSTD_CCtx_loadDictionary_advanced() : + * Same as ZSTD_CCtx_loadDictionary(), but gives finer control over + * how to load the dictionary (by copy ? by reference ?) + * and how to interpret it (automatic ? force raw mode ? full mode only ?) */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_CCtx_refPrefix_advanced() : + * Same as ZSTD_CCtx_refPrefix(), but gives finer control over + * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); + +/* === experimental parameters === */ +/* these parameters can be used with ZSTD_setParameter() + * they are not guaranteed to remain supported in the future */ + + /* Enables rsyncable mode, + * which makes compressed files more rsync friendly + * by adding periodic synchronization points to the compressed data. + * The target average block size is ZSTD_c_jobSize / 2. + * It's possible to modify the job size to increase or decrease + * the granularity of the synchronization point. + * Once the jobSize is smaller than the window size, + * it will result in compression ratio degradation. + * NOTE 1: rsyncable mode only works when multithreading is enabled. + * NOTE 2: rsyncable performs poorly in combination with long range mode, + * since it will decrease the effectiveness of synchronization points, + * though mileage may vary. + * NOTE 3: Rsyncable mode limits maximum compression speed to ~400 MB/s. + * If the selected compression level is already running significantly slower, + * the overall speed won't be significantly impacted. + */ + #define ZSTD_c_rsyncable ZSTD_c_experimentalParam1 -/* ************************************* - * Constants - ***************************************/ +/* Select a compression format. + * The value must be of type ZSTD_format_e. + * See ZSTD_format_e enum definition for details */ +#define ZSTD_c_format ZSTD_c_experimentalParam2 -/* All magic numbers are supposed read/written to/from files/memory using little-endian convention */ -#define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */ -#define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* valid since v0.7.0 */ -#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50 /* all 16 values, from 0x184D2A50 to 0x184D2A5F, signal the beginning of a skippable frame */ -#define ZSTD_MAGIC_SKIPPABLE_MASK 0xFFFFFFF0 +/* Force back-reference distances to remain < windowSize, + * even when referencing into Dictionary content (default:0) */ +#define ZSTD_c_forceMaxWindow ZSTD_c_experimentalParam3 -#define ZSTD_BLOCKSIZELOG_MAX 17 -#define ZSTD_BLOCKSIZE_MAX (1<=10 enables). + */ +#define ZSTD_c_repcodeResolution ZSTD_c_experimentalParam19 +#define ZSTD_c_searchForExternalRepcodes ZSTD_c_experimentalParam19 /* older name */ + + +/*! ZSTD_CCtx_getParameter() : + * Get the requested compression parameter value, selected by enum ZSTD_cParameter, + * and store it into int* value. + * @return : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value); + + +/*! ZSTD_CCtx_params : + * Quick howto : + * - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure + * - ZSTD_CCtxParams_setParameter() : Push parameters one by one into + * an existing ZSTD_CCtx_params structure. + * This is similar to + * ZSTD_CCtx_setParameter(). + * - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to + * an existing CCtx. + * These parameters will be applied to + * all subsequent frames. + * - ZSTD_compressStream2() : Do compression using the CCtx. + * - ZSTD_freeCCtxParams() : Free the memory, accept NULL pointer. + * + * This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams() + * for static allocation of CCtx for single-threaded compression. + */ +ZSTDLIB_STATIC_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void); +ZSTDLIB_STATIC_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); /* accept NULL pointer */ + +/*! ZSTD_CCtxParams_reset() : + * Reset params to default values. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params); + +/*! ZSTD_CCtxParams_init() : + * Initializes the compression parameters of cctxParams according to + * compression level. All other parameters are reset to their default values. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel); + +/*! ZSTD_CCtxParams_init_advanced() : + * Initializes the compression and frame parameters of cctxParams according to + * params. All other parameters are reset to their default values. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params); + +/*! ZSTD_CCtxParams_setParameter() : Requires v1.4.0+ + * Similar to ZSTD_CCtx_setParameter. + * Set one compression parameter, selected by enum ZSTD_cParameter. + * Parameters must be applied to a ZSTD_CCtx using + * ZSTD_CCtx_setParametersUsingCCtxParams(). + * @result : a code representing success or failure (which can be tested with + * ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value); + +/*! ZSTD_CCtxParams_getParameter() : + * Similar to ZSTD_CCtx_getParameter. + * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value); + +/*! ZSTD_CCtx_setParametersUsingCCtxParams() : + * Apply a set of ZSTD_CCtx_params to the compression context. + * This can be done even after compression is started, + * if nbWorkers==0, this will have no impact until a new compression is started. + * if nbWorkers>=1, new parameters will be picked up at next job, + * with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParametersUsingCCtxParams( + ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params); +/*! ZSTD_compressStream2_simpleArgs() : + * Same as ZSTD_compressStream2(), + * but using only integral types as arguments. + * This variant might be helpful for binders from dynamic languages + * which have troubles handling structures containing memory pointers. + */ +ZSTDLIB_STATIC_API size_t ZSTD_compressStream2_simpleArgs ( + ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos, + ZSTD_EndDirective endOp); /*************************************** -* Simple API +* Advanced decompression functions ***************************************/ -/*! ZSTD_compress() : - * Compresses `src` content as a single zstd compressed frame into already allocated `dst`. - * Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`. - * @return : compressed size written into `dst` (<= `dstCapacity), - * or an error code if it fails (which can be tested using ZSTD_isError()). */ -ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - int compressionLevel); - -/*! ZSTD_decompress() : - * `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames. - * `dstCapacity` is an upper bound of originalSize to regenerate. - * If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data. - * @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), - * or an errorCode if it fails (which can be tested using ZSTD_isError()). */ -ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, - const void* src, size_t compressedSize); - -/*! ZSTD_getFrameContentSize() : requires v1.3.0+ - * `src` should point to the start of a ZSTD encoded frame. - * `srcSize` must be at least as large as the frame header. - * hint : any size >= `ZSTD_frameHeaderSize_max` is large enough. - * @return : - decompressed size of `src` frame content, if known - * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined - * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) - * note 1 : a 0 return value means the frame is valid but "empty". - * note 2 : decompressed size is an optional field, it may not be present, typically in streaming mode. - * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. - * In which case, it's necessary to use streaming mode to decompress data. - * Optionally, application can rely on some implicit limit, - * as ZSTD_decompress() only needs an upper bound of decompressed size. - * (For example, data could be necessarily cut into blocks <= 16 KB). - * note 3 : decompressed size is always present when compression is completed using single-pass functions, - * such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict(). - * note 4 : decompressed size can be very large (64-bits value), - * potentially larger than what local system can handle as a single memory segment. - * In which case, it's necessary to use streaming mode to decompress data. - * note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified. - * Always ensure return value fits within application's authorized limits. - * Each application can set its own limits. - * note 6 : This function replaces ZSTD_getDecompressedSize() */ -#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1) -#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2) -ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); -/*! ZSTD_getDecompressedSize() : - * NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize(). - * Both functions work the same way, but ZSTD_getDecompressedSize() blends - * "empty", "unknown" and "error" results to the same return value (0), - * while ZSTD_getFrameContentSize() gives them separate return values. - * @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */ -ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); +/*! ZSTD_isFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier. + * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. + * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. + * Note 3 : Skippable Frame Identifiers are considered valid. */ +ZSTDLIB_STATIC_API unsigned ZSTD_isFrame(const void* buffer, size_t size); -/*! ZSTD_findFrameCompressedSize() : - * `src` should point to the start of a ZSTD frame or skippable frame. - * `srcSize` must be >= first frame size - * @return : the compressed size of the first frame starting at `src`, - * suitable to pass as `srcSize` to `ZSTD_decompress` or similar, - * or an error code if input is invalid */ -ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize); +/*! ZSTD_createDDict_byReference() : + * Create a digested dictionary, ready to start decompression operation without startup delay. + * Dictionary content is referenced, and therefore stays in dictBuffer. + * It is important that dictBuffer outlives DDict, + * it must remain read accessible throughout the lifetime of DDict */ +ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize); +/*! ZSTD_DCtx_loadDictionary_byReference() : + * Same as ZSTD_DCtx_loadDictionary(), + * but references `dict` content instead of copying it into `dctx`. + * This saves memory if `dict` remains around., + * However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression. */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); -/*====== Helper functions ======*/ -#define ZSTD_COMPRESSBOUND(srcSize) ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */ -ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */ -ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ -ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */ -ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed */ -ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */ +/*! ZSTD_DCtx_loadDictionary_advanced() : + * Same as ZSTD_DCtx_loadDictionary(), + * but gives direct control over + * how to load the dictionary (by copy ? by reference ?) + * and how to interpret it (automatic ? force raw mode ? full mode only ?). */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); +/*! ZSTD_DCtx_refPrefix_advanced() : + * Same as ZSTD_DCtx_refPrefix(), but gives finer control over + * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); -/*************************************** -* Explicit context -***************************************/ -/*= Compression context - * When compressing many times, - * it is recommended to allocate a context just once, - * and re-use it for each successive compression operation. - * This will make workload friendlier for system's memory. - * Note : re-using context is just a speed / resource optimization. - * It doesn't change the compression ratio, which remains identical. - * Note 2 : In multi-threaded environments, - * use one different context per thread for parallel execution. +/*! ZSTD_DCtx_setMaxWindowSize() : + * Refuses allocating internal buffers for frames requiring a window size larger than provided limit. + * This protects a decoder context from reserving too much memory for itself (potential attack scenario). + * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. + * By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + * @return : 0, or an error code (which can be tested using ZSTD_isError()). */ -typedef struct ZSTD_CCtx_s ZSTD_CCtx; -ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); -ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize); -/*! ZSTD_compressCCtx() : - * Same as ZSTD_compress(), using an explicit ZSTD_CCtx. - * Important : in order to behave similarly to `ZSTD_compress()`, - * this function compresses at requested compression level, - * __ignoring any other parameter__ . - * If any advanced parameter was set using the advanced API, - * they will all be reset. Only `compressionLevel` remains. +/*! ZSTD_DCtx_getParameter() : + * Get the requested decompression parameter value, selected by enum ZSTD_dParameter, + * and store it into int* value. + * @return : 0, or an error code (which can be tested with ZSTD_isError()). */ -ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - int compressionLevel); - -/*= Decompression context - * When decompressing many times, - * it is recommended to allocate a context only once, - * and re-use it for each successive compression operation. - * This will make workload friendlier for system's memory. - * Use one context per thread for parallel execution. */ -typedef struct ZSTD_DCtx_s ZSTD_DCtx; -ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void); -ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value); -/*! ZSTD_decompressDCtx() : - * Same as ZSTD_decompress(), - * requires an allocated ZSTD_DCtx. - * Compatible with sticky parameters. +/* ZSTD_d_format + * experimental parameter, + * allowing selection between ZSTD_format_e input compression formats */ -ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize); - - -/*************************************** -* Advanced compression API -***************************************/ - -/* API design : - * Parameters are pushed one by one into an existing context, - * using ZSTD_CCtx_set*() functions. - * Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame. - * "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` ! - * __They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx()__ . +#define ZSTD_d_format ZSTD_d_experimentalParam1 +/* ZSTD_d_stableOutBuffer + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable. * - * It's possible to reset all parameters to "default" using ZSTD_CCtx_reset(). + * Tells the decompressor that the ZSTD_outBuffer will ALWAYS be the same + * between calls, except for the modifications that zstd makes to pos (the + * caller must not modify pos). This is checked by the decompressor, and + * decompression will fail if it ever changes. Therefore the ZSTD_outBuffer + * MUST be large enough to fit the entire decompressed frame. This will be + * checked when the frame content size is known. The data in the ZSTD_outBuffer + * in the range [dst, dst + pos) MUST not be modified during decompression + * or you will get data corruption. + * + * When this flag is enabled zstd won't allocate an output buffer, because + * it can write directly to the ZSTD_outBuffer, but it will still allocate + * an input buffer large enough to fit any compressed block. This will also + * avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer. + * If you need to avoid the input buffer allocation use the buffer-less + * streaming API. + * + * NOTE: So long as the ZSTD_outBuffer always points to valid memory, using + * this flag is ALWAYS memory safe, and will never access out-of-bounds + * memory. However, decompression WILL fail if you violate the preconditions. * - * This API supercedes all other "advanced" API entry points in the experimental section. - * In the future, we expect to remove from experimental API entry points which are redundant with this API. + * WARNING: The data in the ZSTD_outBuffer in the range [dst, dst + pos) MUST + * not be modified during decompression or you will get data corruption. This + * is because zstd needs to reference data in the ZSTD_outBuffer to regenerate + * matches. Normally zstd maintains its own buffer for this purpose, but passing + * this flag tells zstd to use the user provided buffer. */ +#define ZSTD_d_stableOutBuffer ZSTD_d_experimentalParam2 +/* ZSTD_d_forceIgnoreChecksum + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable + * + * Tells the decompressor to skip checksum validation during decompression, regardless + * of whether checksumming was specified during compression. This offers some + * slight performance benefits, and may be useful for debugging. + * Param has values of type ZSTD_forceIgnoreChecksum_e + */ +#define ZSTD_d_forceIgnoreChecksum ZSTD_d_experimentalParam3 -/* Compression strategies, listed from fastest to strongest */ -typedef enum { ZSTD_fast=1, - ZSTD_dfast=2, - ZSTD_greedy=3, - ZSTD_lazy=4, - ZSTD_lazy2=5, - ZSTD_btlazy2=6, - ZSTD_btopt=7, - ZSTD_btultra=8, - ZSTD_btultra2=9 - /* note : new strategies _might_ be added in the future. - Only the order (from fast to strong) is guaranteed */ -} ZSTD_strategy; - - -typedef enum { - - /* compression parameters - * Note: When compressing with a ZSTD_CDict these parameters are superseded - * by the parameters used to construct the ZSTD_CDict. - * See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict). */ - ZSTD_c_compressionLevel=100, /* Set compression parameters according to pre-defined cLevel table. - * Note that exact compression parameters are dynamically determined, - * depending on both compression level and srcSize (when known). - * Default level is ZSTD_CLEVEL_DEFAULT==3. - * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT. - * Note 1 : it's possible to pass a negative compression level. - * Note 2 : setting a level does not automatically set all other compression parameters - * to default. Setting this will however eventually dynamically impact the compression - * parameters which have not been manually set. The manually set - * ones will 'stick'. */ - /* Advanced compression parameters : - * It's possible to pin down compression parameters to some specific values. - * In which case, these values are no longer dynamically selected by the compressor */ - ZSTD_c_windowLog=101, /* Maximum allowed back-reference distance, expressed as power of 2. - * This will set a memory budget for streaming decompression, - * with larger values requiring more memory - * and typically compressing more. - * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX. - * Special: value 0 means "use default windowLog". - * Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT - * requires explicitly allowing such size at streaming decompression stage. */ - ZSTD_c_hashLog=102, /* Size of the initial probe table, as a power of 2. - * Resulting memory usage is (1 << (hashLog+2)). - * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX. - * Larger tables improve compression ratio of strategies <= dFast, - * and improve speed of strategies > dFast. - * Special: value 0 means "use default hashLog". */ - ZSTD_c_chainLog=103, /* Size of the multi-probe search table, as a power of 2. - * Resulting memory usage is (1 << (chainLog+2)). - * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX. - * Larger tables result in better and slower compression. - * This parameter is useless for "fast" strategy. - * It's still useful when using "dfast" strategy, - * in which case it defines a secondary probe table. - * Special: value 0 means "use default chainLog". */ - ZSTD_c_searchLog=104, /* Number of search attempts, as a power of 2. - * More attempts result in better and slower compression. - * This parameter is useless for "fast" and "dFast" strategies. - * Special: value 0 means "use default searchLog". */ - ZSTD_c_minMatch=105, /* Minimum size of searched matches. - * Note that Zstandard can still find matches of smaller size, - * it just tweaks its search algorithm to look for this size and larger. - * Larger values increase compression and decompression speed, but decrease ratio. - * Must be clamped between ZSTD_MINMATCH_MIN and ZSTD_MINMATCH_MAX. - * Note that currently, for all strategies < btopt, effective minimum is 4. - * , for all strategies > fast, effective maximum is 6. - * Special: value 0 means "use default minMatchLength". */ - ZSTD_c_targetLength=106, /* Impact of this field depends on strategy. - * For strategies btopt, btultra & btultra2: - * Length of Match considered "good enough" to stop search. - * Larger values make compression stronger, and slower. - * For strategy fast: - * Distance between match sampling. - * Larger values make compression faster, and weaker. - * Special: value 0 means "use default targetLength". */ - ZSTD_c_strategy=107, /* See ZSTD_strategy enum definition. - * The higher the value of selected strategy, the more complex it is, - * resulting in stronger and slower compression. - * Special: value 0 means "use default strategy". */ +/* ZSTD_d_refMultipleDDicts + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable + * + * If enabled and dctx is allocated on the heap, then additional memory will be allocated + * to store references to multiple ZSTD_DDict. That is, multiple calls of ZSTD_refDDict() + * using a given ZSTD_DCtx, rather than overwriting the previous DDict reference, will instead + * store all references. At decompression time, the appropriate dictID is selected + * from the set of DDicts based on the dictID in the frame. + * + * Usage is simply calling ZSTD_refDDict() on multiple dict buffers. + * + * Param has values of byte ZSTD_refMultipleDDicts_e + * + * WARNING: Enabling this parameter and calling ZSTD_DCtx_refDDict(), will trigger memory + * allocation for the hash table. ZSTD_freeDCtx() also frees this memory. + * Memory is allocated as per ZSTD_DCtx::customMem. + * + * Although this function allocates memory for the table, the user is still responsible for + * memory management of the underlying ZSTD_DDict* themselves. + */ +#define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4 - /* LDM mode parameters */ - ZSTD_c_enableLongDistanceMatching=160, /* Enable long distance matching. - * This parameter is designed to improve compression ratio - * for large inputs, by finding large matches at long distance. - * It increases memory usage and window size. - * Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB - * except when expressly set to a different value. - * Note: will be enabled by default if ZSTD_c_windowLog >= 128 MB and - * compression strategy >= ZSTD_btopt (== compression level 16+) */ - ZSTD_c_ldmHashLog=161, /* Size of the table for long distance matching, as a power of 2. - * Larger values increase memory usage and compression ratio, - * but decrease compression speed. - * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX - * default: windowlog - 7. - * Special: value 0 means "automatically determine hashlog". */ - ZSTD_c_ldmMinMatch=162, /* Minimum match size for long distance matcher. - * Larger/too small values usually decrease compression ratio. - * Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX. - * Special: value 0 means "use default value" (default: 64). */ - ZSTD_c_ldmBucketSizeLog=163, /* Log size of each bucket in the LDM hash table for collision resolution. - * Larger values improve collision resolution but decrease compression speed. - * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX. - * Special: value 0 means "use default value" (default: 3). */ - ZSTD_c_ldmHashRateLog=164, /* Frequency of inserting/looking up entries into the LDM hash table. - * Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN). - * Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage. - * Larger values improve compression speed. - * Deviating far from default value will likely result in a compression ratio decrease. - * Special: value 0 means "automatically determine hashRateLog". */ +/* ZSTD_d_disableHuffmanAssembly + * Set to 1 to disable the Huffman assembly implementation. + * The default value is 0, which allows zstd to use the Huffman assembly + * implementation if available. + * + * This parameter can be used to disable Huffman assembly at runtime. + * If you want to disable it at compile time you can define the macro + * ZSTD_DISABLE_ASM. + */ +#define ZSTD_d_disableHuffmanAssembly ZSTD_d_experimentalParam5 - /* frame parameters */ - ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1) - * Content size must be known at the beginning of compression. - * This is automatically the case when using ZSTD_compress2(), - * For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */ - ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */ - ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */ +/* ZSTD_d_maxBlockSize + * Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB). + * The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default. + * + * Forces the decompressor to reject blocks whose content size is + * larger than the configured maxBlockSize. When maxBlockSize is + * larger than the windowSize, the windowSize is used instead. + * This saves memory on the decoder when you know all blocks are small. + * + * This option is typically used in conjunction with ZSTD_c_maxBlockSize. + * + * WARNING: This causes the decoder to reject otherwise valid frames + * that have block sizes larger than the configured maxBlockSize. + */ +#define ZSTD_d_maxBlockSize ZSTD_d_experimentalParam6 - /* multi-threading parameters */ - /* These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD). - * Otherwise, trying to set any other value than default (0) will be a no-op and return an error. - * In a situation where it's unknown if the linked library supports multi-threading or not, - * setting ZSTD_c_nbWorkers to any value >= 1 and consulting the return value provides a quick way to check this property. - */ - ZSTD_c_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel. - * When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() : - * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller, - * while compression is performed in parallel, within worker thread(s). - * (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end : - * in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call). - * More workers improve speed, but also increase memory usage. - * Default value is `0`, aka "single-threaded mode" : no worker is spawned, - * compression is performed inside Caller's thread, and all invocations are blocking */ - ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1. - * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads. - * 0 means default, which is dynamically determined based on compression parameters. - * Job size must be a minimum of overlap size, or 1 MB, whichever is largest. - * The minimum size is automatically and transparently enforced. */ - ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size. - * The overlap size is an amount of data reloaded from previous job at the beginning of a new job. - * It helps preserve compression ratio, while each job is compressed in parallel. - * This value is enforced only when nbWorkers >= 1. - * Larger values increase compression ratio, but decrease speed. - * Possible values range from 0 to 9 : - * - 0 means "default" : value will be determined by the library, depending on strategy - * - 1 means "no overlap" - * - 9 means "full overlap", using a full window size. - * Each intermediate rank increases/decreases load size by a factor 2 : - * 9: full window; 8: w/2; 7: w/4; 6: w/8; 5:w/16; 4: w/32; 3:w/64; 2:w/128; 1:no overlap; 0:default - * default value varies between 6 and 9, depending on strategy */ - /* note : additional experimental parameters are also available - * within the experimental section of the API. - * At the time of this writing, they include : - * ZSTD_c_rsyncable - * ZSTD_c_format - * ZSTD_c_forceMaxWindow - * ZSTD_c_forceAttachDict - * ZSTD_c_literalCompressionMode - * ZSTD_c_targetCBlockSize - * ZSTD_c_srcSizeHint - * ZSTD_c_enableDedicatedDictSearch - * ZSTD_c_stableInBuffer - * ZSTD_c_stableOutBuffer - * ZSTD_c_blockDelimiters - * ZSTD_c_validateSequences - * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. - * note : never ever use experimentalParam? names directly; - * also, the enums values themselves are unstable and can still change. - */ - ZSTD_c_experimentalParam1=500, - ZSTD_c_experimentalParam2=10, - ZSTD_c_experimentalParam3=1000, - ZSTD_c_experimentalParam4=1001, - ZSTD_c_experimentalParam5=1002, - ZSTD_c_experimentalParam6=1003, - ZSTD_c_experimentalParam7=1004, - ZSTD_c_experimentalParam8=1005, - ZSTD_c_experimentalParam9=1006, - ZSTD_c_experimentalParam10=1007, - ZSTD_c_experimentalParam11=1008, - ZSTD_c_experimentalParam12=1009 -} ZSTD_cParameter; +/*! ZSTD_DCtx_setFormat() : + * This function is REDUNDANT. Prefer ZSTD_DCtx_setParameter(). + * Instruct the decoder context about what kind of data to decode next. + * This instruction is mandatory to decode data without a fully-formed header, + * such ZSTD_f_zstd1_magicless for example. + * @return : 0, or an error code (which can be tested using ZSTD_isError()). */ +ZSTD_DEPRECATED("use ZSTD_DCtx_setParameter() instead") +ZSTDLIB_STATIC_API +size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); -typedef struct { - size_t error; - int lowerBound; - int upperBound; -} ZSTD_bounds; +/*! ZSTD_decompressStream_simpleArgs() : + * Same as ZSTD_decompressStream(), + * but using only integral types as arguments. + * This can be helpful for binders from dynamic languages + * which have troubles handling structures containing memory pointers. + */ +ZSTDLIB_STATIC_API size_t ZSTD_decompressStream_simpleArgs ( + ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos); -/*! ZSTD_cParam_getBounds() : - * All parameters must belong to an interval with lower and upper bounds, - * otherwise they will either trigger an error or be automatically clamped. - * @return : a structure, ZSTD_bounds, which contains - * - an error status field, which must be tested using ZSTD_isError() - * - lower and upper bounds, both inclusive + +/******************************************************************** +* Advanced streaming functions +* Warning : most of these functions are now redundant with the Advanced API. +* Once Advanced API reaches "stable" status, +* redundant functions will be deprecated, and then at some point removed. +********************************************************************/ + +/*===== Advanced Streaming compression functions =====*/ + +/*! ZSTD_initCStream_srcSize() : + * This function is DEPRECATED, and equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * + * pledgedSrcSize must be correct. If it is not known at init time, use + * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, + * "0" also disables frame content size field. It may be enabled in the future. + * This prototype will generate compilation warnings. */ -ZSTDLIB_API ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter cParam); +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, + int compressionLevel, + unsigned long long pledgedSrcSize); -/*! ZSTD_CCtx_setParameter() : - * Set one compression parameter, selected by enum ZSTD_cParameter. - * All parameters have valid bounds. Bounds can be queried using ZSTD_cParam_getBounds(). - * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). - * Setting a parameter is generally only possible during frame initialization (before starting compression). - * Exception : when using multi-threading mode (nbWorkers >= 1), - * the following parameters can be updated _during_ compression (within same frame): - * => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy. - * new parameters will be active for next job only (after a flush()). - * @return : an error code (which can be tested using ZSTD_isError()). +/*! ZSTD_initCStream_usingDict() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); + * + * Creates of an internal CDict (incompatible with static CCtx), except if + * dict == NULL or dictSize < 8, in which case no dict is used. + * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if + * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy. + * This prototype will generate compilation warnings. */ -ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value); +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + int compressionLevel); -/*! ZSTD_CCtx_setPledgedSrcSize() : - * Total input data size to be compressed as a single frame. - * Value will be written in frame header, unless if explicitly forbidden using ZSTD_c_contentSizeFlag. - * This value will also be controlled at end of frame, and trigger an error if not respected. - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame. - * In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN. - * ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame. - * Note 2 : pledgedSrcSize is only valid once, for the next frame. - * It's discarded at the end of the frame, and replaced by ZSTD_CONTENTSIZE_UNKNOWN. - * Note 3 : Whenever all input data is provided and consumed in a single round, - * for example with ZSTD_compress2(), - * or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end), - * this value is automatically overridden by srcSize instead. +/*! ZSTD_initCStream_advanced() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setParams(zcs, params); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); + * + * dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy. + * pledgedSrcSize must be correct. + * If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. + * This prototype will generate compilation warnings. */ -ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize); +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + ZSTD_parameters params, + unsigned long long pledgedSrcSize); -typedef enum { - ZSTD_reset_session_only = 1, - ZSTD_reset_parameters = 2, - ZSTD_reset_session_and_parameters = 3 -} ZSTD_ResetDirective; +/*! ZSTD_initCStream_usingCDict() : + * This function is DEPRECATED, and equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, cdict); + * + * note : cdict will just be referenced, and must outlive compression session + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); -/*! ZSTD_CCtx_reset() : - * There are 2 different things that can be reset, independently or jointly : - * - The session : will stop compressing current frame, and make CCtx ready to start a new one. - * Useful after an error, or to interrupt any ongoing compression. - * Any internal data not yet flushed is cancelled. - * Compression parameters and dictionary remain unchanged. - * They will be used to compress next frame. - * Resetting session never fails. - * - The parameters : changes all parameters back to "default". - * This removes any reference to any dictionary too. - * Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing) - * otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError()) - * - Both : similar to resetting the session, followed by resetting parameters. +/*! ZSTD_initCStream_usingCDict_advanced() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setFParams(zcs, fParams); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * ZSTD_CCtx_refCDict(zcs, cdict); + * + * same as ZSTD_initCStream_usingCDict(), with control over frame parameters. + * pledgedSrcSize must be correct. If srcSize is not known at init time, use + * value ZSTD_CONTENTSIZE_UNKNOWN. + * This prototype will generate compilation warnings. */ -ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset); +ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fParams, + unsigned long long pledgedSrcSize); -/*! ZSTD_compress2() : - * Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API. - * ZSTD_compress2() always starts a new frame. - * Should cctx hold data from a previously unfinished frame, everything about it is forgotten. - * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() - * - The function is always blocking, returns when compression is completed. - * Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`. - * @return : compressed size written into `dst` (<= `dstCapacity), - * or an error code if it fails (which can be tested using ZSTD_isError()). +/*! ZSTD_resetCStream() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * Note: ZSTD_resetCStream() interprets pledgedSrcSize == 0 as ZSTD_CONTENTSIZE_UNKNOWN, but + * ZSTD_CCtx_setPledgedSrcSize() does not do the same, so ZSTD_CONTENTSIZE_UNKNOWN must be + * explicitly specified. + * + * start a new frame, using same parameters from previous frame. + * This is typically useful to skip dictionary loading stage, since it will reuse it in-place. + * Note that zcs must be init at least once before using ZSTD_resetCStream(). + * If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN. + * If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end. + * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs, + * but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead. + * @return : 0, or an error code (which can be tested using ZSTD_isError()) + * This prototype will generate compilation warnings. */ -ZSTDLIB_API size_t ZSTD_compress2( ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize); +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); -/*************************************** -* Advanced decompression API -***************************************/ +typedef struct { + unsigned long long ingested; /* nb input bytes read and buffered */ + unsigned long long consumed; /* nb input bytes actually compressed */ + unsigned long long produced; /* nb of compressed bytes generated and buffered */ + unsigned long long flushed; /* nb of compressed bytes flushed : not provided; can be tracked from caller side */ + unsigned currentJobID; /* MT only : latest started job nb */ + unsigned nbActiveWorkers; /* MT only : nb of workers actively compressing at probe time */ +} ZSTD_frameProgression; -/* The advanced API pushes parameters one by one into an existing DCtx context. - * Parameters are sticky, and remain valid for all following frames - * using the same DCtx context. - * It's possible to reset parameters to default values using ZSTD_DCtx_reset(). - * Note : This API is compatible with existing ZSTD_decompressDCtx() and ZSTD_decompressStream(). - * Therefore, no new decompression function is necessary. +/* ZSTD_getFrameProgression() : + * tells how much data has been ingested (read from input) + * consumed (input actually compressed) and produced (output) for current frame. + * Note : (ingested - consumed) is amount of input data buffered internally, not yet compressed. + * Aggregates progression inside active worker threads. */ +ZSTDLIB_STATIC_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx); -typedef enum { - - ZSTD_d_windowLogMax=100, /* Select a size limit (in power of 2) beyond which - * the streaming API will refuse to allocate memory buffer - * in order to protect the host from unreasonable memory requirements. - * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. - * By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT). - * Special: value 0 means "use default maximum windowLog". */ +/*! ZSTD_toFlushNow() : + * Tell how many bytes are ready to be flushed immediately. + * Useful for multithreading scenarios (nbWorkers >= 1). + * Probe the oldest active job, defined as oldest job not yet entirely flushed, + * and check its output buffer. + * @return : amount of data stored in oldest job and ready to be flushed immediately. + * if @return == 0, it means either : + * + there is no active job (could be checked with ZSTD_frameProgression()), or + * + oldest job is still actively compressing data, + * but everything it has produced has also been flushed so far, + * therefore flush speed is limited by production speed of oldest job + * irrespective of the speed of concurrent (and newer) jobs. + */ +ZSTDLIB_STATIC_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx); - /* note : additional experimental parameters are also available - * within the experimental section of the API. - * At the time of this writing, they include : - * ZSTD_d_format - * ZSTD_d_stableOutBuffer - * ZSTD_d_forceIgnoreChecksum - * ZSTD_d_refMultipleDDicts - * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. - * note : never ever use experimentalParam? names directly - */ - ZSTD_d_experimentalParam1=1000, - ZSTD_d_experimentalParam2=1001, - ZSTD_d_experimentalParam3=1002, - ZSTD_d_experimentalParam4=1003 -} ZSTD_dParameter; +/*===== Advanced Streaming decompression functions =====*/ -/*! ZSTD_dParam_getBounds() : - * All parameters must belong to an interval with lower and upper bounds, - * otherwise they will either trigger an error or be automatically clamped. - * @return : a structure, ZSTD_bounds, which contains - * - an error status field, which must be tested using ZSTD_isError() - * - both lower and upper bounds, inclusive +/*! + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_loadDictionary(zds, dict, dictSize); + * + * note: no dictionary will be used if dict == NULL or dictSize < 8 */ -ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam); +ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_loadDictionary, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); -/*! ZSTD_DCtx_setParameter() : - * Set one compression parameter, selected by enum ZSTD_dParameter. - * All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds(). - * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). - * Setting a parameter is only possible during frame initialization (before starting decompression). - * @return : 0, or an error code (which can be tested using ZSTD_isError()). +/*! + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_refDDict(zds, ddict); + * + * note : ddict is referenced, it must outlive decompression session */ -ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value); +ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_refDDict, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); -/*! ZSTD_DCtx_reset() : - * Return a DCtx to clean state. - * Session and parameters can be reset jointly or separately. - * Parameters can only be reset when no active frame is being decompressed. - * @return : 0, or an error code, which can be tested with ZSTD_isError() +/*! + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * + * reuse decompression parameters from previous init; saves dictionary loading */ -ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset); +ZSTD_DEPRECATED("use ZSTD_DCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); -/**************************** -* Streaming -****************************/ +/* ********************* BLOCK-LEVEL SEQUENCE PRODUCER API ********************* + * + * *** OVERVIEW *** + * The Block-Level Sequence Producer API allows users to provide their own custom + * sequence producer which libzstd invokes to process each block. The produced list + * of sequences (literals and matches) is then post-processed by libzstd to produce + * valid compressed blocks. + * + * This block-level offload API is a more granular complement of the existing + * frame-level offload API compressSequences() (introduced in v1.5.1). It offers + * an easier migration story for applications already integrated with libzstd: the + * user application continues to invoke the same compression functions + * ZSTD_compress2() or ZSTD_compressStream2() as usual, and transparently benefits + * from the specific advantages of the external sequence producer. For example, + * the sequence producer could be tuned to take advantage of known characteristics + * of the input, to offer better speed / ratio, or could leverage hardware + * acceleration not available within libzstd itself. + * + * See contrib/externalSequenceProducer for an example program employing the + * Block-Level Sequence Producer API. + * + * *** USAGE *** + * The user is responsible for implementing a function of type + * ZSTD_sequenceProducer_F. For each block, zstd will pass the following + * arguments to the user-provided function: + * + * - sequenceProducerState: a pointer to a user-managed state for the sequence + * producer. + * + * - outSeqs, outSeqsCapacity: an output buffer for the sequence producer. + * outSeqsCapacity is guaranteed >= ZSTD_sequenceBound(srcSize). The memory + * backing outSeqs is managed by the CCtx. + * + * - src, srcSize: an input buffer for the sequence producer to parse. + * srcSize is guaranteed to be <= ZSTD_BLOCKSIZE_MAX. + * + * - dict, dictSize: a history buffer, which may be empty, which the sequence + * producer may reference as it parses the src buffer. Currently, zstd will + * always pass dictSize == 0 into external sequence producers, but this will + * change in the future. + * + * - compressionLevel: a signed integer representing the zstd compression level + * set by the user for the current operation. The sequence producer may choose + * to use this information to change its compression strategy and speed/ratio + * tradeoff. Note: the compression level does not reflect zstd parameters set + * through the advanced API. + * + * - windowSize: a size_t representing the maximum allowed offset for external + * sequences. Note that sequence offsets are sometimes allowed to exceed the + * windowSize if a dictionary is present, see doc/zstd_compression_format.md + * for details. + * + * The user-provided function shall return a size_t representing the number of + * sequences written to outSeqs. This return value will be treated as an error + * code if it is greater than outSeqsCapacity. The return value must be non-zero + * if srcSize is non-zero. The ZSTD_SEQUENCE_PRODUCER_ERROR macro is provided + * for convenience, but any value greater than outSeqsCapacity will be treated as + * an error code. + * + * If the user-provided function does not return an error code, the sequences + * written to outSeqs must be a valid parse of the src buffer. Data corruption may + * occur if the parse is not valid. A parse is defined to be valid if the + * following conditions hold: + * - The sum of matchLengths and literalLengths must equal srcSize. + * - All sequences in the parse, except for the final sequence, must have + * matchLength >= ZSTD_MINMATCH_MIN. The final sequence must have + * matchLength >= ZSTD_MINMATCH_MIN or matchLength == 0. + * - All offsets must respect the windowSize parameter as specified in + * doc/zstd_compression_format.md. + * - If the final sequence has matchLength == 0, it must also have offset == 0. + * + * zstd will only validate these conditions (and fail compression if they do not + * hold) if the ZSTD_c_validateSequences cParam is enabled. Note that sequence + * validation has a performance cost. + * + * If the user-provided function returns an error, zstd will either fall back + * to an internal sequence producer or fail the compression operation. The user can + * choose between the two behaviors by setting the ZSTD_c_enableSeqProducerFallback + * cParam. Fallback compression will follow any other cParam settings, such as + * compression level, the same as in a normal compression operation. + * + * The user shall instruct zstd to use a particular ZSTD_sequenceProducer_F + * function by calling + * ZSTD_registerSequenceProducer(cctx, + * sequenceProducerState, + * sequenceProducer) + * This setting will persist until the next parameter reset of the CCtx. + * + * The sequenceProducerState must be initialized by the user before calling + * ZSTD_registerSequenceProducer(). The user is responsible for destroying the + * sequenceProducerState. + * + * *** LIMITATIONS *** + * This API is compatible with all zstd compression APIs which respect advanced parameters. + * However, there are three limitations: + * + * First, the ZSTD_c_enableLongDistanceMatching cParam is not currently supported. + * COMPRESSION WILL FAIL if it is enabled and the user tries to compress with a block-level + * external sequence producer. + * - Note that ZSTD_c_enableLongDistanceMatching is auto-enabled by default in some + * cases (see its documentation for details). Users must explicitly set + * ZSTD_c_enableLongDistanceMatching to ZSTD_ps_disable in such cases if an external + * sequence producer is registered. + * - As of this writing, ZSTD_c_enableLongDistanceMatching is disabled by default + * whenever ZSTD_c_windowLog < 128MB, but that's subject to change. Users should + * check the docs on ZSTD_c_enableLongDistanceMatching whenever the Block-Level Sequence + * Producer API is used in conjunction with advanced settings (like ZSTD_c_windowLog). + * + * Second, history buffers are not currently supported. Concretely, zstd will always pass + * dictSize == 0 to the external sequence producer (for now). This has two implications: + * - Dictionaries are not currently supported. Compression will *not* fail if the user + * references a dictionary, but the dictionary won't have any effect. + * - Stream history is not currently supported. All advanced compression APIs, including + * streaming APIs, work with external sequence producers, but each block is treated as + * an independent chunk without history from previous blocks. + * + * Third, multi-threading within a single compression is not currently supported. In other words, + * COMPRESSION WILL FAIL if ZSTD_c_nbWorkers > 0 and an external sequence producer is registered. + * Multi-threading across compressions is fine: simply create one CCtx per thread. + * + * Long-term, we plan to overcome all three limitations. There is no technical blocker to + * overcoming them. It is purely a question of engineering effort. + */ -typedef struct ZSTD_inBuffer_s { - const void* src; /**< start of input buffer */ - size_t size; /**< size of input buffer */ - size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */ -} ZSTD_inBuffer; +#define ZSTD_SEQUENCE_PRODUCER_ERROR ((size_t)(-1)) -typedef struct ZSTD_outBuffer_s { - void* dst; /**< start of output buffer */ - size_t size; /**< size of output buffer */ - size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */ -} ZSTD_outBuffer; +typedef size_t (*ZSTD_sequenceProducer_F) ( + void* sequenceProducerState, + ZSTD_Sequence* outSeqs, size_t outSeqsCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize, + int compressionLevel, + size_t windowSize +); +/*! ZSTD_registerSequenceProducer() : + * Instruct zstd to use a block-level external sequence producer function. + * + * The sequenceProducerState must be initialized by the caller, and the caller is + * responsible for managing its lifetime. This parameter is sticky across + * compressions. It will remain set until the user explicitly resets compression + * parameters. + * + * Sequence producer registration is considered to be an "advanced parameter", + * part of the "advanced API". This means it will only have an effect on compression + * APIs which respect advanced parameters, such as compress2() and compressStream2(). + * Older compression APIs such as compressCCtx(), which predate the introduction of + * "advanced parameters", will ignore any external sequence producer setting. + * + * The sequence producer can be "cleared" by registering a NULL function pointer. This + * removes all limitations described above in the "LIMITATIONS" section of the API docs. + * + * The user is strongly encouraged to read the full API documentation (above) before + * calling this function. */ +ZSTDLIB_STATIC_API void +ZSTD_registerSequenceProducer( + ZSTD_CCtx* cctx, + void* sequenceProducerState, + ZSTD_sequenceProducer_F sequenceProducer +); + +/*! ZSTD_CCtxParams_registerSequenceProducer() : + * Same as ZSTD_registerSequenceProducer(), but operates on ZSTD_CCtx_params. + * This is used for accurate size estimation with ZSTD_estimateCCtxSize_usingCCtxParams(), + * which is needed when creating a ZSTD_CCtx with ZSTD_initStaticCCtx(). + * + * If you are using the external sequence producer API in a scenario where ZSTD_initStaticCCtx() + * is required, then this function is for you. Otherwise, you probably don't need it. + * + * See tests/zstreamtest.c for example usage. */ +ZSTDLIB_STATIC_API void +ZSTD_CCtxParams_registerSequenceProducer( + ZSTD_CCtx_params* params, + void* sequenceProducerState, + ZSTD_sequenceProducer_F sequenceProducer +); -/*-*********************************************************************** -* Streaming compression - HowTo -* -* A ZSTD_CStream object is required to track streaming operation. -* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. -* ZSTD_CStream objects can be reused multiple times on consecutive compression operations. -* It is recommended to re-use ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory. -* -* For parallel execution, use one separate ZSTD_CStream per thread. -* -* note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing. -* -* Parameters are sticky : when starting a new compression on the same context, -* it will re-use the same sticky parameters as previous compression session. -* When in doubt, it's recommended to fully initialize the context before usage. -* Use ZSTD_CCtx_reset() to reset the context and ZSTD_CCtx_setParameter(), -* ZSTD_CCtx_setPledgedSrcSize(), or ZSTD_CCtx_loadDictionary() and friends to -* set more specific parameters, the pledged source size, or load a dictionary. -* -* Use ZSTD_compressStream2() with ZSTD_e_continue as many times as necessary to -* consume input stream. The function will automatically update both `pos` -* fields within `input` and `output`. -* Note that the function may not consume the entire input, for example, because -* the output buffer is already full, in which case `input.pos < input.size`. -* The caller must check if input has been entirely consumed. -* If not, the caller must make some room to receive more compressed data, -* and then present again remaining input data. -* note: ZSTD_e_continue is guaranteed to make some forward progress when called, -* but doesn't guarantee maximal forward progress. This is especially relevant -* when compressing with multiple threads. The call won't block if it can -* consume some input, but if it can't it will wait for some, but not all, -* output to be flushed. -* @return : provides a minimum amount of data remaining to be flushed from internal buffers -* or an error code, which can be tested using ZSTD_isError(). -* -* At any moment, it's possible to flush whatever data might remain stuck within internal buffer, -* using ZSTD_compressStream2() with ZSTD_e_flush. `output->pos` will be updated. -* Note that, if `output->size` is too small, a single invocation with ZSTD_e_flush might not be enough (return code > 0). -* In which case, make some room to receive more compressed data, and call again ZSTD_compressStream2() with ZSTD_e_flush. -* You must continue calling ZSTD_compressStream2() with ZSTD_e_flush until it returns 0, at which point you can change the -* operation. -* note: ZSTD_e_flush will flush as much output as possible, meaning when compressing with multiple threads, it will -* block until the flush is complete or the output buffer is full. -* @return : 0 if internal buffers are entirely flushed, -* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), -* or an error code, which can be tested using ZSTD_isError(). +/********************************************************************* +* Buffer-less and synchronous inner streaming functions (DEPRECATED) * -* Calling ZSTD_compressStream2() with ZSTD_e_end instructs to finish a frame. -* It will perform a flush and write frame epilogue. -* The epilogue is required for decoders to consider a frame completed. -* flush operation is the same, and follows same rules as calling ZSTD_compressStream2() with ZSTD_e_flush. -* You must continue calling ZSTD_compressStream2() with ZSTD_e_end until it returns 0, at which point you are free to -* start a new frame. -* note: ZSTD_e_end will flush as much output as possible, meaning when compressing with multiple threads, it will -* block until the flush is complete or the output buffer is full. -* @return : 0 if frame fully completed and fully flushed, -* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), -* or an error code, which can be tested using ZSTD_isError(). +* This API is deprecated, and will be removed in a future version. +* It allows streaming (de)compression with user allocated buffers. +* However, it is hard to use, and not as well tested as the rest of +* our API. * -* *******************************************************************/ +* Please use the normal streaming API instead: ZSTD_compressStream2, +* and ZSTD_decompressStream. +* If there is functionality that you need, but it doesn't provide, +* please open an issue on our GitHub. +********************************************************************* */ -typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are now effectively same object (>= v1.3.0) */ - /* Continue to distinguish them for compatibility with older versions <= v1.2.0 */ -/*===== ZSTD_CStream management functions =====*/ -ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void); -ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); +/** + Buffer-less streaming compression (synchronous mode) -/*===== Streaming compression functions =====*/ -typedef enum { - ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */ - ZSTD_e_flush=1, /* flush any data provided so far, - * it creates (at least) one new block, that can be decoded immediately on reception; - * frame will continue: any future data can still reference previously compressed data, improving compression. - * note : multithreaded compression will block to flush as much output as possible. */ - ZSTD_e_end=2 /* flush any remaining data _and_ close current frame. - * note that frame is only closed after compressed data is fully flushed (return value == 0). - * After that point, any additional data starts a new frame. - * note : each frame is independent (does not reference any content from previous frame). - : note : multithreaded compression will block to flush as much output as possible. */ -} ZSTD_EndDirective; + A ZSTD_CCtx object is required to track streaming operations. + Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource. + ZSTD_CCtx object can be reused multiple times within successive compression operations. + + Start by initializing a context. + Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression. + + Then, consume your input using ZSTD_compressContinue(). + There are some important considerations to keep in mind when using this advanced function : + - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffers only. + - Interface is synchronous : input is consumed entirely and produces 1+ compressed blocks. + - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario. + Worst case evaluation is provided by ZSTD_compressBound(). + ZSTD_compressContinue() doesn't guarantee recover after a failed compression. + - ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog). + It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks) + - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps. + In which case, it will "discard" the relevant memory section from its history. + + Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum. + It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame. + Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders. + + `ZSTD_CCtx` object can be reused (ZSTD_compressBegin()) to compress again. +*/ + +/*===== Buffer-less streaming compression functions =====*/ +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */ + +ZSTD_DEPRECATED("This function will likely be removed in a future release. It is misleading and has very limited utility.") +ZSTDLIB_STATIC_API +size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */ + +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +/* The ZSTD_compressBegin_advanced() and ZSTD_compressBegin_usingCDict_advanced() are now DEPRECATED and will generate a compiler warning */ +ZSTD_DEPRECATED("use advanced API to access custom parameters") +ZSTDLIB_STATIC_API +size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */ +ZSTD_DEPRECATED("use advanced API to access custom parameters") +ZSTDLIB_STATIC_API +size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */ +/** + Buffer-less streaming decompression (synchronous mode) + + A ZSTD_DCtx object is required to track streaming operations. + Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. + A ZSTD_DCtx object can be reused multiple times. + + First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader(). + Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough. + Data fragment must be large enough to ensure successful decoding. + `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough. + result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled. + >0 : `srcSize` is too small, please provide at least result bytes on next attempt. + errorCode, which can be tested using ZSTD_isError(). + + It fills a ZSTD_FrameHeader structure with important information to correctly decode the frame, + such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`). + Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information. + As a consequence, check that values remain within valid application range. + For example, do not allocate memory blindly, check that `windowSize` is within expectation. + Each application can set its own limits, depending on local restrictions. + For extended interoperability, it is recommended to support `windowSize` of at least 8 MB. -/*! ZSTD_compressStream2() : - * Behaves about the same as ZSTD_compressStream, with additional control on end directive. - * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() - * - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode) - * - output->pos must be <= dstCapacity, input->pos must be <= srcSize - * - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit. - * - endOp must be a valid directive - * - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller. - * - When nbWorkers>=1, function is non-blocking : it copies a portion of input, distributes jobs to internal worker threads, flush to output whatever is available, - * and then immediately returns, just indicating that there is some data remaining to be flushed. - * The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte. - * - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking. - * - @return provides a minimum amount of data remaining to be flushed from internal buffers - * or an error code, which can be tested using ZSTD_isError(). - * if @return != 0, flush is not fully completed, there is still some data left within internal buffers. - * This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers. - * For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed. - * - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0), - * only ZSTD_e_end or ZSTD_e_flush operations are allowed. - * Before starting a new compression job, or changing compression parameters, - * it is required to fully flush internal buffers. - */ -ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, - ZSTD_outBuffer* output, - ZSTD_inBuffer* input, - ZSTD_EndDirective endOp); + ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize` bytes. + ZSTD_decompressContinue() is very sensitive to contiguity, + if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place, + or that previous contiguous segment is large enough to properly handle maximum back-reference distance. + There are multiple ways to guarantee this condition. + The most memory efficient way is to use a round buffer of sufficient size. + Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(), + which can return an error code if required value is too large for current system (in 32-bits mode). + In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one, + up to the moment there is not enough room left in the buffer to guarantee decoding another full block, + which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`. + At which point, decoding can resume from the beginning of the buffer. + Note that already decoded data stored in the buffer should be flushed before being overwritten. -/* These buffer sizes are softly recommended. - * They are not required : ZSTD_compressStream*() happily accepts any buffer size, for both input and output. - * Respecting the recommended size just makes it a bit easier for ZSTD_compressStream*(), - * reducing the amount of memory shuffling and buffering, resulting in minor performance savings. - * - * However, note that these recommendations are from the perspective of a C caller program. - * If the streaming interface is invoked from some other language, - * especially managed ones such as Java or Go, through a foreign function interface such as jni or cgo, - * a major performance rule is to reduce crossing such interface to an absolute minimum. - * It's not rare that performance ends being spent more into the interface, rather than compression itself. - * In which cases, prefer using large buffers, as large as practical, - * for both input and output, to reduce the nb of roundtrips. - */ -ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ -ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block. */ + There are alternatives possible, for example using two or more buffers of size `windowSize` each, though they consume more memory. + Finally, if you control the compression process, you can also ignore all buffer size rules, + as long as the encoder and decoder progress in "lock-step", + aka use exactly the same buffer sizes, break contiguity at the same place, etc. -/* ***************************************************************************** - * This following is a legacy streaming API. - * It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2(). - * It is redundant, but remains fully supported. - * Advanced parameters and dictionary compression can only be used through the - * new API. - ******************************************************************************/ + Once buffers are setup, start decompression, with ZSTD_decompressBegin(). + If decompression requires a dictionary, use ZSTD_decompressBegin_usingDict() or ZSTD_decompressBegin_usingDDict(). -/*! - * Equivalent to: - * - * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) - * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); - */ -ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); -/*! - * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue). - * NOTE: The return value is different. ZSTD_compressStream() returns a hint for - * the next read size (if non-zero and not an error). ZSTD_compressStream2() - * returns the minimum nb of bytes left to flush (if non-zero and not an error). - */ -ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); -/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */ -ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); -/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */ -ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); + Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively. + ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue(). + ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail. + result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). + It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item. + It can also be an error code, which can be tested with ZSTD_isError(). -/*-*************************************************************************** -* Streaming decompression - HowTo -* -* A ZSTD_DStream object is required to track streaming operations. -* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. -* ZSTD_DStream objects can be re-used multiple times. -* -* Use ZSTD_initDStream() to start a new decompression operation. -* @return : recommended first input size -* Alternatively, use advanced API to set specific properties. -* -* Use ZSTD_decompressStream() repetitively to consume your input. -* The function will update both `pos` fields. -* If `input.pos < input.size`, some input has not been consumed. -* It's up to the caller to present again remaining data. -* The function tries to flush all data decoded immediately, respecting output buffer size. -* If `output.pos < output.size`, decoder has flushed everything it could. -* But if `output.pos == output.size`, there might be some data left within internal buffers., -* In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer. -* Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX. -* @return : 0 when a frame is completely decoded and fully flushed, -* or an error code, which can be tested using ZSTD_isError(), -* or any other value > 0, which means there is still some decoding or flushing to do to complete current frame : -* the return value is a suggested next input size (just a hint for better latency) -* that will never request more than the remaining frame size. -* *******************************************************************************/ + A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. + Context can then be reset to start a new decompression. -typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */ - /* For compatibility with versions <= v1.2.0, prefer differentiating them. */ -/*===== ZSTD_DStream management functions =====*/ -ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void); -ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); + Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType(). + This information is not required to properly decode a frame. -/*===== Streaming decompression functions =====*/ + == Special case : skippable frames == -/* This function is redundant with the advanced API and equivalent to: - * - * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); - * ZSTD_DCtx_refDDict(zds, NULL); - */ -ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); + Skippable frames allow integration of user-defined data into a flow of concatenated frames. + Skippable frames will be ignored (skipped) by decompressor. + The format of skippable frames is as follows : + a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F + b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits + c) Frame Content - any content (User Data) of length equal to Frame Size + For skippable frames ZSTD_getFrameHeader() returns zfhPtr->frameType==ZSTD_skippableFrame. + For skippable frames ZSTD_decompressContinue() always returns 0 : it only skips the content. +*/ -ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); +/*===== Buffer-less streaming decompression functions =====*/ -ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ -ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */ +ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */ +ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); +ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); +ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); -/************************** -* Simple dictionary API -***************************/ -/*! ZSTD_compress_usingDict() : - * Compression at an explicit compression level using a Dictionary. - * A dictionary can be any arbitrary data segment (also called a prefix), - * or a buffer with specified information (see dictBuilder/zdict.h). - * Note : This function loads the dictionary, resulting in significant startup delay. - * It's intended for a dictionary used only once. - * Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */ -ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize, - int compressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); +ZSTDLIB_STATIC_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -/*! ZSTD_decompress_usingDict() : - * Decompression using a known Dictionary. - * Dictionary must be identical to the one used during compression. - * Note : This function loads the dictionary, resulting in significant startup delay. - * It's intended for a dictionary used only once. - * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */ -ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize); +/* misc */ +ZSTD_DEPRECATED("This function will likely be removed in the next minor release. It is misleading and has very limited utility.") +ZSTDLIB_STATIC_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); +typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; +ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); -/*********************************** - * Bulk processing dictionary API - **********************************/ -typedef struct ZSTD_CDict_s ZSTD_CDict; -/*! ZSTD_createCDict() : - * When compressing multiple messages or blocks using the same dictionary, - * it's recommended to digest the dictionary only once, since it's a costly operation. - * ZSTD_createCDict() will create a state from digesting a dictionary. - * The resulting state can be used for future compression operations with very limited startup cost. - * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. - * @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict. - * Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content. - * Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer, - * in which case the only thing that it transports is the @compressionLevel. - * This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively, - * expecting a ZSTD_CDict parameter with any data, including those without a known dictionary. */ -ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, - int compressionLevel); -/*! ZSTD_freeCDict() : - * Function frees memory allocated by ZSTD_createCDict(). */ -ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); +/* ========================================= */ +/** Block level API (DEPRECATED) */ +/* ========================================= */ -/*! ZSTD_compress_usingCDict() : - * Compression using a digested Dictionary. - * Recommended when same dictionary is used multiple times. - * Note : compression level is _decided at dictionary creation time_, - * and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */ -ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTD_CDict* cdict); +/*! + This API is deprecated in favor of the regular compression API. + You can get the frame header down to 2 bytes by setting: + - ZSTD_c_format = ZSTD_f_zstd1_magicless + - ZSTD_c_contentSizeFlag = 0 + - ZSTD_c_checksumFlag = 0 + - ZSTD_c_dictIDFlag = 0 -typedef struct ZSTD_DDict_s ZSTD_DDict; + This API is not as well tested as our normal API, so we recommend not using it. + We will be removing it in a future version. If the normal API doesn't provide + the functionality you need, please open a GitHub issue. -/*! ZSTD_createDDict() : - * Create a digested dictionary, ready to start decompression operation without startup delay. - * dictBuffer can be released after DDict creation, as its content is copied inside DDict. */ -ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize); + Block functions produce and decode raw zstd blocks, without frame metadata. + Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes). + But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes. -/*! ZSTD_freeDDict() : - * Function frees memory allocated with ZSTD_createDDict() */ -ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict); + A few rules to respect : + - Compressing and decompressing require a context structure + + Use ZSTD_createCCtx() and ZSTD_createDCtx() + - It is necessary to init context before starting + + compression : any ZSTD_compressBegin*() variant, including with dictionary + + decompression : any ZSTD_decompressBegin*() variant, including with dictionary + - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB + + If input is larger than a block size, it's necessary to split input data into multiple blocks + + For inputs larger than a single block, consider using regular ZSTD_compress() instead. + Frame metadata is not that costly, and quickly becomes negligible as source size grows larger than a block. + - When a block is considered not compressible enough, ZSTD_compressBlock() result will be 0 (zero) ! + ===> In which case, nothing is produced into `dst` ! + + User __must__ test for such outcome and deal directly with uncompressed data + + A block cannot be declared incompressible if ZSTD_compressBlock() return value was != 0. + Doing so would mess up with statistics history, leading to potential data corruption. + + ZSTD_decompressBlock() _doesn't accept uncompressed data as input_ !! + + In case of multiple successive blocks, should some of them be uncompressed, + decoder must be informed of their existence in order to follow proper history. + Use ZSTD_insertBlock() for such a case. +*/ -/*! ZSTD_decompress_usingDDict() : - * Decompression using a digested Dictionary. - * Recommended when same dictionary is used multiple times. */ -ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTD_DDict* ddict); +/*===== Raw zstd block functions =====*/ +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx); +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */ + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */ +/**** ended inlining ../zstd.h ****/ +#define FSE_STATIC_LINKING_ONLY +/**** skipping file: fse.h ****/ +/**** skipping file: huf.h ****/ +#ifndef XXH_STATIC_LINKING_ONLY +# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ +#endif +/**** start inlining xxhash.h ****/ +/* + * xxHash - Extremely Fast Hash algorithm + * Header File + * Copyright (c) Yann Collet - Meta Platforms, Inc + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ +/* Local adaptations for Zstandard */ -/******************************** - * Dictionary helper functions - *******************************/ +#ifndef XXH_NO_XXH3 +# define XXH_NO_XXH3 +#endif -/*! ZSTD_getDictID_fromDict() : - * Provides the dictID stored within dictionary. - * if @return == 0, the dictionary is not conformant with Zstandard specification. - * It can still be loaded, but as a content-only dictionary. */ -ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize); +#ifndef XXH_NAMESPACE +# define XXH_NAMESPACE ZSTD_ +#endif -/*! ZSTD_getDictID_fromDDict() : - * Provides the dictID of the dictionary loaded into `ddict`. - * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. - * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ -ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict); +/*! + * @mainpage xxHash + * + * xxHash is an extremely fast non-cryptographic hash algorithm, working at RAM speed + * limits. + * + * It is proposed in four flavors, in three families: + * 1. @ref XXH32_family + * - Classic 32-bit hash function. Simple, compact, and runs on almost all + * 32-bit and 64-bit systems. + * 2. @ref XXH64_family + * - Classic 64-bit adaptation of XXH32. Just as simple, and runs well on most + * 64-bit systems (but _not_ 32-bit systems). + * 3. @ref XXH3_family + * - Modern 64-bit and 128-bit hash function family which features improved + * strength and performance across the board, especially on smaller data. + * It benefits greatly from SIMD and 64-bit without requiring it. + * + * Benchmarks + * --- + * The reference system uses an Intel i7-9700K CPU, and runs Ubuntu x64 20.04. + * The open source benchmark program is compiled with clang v10.0 using -O3 flag. + * + * | Hash Name | ISA ext | Width | Large Data Speed | Small Data Velocity | + * | -------------------- | ------- | ----: | ---------------: | ------------------: | + * | XXH3_64bits() | @b AVX2 | 64 | 59.4 GB/s | 133.1 | + * | MeowHash | AES-NI | 128 | 58.2 GB/s | 52.5 | + * | XXH3_128bits() | @b AVX2 | 128 | 57.9 GB/s | 118.1 | + * | CLHash | PCLMUL | 64 | 37.1 GB/s | 58.1 | + * | XXH3_64bits() | @b SSE2 | 64 | 31.5 GB/s | 133.1 | + * | XXH3_128bits() | @b SSE2 | 128 | 29.6 GB/s | 118.1 | + * | RAM sequential read | | N/A | 28.0 GB/s | N/A | + * | ahash | AES-NI | 64 | 22.5 GB/s | 107.2 | + * | City64 | | 64 | 22.0 GB/s | 76.6 | + * | T1ha2 | | 64 | 22.0 GB/s | 99.0 | + * | City128 | | 128 | 21.7 GB/s | 57.7 | + * | FarmHash | AES-NI | 64 | 21.3 GB/s | 71.9 | + * | XXH64() | | 64 | 19.4 GB/s | 71.0 | + * | SpookyHash | | 64 | 19.3 GB/s | 53.2 | + * | Mum | | 64 | 18.0 GB/s | 67.0 | + * | CRC32C | SSE4.2 | 32 | 13.0 GB/s | 57.9 | + * | XXH32() | | 32 | 9.7 GB/s | 71.9 | + * | City32 | | 32 | 9.1 GB/s | 66.0 | + * | Blake3* | @b AVX2 | 256 | 4.4 GB/s | 8.1 | + * | Murmur3 | | 32 | 3.9 GB/s | 56.1 | + * | SipHash* | | 64 | 3.0 GB/s | 43.2 | + * | Blake3* | @b SSE2 | 256 | 2.4 GB/s | 8.1 | + * | HighwayHash | | 64 | 1.4 GB/s | 6.0 | + * | FNV64 | | 64 | 1.2 GB/s | 62.7 | + * | Blake2* | | 256 | 1.1 GB/s | 5.1 | + * | SHA1* | | 160 | 0.8 GB/s | 5.6 | + * | MD5* | | 128 | 0.6 GB/s | 7.8 | + * @note + * - Hashes which require a specific ISA extension are noted. SSE2 is also noted, + * even though it is mandatory on x64. + * - Hashes with an asterisk are cryptographic. Note that MD5 is non-cryptographic + * by modern standards. + * - Small data velocity is a rough average of algorithm's efficiency for small + * data. For more accurate information, see the wiki. + * - More benchmarks and strength tests are found on the wiki: + * https://github.com/Cyan4973/xxHash/wiki + * + * Usage + * ------ + * All xxHash variants use a similar API. Changing the algorithm is a trivial + * substitution. + * + * @pre + * For functions which take an input and length parameter, the following + * requirements are assumed: + * - The range from [`input`, `input + length`) is valid, readable memory. + * - The only exception is if the `length` is `0`, `input` may be `NULL`. + * - For C++, the objects must have the *TriviallyCopyable* property, as the + * functions access bytes directly as if it was an array of `unsigned char`. + * + * @anchor single_shot_example + * **Single Shot** + * + * These functions are stateless functions which hash a contiguous block of memory, + * immediately returning the result. They are the easiest and usually the fastest + * option. + * + * XXH32(), XXH64(), XXH3_64bits(), XXH3_128bits() + * + * @code{.c} + * #include + * #include "xxhash.h" + * + * // Example for a function which hashes a null terminated string with XXH32(). + * XXH32_hash_t hash_string(const char* string, XXH32_hash_t seed) + * { + * // NULL pointers are only valid if the length is zero + * size_t length = (string == NULL) ? 0 : strlen(string); + * return XXH32(string, length, seed); + * } + * @endcode + * + * + * @anchor streaming_example + * **Streaming** + * + * These groups of functions allow incremental hashing of unknown size, even + * more than what would fit in a size_t. + * + * XXH32_reset(), XXH64_reset(), XXH3_64bits_reset(), XXH3_128bits_reset() + * + * @code{.c} + * #include + * #include + * #include "xxhash.h" + * // Example for a function which hashes a FILE incrementally with XXH3_64bits(). + * XXH64_hash_t hashFile(FILE* f) + * { + * // Allocate a state struct. Do not just use malloc() or new. + * XXH3_state_t* state = XXH3_createState(); + * assert(state != NULL && "Out of memory!"); + * // Reset the state to start a new hashing session. + * XXH3_64bits_reset(state); + * char buffer[4096]; + * size_t count; + * // Read the file in chunks + * while ((count = fread(buffer, 1, sizeof(buffer), f)) != 0) { + * // Run update() as many times as necessary to process the data + * XXH3_64bits_update(state, buffer, count); + * } + * // Retrieve the finalized hash. This will not change the state. + * XXH64_hash_t result = XXH3_64bits_digest(state); + * // Free the state. Do not use free(). + * XXH3_freeState(state); + * return result; + * } + * @endcode + * + * Streaming functions generate the xxHash value from an incremental input. + * This method is slower than single-call functions, due to state management. + * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized. + * + * An XXH state must first be allocated using `XXH*_createState()`. + * + * Start a new hash by initializing the state with a seed using `XXH*_reset()`. + * + * Then, feed the hash state by calling `XXH*_update()` as many times as necessary. + * + * The function returns an error code, with 0 meaning OK, and any other value + * meaning there is an error. + * + * Finally, a hash value can be produced anytime, by using `XXH*_digest()`. + * This function returns the nn-bits hash as an int or long long. + * + * It's still possible to continue inserting input into the hash state after a + * digest, and generate new hash values later on by invoking `XXH*_digest()`. + * + * When done, release the state using `XXH*_freeState()`. + * + * + * @anchor canonical_representation_example + * **Canonical Representation** + * + * The default return values from XXH functions are unsigned 32, 64 and 128 bit + * integers. + * This the simplest and fastest format for further post-processing. + * + * However, this leaves open the question of what is the order on the byte level, + * since little and big endian conventions will store the same number differently. + * + * The canonical representation settles this issue by mandating big-endian + * convention, the same convention as human-readable numbers (large digits first). + * + * When writing hash values to storage, sending them over a network, or printing + * them, it's highly recommended to use the canonical representation to ensure + * portability across a wider range of systems, present and future. + * + * The following functions allow transformation of hash values to and from + * canonical format. + * + * XXH32_canonicalFromHash(), XXH32_hashFromCanonical(), + * XXH64_canonicalFromHash(), XXH64_hashFromCanonical(), + * XXH128_canonicalFromHash(), XXH128_hashFromCanonical(), + * + * @code{.c} + * #include + * #include "xxhash.h" + * + * // Example for a function which prints XXH32_hash_t in human readable format + * void printXxh32(XXH32_hash_t hash) + * { + * XXH32_canonical_t cano; + * XXH32_canonicalFromHash(&cano, hash); + * size_t i; + * for(i = 0; i < sizeof(cano.digest); ++i) { + * printf("%02x", cano.digest[i]); + * } + * printf("\n"); + * } + * + * // Example for a function which converts XXH32_canonical_t to XXH32_hash_t + * XXH32_hash_t convertCanonicalToXxh32(XXH32_canonical_t cano) + * { + * XXH32_hash_t hash = XXH32_hashFromCanonical(&cano); + * return hash; + * } + * @endcode + * + * + * @file xxhash.h + * xxHash prototypes and implementation + */ -/*! ZSTD_getDictID_fromFrame() : - * Provides the dictID required to decompressed the frame stored within `src`. - * If @return == 0, the dictID could not be decoded. - * This could for one of the following reasons : - * - The frame does not require a dictionary to be decoded (most common case). - * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information. - * Note : this use case also happens when using a non-conformant dictionary. - * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). - * - This is not a Zstandard frame. - * When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */ -ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); +/* **************************** + * INLINE mode + ******************************/ +/*! + * @defgroup public Public API + * Contains details on the public xxHash functions. + * @{ + */ +#ifdef XXH_DOXYGEN +/*! + * @brief Gives access to internal state declaration, required for static allocation. + * + * Incompatible with dynamic linking, due to risks of ABI changes. + * + * Usage: + * @code{.c} + * #define XXH_STATIC_LINKING_ONLY + * #include "xxhash.h" + * @endcode + */ +# define XXH_STATIC_LINKING_ONLY +/* Do not undef XXH_STATIC_LINKING_ONLY for Doxygen */ +/*! + * @brief Gives access to internal definitions. + * + * Usage: + * @code{.c} + * #define XXH_STATIC_LINKING_ONLY + * #define XXH_IMPLEMENTATION + * #include "xxhash.h" + * @endcode + */ +# define XXH_IMPLEMENTATION +/* Do not undef XXH_IMPLEMENTATION for Doxygen */ -/******************************************************************************* - * Advanced dictionary and prefix API +/*! + * @brief Exposes the implementation and marks all functions as `inline`. * - * This API allows dictionaries to be used with ZSTD_compress2(), - * ZSTD_compressStream2(), and ZSTD_decompress(). Dictionaries are sticky, and - * only reset with the context is reset with ZSTD_reset_parameters or - * ZSTD_reset_session_and_parameters. Prefixes are single-use. - ******************************************************************************/ + * Use these build macros to inline xxhash into the target unit. + * Inlining improves performance on small inputs, especially when the length is + * expressed as a compile-time constant: + * + * https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html + * + * It also keeps xxHash symbols private to the unit, so they are not exported. + * + * Usage: + * @code{.c} + * #define XXH_INLINE_ALL + * #include "xxhash.h" + * @endcode + * Do not compile and link xxhash.o as a separate object, as it is not useful. + */ +# define XXH_INLINE_ALL +# undef XXH_INLINE_ALL +/*! + * @brief Exposes the implementation without marking functions as inline. + */ +# define XXH_PRIVATE_API +# undef XXH_PRIVATE_API +/*! + * @brief Emulate a namespace by transparently prefixing all symbols. + * + * If you want to include _and expose_ xxHash functions from within your own + * library, but also want to avoid symbol collisions with other libraries which + * may also include xxHash, you can use @ref XXH_NAMESPACE to automatically prefix + * any public symbol from xxhash library with the value of @ref XXH_NAMESPACE + * (therefore, avoid empty or numeric values). + * + * Note that no change is required within the calling program as long as it + * includes `xxhash.h`: Regular symbol names will be automatically translated + * by this header. + */ +# define XXH_NAMESPACE /* YOUR NAME HERE */ +# undef XXH_NAMESPACE +#endif +#if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) \ + && !defined(XXH_INLINE_ALL_31684351384) + /* this section should be traversed only once */ +# define XXH_INLINE_ALL_31684351384 + /* give access to the advanced API, required to compile implementations */ +# undef XXH_STATIC_LINKING_ONLY /* avoid macro redef */ +# define XXH_STATIC_LINKING_ONLY + /* make all functions private */ +# undef XXH_PUBLIC_API +# if defined(__GNUC__) +# define XXH_PUBLIC_API static __inline __attribute__((unused)) +# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define XXH_PUBLIC_API static inline +# elif defined(_MSC_VER) +# define XXH_PUBLIC_API static __inline +# else + /* note: this version may generate warnings for unused static functions */ +# define XXH_PUBLIC_API static +# endif -/*! ZSTD_CCtx_loadDictionary() : - * Create an internal CDict from `dict` buffer. - * Decompression will have to use same dictionary. - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary, - * meaning "return to no-dictionary mode". - * Note 1 : Dictionary is sticky, it will be used for all future compressed frames. - * To return to "no-dictionary" situation, load a NULL dictionary (or reset parameters). - * Note 2 : Loading a dictionary involves building tables. - * It's also a CPU consuming operation, with non-negligible impact on latency. - * Tables are dependent on compression parameters, and for this reason, - * compression parameters can no longer be changed after loading a dictionary. - * Note 3 :`dict` content will be copied internally. - * Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead. - * In such a case, dictionary buffer must outlive its users. - * Note 4 : Use ZSTD_CCtx_loadDictionary_advanced() - * to precisely select how dictionary content must be interpreted. */ -ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); + /* + * This part deals with the special case where a unit wants to inline xxHash, + * but "xxhash.h" has previously been included without XXH_INLINE_ALL, + * such as part of some previously included *.h header file. + * Without further action, the new include would just be ignored, + * and functions would effectively _not_ be inlined (silent failure). + * The following macros solve this situation by prefixing all inlined names, + * avoiding naming collision with previous inclusions. + */ + /* Before that, we unconditionally #undef all symbols, + * in case they were already defined with XXH_NAMESPACE. + * They will then be redefined for XXH_INLINE_ALL + */ +# undef XXH_versionNumber + /* XXH32 */ +# undef XXH32 +# undef XXH32_createState +# undef XXH32_freeState +# undef XXH32_reset +# undef XXH32_update +# undef XXH32_digest +# undef XXH32_copyState +# undef XXH32_canonicalFromHash +# undef XXH32_hashFromCanonical + /* XXH64 */ +# undef XXH64 +# undef XXH64_createState +# undef XXH64_freeState +# undef XXH64_reset +# undef XXH64_update +# undef XXH64_digest +# undef XXH64_copyState +# undef XXH64_canonicalFromHash +# undef XXH64_hashFromCanonical + /* XXH3_64bits */ +# undef XXH3_64bits +# undef XXH3_64bits_withSecret +# undef XXH3_64bits_withSeed +# undef XXH3_64bits_withSecretandSeed +# undef XXH3_createState +# undef XXH3_freeState +# undef XXH3_copyState +# undef XXH3_64bits_reset +# undef XXH3_64bits_reset_withSeed +# undef XXH3_64bits_reset_withSecret +# undef XXH3_64bits_update +# undef XXH3_64bits_digest +# undef XXH3_generateSecret + /* XXH3_128bits */ +# undef XXH128 +# undef XXH3_128bits +# undef XXH3_128bits_withSeed +# undef XXH3_128bits_withSecret +# undef XXH3_128bits_reset +# undef XXH3_128bits_reset_withSeed +# undef XXH3_128bits_reset_withSecret +# undef XXH3_128bits_reset_withSecretandSeed +# undef XXH3_128bits_update +# undef XXH3_128bits_digest +# undef XXH128_isEqual +# undef XXH128_cmp +# undef XXH128_canonicalFromHash +# undef XXH128_hashFromCanonical + /* Finally, free the namespace itself */ +# undef XXH_NAMESPACE + + /* employ the namespace for XXH_INLINE_ALL */ +# define XXH_NAMESPACE XXH_INLINE_ + /* + * Some identifiers (enums, type names) are not symbols, + * but they must nonetheless be renamed to avoid redeclaration. + * Alternative solution: do not redeclare them. + * However, this requires some #ifdefs, and has a more dispersed impact. + * Meanwhile, renaming can be achieved in a single place. + */ +# define XXH_IPREF(Id) XXH_NAMESPACE ## Id +# define XXH_OK XXH_IPREF(XXH_OK) +# define XXH_ERROR XXH_IPREF(XXH_ERROR) +# define XXH_errorcode XXH_IPREF(XXH_errorcode) +# define XXH32_canonical_t XXH_IPREF(XXH32_canonical_t) +# define XXH64_canonical_t XXH_IPREF(XXH64_canonical_t) +# define XXH128_canonical_t XXH_IPREF(XXH128_canonical_t) +# define XXH32_state_s XXH_IPREF(XXH32_state_s) +# define XXH32_state_t XXH_IPREF(XXH32_state_t) +# define XXH64_state_s XXH_IPREF(XXH64_state_s) +# define XXH64_state_t XXH_IPREF(XXH64_state_t) +# define XXH3_state_s XXH_IPREF(XXH3_state_s) +# define XXH3_state_t XXH_IPREF(XXH3_state_t) +# define XXH128_hash_t XXH_IPREF(XXH128_hash_t) + /* Ensure the header is parsed again, even if it was previously included */ +# undef XXHASH_H_5627135585666179 +# undef XXHASH_H_STATIC_13879238742 +#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */ + +/* **************************************************************** + * Stable API + *****************************************************************/ +#ifndef XXHASH_H_5627135585666179 +#define XXHASH_H_5627135585666179 1 -/*! ZSTD_CCtx_refCDict() : - * Reference a prepared dictionary, to be used for all next compressed frames. - * Note that compression parameters are enforced from within CDict, - * and supersede any compression parameter previously set within CCtx. - * The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs. - * The ignored parameters will be used again if the CCtx is returned to no-dictionary mode. - * The dictionary will remain valid for future compressed frames using same CCtx. - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Special : Referencing a NULL CDict means "return to no-dictionary mode". - * Note 1 : Currently, only one dictionary can be managed. - * Referencing a new dictionary effectively "discards" any previous one. - * Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */ -ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); +/*! @brief Marks a global symbol. */ +#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) +# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) +# ifdef XXH_EXPORT +# define XXH_PUBLIC_API __declspec(dllexport) +# elif XXH_IMPORT +# define XXH_PUBLIC_API __declspec(dllimport) +# endif +# else +# define XXH_PUBLIC_API /* do nothing */ +# endif +#endif -/*! ZSTD_CCtx_refPrefix() : - * Reference a prefix (single-usage dictionary) for next compressed frame. - * A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end). - * Decompression will need same prefix to properly regenerate data. - * Compressing with a prefix is similar in outcome as performing a diff and compressing it, - * but performs much faster, especially during decompression (compression speed is tunable with compression level). - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary - * Note 1 : Prefix buffer is referenced. It **must** outlive compression. - * Its content must remain unmodified during compression. - * Note 2 : If the intention is to diff some large src data blob with some prior version of itself, - * ensure that the window size is large enough to contain the entire source. - * See ZSTD_c_windowLog. - * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters. - * It's a CPU consuming operation, with non-negligible impact on latency. - * If there is a need to use the same prefix multiple times, consider loadDictionary instead. - * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent). - * Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */ -ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, - const void* prefix, size_t prefixSize); +#ifdef XXH_NAMESPACE +# define XXH_CAT(A,B) A##B +# define XXH_NAME2(A,B) XXH_CAT(A,B) +# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) +/* XXH32 */ +# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) +# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) +# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) +# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) +# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) +# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) +# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) +# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) +# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) +/* XXH64 */ +# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) +# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) +# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) +# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) +# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) +# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) +# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) +# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) +# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) +/* XXH3_64bits */ +# define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits) +# define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret) +# define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed) +# define XXH3_64bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecretandSeed) +# define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState) +# define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState) +# define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState) +# define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset) +# define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed) +# define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret) +# define XXH3_64bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecretandSeed) +# define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update) +# define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest) +# define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret) +# define XXH3_generateSecret_fromSeed XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret_fromSeed) +/* XXH3_128bits */ +# define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128) +# define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits) +# define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed) +# define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret) +# define XXH3_128bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecretandSeed) +# define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset) +# define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed) +# define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret) +# define XXH3_128bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecretandSeed) +# define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update) +# define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest) +# define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual) +# define XXH128_cmp XXH_NAME2(XXH_NAMESPACE, XXH128_cmp) +# define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash) +# define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical) +#endif -/*! ZSTD_DCtx_loadDictionary() : - * Create an internal DDict from dict buffer, - * to be used to decompress next frames. - * The dictionary remains valid for all future frames, until explicitly invalidated. - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary, - * meaning "return to no-dictionary mode". - * Note 1 : Loading a dictionary involves building tables, - * which has a non-negligible impact on CPU usage and latency. - * It's recommended to "load once, use many times", to amortize the cost - * Note 2 :`dict` content will be copied internally, so `dict` can be released after loading. - * Use ZSTD_DCtx_loadDictionary_byReference() to reference dictionary content instead. - * Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to take control of - * how dictionary content is loaded and interpreted. - */ -ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); -/*! ZSTD_DCtx_refDDict() : - * Reference a prepared dictionary, to be used to decompress next frames. - * The dictionary remains active for decompression of future frames using same DCtx. +/* ************************************* +* Compiler specifics +***************************************/ + +/* specific declaration modes for Windows */ +#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) +# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) +# ifdef XXH_EXPORT +# define XXH_PUBLIC_API __declspec(dllexport) +# elif XXH_IMPORT +# define XXH_PUBLIC_API __declspec(dllimport) +# endif +# else +# define XXH_PUBLIC_API /* do nothing */ +# endif +#endif + +#if defined (__GNUC__) +# define XXH_CONSTF __attribute__((const)) +# define XXH_PUREF __attribute__((pure)) +# define XXH_MALLOCF __attribute__((malloc)) +#else +# define XXH_CONSTF /* disable */ +# define XXH_PUREF +# define XXH_MALLOCF +#endif + +/* ************************************* +* Version +***************************************/ +#define XXH_VERSION_MAJOR 0 +#define XXH_VERSION_MINOR 8 +#define XXH_VERSION_RELEASE 2 +/*! @brief Version number, encoded as two digits each */ +#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) + +#if defined (__cplusplus) +extern "C" { +#endif +/*! + * @brief Obtains the xxHash version. * - * If called with ZSTD_d_refMultipleDDicts enabled, repeated calls of this function - * will store the DDict references in a table, and the DDict used for decompression - * will be determined at decompression time, as per the dict ID in the frame. - * The memory for the table is allocated on the first call to refDDict, and can be - * freed with ZSTD_freeDCtx(). + * This is mostly useful when xxHash is compiled as a shared library, + * since the returned value comes from the library, as opposed to header file. * - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Note 1 : Currently, only one dictionary can be managed. - * Referencing a new dictionary effectively "discards" any previous one. - * Special: referencing a NULL DDict means "return to no-dictionary mode". - * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx. - */ -ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); - -/*! ZSTD_DCtx_refPrefix() : - * Reference a prefix (single-usage dictionary) to decompress next frame. - * This is the reverse operation of ZSTD_CCtx_refPrefix(), - * and must use the same prefix as the one used during compression. - * Prefix is **only used once**. Reference is discarded at end of frame. - * End of frame is reached when ZSTD_decompressStream() returns 0. - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary - * Note 2 : Prefix buffer is referenced. It **must** outlive decompression. - * Prefix buffer must remain unmodified up to the end of frame, - * reached when ZSTD_decompressStream() returns 0. - * Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent). - * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section) - * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost. - * A full dictionary is more costly, as it requires building tables. + * @return @ref XXH_VERSION_NUMBER of the invoked library. */ -ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, - const void* prefix, size_t prefixSize); +XXH_PUBLIC_API XXH_CONSTF unsigned XXH_versionNumber (void); -/* === Memory management === */ - -/*! ZSTD_sizeof_*() : - * These functions give the _current_ memory usage of selected object. - * Note that object memory usage can evolve (increase or decrease) over time. */ -ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); -ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); -ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); -ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); -ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict); -ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); +#if defined (__cplusplus) +} +#endif -#endif /* ZSTD_H_235446 */ +/* **************************** +* Common basic types +******************************/ +#include /* size_t */ +/*! + * @brief Exit code for the streaming API. + */ +typedef enum { + XXH_OK = 0, /*!< OK */ + XXH_ERROR /*!< Error */ +} XXH_errorcode; -/* ************************************************************************************** - * ADVANCED AND EXPERIMENTAL FUNCTIONS - **************************************************************************************** - * The definitions in the following section are considered experimental. - * They are provided for advanced scenarios. - * They should never be used with a dynamic library, as prototypes may change in the future. - * Use them only in association with static linking. - * ***************************************************************************************/ +/*-********************************************************************** +* 32-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* Don't show include */ +/*! + * @brief An unsigned 32-bit integer. + * + * Not necessarily defined to `uint32_t` but functionally equivalent. + */ +typedef uint32_t XXH32_hash_t; + +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# ifdef _AIX +# include +# else +# include +# endif + typedef uint32_t XXH32_hash_t; -#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY) -#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY +#else +# include +# if UINT_MAX == 0xFFFFFFFFUL + typedef unsigned int XXH32_hash_t; +# elif ULONG_MAX == 0xFFFFFFFFUL + typedef unsigned long XXH32_hash_t; +# else +# error "unsupported platform: need a 32-bit type" +# endif +#endif -/**************************************************************************************** - * experimental API (static linking only) - **************************************************************************************** - * The following symbols and constants - * are not planned to join "stable API" status in the near future. - * They can still change in future versions. - * Some of them are planned to remain in the static_only section indefinitely. - * Some of them might be removed in the future (especially when redundant with existing stable functions) - * ***************************************************************************************/ +#if defined (__cplusplus) +extern "C" { +#endif -#define ZSTD_FRAMEHEADERSIZE_PREFIX(format) ((format) == ZSTD_f_zstd1 ? 5 : 1) /* minimum input size required to query frame header size */ -#define ZSTD_FRAMEHEADERSIZE_MIN(format) ((format) == ZSTD_f_zstd1 ? 6 : 2) -#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */ -#define ZSTD_SKIPPABLEHEADERSIZE 8 +/*! + * @} + * + * @defgroup XXH32_family XXH32 family + * @ingroup public + * Contains functions used in the classic 32-bit xxHash algorithm. + * + * @note + * XXH32 is useful for older platforms, with no or poor 64-bit performance. + * Note that the @ref XXH3_family provides competitive speed for both 32-bit + * and 64-bit systems, and offers true 64/128 bit hash results. + * + * @see @ref XXH64_family, @ref XXH3_family : Other xxHash families + * @see @ref XXH32_impl for implementation details + * @{ + */ -/* compression parameter bounds */ -#define ZSTD_WINDOWLOG_MAX_32 30 -#define ZSTD_WINDOWLOG_MAX_64 31 -#define ZSTD_WINDOWLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64)) -#define ZSTD_WINDOWLOG_MIN 10 -#define ZSTD_HASHLOG_MAX ((ZSTD_WINDOWLOG_MAX < 30) ? ZSTD_WINDOWLOG_MAX : 30) -#define ZSTD_HASHLOG_MIN 6 -#define ZSTD_CHAINLOG_MAX_32 29 -#define ZSTD_CHAINLOG_MAX_64 30 -#define ZSTD_CHAINLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_CHAINLOG_MAX_32 : ZSTD_CHAINLOG_MAX_64)) -#define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN -#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1) -#define ZSTD_SEARCHLOG_MIN 1 -#define ZSTD_MINMATCH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */ -#define ZSTD_MINMATCH_MIN 3 /* only for ZSTD_btopt+, faster strategies are limited to 4 */ -#define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX -#define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */ -#define ZSTD_STRATEGY_MIN ZSTD_fast -#define ZSTD_STRATEGY_MAX ZSTD_btultra2 +/*! + * @brief Calculates the 32-bit hash of @p input using xxHash32. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 32-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 32-bit xxHash32 value. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32 (const void* input, size_t length, XXH32_hash_t seed); +#ifndef XXH_NO_STREAM +/*! + * @typedef struct XXH32_state_s XXH32_state_t + * @brief The opaque state struct for the XXH32 streaming API. + * + * @see XXH32_state_s for details. + */ +typedef struct XXH32_state_s XXH32_state_t; -#define ZSTD_OVERLAPLOG_MIN 0 -#define ZSTD_OVERLAPLOG_MAX 9 +/*! + * @brief Allocates an @ref XXH32_state_t. + * + * @return An allocated pointer of @ref XXH32_state_t on success. + * @return `NULL` on failure. + * + * @note Must be freed with XXH32_freeState(). + */ +XXH_PUBLIC_API XXH_MALLOCF XXH32_state_t* XXH32_createState(void); +/*! + * @brief Frees an @ref XXH32_state_t. + * + * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref XXH32_createState(). + * + * @return @ref XXH_OK. + * + * @note @p statePtr must be allocated with XXH32_createState(). + * + */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); +/*! + * @brief Copies one @ref XXH32_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state); -#define ZSTD_WINDOWLOG_LIMIT_DEFAULT 27 /* by default, the streaming decoder will refuse any frame - * requiring larger than (1<= XXH_C23_VN) && defined(__has_c_attribute) +# define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) +#else +# define XXH_HAS_C_ATTRIBUTE(x) 0 +#endif +/*! @endcond */ - unsigned int rep; /* Represents which repeat offset is represented by the field 'offset'. - * Ranges from [0, 3]. - * - * Repeat offsets are essentially previous offsets from previous sequences sorted in - * recency order. For more detail, see doc/zstd_compression_format.md - * - * If rep == 0, then 'offset' does not contain a repeat offset. - * If rep > 0: - * If litLength != 0: - * rep == 1 --> offset == repeat_offset_1 - * rep == 2 --> offset == repeat_offset_2 - * rep == 3 --> offset == repeat_offset_3 - * If litLength == 0: - * rep == 1 --> offset == repeat_offset_2 - * rep == 2 --> offset == repeat_offset_3 - * rep == 3 --> offset == repeat_offset_1 - 1 - * - * Note: This field is optional. ZSTD_generateSequences() will calculate the value of - * 'rep', but repeat offsets do not necessarily need to be calculated from an external - * sequence provider's perspective. For example, ZSTD_compressSequences() does not - * use this 'rep' field at all (as of now). - */ -} ZSTD_Sequence; +/*! @cond Doxygen ignores this part */ +#if defined(__cplusplus) && defined(__has_cpp_attribute) +# define XXH_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) +#else +# define XXH_HAS_CPP_ATTRIBUTE(x) 0 +#endif +/*! @endcond */ -typedef struct { - unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */ - unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */ - unsigned hashLog; /**< dispatch table : larger == faster, more memory */ - unsigned searchLog; /**< nb of searches : larger == more compression, slower */ - unsigned minMatch; /**< match length searched : larger == faster decompression, sometimes less compression */ - unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */ - ZSTD_strategy strategy; /**< see ZSTD_strategy definition above */ -} ZSTD_compressionParameters; +/*! @cond Doxygen ignores this part */ +/* + * Define XXH_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute + * introduced in CPP17 and C23. + * CPP17 : https://en.cppreference.com/w/cpp/language/attributes/fallthrough + * C23 : https://en.cppreference.com/w/c/language/attributes/fallthrough + */ +#if XXH_HAS_C_ATTRIBUTE(fallthrough) || XXH_HAS_CPP_ATTRIBUTE(fallthrough) +# define XXH_FALLTHROUGH [[fallthrough]] +#elif XXH_HAS_ATTRIBUTE(__fallthrough__) +# define XXH_FALLTHROUGH __attribute__ ((__fallthrough__)) +#else +# define XXH_FALLTHROUGH /* fallthrough */ +#endif +/*! @endcond */ -typedef struct { - int contentSizeFlag; /**< 1: content size will be in frame header (when known) */ - int checksumFlag; /**< 1: generate a 32-bits checksum using XXH64 algorithm at end of frame, for error detection */ - int noDictIDFlag; /**< 1: no dictID will be saved into frame header (dictID is only useful for dictionary compression) */ -} ZSTD_frameParameters; +/*! @cond Doxygen ignores this part */ +/* + * Define XXH_NOESCAPE for annotated pointers in public API. + * https://clang.llvm.org/docs/AttributeReference.html#noescape + * As of writing this, only supported by clang. + */ +#if XXH_HAS_ATTRIBUTE(noescape) +# define XXH_NOESCAPE __attribute__((noescape)) +#else +# define XXH_NOESCAPE +#endif +/*! @endcond */ -typedef struct { - ZSTD_compressionParameters cParams; - ZSTD_frameParameters fParams; -} ZSTD_parameters; +#if defined (__cplusplus) +} /* end of extern "C" */ +#endif -typedef enum { - ZSTD_dct_auto = 0, /* dictionary is "full" when starting with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */ - ZSTD_dct_rawContent = 1, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */ - ZSTD_dct_fullDict = 2 /* refuses to load a dictionary if it does not respect Zstandard's specification, starting with ZSTD_MAGIC_DICTIONARY */ -} ZSTD_dictContentType_e; +/*! + * @} + * @ingroup public + * @{ + */ -typedef enum { - ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */ - ZSTD_dlm_byRef = 1 /**< Reference dictionary content -- the dictionary buffer must outlive its users. */ -} ZSTD_dictLoadMethod_e; +#ifndef XXH_NO_LONG_LONG +/*-********************************************************************** +* 64-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* don't include */ +/*! + * @brief An unsigned 64-bit integer. + * + * Not necessarily defined to `uint64_t` but functionally equivalent. + */ +typedef uint64_t XXH64_hash_t; +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# ifdef _AIX +# include +# else +# include +# endif + typedef uint64_t XXH64_hash_t; +#else +# include +# if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL + /* LP64 ABI says uint64_t is unsigned long */ + typedef unsigned long XXH64_hash_t; +# else + /* the following type must have a width of 64-bit */ + typedef unsigned long long XXH64_hash_t; +# endif +#endif -typedef enum { - ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */ - ZSTD_f_zstd1_magicless = 1 /* Variant of zstd frame format, without initial 4-bytes magic number. - * Useful to save 4 bytes per generated frame. - * Decoder cannot recognise automatically this format, requiring this instruction. */ -} ZSTD_format_e; +#if defined (__cplusplus) +extern "C" { +#endif +/*! + * @} + * + * @defgroup XXH64_family XXH64 family + * @ingroup public + * @{ + * Contains functions used in the classic 64-bit xxHash algorithm. + * + * @note + * XXH3 provides competitive speed for both 32-bit and 64-bit systems, + * and offers true 64/128 bit hash results. + * It provides better speed for systems with vector processing capabilities. + */ -typedef enum { - /* Note: this enum controls ZSTD_d_forceIgnoreChecksum */ - ZSTD_d_validateChecksum = 0, - ZSTD_d_ignoreChecksum = 1 -} ZSTD_forceIgnoreChecksum_e; +/*! + * @brief Calculates the 64-bit hash of @p input using xxHash64. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 64-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit xxHash64 value. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed); -typedef enum { - /* Note: this enum controls ZSTD_d_refMultipleDDicts */ - ZSTD_rmd_refSingleDDict = 0, - ZSTD_rmd_refMultipleDDicts = 1 -} ZSTD_refMultipleDDicts_e; +/******* Streaming *******/ +#ifndef XXH_NO_STREAM +/*! + * @brief The opaque state struct for the XXH64 streaming API. + * + * @see XXH64_state_s for details. + */ +typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ -typedef enum { - /* Note: this enum and the behavior it controls are effectively internal - * implementation details of the compressor. They are expected to continue - * to evolve and should be considered only in the context of extremely - * advanced performance tuning. - * - * Zstd currently supports the use of a CDict in three ways: - * - * - The contents of the CDict can be copied into the working context. This - * means that the compression can search both the dictionary and input - * while operating on a single set of internal tables. This makes - * the compression faster per-byte of input. However, the initial copy of - * the CDict's tables incurs a fixed cost at the beginning of the - * compression. For small compressions (< 8 KB), that copy can dominate - * the cost of the compression. - * - * - The CDict's tables can be used in-place. In this model, compression is - * slower per input byte, because the compressor has to search two sets of - * tables. However, this model incurs no start-up cost (as long as the - * working context's tables can be reused). For small inputs, this can be - * faster than copying the CDict's tables. - * - * - The CDict's tables are not used at all, and instead we use the working - * context alone to reload the dictionary and use params based on the source - * size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict(). - * This method is effective when the dictionary sizes are very small relative - * to the input size, and the input size is fairly large to begin with. - * - * Zstd has a simple internal heuristic that selects which strategy to use - * at the beginning of a compression. However, if experimentation shows that - * Zstd is making poor choices, it is possible to override that choice with - * this enum. - */ - ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */ - ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */ - ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */ - ZSTD_dictForceLoad = 3 /* Always reload the dictionary */ -} ZSTD_dictAttachPref_e; +/*! + * @brief Allocates an @ref XXH64_state_t. + * + * @return An allocated pointer of @ref XXH64_state_t on success. + * @return `NULL` on failure. + * + * @note Must be freed with XXH64_freeState(). + */ +XXH_PUBLIC_API XXH_MALLOCF XXH64_state_t* XXH64_createState(void); -typedef enum { - ZSTD_lcm_auto = 0, /**< Automatically determine the compression mode based on the compression level. - * Negative compression levels will be uncompressed, and positive compression - * levels will be compressed. */ - ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be - * emitted if Huffman compression is not profitable. */ - ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */ -} ZSTD_literalCompressionMode_e; +/*! + * @brief Frees an @ref XXH64_state_t. + * + * @param statePtr A pointer to an @ref XXH64_state_t allocated with @ref XXH64_createState(). + * + * @return @ref XXH_OK. + * + * @note @p statePtr must be allocated with XXH64_createState(). + */ +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); +/*! + * @brief Copies one @ref XXH64_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dst_state, const XXH64_state_t* src_state); -/*************************************** -* Frame size functions -***************************************/ +/*! + * @brief Resets an @ref XXH64_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note This function resets and seeds a state. Call it before @ref XXH64_update(). + */ +XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed); -/*! ZSTD_findDecompressedSize() : - * `src` should point to the start of a series of ZSTD encoded and/or skippable frames - * `srcSize` must be the _exact_ size of this series - * (i.e. there should be a frame boundary at `src + srcSize`) - * @return : - decompressed size of all data in all successive frames - * - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN - * - if an error occurred: ZSTD_CONTENTSIZE_ERROR +/*! + * @brief Consumes a block of @p input to an @ref XXH64_state_t. * - * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. - * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. - * In which case, it's necessary to use streaming mode to decompress data. - * note 2 : decompressed size is always present when compression is done with ZSTD_compress() - * note 3 : decompressed size can be very large (64-bits value), - * potentially larger than what local system can handle as a single memory segment. - * In which case, it's necessary to use streaming mode to decompress data. - * note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified. - * Always ensure result fits within application's authorized limits. - * Each application can set its own limits. - * note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to - * read each contained frame header. This is fast as most of the data is skipped, - * however it does mean that all frame data must be present and valid. */ -ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note Call this to incrementally consume blocks of data. + */ +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH_NOESCAPE XXH64_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); -/*! ZSTD_decompressBound() : - * `src` should point to the start of a series of ZSTD encoded and/or skippable frames - * `srcSize` must be the _exact_ size of this series - * (i.e. there should be a frame boundary at `src + srcSize`) - * @return : - upper-bound for the decompressed size of all data in all successive frames - * - if an error occurred: ZSTD_CONTENTSIZE_ERROR +/*! + * @brief Returns the calculated hash value from an @ref XXH64_state_t. * - * note 1 : an error can occur if `src` contains an invalid or incorrectly formatted frame. - * note 2 : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`. - * in this case, `ZSTD_findDecompressedSize` and `ZSTD_decompressBound` return the same value. - * note 3 : when the decompressed size field isn't available, the upper-bound for that frame is calculated by: - * upper-bound = # blocks * min(128 KB, Window_Size) + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated 64-bit xxHash64 value from that state. + * + * @note + * Calling XXH64_digest() will not affect @p statePtr, so you can update, + * digest, and update again. */ -ZSTDLIB_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize); +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_digest (XXH_NOESCAPE const XXH64_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ +/******* Canonical representation *******/ -/*! ZSTD_frameHeaderSize() : - * srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX. - * @return : size of the Frame Header, - * or an error code (if srcSize is too small) */ -ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); +/*! + * @brief Canonical (big endian) representation of @ref XXH64_hash_t. + */ +typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t; -typedef enum { - ZSTD_sf_noBlockDelimiters = 0, /* Representation of ZSTD_Sequence has no block delimiters, sequences only */ - ZSTD_sf_explicitBlockDelimiters = 1 /* Representation of ZSTD_Sequence contains explicit block delimiters */ -} ZSTD_sequenceFormat_e; +/*! + * @brief Converts an @ref XXH64_hash_t to a big endian @ref XXH64_canonical_t. + * + * @param dst The @ref XXH64_canonical_t pointer to be stored to. + * @param hash The @ref XXH64_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash); -/*! ZSTD_generateSequences() : - * Generate sequences using ZSTD_compress2, given a source buffer. +/*! + * @brief Converts an @ref XXH64_canonical_t to a native @ref XXH64_hash_t. * - * Each block will end with a dummy sequence - * with offset == 0, matchLength == 0, and litLength == length of last literals. - * litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0) - * simply acts as a block delimiter. + * @param src The @ref XXH64_canonical_t to convert. * - * zc can be used to insert custom compression params. - * This function invokes ZSTD_compress2 + * @pre + * @p src must not be `NULL`. * - * The output of this function can be fed into ZSTD_compressSequences() with CCtx - * setting of ZSTD_c_blockDelimiters as ZSTD_sf_explicitBlockDelimiters - * @return : number of sequences generated + * @return The converted hash. + * + * @see @ref canonical_representation_example "Canonical Representation Example" */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src); -ZSTDLIB_API size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, - size_t outSeqsSize, const void* src, size_t srcSize); +#ifndef XXH_NO_XXH3 -/*! ZSTD_mergeBlockDelimiters() : - * Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals - * by merging them into into the literals of the next sequence. +/*! + * @} + * ************************************************************************ + * @defgroup XXH3_family XXH3 family + * @ingroup public + * @{ * - * As such, the final generated result has no explicit representation of block boundaries, - * and the final last literals segment is not represented in the sequences. + * XXH3 is a more recent hash algorithm featuring: + * - Improved speed for both small and large inputs + * - True 64-bit and 128-bit outputs + * - SIMD acceleration + * - Improved 32-bit viability * - * The output of this function can be fed into ZSTD_compressSequences() with CCtx - * setting of ZSTD_c_blockDelimiters as ZSTD_sf_noBlockDelimiters - * @return : number of sequences left after merging + * Speed analysis methodology is explained here: + * + * https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html + * + * Compared to XXH64, expect XXH3 to run approximately + * ~2x faster on large inputs and >3x faster on small ones, + * exact differences vary depending on platform. + * + * XXH3's speed benefits greatly from SIMD and 64-bit arithmetic, + * but does not require it. + * Most 32-bit and 64-bit targets that can run XXH32 smoothly can run XXH3 + * at competitive speeds, even without vector support. Further details are + * explained in the implementation. + * + * XXH3 has a fast scalar implementation, but it also includes accelerated SIMD + * implementations for many common platforms: + * - AVX512 + * - AVX2 + * - SSE2 + * - ARM NEON + * - WebAssembly SIMD128 + * - POWER8 VSX + * - s390x ZVector + * This can be controlled via the @ref XXH_VECTOR macro, but it automatically + * selects the best version according to predefined macros. For the x86 family, an + * automatic runtime dispatcher is included separately in @ref xxh_x86dispatch.c. + * + * XXH3 implementation is portable: + * it has a generic C90 formulation that can be compiled on any platform, + * all implementations generate exactly the same hash value on all platforms. + * Starting from v0.8.0, it's also labelled "stable", meaning that + * any future version will also generate the same hash value. + * + * XXH3 offers 2 variants, _64bits and _128bits. + * + * When only 64 bits are needed, prefer invoking the _64bits variant, as it + * reduces the amount of mixing, resulting in faster speed on small inputs. + * It's also generally simpler to manipulate a scalar return type than a struct. + * + * The API supports one-shot hashing, streaming mode, and custom secrets. */ -ZSTDLIB_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize); +/*-********************************************************************** +* XXH3 64-bit variant +************************************************************************/ -/*! ZSTD_compressSequences() : - * Compress an array of ZSTD_Sequence, generated from the original source buffer, into dst. - * If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.) - * The entire source is compressed into a single frame. +/*! + * @brief Calculates 64-bit unseeded variant of XXH3 hash of @p input. * - * The compression behavior changes based on cctx params. In particular: - * If ZSTD_c_blockDelimiters == ZSTD_sf_noBlockDelimiters, the array of ZSTD_Sequence is expected to contain - * no block delimiters (defined in ZSTD_Sequence). Block boundaries are roughly determined based on - * the block size derived from the cctx, and sequences may be split. This is the default setting. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. * - * If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain - * block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. * - * If ZSTD_c_validateSequences == 0, this function will blindly accept the sequences provided. Invalid sequences cause undefined - * behavior. If ZSTD_c_validateSequences == 1, then if sequence is invalid (see doc/zstd_compression_format.md for - * specifics regarding offset/matchlength requirements) then the function will bail out and return an error. + * @return The calculated 64-bit XXH3 hash value. * - * In addition to the two adjustable experimental params, there are other important cctx params. - * - ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN. - * - ZSTD_c_compressionLevel accordingly adjusts the strength of the entropy coder, as it would in typical compression. - * - ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset - * is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md + * @note + * This is equivalent to @ref XXH3_64bits_withSeed() with a seed of `0`, however + * it may have slightly better performance due to constant propagation of the + * defaults. + * + * @see + * XXH3_64bits_withSeed(), XXH3_64bits_withSecret(): other seeding variants + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length); + +/*! + * @brief Calculates 64-bit seeded variant of XXH3 hash of @p input. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit XXH3 hash value. + * + * @note + * seed == 0 produces the same results as @ref XXH3_64bits(). + * + * This variant generates a custom secret on the fly based on default secret + * altered using the @p seed value. * - * Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused. - * Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly, - * and cannot emit an RLE block that disagrees with the repcode history - * @return : final compressed size or a ZSTD error. + * While this operation is decently fast, note that it's not completely free. + * + * @see @ref single_shot_example "Single Shot Example" for an example. */ -ZSTDLIB_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstSize, - const ZSTD_Sequence* inSeqs, size_t inSeqsSize, - const void* src, size_t srcSize); +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed); +/*! + * The bare minimum size for a custom secret. + * + * @see + * XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(), + * XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret(). + */ +#define XXH3_SECRET_SIZE_MIN 136 -/*! ZSTD_writeSkippableFrame() : - * Generates a zstd skippable frame containing data given by src, and writes it to dst buffer. +/*! + * @brief Calculates 64-bit variant of XXH3 with a custom "secret". * - * Skippable frames begin with a a 4-byte magic number. There are 16 possible choices of magic number, - * ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15. - * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so - * the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant. + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. * - * Returns an error if destination buffer is not large enough, if the source size is not representable - * with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid). + * @return The calculated 64-bit XXH3 hash value. * - * @return : number of bytes written or a ZSTD error. + * @pre + * The memory between @p data and @p data + @p len must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p data may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * It's possible to provide any blob of bytes as a "secret" to generate the hash. + * This makes it more difficult for an external actor to prepare an intentional collision. + * The main condition is that @p secretSize *must* be large enough (>= @ref XXH3_SECRET_SIZE_MIN). + * However, the quality of the secret impacts the dispersion of the hash algorithm. + * Therefore, the secret _must_ look like a bunch of random bytes. + * Avoid "trivial" or structured data such as repeated sequences or a text document. + * Whenever in doubt about the "randomness" of the blob of bytes, + * consider employing @ref XXH3_generateSecret() instead (see below). + * It will generate a proper high entropy secret derived from the blob of bytes. + * Another advantage of using XXH3_generateSecret() is that + * it guarantees that all bits within the initial blob of bytes + * will impact every bit of the output. + * This is not necessarily the case when using the blob of bytes directly + * because, when hashing _small_ inputs, only a portion of the secret is employed. + * + * @see @ref single_shot_example "Single Shot Example" for an example. */ -ZSTDLIB_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity, - const void* src, size_t srcSize, unsigned magicVariant); +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize); -/*************************************** -* Memory management -***************************************/ +/******* Streaming *******/ +#ifndef XXH_NO_STREAM +/* + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + */ -/*! ZSTD_estimate*() : - * These functions make it possible to estimate memory usage - * of a future {D,C}Ctx, before its creation. +/*! + * @brief The opaque state struct for the XXH3 streaming API. * - * ZSTD_estimateCCtxSize() will provide a memory budget large enough - * for any compression level up to selected one. - * Note : Unlike ZSTD_estimateCStreamSize*(), this estimate - * does not include space for a window buffer. - * Therefore, the estimation is only guaranteed for single-shot compressions, not streaming. - * The estimate will assume the input may be arbitrarily large, - * which is the worst case. + * @see XXH3_state_s for details. + */ +typedef struct XXH3_state_s XXH3_state_t; +XXH_PUBLIC_API XXH_MALLOCF XXH3_state_t* XXH3_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr); + +/*! + * @brief Copies one @ref XXH3_state_t to another. * - * When srcSize can be bound by a known and rather "small" value, - * this fact can be used to provide a tighter estimation - * because the CCtx compression context will need less memory. - * This tighter estimation can be provided by more advanced functions - * ZSTD_estimateCCtxSize_usingCParams(), which can be used in tandem with ZSTD_getCParams(), - * and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter(). - * Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits. + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state); + +/*! + * @brief Resets an @ref XXH3_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret with default parameters. + * - Call this function before @ref XXH3_64bits_update(). + * - Digest will be equivalent to `XXH3_64bits()`. * - * Note 2 : only single-threaded compression is supported. - * ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1. */ -ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel); -ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); -ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params); -ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void); +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr); -/*! ZSTD_estimateCStreamSize() : - * ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one. - * It will also consider src size to be arbitrarily "large", which is worst case. - * If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation. - * ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. - * ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1. - * Note : CStream size estimation is only correct for single-threaded compression. - * ZSTD_DStream memory budget depends on window Size. - * This information can be passed manually, using ZSTD_estimateDStreamSize, - * or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame(); - * Note : if streaming is init with function ZSTD_init?Stream_usingDict(), - * an internal ?Dict will be created, which additional size is not estimated here. - * In this case, get total size by adding ZSTD_estimate?DictSize */ -ZSTDLIB_API size_t ZSTD_estimateCStreamSize(int compressionLevel); -ZSTDLIB_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams); -ZSTDLIB_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params); -ZSTDLIB_API size_t ZSTD_estimateDStreamSize(size_t windowSize); -ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize); +/*! + * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret from `seed`. + * - Call this function before @ref XXH3_64bits_update(). + * - Digest will be equivalent to `XXH3_64bits_withSeed()`. + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed); -/*! ZSTD_estimate?DictSize() : - * ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict(). - * ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, like ZSTD_createCDict_advanced(). - * Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller. +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * `secret` is referenced, it _must outlive_ the hash streaming session. + * + * Similar to one-shot API, `secretSize` must be >= @ref XXH3_SECRET_SIZE_MIN, + * and the quality of produced hash values depends on secret's entropy + * (secret's content should look like a bunch of random bytes). + * When in doubt about the randomness of a candidate `secret`, + * consider employing `XXH3_generateSecret()` instead (see below). */ -ZSTDLIB_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel); -ZSTDLIB_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod); -ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod); +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize); -/*! ZSTD_initStatic*() : - * Initialize an object using a pre-allocated fixed-size buffer. - * workspace: The memory area to emplace the object into. - * Provided pointer *must be 8-bytes aligned*. - * Buffer must outlive object. - * workspaceSize: Use ZSTD_estimate*Size() to determine - * how large workspace must be to support target scenario. - * @return : pointer to object (same address as workspace, just different type), - * or NULL if error (size too small, incorrect alignment, etc.) - * Note : zstd will never resize nor malloc() when using a static buffer. - * If the object requires more memory than available, - * zstd will just error out (typically ZSTD_error_memory_allocation). - * Note 2 : there is no corresponding "free" function. - * Since workspace is allocated externally, it must be freed externally too. - * Note 3 : cParams : use ZSTD_getCParams() to convert a compression level - * into its associated cParams. - * Limitation 1 : currently not compatible with internal dictionary creation, triggered by - * ZSTD_CCtx_loadDictionary(), ZSTD_initCStream_usingDict() or ZSTD_initDStream_usingDict(). - * Limitation 2 : static cctx currently not compatible with multi-threading. - * Limitation 3 : static dctx is incompatible with legacy support. +/*! + * @brief Consumes a block of @p input to an @ref XXH3_state_t. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note Call this to incrementally consume blocks of data. */ -ZSTDLIB_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize); -ZSTDLIB_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); -ZSTDLIB_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize); -ZSTDLIB_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */ +/*! + * @brief Returns the calculated XXH3 64-bit hash value from an @ref XXH3_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated XXH3 64-bit hash value from that state. + * + * @note + * Calling XXH3_64bits_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ -ZSTDLIB_API const ZSTD_CDict* ZSTD_initStaticCDict( - void* workspace, size_t workspaceSize, - const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, - ZSTD_dictContentType_e dictContentType, - ZSTD_compressionParameters cParams); +/* note : canonical representation of XXH3 is the same as XXH64 + * since they both produce XXH64_hash_t values */ -ZSTDLIB_API const ZSTD_DDict* ZSTD_initStaticDDict( - void* workspace, size_t workspaceSize, - const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, - ZSTD_dictContentType_e dictContentType); +/*-********************************************************************** +* XXH3 128-bit variant +************************************************************************/ -/*! Custom memory allocation : - * These prototypes make it possible to pass your own allocation/free functions. - * ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below. - * All allocation/free operations will be completed using these custom variants instead of regular ones. +/*! + * @brief The return value from 128-bit hashes. + * + * Stored in little endian order, although the fields themselves are in native + * endianness. */ -typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size); -typedef void (*ZSTD_freeFunction) (void* opaque, void* address); -typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; -static -#ifdef __GNUC__ -__attribute__((__unused__)) -#endif -ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */ - -ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); -ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); -ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); -ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); - -ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, - ZSTD_dictContentType_e dictContentType, - ZSTD_compressionParameters cParams, - ZSTD_customMem customMem); +typedef struct { + XXH64_hash_t low64; /*!< `value & 0xFFFFFFFFFFFFFFFF` */ + XXH64_hash_t high64; /*!< `value >> 64` */ +} XXH128_hash_t; -/* ! Thread pool : - * These prototypes make it possible to share a thread pool among multiple compression contexts. - * This can limit resources for applications with multiple threads where each one uses - * a threaded compression mode (via ZSTD_c_nbWorkers parameter). - * ZSTD_createThreadPool creates a new thread pool with a given number of threads. - * Note that the lifetime of such pool must exist while being used. - * ZSTD_CCtx_refThreadPool assigns a thread pool to a context (use NULL argument value - * to use an internal thread pool). - * ZSTD_freeThreadPool frees a thread pool. +/*! + * @brief Calculates 128-bit unseeded variant of XXH3 of @p data. + * + * @param data The block of data to be hashed, at least @p length bytes in size. + * @param len The length of @p data, in bytes. + * + * @return The calculated 128-bit variant of XXH3 value. + * + * The 128-bit variant of XXH3 has more strength, but it has a bit of overhead + * for shorter inputs. + * + * This is equivalent to @ref XXH3_128bits_withSeed() with a seed of `0`, however + * it may have slightly better performance due to constant propagation of the + * defaults. + * + * @see XXH3_128bits_withSeed(), XXH3_128bits_withSecret(): other seeding variants + * @see @ref single_shot_example "Single Shot Example" for an example. */ -typedef struct POOL_ctx_s ZSTD_threadPool; -ZSTDLIB_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads); -ZSTDLIB_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool); -ZSTDLIB_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool); - +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* data, size_t len); +/*! @brief Calculates 128-bit seeded variant of XXH3 hash of @p data. + * + * @param data The block of data to be hashed, at least @p length bytes in size. + * @param len The length of @p data, in bytes. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @return The calculated 128-bit variant of XXH3 value. + * + * @note + * seed == 0 produces the same results as @ref XXH3_64bits(). + * + * This variant generates a custom secret on the fly based on default secret + * altered using the @p seed value. + * + * While this operation is decently fast, note that it's not completely free. + * + * @see XXH3_128bits(), XXH3_128bits_withSecret(): other seeding variants + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSeed(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed); +/*! + * @brief Calculates 128-bit variant of XXH3 with a custom "secret". + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @return The calculated 128-bit variant of XXH3 value. + * + * It's possible to provide any blob of bytes as a "secret" to generate the hash. + * This makes it more difficult for an external actor to prepare an intentional collision. + * The main condition is that @p secretSize *must* be large enough (>= @ref XXH3_SECRET_SIZE_MIN). + * However, the quality of the secret impacts the dispersion of the hash algorithm. + * Therefore, the secret _must_ look like a bunch of random bytes. + * Avoid "trivial" or structured data such as repeated sequences or a text document. + * Whenever in doubt about the "randomness" of the blob of bytes, + * consider employing @ref XXH3_generateSecret() instead (see below). + * It will generate a proper high entropy secret derived from the blob of bytes. + * Another advantage of using XXH3_generateSecret() is that + * it guarantees that all bits within the initial blob of bytes + * will impact every bit of the output. + * This is not necessarily the case when using the blob of bytes directly + * because, when hashing _small_ inputs, only a portion of the secret is employed. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize); +/******* Streaming *******/ +#ifndef XXH_NO_STREAM /* - * This API is temporary and is expected to change or disappear in the future! + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + * + * XXH3_128bits uses the same XXH3_state_t as XXH3_64bits(). + * Use already declared XXH3_createState() and XXH3_freeState(). + * + * All reset and streaming functions have same meaning as their 64-bit counterpart. */ -ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced2( - const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, - ZSTD_dictContentType_e dictContentType, - const ZSTD_CCtx_params* cctxParams, - ZSTD_customMem customMem); -ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced( - const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, - ZSTD_dictContentType_e dictContentType, - ZSTD_customMem customMem); +/*! + * @brief Resets an @ref XXH3_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret with default parameters. + * - Call it before @ref XXH3_128bits_update(). + * - Digest will be equivalent to `XXH3_128bits()`. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr); +/*! + * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret from `seed`. + * - Call it before @ref XXH3_128bits_update(). + * - Digest will be equivalent to `XXH3_128bits_withSeed()`. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed); +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * `secret` is referenced, it _must outlive_ the hash streaming session. + * Similar to one-shot API, `secretSize` must be >= @ref XXH3_SECRET_SIZE_MIN, + * and the quality of produced hash values depends on secret's entropy + * (secret's content should look like a bunch of random bytes). + * When in doubt about the randomness of a candidate `secret`, + * consider employing `XXH3_generateSecret()` instead (see below). + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize); -/*************************************** -* Advanced compression functions -***************************************/ +/*! + * @brief Consumes a block of @p input to an @ref XXH3_state_t. + * + * Call this to incrementally consume blocks of data. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); -/*! ZSTD_createCDict_byReference() : - * Create a digested dictionary for compression - * Dictionary content is just referenced, not duplicated. - * As a consequence, `dictBuffer` **must** outlive CDict, - * and its content must remain unmodified throughout the lifetime of CDict. - * note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */ -ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); +/*! + * @brief Returns the calculated XXH3 128-bit hash value from an @ref XXH3_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated XXH3 128-bit hash value from that state. + * + * @note + * Calling XXH3_128bits_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ -/*! ZSTD_getDictID_fromCDict() : - * Provides the dictID of the dictionary loaded into `cdict`. - * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. - * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ -ZSTDLIB_API unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict); +/* Following helper functions make it possible to compare XXH128_hast_t values. + * Since XXH128_hash_t is a structure, this capability is not offered by the language. + * Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */ -/*! ZSTD_getCParams() : - * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize. - * `estimatedSrcSize` value is optional, select 0 if not known */ -ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); +/*! + * @brief Check equality of two XXH128_hash_t values + * + * @param h1 The 128-bit hash value. + * @param h2 Another 128-bit hash value. + * + * @return `1` if `h1` and `h2` are equal. + * @return `0` if they are not. + */ +XXH_PUBLIC_API XXH_PUREF int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2); -/*! ZSTD_getParams() : - * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`. - * All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */ -ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); +/*! + * @brief Compares two @ref XXH128_hash_t + * + * This comparator is compatible with stdlib's `qsort()`/`bsearch()`. + * + * @param h128_1 Left-hand side value + * @param h128_2 Right-hand side value + * + * @return >0 if @p h128_1 > @p h128_2 + * @return =0 if @p h128_1 == @p h128_2 + * @return <0 if @p h128_1 < @p h128_2 + */ +XXH_PUBLIC_API XXH_PUREF int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2); -/*! ZSTD_checkCParams() : - * Ensure param values remain within authorized range. - * @return 0 on success, or an error code (can be checked with ZSTD_isError()) */ -ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); -/*! ZSTD_adjustCParams() : - * optimize params for a given `srcSize` and `dictSize`. - * `srcSize` can be unknown, in which case use ZSTD_CONTENTSIZE_UNKNOWN. - * `dictSize` must be `0` when there is no dictionary. - * cPar can be invalid : all parameters will be clamped within valid range in the @return struct. - * This function never fails (wide contract) */ -ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); +/******* Canonical representation *******/ +typedef struct { unsigned char digest[sizeof(XXH128_hash_t)]; } XXH128_canonical_t; -/*! ZSTD_compress_advanced() : - * Note : this function is now DEPRECATED. - * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters. - * This prototype will be marked as deprecated and generate compilation warning on reaching v1.5.x */ -ZSTDLIB_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize, - ZSTD_parameters params); -/*! ZSTD_compress_usingCDict_advanced() : - * Note : this function is now REDUNDANT. - * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters. - * This prototype will be marked as deprecated and generate compilation warning in some future version */ -ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTD_CDict* cdict, - ZSTD_frameParameters fParams); +/*! + * @brief Converts an @ref XXH128_hash_t to a big endian @ref XXH128_canonical_t. + * + * @param dst The @ref XXH128_canonical_t pointer to be stored to. + * @param hash The @ref XXH128_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash); +/*! + * @brief Converts an @ref XXH128_canonical_t to a native @ref XXH128_hash_t. + * + * @param src The @ref XXH128_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src); -/*! ZSTD_CCtx_loadDictionary_byReference() : - * Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx. - * It saves some memory, but also requires that `dict` outlives its usage within `cctx` */ -ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); -/*! ZSTD_CCtx_loadDictionary_advanced() : - * Same as ZSTD_CCtx_loadDictionary(), but gives finer control over - * how to load the dictionary (by copy ? by reference ?) - * and how to interpret it (automatic ? force raw mode ? full mode only ?) */ -ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); +#endif /* !XXH_NO_XXH3 */ -/*! ZSTD_CCtx_refPrefix_advanced() : - * Same as ZSTD_CCtx_refPrefix(), but gives finer control over - * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ -ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); +#if defined (__cplusplus) +} /* extern "C" */ +#endif -/* === experimental parameters === */ -/* these parameters can be used with ZSTD_setParameter() - * they are not guaranteed to remain supported in the future */ +#endif /* XXH_NO_LONG_LONG */ - /* Enables rsyncable mode, - * which makes compressed files more rsync friendly - * by adding periodic synchronization points to the compressed data. - * The target average block size is ZSTD_c_jobSize / 2. - * It's possible to modify the job size to increase or decrease - * the granularity of the synchronization point. - * Once the jobSize is smaller than the window size, - * it will result in compression ratio degradation. - * NOTE 1: rsyncable mode only works when multithreading is enabled. - * NOTE 2: rsyncable performs poorly in combination with long range mode, - * since it will decrease the effectiveness of synchronization points, - * though mileage may vary. - * NOTE 3: Rsyncable mode limits maximum compression speed to ~400 MB/s. - * If the selected compression level is already running significantly slower, - * the overall speed won't be significantly impacted. - */ - #define ZSTD_c_rsyncable ZSTD_c_experimentalParam1 +/*! + * @} + */ +#endif /* XXHASH_H_5627135585666179 */ -/* Select a compression format. - * The value must be of type ZSTD_format_e. - * See ZSTD_format_e enum definition for details */ -#define ZSTD_c_format ZSTD_c_experimentalParam2 -/* Force back-reference distances to remain < windowSize, - * even when referencing into Dictionary content (default:0) */ -#define ZSTD_c_forceMaxWindow ZSTD_c_experimentalParam3 -/* Controls whether the contents of a CDict - * are used in place, or copied into the working context. - * Accepts values from the ZSTD_dictAttachPref_e enum. - * See the comments on that enum for an explanation of the feature. */ -#define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4 +#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) +#define XXHASH_H_STATIC_13879238742 +/* **************************************************************************** + * This section contains declarations which are not guaranteed to remain stable. + * They may change in future versions, becoming incompatible with a different + * version of the library. + * These declarations should only be used with static linking. + * Never use them in association with dynamic linking! + ***************************************************************************** */ -/* Controls how the literals are compressed (default is auto). - * The value must be of type ZSTD_literalCompressionMode_e. - * See ZSTD_literalCompressionMode_t enum definition for details. +/* + * These definitions are only present to allow static allocation + * of XXH states, on stack or in a struct, for example. + * Never **ever** access their members directly. */ -#define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5 - -/* Tries to fit compressed block size to be around targetCBlockSize. - * No target when targetCBlockSize == 0. - * There is no guarantee on compressed block size (default:0) */ -#define ZSTD_c_targetCBlockSize ZSTD_c_experimentalParam6 - -/* User's best guess of source size. - * Hint is not valid when srcSizeHint == 0. - * There is no guarantee that hint is close to actual source size, - * but compression ratio may regress significantly if guess considerably underestimates */ -#define ZSTD_c_srcSizeHint ZSTD_c_experimentalParam7 - -/* Controls whether the new and experimental "dedicated dictionary search - * structure" can be used. This feature is still rough around the edges, be - * prepared for surprising behavior! - * - * How to use it: - * - * When using a CDict, whether to use this feature or not is controlled at - * CDict creation, and it must be set in a CCtxParams set passed into that - * construction (via ZSTD_createCDict_advanced2()). A compression will then - * use the feature or not based on how the CDict was constructed; the value of - * this param, set in the CCtx, will have no effect. - * - * However, when a dictionary buffer is passed into a CCtx, such as via - * ZSTD_CCtx_loadDictionary(), this param can be set on the CCtx to control - * whether the CDict that is created internally can use the feature or not. - * - * What it does: - * - * Normally, the internal data structures of the CDict are analogous to what - * would be stored in a CCtx after compressing the contents of a dictionary. - * To an approximation, a compression using a dictionary can then use those - * data structures to simply continue what is effectively a streaming - * compression where the simulated compression of the dictionary left off. - * Which is to say, the search structures in the CDict are normally the same - * format as in the CCtx. - * - * It is possible to do better, since the CDict is not like a CCtx: the search - * structures are written once during CDict creation, and then are only read - * after that, while the search structures in the CCtx are both read and - * written as the compression goes along. This means we can choose a search - * structure for the dictionary that is read-optimized. - * - * This feature enables the use of that different structure. - * - * Note that some of the members of the ZSTD_compressionParameters struct have - * different semantics and constraints in the dedicated search structure. It is - * highly recommended that you simply set a compression level in the CCtxParams - * you pass into the CDict creation call, and avoid messing with the cParams - * directly. + +/*! + * @internal + * @brief Structure for XXH32 streaming API. * - * Effects: + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. * - * This will only have any effect when the selected ZSTD_strategy - * implementation supports this feature. Currently, that's limited to - * ZSTD_greedy, ZSTD_lazy, and ZSTD_lazy2. + * Typedef'd to @ref XXH32_state_t. + * Do not access the members of this struct directly. + * @see XXH64_state_s, XXH3_state_s + */ +struct XXH32_state_s { + XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */ + XXH32_hash_t large_len; /*!< Whether the hash is >= 16 (handles @ref total_len_32 overflow) */ + XXH32_hash_t v[4]; /*!< Accumulator lanes */ + XXH32_hash_t mem32[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[16]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem32 */ + XXH32_hash_t reserved; /*!< Reserved field. Do not read nor write to it. */ +}; /* typedef'd to XXH32_state_t */ + + +#ifndef XXH_NO_LONG_LONG /* defined when there is no 64-bit support */ + +/*! + * @internal + * @brief Structure for XXH64 streaming API. * - * Note that this means that the CDict tables can no longer be copied into the - * CCtx, so the dict attachment mode ZSTD_dictForceCopy will no longer be - * useable. The dictionary can only be attached or reloaded. + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. * - * In general, you should expect compression to be faster--sometimes very much - * so--and CDict creation to be slightly slower. Eventually, we will probably - * make this mode the default. + * Typedef'd to @ref XXH64_state_t. + * Do not access the members of this struct directly. + * @see XXH32_state_s, XXH3_state_s */ -#define ZSTD_c_enableDedicatedDictSearch ZSTD_c_experimentalParam8 +struct XXH64_state_s { + XXH64_hash_t total_len; /*!< Total length hashed. This is always 64-bit. */ + XXH64_hash_t v[4]; /*!< Accumulator lanes */ + XXH64_hash_t mem64[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[32]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem64 */ + XXH32_hash_t reserved32; /*!< Reserved field, needed for padding anyways*/ + XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it. */ +}; /* typedef'd to XXH64_state_t */ + +#ifndef XXH_NO_XXH3 + +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */ +# include +# define XXH_ALIGN(n) alignas(n) +#elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */ +/* In C++ alignas() is a keyword */ +# define XXH_ALIGN(n) alignas(n) +#elif defined(__GNUC__) +# define XXH_ALIGN(n) __attribute__ ((aligned(n))) +#elif defined(_MSC_VER) +# define XXH_ALIGN(n) __declspec(align(n)) +#else +# define XXH_ALIGN(n) /* disabled */ +#endif -/* ZSTD_c_stableInBuffer - * Experimental parameter. - * Default is 0 == disabled. Set to 1 to enable. - * - * Tells the compressor that the ZSTD_inBuffer will ALWAYS be the same - * between calls, except for the modifications that zstd makes to pos (the - * caller must not modify pos). This is checked by the compressor, and - * compression will fail if it ever changes. This means the only flush - * mode that makes sense is ZSTD_e_end, so zstd will error if ZSTD_e_end - * is not used. The data in the ZSTD_inBuffer in the range [src, src + pos) - * MUST not be modified during compression or you will get data corruption. +/* Old GCC versions only accept the attribute after the type in structures. */ +#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) /* C11+ */ \ + && ! (defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \ + && defined(__GNUC__) +# define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align) +#else +# define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type +#endif + +/*! + * @brief The size of the internal XXH3 buffer. * - * When this flag is enabled zstd won't allocate an input window buffer, - * because the user guarantees it can reference the ZSTD_inBuffer until - * the frame is complete. But, it will still allocate an output buffer - * large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also - * avoid the memcpy() from the input buffer to the input window buffer. + * This is the optimal update size for incremental hashing. * - * NOTE: ZSTD_compressStream2() will error if ZSTD_e_end is not used. - * That means this flag cannot be used with ZSTD_compressStream(). + * @see XXH3_64b_update(), XXH3_128b_update(). + */ +#define XXH3_INTERNALBUFFER_SIZE 256 + +/*! + * @internal + * @brief Default size of the secret buffer (and @ref XXH3_kSecret). * - * NOTE: So long as the ZSTD_inBuffer always points to valid memory, using - * this flag is ALWAYS memory safe, and will never access out-of-bounds - * memory. However, compression WILL fail if you violate the preconditions. + * This is the size used in @ref XXH3_kSecret and the seeded functions. * - * WARNING: The data in the ZSTD_inBuffer in the range [dst, dst + pos) MUST - * not be modified during compression or you will get data corruption. This - * is because zstd needs to reference data in the ZSTD_inBuffer to find - * matches. Normally zstd maintains its own window buffer for this purpose, - * but passing this flag tells zstd to use the user provided buffer. + * Not to be confused with @ref XXH3_SECRET_SIZE_MIN. */ -#define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9 +#define XXH3_SECRET_DEFAULT_SIZE 192 -/* ZSTD_c_stableOutBuffer - * Experimental parameter. - * Default is 0 == disabled. Set to 1 to enable. +/*! + * @internal + * @brief Structure for XXH3 streaming API. * - * Tells he compressor that the ZSTD_outBuffer will not be resized between - * calls. Specifically: (out.size - out.pos) will never grow. This gives the - * compressor the freedom to say: If the compressed data doesn't fit in the - * output buffer then return ZSTD_error_dstSizeTooSmall. This allows us to - * always decompress directly into the output buffer, instead of decompressing - * into an internal buffer and copying to the output buffer. + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. + * Otherwise it is an opaque type. + * Never use this definition in combination with dynamic library. + * This allows fields to safely be changed in the future. * - * When this flag is enabled zstd won't allocate an output buffer, because - * it can write directly to the ZSTD_outBuffer. It will still allocate the - * input window buffer (see ZSTD_c_stableInBuffer). + * @note ** This structure has a strict alignment requirement of 64 bytes!! ** + * Do not allocate this with `malloc()` or `new`, + * it will not be sufficiently aligned. + * Use @ref XXH3_createState() and @ref XXH3_freeState(), or stack allocation. * - * Zstd will check that (out.size - out.pos) never grows and return an error - * if it does. While not strictly necessary, this should prevent surprises. + * Typedef'd to @ref XXH3_state_t. + * Do never access the members of this struct directly. + * + * @see XXH3_INITSTATE() for stack initialization. + * @see XXH3_createState(), XXH3_freeState(). + * @see XXH32_state_s, XXH64_state_s */ -#define ZSTD_c_stableOutBuffer ZSTD_c_experimentalParam10 +struct XXH3_state_s { + XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]); + /*!< The 8 accumulators. See @ref XXH32_state_s::v and @ref XXH64_state_s::v */ + XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]); + /*!< Used to store a custom secret generated from a seed. */ + XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]); + /*!< The internal buffer. @see XXH32_state_s::mem32 */ + XXH32_hash_t bufferedSize; + /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */ + XXH32_hash_t useSeed; + /*!< Reserved field. Needed for padding on 64-bit. */ + size_t nbStripesSoFar; + /*!< Number or stripes processed. */ + XXH64_hash_t totalLen; + /*!< Total length hashed. 64-bit even on 32-bit targets. */ + size_t nbStripesPerBlock; + /*!< Number of stripes per block. */ + size_t secretLimit; + /*!< Size of @ref customSecret or @ref extSecret */ + XXH64_hash_t seed; + /*!< Seed for _withSeed variants. Must be zero otherwise, @see XXH3_INITSTATE() */ + XXH64_hash_t reserved64; + /*!< Reserved field. */ + const unsigned char* extSecret; + /*!< Reference to an external secret for the _withSecret variants, NULL + * for other variants. */ + /* note: there may be some padding at the end due to alignment on 64 bytes */ +}; /* typedef'd to XXH3_state_t */ + +#undef XXH_ALIGN_MEMBER -/* ZSTD_c_blockDelimiters - * Default is 0 == ZSTD_sf_noBlockDelimiters. - * - * For use with sequence compression API: ZSTD_compressSequences(). +/*! + * @brief Initializes a stack-allocated `XXH3_state_s`. * - * Designates whether or not the given array of ZSTD_Sequence contains block delimiters - * and last literals, which are defined as sequences with offset == 0 and matchLength == 0. - * See the definition of ZSTD_Sequence for more specifics. + * When the @ref XXH3_state_t structure is merely emplaced on stack, + * it should be initialized with XXH3_INITSTATE() or a memset() + * in case its first reset uses XXH3_NNbits_reset_withSeed(). + * This init can be omitted if the first reset uses default or _withSecret mode. + * This operation isn't necessary when the state is created with XXH3_createState(). + * Note that this doesn't prepare the state for a streaming operation, + * it's still necessary to use XXH3_NNbits_reset*() afterwards. */ -#define ZSTD_c_blockDelimiters ZSTD_c_experimentalParam11 +#define XXH3_INITSTATE(XXH3_state_ptr) \ + do { \ + XXH3_state_t* tmp_xxh3_state_ptr = (XXH3_state_ptr); \ + tmp_xxh3_state_ptr->seed = 0; \ + tmp_xxh3_state_ptr->extSecret = NULL; \ + } while(0) -/* ZSTD_c_validateSequences - * Default is 0 == disabled. Set to 1 to enable sequence validation. + +#if defined (__cplusplus) +extern "C" { +#endif + +/*! + * @brief Calculates the 128-bit hash of @p data using XXH3. * - * For use with sequence compression API: ZSTD_compressSequences(). - * Designates whether or not we validate sequences provided to ZSTD_compressSequences() - * during function execution. + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param seed The 64-bit seed to alter the hash's output predictably. * - * Without validation, providing a sequence that does not conform to the zstd spec will cause - * undefined behavior, and may produce a corrupted block. + * @pre + * The memory between @p data and @p data + @p len must be valid, + * readable, contiguous memory. However, if @p len is `0`, @p data may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. * - * With validation enabled, a if sequence is invalid (see doc/zstd_compression_format.md for - * specifics regarding offset/matchlength requirements) then the function will bail out and - * return an error. + * @return The calculated 128-bit XXH3 value. * + * @see @ref single_shot_example "Single Shot Example" for an example. */ -#define ZSTD_c_validateSequences ZSTD_c_experimentalParam12 +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed); -/*! ZSTD_CCtx_getParameter() : - * Get the requested compression parameter value, selected by enum ZSTD_cParameter, - * and store it into int* value. - * @return : 0, or an error code (which can be tested with ZSTD_isError()). - */ -ZSTDLIB_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value); +/* === Experimental API === */ +/* Symbols defined below must be considered tied to a specific library version. */ -/*! ZSTD_CCtx_params : - * Quick howto : - * - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure - * - ZSTD_CCtxParams_setParameter() : Push parameters one by one into - * an existing ZSTD_CCtx_params structure. - * This is similar to - * ZSTD_CCtx_setParameter(). - * - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to - * an existing CCtx. - * These parameters will be applied to - * all subsequent frames. - * - ZSTD_compressStream2() : Do compression using the CCtx. - * - ZSTD_freeCCtxParams() : Free the memory. +/*! + * @brief Derive a high-entropy secret from any user-defined content, named customSeed. * - * This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams() - * for static allocation of CCtx for single-threaded compression. + * @param secretBuffer A writable buffer for derived high-entropy secret data. + * @param secretSize Size of secretBuffer, in bytes. Must be >= XXH3_SECRET_DEFAULT_SIZE. + * @param customSeed A user-defined content. + * @param customSeedSize Size of customSeed, in bytes. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * The generated secret can be used in combination with `*_withSecret()` functions. + * The `_withSecret()` variants are useful to provide a higher level of protection + * than 64-bit seed, as it becomes much more difficult for an external actor to + * guess how to impact the calculation logic. + * + * The function accepts as input a custom seed of any length and any content, + * and derives from it a high-entropy secret of length @p secretSize into an + * already allocated buffer @p secretBuffer. + * + * The generated secret can then be used with any `*_withSecret()` variant. + * The functions @ref XXH3_128bits_withSecret(), @ref XXH3_64bits_withSecret(), + * @ref XXH3_128bits_reset_withSecret() and @ref XXH3_64bits_reset_withSecret() + * are part of this list. They all accept a `secret` parameter + * which must be large enough for implementation reasons (>= @ref XXH3_SECRET_SIZE_MIN) + * _and_ feature very high entropy (consist of random-looking bytes). + * These conditions can be a high bar to meet, so @ref XXH3_generateSecret() can + * be employed to ensure proper quality. + * + * @p customSeed can be anything. It can have any size, even small ones, + * and its content can be anything, even "poor entropy" sources such as a bunch + * of zeroes. The resulting `secret` will nonetheless provide all required qualities. + * + * @pre + * - @p secretSize must be >= @ref XXH3_SECRET_SIZE_MIN + * - When @p customSeedSize > 0, supplying NULL as customSeed is undefined behavior. + * + * Example code: + * @code{.c} + * #include + * #include + * #include + * #define XXH_STATIC_LINKING_ONLY // expose unstable API + * #include "xxhash.h" + * // Hashes argv[2] using the entropy from argv[1]. + * int main(int argc, char* argv[]) + * { + * char secret[XXH3_SECRET_SIZE_MIN]; + * if (argv != 3) { return 1; } + * XXH3_generateSecret(secret, sizeof(secret), argv[1], strlen(argv[1])); + * XXH64_hash_t h = XXH3_64bits_withSecret( + * argv[2], strlen(argv[2]), + * secret, sizeof(secret) + * ); + * printf("%016llx\n", (unsigned long long) h); + * } + * @endcode */ -ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void); -ZSTDLIB_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); +XXH_PUBLIC_API XXH_errorcode XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize); -/*! ZSTD_CCtxParams_reset() : - * Reset params to default values. +/*! + * @brief Generate the same secret as the _withSeed() variants. + * + * @param secretBuffer A writable buffer of @ref XXH3_SECRET_SIZE_MIN bytes + * @param seed The 64-bit seed to alter the hash result predictably. + * + * The generated secret can be used in combination with + *`*_withSecret()` and `_withSecretandSeed()` variants. + * + * Example C++ `std::string` hash class: + * @code{.cpp} + * #include + * #define XXH_STATIC_LINKING_ONLY // expose unstable API + * #include "xxhash.h" + * // Slow, seeds each time + * class HashSlow { + * XXH64_hash_t seed; + * public: + * HashSlow(XXH64_hash_t s) : seed{s} {} + * size_t operator()(const std::string& x) const { + * return size_t{XXH3_64bits_withSeed(x.c_str(), x.length(), seed)}; + * } + * }; + * // Fast, caches the seeded secret for future uses. + * class HashFast { + * unsigned char secret[XXH3_SECRET_SIZE_MIN]; + * public: + * HashFast(XXH64_hash_t s) { + * XXH3_generateSecret_fromSeed(secret, seed); + * } + * size_t operator()(const std::string& x) const { + * return size_t{ + * XXH3_64bits_withSecret(x.c_str(), x.length(), secret, sizeof(secret)) + * }; + * } + * }; + * @endcode */ -ZSTDLIB_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params); +XXH_PUBLIC_API void XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer, XXH64_hash_t seed); -/*! ZSTD_CCtxParams_init() : - * Initializes the compression parameters of cctxParams according to - * compression level. All other parameters are reset to their default values. +/*! + * @brief Calculates 64/128-bit seeded variant of XXH3 hash of @p data. + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * These variants generate hash values using either + * @p seed for "short" keys (< @ref XXH3_MIDSIZE_MAX = 240 bytes) + * or @p secret for "large" keys (>= @ref XXH3_MIDSIZE_MAX). + * + * This generally benefits speed, compared to `_withSeed()` or `_withSecret()`. + * `_withSeed()` has to generate the secret on the fly for "large" keys. + * It's fast, but can be perceptible for "not so large" keys (< 1 KB). + * `_withSecret()` has to generate the masks on the fly for "small" keys, + * which requires more instructions than _withSeed() variants. + * Therefore, _withSecretandSeed variant combines the best of both worlds. + * + * When @p secret has been generated by XXH3_generateSecret_fromSeed(), + * this variant produces *exactly* the same results as `_withSeed()` variant, + * hence offering only a pure speed benefit on "large" input, + * by skipping the need to regenerate the secret for every large input. + * + * Another usage scenario is to hash the secret to a 64-bit hash value, + * for example with XXH3_64bits(), which then becomes the seed, + * and then employ both the seed and the secret in _withSecretandSeed(). + * On top of speed, an added benefit is that each bit in the secret + * has a 50% chance to swap each bit in the output, via its impact to the seed. + * + * This is not guaranteed when using the secret directly in "small data" scenarios, + * because only portions of the secret are employed for small data. */ -ZSTDLIB_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel); - -/*! ZSTD_CCtxParams_init_advanced() : - * Initializes the compression and frame parameters of cctxParams according to - * params. All other parameters are reset to their default values. +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t +XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* data, size_t len, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed); +/*! + * @brief Calculates 128-bit seeded variant of XXH3 hash of @p data. + * + * @param input The block of data to be hashed, at least @p len bytes in size. + * @param length The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed64 The 64-bit seed to alter the hash result predictably. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @see XXH3_64bits_withSecretandSeed() */ -ZSTDLIB_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params); - -/*! ZSTD_CCtxParams_setParameter() : - * Similar to ZSTD_CCtx_setParameter. - * Set one compression parameter, selected by enum ZSTD_cParameter. - * Parameters must be applied to a ZSTD_CCtx using - * ZSTD_CCtx_setParametersUsingCCtxParams(). - * @result : a code representing success or failure (which can be tested with - * ZSTD_isError()). +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t +XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed64); +#ifndef XXH_NO_STREAM +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed64 The 64-bit seed to alter the hash result predictably. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @see XXH3_64bits_withSecretandSeed() */ -ZSTDLIB_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value); - -/*! ZSTD_CCtxParams_getParameter() : - * Similar to ZSTD_CCtx_getParameter. - * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter. - * @result : 0, or an error code (which can be tested with ZSTD_isError()). +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed64); +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed64 The 64-bit seed to alter the hash result predictably. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @see XXH3_64bits_withSecretandSeed() */ -ZSTDLIB_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value); +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed64); +#endif /* !XXH_NO_STREAM */ -/*! ZSTD_CCtx_setParametersUsingCCtxParams() : - * Apply a set of ZSTD_CCtx_params to the compression context. - * This can be done even after compression is started, - * if nbWorkers==0, this will have no impact until a new compression is started. - * if nbWorkers>=1, new parameters will be picked up at next job, - * with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated). - */ -ZSTDLIB_API size_t ZSTD_CCtx_setParametersUsingCCtxParams( - ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params); +#if defined (__cplusplus) +} /* extern "C" */ +#endif -/*! ZSTD_compressStream2_simpleArgs() : - * Same as ZSTD_compressStream2(), - * but using only integral types as arguments. - * This variant might be helpful for binders from dynamic languages - * which have troubles handling structures containing memory pointers. - */ -ZSTDLIB_API size_t ZSTD_compressStream2_simpleArgs ( - ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, size_t* dstPos, - const void* src, size_t srcSize, size_t* srcPos, - ZSTD_EndDirective endOp); +#endif /* !XXH_NO_XXH3 */ +#endif /* XXH_NO_LONG_LONG */ +#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) +# define XXH_IMPLEMENTATION +#endif -/*************************************** -* Advanced decompression functions -***************************************/ +#endif /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */ -/*! ZSTD_isFrame() : - * Tells if the content of `buffer` starts with a valid Frame Identifier. - * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. - * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. - * Note 3 : Skippable Frame Identifiers are considered valid. */ -ZSTDLIB_API unsigned ZSTD_isFrame(const void* buffer, size_t size); -/*! ZSTD_createDDict_byReference() : - * Create a digested dictionary, ready to start decompression operation without startup delay. - * Dictionary content is referenced, and therefore stays in dictBuffer. - * It is important that dictBuffer outlives DDict, - * it must remain read accessible throughout the lifetime of DDict */ -ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize); +/* ======================================================================== */ +/* ======================================================================== */ +/* ======================================================================== */ -/*! ZSTD_DCtx_loadDictionary_byReference() : - * Same as ZSTD_DCtx_loadDictionary(), - * but references `dict` content instead of copying it into `dctx`. - * This saves memory if `dict` remains around., - * However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression. */ -ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); -/*! ZSTD_DCtx_loadDictionary_advanced() : - * Same as ZSTD_DCtx_loadDictionary(), - * but gives direct control over - * how to load the dictionary (by copy ? by reference ?) - * and how to interpret it (automatic ? force raw mode ? full mode only ?). */ -ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); +/*-********************************************************************** + * xxHash implementation + *-********************************************************************** + * xxHash's implementation used to be hosted inside xxhash.c. + * + * However, inlining requires implementation to be visible to the compiler, + * hence be included alongside the header. + * Previously, implementation was hosted inside xxhash.c, + * which was then #included when inlining was activated. + * This construction created issues with a few build and install systems, + * as it required xxhash.c to be stored in /include directory. + * + * xxHash implementation is now directly integrated within xxhash.h. + * As a consequence, xxhash.c is no longer needed in /include. + * + * xxhash.c is still available and is still useful. + * In a "normal" setup, when xxhash is not inlined, + * xxhash.h only exposes the prototypes and public symbols, + * while xxhash.c can be built into an object file xxhash.o + * which can then be linked into the final binary. + ************************************************************************/ -/*! ZSTD_DCtx_refPrefix_advanced() : - * Same as ZSTD_DCtx_refPrefix(), but gives finer control over - * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ -ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); +#if ( defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) \ + || defined(XXH_IMPLEMENTATION) ) && !defined(XXH_IMPLEM_13a8737387) +# define XXH_IMPLEM_13a8737387 -/*! ZSTD_DCtx_setMaxWindowSize() : - * Refuses allocating internal buffers for frames requiring a window size larger than provided limit. - * This protects a decoder context from reserving too much memory for itself (potential attack scenario). - * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. - * By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) - * @return : 0, or an error code (which can be tested using ZSTD_isError()). - */ -ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize); +/* ************************************* +* Tuning parameters +***************************************/ -/*! ZSTD_DCtx_getParameter() : - * Get the requested decompression parameter value, selected by enum ZSTD_dParameter, - * and store it into int* value. - * @return : 0, or an error code (which can be tested with ZSTD_isError()). +/*! + * @defgroup tuning Tuning parameters + * @{ + * + * Various macros to control xxHash's behavior. */ -ZSTDLIB_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value); - -/* ZSTD_d_format - * experimental parameter, - * allowing selection between ZSTD_format_e input compression formats +#ifdef XXH_DOXYGEN +/*! + * @brief Define this to disable 64-bit code. + * + * Useful if only using the @ref XXH32_family and you have a strict C90 compiler. */ -#define ZSTD_d_format ZSTD_d_experimentalParam1 -/* ZSTD_d_stableOutBuffer - * Experimental parameter. - * Default is 0 == disabled. Set to 1 to enable. +# define XXH_NO_LONG_LONG +# undef XXH_NO_LONG_LONG /* don't actually */ +/*! + * @brief Controls how unaligned memory is accessed. * - * Tells the decompressor that the ZSTD_outBuffer will ALWAYS be the same - * between calls, except for the modifications that zstd makes to pos (the - * caller must not modify pos). This is checked by the decompressor, and - * decompression will fail if it ever changes. Therefore the ZSTD_outBuffer - * MUST be large enough to fit the entire decompressed frame. This will be - * checked when the frame content size is known. The data in the ZSTD_outBuffer - * in the range [dst, dst + pos) MUST not be modified during decompression - * or you will get data corruption. + * By default, access to unaligned memory is controlled by `memcpy()`, which is + * safe and portable. * - * When this flags is enabled zstd won't allocate an output buffer, because - * it can write directly to the ZSTD_outBuffer, but it will still allocate - * an input buffer large enough to fit any compressed block. This will also - * avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer. - * If you need to avoid the input buffer allocation use the buffer-less - * streaming API. + * Unfortunately, on some target/compiler combinations, the generated assembly + * is sub-optimal. + * + * The below switch allow selection of a different access method + * in the search for improved performance. + * + * @par Possible options: + * + * - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy` + * @par + * Use `memcpy()`. Safe and portable. Note that most modern compilers will + * eliminate the function call and treat it as an unaligned access. + * + * - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((aligned(1)))` + * @par + * Depends on compiler extensions and is therefore not portable. + * This method is safe _if_ your compiler supports it, + * and *generally* as fast or faster than `memcpy`. + * + * - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast + * @par + * Casts directly and dereferences. This method doesn't depend on the + * compiler, but it violates the C standard as it directly dereferences an + * unaligned pointer. It can generate buggy code on targets which do not + * support unaligned memory accesses, but in some circumstances, it's the + * only known way to get the most performance. + * + * - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift + * @par + * Also portable. This can generate the best code on old compilers which don't + * inline small `memcpy()` calls, and it might also be faster on big-endian + * systems which lack a native byteswap instruction. However, some compilers + * will emit literal byteshifts even if the target supports unaligned access. + * + * + * @warning + * Methods 1 and 2 rely on implementation-defined behavior. Use these with + * care, as what works on one compiler/platform/optimization level may cause + * another to read garbage data or even crash. + * + * See https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details. + * + * Prefer these methods in priority order (0 > 3 > 1 > 2) + */ +# define XXH_FORCE_MEMORY_ACCESS 0 + +/*! + * @def XXH_SIZE_OPT + * @brief Controls how much xxHash optimizes for size. + * + * xxHash, when compiled, tends to result in a rather large binary size. This + * is mostly due to heavy usage to forced inlining and constant folding of the + * @ref XXH3_family to increase performance. + * + * However, some developers prefer size over speed. This option can + * significantly reduce the size of the generated code. When using the `-Os` + * or `-Oz` options on GCC or Clang, this is defined to 1 by default, + * otherwise it is defined to 0. * - * NOTE: So long as the ZSTD_outBuffer always points to valid memory, using - * this flag is ALWAYS memory safe, and will never access out-of-bounds - * memory. However, decompression WILL fail if you violate the preconditions. + * Most of these size optimizations can be controlled manually. * - * WARNING: The data in the ZSTD_outBuffer in the range [dst, dst + pos) MUST - * not be modified during decompression or you will get data corruption. This - * is because zstd needs to reference data in the ZSTD_outBuffer to regenerate - * matches. Normally zstd maintains its own buffer for this purpose, but passing - * this flag tells zstd to use the user provided buffer. + * This is a number from 0-2. + * - `XXH_SIZE_OPT` == 0: Default. xxHash makes no size optimizations. Speed + * comes first. + * - `XXH_SIZE_OPT` == 1: Default for `-Os` and `-Oz`. xxHash is more + * conservative and disables hacks that increase code size. It implies the + * options @ref XXH_NO_INLINE_HINTS == 1, @ref XXH_FORCE_ALIGN_CHECK == 0, + * and @ref XXH3_NEON_LANES == 8 if they are not already defined. + * - `XXH_SIZE_OPT` == 2: xxHash tries to make itself as small as possible. + * Performance may cry. For example, the single shot functions just use the + * streaming API. */ -#define ZSTD_d_stableOutBuffer ZSTD_d_experimentalParam2 +# define XXH_SIZE_OPT 0 -/* ZSTD_d_forceIgnoreChecksum - * Experimental parameter. - * Default is 0 == disabled. Set to 1 to enable +/*! + * @def XXH_FORCE_ALIGN_CHECK + * @brief If defined to non-zero, adds a special path for aligned inputs (XXH32() + * and XXH64() only). * - * Tells the decompressor to skip checksum validation during decompression, regardless - * of whether checksumming was specified during compression. This offers some - * slight performance benefits, and may be useful for debugging. - * Param has values of type ZSTD_forceIgnoreChecksum_e + * This is an important performance trick for architectures without decent + * unaligned memory access performance. + * + * It checks for input alignment, and when conditions are met, uses a "fast + * path" employing direct 32-bit/64-bit reads, resulting in _dramatically + * faster_ read speed. + * + * The check costs one initial branch per hash, which is generally negligible, + * but not zero. + * + * Moreover, it's not useful to generate an additional code path if memory + * access uses the same instruction for both aligned and unaligned + * addresses (e.g. x86 and aarch64). + * + * In these cases, the alignment check can be removed by setting this macro to 0. + * Then the code will always use unaligned memory access. + * Align check is automatically disabled on x86, x64, ARM64, and some ARM chips + * which are platforms known to offer good unaligned memory accesses performance. + * + * It is also disabled by default when @ref XXH_SIZE_OPT >= 1. + * + * This option does not affect XXH3 (only XXH32 and XXH64). */ -#define ZSTD_d_forceIgnoreChecksum ZSTD_d_experimentalParam3 +# define XXH_FORCE_ALIGN_CHECK 0 -/* ZSTD_d_refMultipleDDicts - * Experimental parameter. - * Default is 0 == disabled. Set to 1 to enable +/*! + * @def XXH_NO_INLINE_HINTS + * @brief When non-zero, sets all functions to `static`. * - * If enabled and dctx is allocated on the heap, then additional memory will be allocated - * to store references to multiple ZSTD_DDict. That is, multiple calls of ZSTD_refDDict() - * using a given ZSTD_DCtx, rather than overwriting the previous DDict reference, will instead - * store all references. At decompression time, the appropriate dictID is selected - * from the set of DDicts based on the dictID in the frame. + * By default, xxHash tries to force the compiler to inline almost all internal + * functions. * - * Usage is simply calling ZSTD_refDDict() on multiple dict buffers. + * This can usually improve performance due to reduced jumping and improved + * constant folding, but significantly increases the size of the binary which + * might not be favorable. * - * Param has values of byte ZSTD_refMultipleDDicts_e + * Additionally, sometimes the forced inlining can be detrimental to performance, + * depending on the architecture. * - * WARNING: Enabling this parameter and calling ZSTD_DCtx_refDDict(), will trigger memory - * allocation for the hash table. ZSTD_freeDCtx() also frees this memory. - * Memory is allocated as per ZSTD_DCtx::customMem. + * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the + * compiler full control on whether to inline or not. * - * Although this function allocates memory for the table, the user is still responsible for - * memory management of the underlying ZSTD_DDict* themselves. + * When not optimizing (-O0), using `-fno-inline` with GCC or Clang, or if + * @ref XXH_SIZE_OPT >= 1, this will automatically be defined. */ -#define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4 +# define XXH_NO_INLINE_HINTS 0 +/*! + * @def XXH3_INLINE_SECRET + * @brief Determines whether to inline the XXH3 withSecret code. + * + * When the secret size is known, the compiler can improve the performance + * of XXH3_64bits_withSecret() and XXH3_128bits_withSecret(). + * + * However, if the secret size is not known, it doesn't have any benefit. This + * happens when xxHash is compiled into a global symbol. Therefore, if + * @ref XXH_INLINE_ALL is *not* defined, this will be defined to 0. + * + * Additionally, this defaults to 0 on GCC 12+, which has an issue with function pointers + * that are *sometimes* force inline on -Og, and it is impossible to automatically + * detect this optimization level. + */ +# define XXH3_INLINE_SECRET 0 -/*! ZSTD_DCtx_setFormat() : - * Instruct the decoder context about what kind of data to decode next. - * This instruction is mandatory to decode data without a fully-formed header, - * such ZSTD_f_zstd1_magicless for example. - * @return : 0, or an error code (which can be tested using ZSTD_isError()). */ -ZSTDLIB_API size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); +/*! + * @def XXH32_ENDJMP + * @brief Whether to use a jump for `XXH32_finalize`. + * + * For performance, `XXH32_finalize` uses multiple branches in the finalizer. + * This is generally preferable for performance, + * but depending on exact architecture, a jmp may be preferable. + * + * This setting is only possibly making a difference for very small inputs. + */ +# define XXH32_ENDJMP 0 -/*! ZSTD_decompressStream_simpleArgs() : - * Same as ZSTD_decompressStream(), - * but using only integral types as arguments. - * This can be helpful for binders from dynamic languages - * which have troubles handling structures containing memory pointers. +/*! + * @internal + * @brief Redefines old internal names. + * + * For compatibility with code that uses xxHash's internals before the names + * were changed to improve namespacing. There is no other reason to use this. */ -ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs ( - ZSTD_DCtx* dctx, - void* dst, size_t dstCapacity, size_t* dstPos, - const void* src, size_t srcSize, size_t* srcPos); +# define XXH_OLD_NAMES +# undef XXH_OLD_NAMES /* don't actually use, it is ugly. */ +/*! + * @def XXH_NO_STREAM + * @brief Disables the streaming API. + * + * When xxHash is not inlined and the streaming functions are not used, disabling + * the streaming functions can improve code size significantly, especially with + * the @ref XXH3_family which tends to make constant folded copies of itself. + */ +# define XXH_NO_STREAM +# undef XXH_NO_STREAM /* don't actually */ +#endif /* XXH_DOXYGEN */ +/*! + * @} + */ -/******************************************************************** -* Advanced streaming functions -* Warning : most of these functions are now redundant with the Advanced API. -* Once Advanced API reaches "stable" status, -* redundant functions will be deprecated, and then at some point removed. -********************************************************************/ +#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ + /* prefer __packed__ structures (method 1) for GCC + * < ARMv7 with unaligned access (e.g. Raspbian armhf) still uses byte shifting, so we use memcpy + * which for some reason does unaligned loads. */ +# if defined(__GNUC__) && !(defined(__ARM_ARCH) && __ARM_ARCH < 7 && defined(__ARM_FEATURE_UNALIGNED)) +# define XXH_FORCE_MEMORY_ACCESS 1 +# endif +#endif -/*===== Advanced Streaming compression functions =====*/ +#ifndef XXH_SIZE_OPT + /* default to 1 for -Os or -Oz */ +# if (defined(__GNUC__) || defined(__clang__)) && defined(__OPTIMIZE_SIZE__) +# define XXH_SIZE_OPT 1 +# else +# define XXH_SIZE_OPT 0 +# endif +#endif -/*! ZSTD_initCStream_srcSize() : - * This function is deprecated, and equivalent to: - * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) - * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); - * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); - * - * pledgedSrcSize must be correct. If it is not known at init time, use - * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, - * "0" also disables frame content size field. It may be enabled in the future. - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x +#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ + /* don't check on sizeopt, x86, aarch64, or arm when unaligned access is available */ +# if XXH_SIZE_OPT >= 1 || \ + defined(__i386) || defined(__x86_64__) || defined(__aarch64__) || defined(__ARM_FEATURE_UNALIGNED) \ + || defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) || defined(_M_ARM) /* visual */ +# define XXH_FORCE_ALIGN_CHECK 0 +# else +# define XXH_FORCE_ALIGN_CHECK 1 +# endif +#endif + +#ifndef XXH_NO_INLINE_HINTS +# if XXH_SIZE_OPT >= 1 || defined(__NO_INLINE__) /* -O0, -fno-inline */ +# define XXH_NO_INLINE_HINTS 1 +# else +# define XXH_NO_INLINE_HINTS 0 +# endif +#endif + +#ifndef XXH3_INLINE_SECRET +# if (defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 12) \ + || !defined(XXH_INLINE_ALL) +# define XXH3_INLINE_SECRET 0 +# else +# define XXH3_INLINE_SECRET 1 +# endif +#endif + +#ifndef XXH32_ENDJMP +/* generally preferable for performance */ +# define XXH32_ENDJMP 0 +#endif + +/*! + * @defgroup impl Implementation + * @{ */ -ZSTDLIB_API size_t -ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, - int compressionLevel, - unsigned long long pledgedSrcSize); -/*! ZSTD_initCStream_usingDict() : - * This function is deprecated, and is equivalent to: - * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); - * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); - * - * Creates of an internal CDict (incompatible with static CCtx), except if - * dict == NULL or dictSize < 8, in which case no dict is used. - * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if - * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy. - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x +/* ************************************* +* Includes & Memory related functions +***************************************/ +#include /* memcmp, memcpy */ +#include /* ULLONG_MAX */ + +#if defined(XXH_NO_STREAM) +/* nothing */ +#elif defined(XXH_NO_STDLIB) + +/* When requesting to disable any mention of stdlib, + * the library loses the ability to invoked malloc / free. + * In practice, it means that functions like `XXH*_createState()` + * will always fail, and return NULL. + * This flag is useful in situations where + * xxhash.h is integrated into some kernel, embedded or limited environment + * without access to dynamic allocation. */ -ZSTDLIB_API size_t -ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, - const void* dict, size_t dictSize, - int compressionLevel); -/*! ZSTD_initCStream_advanced() : - * This function is deprecated, and is approximately equivalent to: - * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * // Pseudocode: Set each zstd parameter and leave the rest as-is. - * for ((param, value) : params) { - * ZSTD_CCtx_setParameter(zcs, param, value); - * } - * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); - * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); - * - * dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy. - * pledgedSrcSize must be correct. - * If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x +#if defined (__cplusplus) +extern "C" { +#endif + +static XXH_CONSTF void* XXH_malloc(size_t s) { (void)s; return NULL; } +static void XXH_free(void* p) { (void)p; } + +#if defined (__cplusplus) +} /* extern "C" */ +#endif + +#else + +/* + * Modify the local functions below should you wish to use + * different memory routines for malloc() and free() */ -ZSTDLIB_API size_t -ZSTD_initCStream_advanced(ZSTD_CStream* zcs, - const void* dict, size_t dictSize, - ZSTD_parameters params, - unsigned long long pledgedSrcSize); +#include -/*! ZSTD_initCStream_usingCDict() : - * This function is deprecated, and equivalent to: - * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * ZSTD_CCtx_refCDict(zcs, cdict); - * - * note : cdict will just be referenced, and must outlive compression session - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x +#if defined (__cplusplus) +extern "C" { +#endif +/*! + * @internal + * @brief Modify this function to use a different routine than malloc(). */ -ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); +static XXH_MALLOCF void* XXH_malloc(size_t s) { return malloc(s); } -/*! ZSTD_initCStream_usingCDict_advanced() : - * This function is DEPRECATED, and is approximately equivalent to: - * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * // Pseudocode: Set each zstd frame parameter and leave the rest as-is. - * for ((fParam, value) : fParams) { - * ZSTD_CCtx_setParameter(zcs, fParam, value); - * } - * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); - * ZSTD_CCtx_refCDict(zcs, cdict); - * - * same as ZSTD_initCStream_usingCDict(), with control over frame parameters. - * pledgedSrcSize must be correct. If srcSize is not known at init time, use - * value ZSTD_CONTENTSIZE_UNKNOWN. - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x +/*! + * @internal + * @brief Modify this function to use a different routine than free(). */ -ZSTDLIB_API size_t -ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, - const ZSTD_CDict* cdict, - ZSTD_frameParameters fParams, - unsigned long long pledgedSrcSize); +static void XXH_free(void* p) { free(p); } -/*! ZSTD_resetCStream() : - * This function is deprecated, and is equivalent to: - * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); - * - * start a new frame, using same parameters from previous frame. - * This is typically useful to skip dictionary loading stage, since it will re-use it in-place. - * Note that zcs must be init at least once before using ZSTD_resetCStream(). - * If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN. - * If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end. - * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs, - * but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead. - * @return : 0, or an error code (which can be tested using ZSTD_isError()) - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x +#if defined (__cplusplus) +} /* extern "C" */ +#endif + +#endif /* XXH_NO_STDLIB */ + +#if defined (__cplusplus) +extern "C" { +#endif +/*! + * @internal + * @brief Modify this function to use a different routine than memcpy(). */ -ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); +static void* XXH_memcpy(void* dest, const void* src, size_t size) +{ + return memcpy(dest,src,size); +} +#if defined (__cplusplus) +} /* extern "C" */ +#endif -typedef struct { - unsigned long long ingested; /* nb input bytes read and buffered */ - unsigned long long consumed; /* nb input bytes actually compressed */ - unsigned long long produced; /* nb of compressed bytes generated and buffered */ - unsigned long long flushed; /* nb of compressed bytes flushed : not provided; can be tracked from caller side */ - unsigned currentJobID; /* MT only : latest started job nb */ - unsigned nbActiveWorkers; /* MT only : nb of workers actively compressing at probe time */ -} ZSTD_frameProgression; +/* ************************************* +* Compiler Specific Options +***************************************/ +#ifdef _MSC_VER /* Visual Studio warning fix */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif -/* ZSTD_getFrameProgression() : - * tells how much data has been ingested (read from input) - * consumed (input actually compressed) and produced (output) for current frame. - * Note : (ingested - consumed) is amount of input data buffered internally, not yet compressed. - * Aggregates progression inside active worker threads. +#if XXH_NO_INLINE_HINTS /* disable inlining hints */ +# if defined(__GNUC__) || defined(__clang__) +# define XXH_FORCE_INLINE static __attribute__((unused)) +# else +# define XXH_FORCE_INLINE static +# endif +# define XXH_NO_INLINE static +/* enable inlining hints */ +#elif defined(__GNUC__) || defined(__clang__) +# define XXH_FORCE_INLINE static __inline__ __attribute__((always_inline, unused)) +# define XXH_NO_INLINE static __attribute__((noinline)) +#elif defined(_MSC_VER) /* Visual Studio */ +# define XXH_FORCE_INLINE static __forceinline +# define XXH_NO_INLINE static __declspec(noinline) +#elif defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* C99 */ +# define XXH_FORCE_INLINE static inline +# define XXH_NO_INLINE static +#else +# define XXH_FORCE_INLINE static +# define XXH_NO_INLINE static +#endif + +#if XXH3_INLINE_SECRET +# define XXH3_WITH_SECRET_INLINE XXH_FORCE_INLINE +#else +# define XXH3_WITH_SECRET_INLINE XXH_NO_INLINE +#endif + + +/* ************************************* +* Debug +***************************************/ +/*! + * @ingroup tuning + * @def XXH_DEBUGLEVEL + * @brief Sets the debugging level. + * + * XXH_DEBUGLEVEL is expected to be defined externally, typically via the + * compiler's command line options. The value must be a number. */ -ZSTDLIB_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx); +#ifndef XXH_DEBUGLEVEL +# ifdef DEBUGLEVEL /* backwards compat */ +# define XXH_DEBUGLEVEL DEBUGLEVEL +# else +# define XXH_DEBUGLEVEL 0 +# endif +#endif -/*! ZSTD_toFlushNow() : - * Tell how many bytes are ready to be flushed immediately. - * Useful for multithreading scenarios (nbWorkers >= 1). - * Probe the oldest active job, defined as oldest job not yet entirely flushed, - * and check its output buffer. - * @return : amount of data stored in oldest job and ready to be flushed immediately. - * if @return == 0, it means either : - * + there is no active job (could be checked with ZSTD_frameProgression()), or - * + oldest job is still actively compressing data, - * but everything it has produced has also been flushed so far, - * therefore flush speed is limited by production speed of oldest job - * irrespective of the speed of concurrent (and newer) jobs. +#if (XXH_DEBUGLEVEL>=1) +# include /* note: can still be disabled with NDEBUG */ +# define XXH_ASSERT(c) assert(c) +#else +# if defined(__INTEL_COMPILER) +# define XXH_ASSERT(c) XXH_ASSUME((unsigned char) (c)) +# else +# define XXH_ASSERT(c) XXH_ASSUME(c) +# endif +#endif + +/* note: use after variable declarations */ +#ifndef XXH_STATIC_ASSERT +# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */ +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { _Static_assert((c),m); } while(0) +# elif defined(__cplusplus) && (__cplusplus >= 201103L) /* C++11 */ +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0) +# else +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { struct xxh_sa { char x[(c) ? 1 : -1]; }; } while(0) +# endif +# define XXH_STATIC_ASSERT(c) XXH_STATIC_ASSERT_WITH_MESSAGE((c),#c) +#endif + +/*! + * @internal + * @def XXH_COMPILER_GUARD(var) + * @brief Used to prevent unwanted optimizations for @p var. + * + * It uses an empty GCC inline assembly statement with a register constraint + * which forces @p var into a general purpose register (eg eax, ebx, ecx + * on x86) and marks it as modified. + * + * This is used in a few places to avoid unwanted autovectorization (e.g. + * XXH32_round()). All vectorization we want is explicit via intrinsics, + * and _usually_ isn't wanted elsewhere. + * + * We also use it to prevent unwanted constant folding for AArch64 in + * XXH3_initCustomSecret_scalar(). */ -ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx); +#if defined(__GNUC__) || defined(__clang__) +# define XXH_COMPILER_GUARD(var) __asm__("" : "+r" (var)) +#else +# define XXH_COMPILER_GUARD(var) ((void)0) +#endif + +/* Specifically for NEON vectors which use the "w" constraint, on + * Clang. */ +#if defined(__clang__) && defined(__ARM_ARCH) && !defined(__wasm__) +# define XXH_COMPILER_GUARD_CLANG_NEON(var) __asm__("" : "+w" (var)) +#else +# define XXH_COMPILER_GUARD_CLANG_NEON(var) ((void)0) +#endif +/* ************************************* +* Basic Types +***************************************/ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# ifdef _AIX +# include +# else +# include +# endif + typedef uint8_t xxh_u8; +#else + typedef unsigned char xxh_u8; +#endif +typedef XXH32_hash_t xxh_u32; -/*===== Advanced Streaming decompression functions =====*/ +#ifdef XXH_OLD_NAMES +# warning "XXH_OLD_NAMES is planned to be removed starting v0.9. If the program depends on it, consider moving away from it by employing newer type names directly" +# define BYTE xxh_u8 +# define U8 xxh_u8 +# define U32 xxh_u32 +#endif + +#if defined (__cplusplus) +extern "C" { +#endif + +/* *** Memory access *** */ /*! - * This function is deprecated, and is equivalent to: + * @internal + * @fn xxh_u32 XXH_read32(const void* ptr) + * @brief Reads an unaligned 32-bit integer from @p ptr in native endianness. * - * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); - * ZSTD_DCtx_loadDictionary(zds, dict, dictSize); + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. * - * note: no dictionary will be used if dict == NULL or dictSize < 8 - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + * @param ptr The pointer to read from. + * @return The 32-bit native endian integer from the bytes at @p ptr. */ -ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /*! - * This function is deprecated, and is equivalent to: + * @internal + * @fn xxh_u32 XXH_readLE32(const void* ptr) + * @brief Reads an unaligned 32-bit little endian integer from @p ptr. * - * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); - * ZSTD_DCtx_refDDict(zds, ddict); + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. * - * note : ddict is referenced, it must outlive decompression session - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + * @param ptr The pointer to read from. + * @return The 32-bit little endian integer from the bytes at @p ptr. */ -ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /*! - * This function is deprecated, and is equivalent to: + * @internal + * @fn xxh_u32 XXH_readBE32(const void* ptr) + * @brief Reads an unaligned 32-bit big endian integer from @p ptr. * - * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. * - * re-use decompression parameters from previous init; saves dictionary loading - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + * @param ptr The pointer to read from. + * @return The 32-bit big endian integer from the bytes at @p ptr. */ -ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); - - -/********************************************************************* -* Buffer-less and synchronous inner streaming functions -* -* This is an advanced API, giving full control over buffer management, for users which need direct control over memory. -* But it's also a complex one, with several restrictions, documented below. -* Prefer normal streaming API for an easier experience. -********************************************************************* */ - -/** - Buffer-less streaming compression (synchronous mode) - A ZSTD_CCtx object is required to track streaming operations. - Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource. - ZSTD_CCtx object can be re-used multiple times within successive compression operations. - - Start by initializing a context. - Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression, - or ZSTD_compressBegin_advanced(), for finer parameter control. - It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx() - - Then, consume your input using ZSTD_compressContinue(). - There are some important considerations to keep in mind when using this advanced function : - - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffers only. - - Interface is synchronous : input is consumed entirely and produces 1+ compressed blocks. - - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario. - Worst case evaluation is provided by ZSTD_compressBound(). - ZSTD_compressContinue() doesn't guarantee recover after a failed compression. - - ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog). - It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks) - - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps. - In which case, it will "discard" the relevant memory section from its history. - - Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum. - It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame. - Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders. - - `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress again. -*/ - -/*===== Buffer-less streaming compression functions =====*/ -ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); -ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); -ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */ -ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */ -ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */ -ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */ - -ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); - - -/** - Buffer-less streaming decompression (synchronous mode) - - A ZSTD_DCtx object is required to track streaming operations. - Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. - A ZSTD_DCtx object can be re-used multiple times. +/*! + * @internal + * @fn xxh_u32 XXH_readLE32_align(const void* ptr, XXH_alignment align) + * @brief Like @ref XXH_readLE32(), but has an option for aligned reads. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * Note that when @ref XXH_FORCE_ALIGN_CHECK == 0, the @p align parameter is + * always @ref XXH_alignment::XXH_unaligned. + * + * @param ptr The pointer to read from. + * @param align Whether @p ptr is aligned. + * @pre + * If @p align == @ref XXH_alignment::XXH_aligned, @p ptr must be 4 byte + * aligned. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ - First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader(). - Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough. - Data fragment must be large enough to ensure successful decoding. - `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough. - @result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled. - >0 : `srcSize` is too small, please provide at least @result bytes on next attempt. - errorCode, which can be tested using ZSTD_isError(). +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE32 and XXH_readBE32. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) - It fills a ZSTD_frameHeader structure with important information to correctly decode the frame, - such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`). - Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information. - As a consequence, check that values remain within valid application range. - For example, do not allocate memory blindly, check that `windowSize` is within expectation. - Each application can set its own limits, depending on local restrictions. - For extended interoperability, it is recommended to support `windowSize` of at least 8 MB. +/* + * Force direct memory access. Only works on CPU which support unaligned memory + * access in hardware. + */ +static xxh_u32 XXH_read32(const void* memPtr) { return *(const xxh_u32*) memPtr; } - ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize` bytes. - ZSTD_decompressContinue() is very sensitive to contiguity, - if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place, - or that previous contiguous segment is large enough to properly handle maximum back-reference distance. - There are multiple ways to guarantee this condition. +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) - The most memory efficient way is to use a round buffer of sufficient size. - Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(), - which can @return an error code if required value is too large for current system (in 32-bits mode). - In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one, - up to the moment there is not enough room left in the buffer to guarantee decoding another full block, - which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`. - At which point, decoding can resume from the beginning of the buffer. - Note that already decoded data stored in the buffer should be flushed before being overwritten. +/* + * __attribute__((aligned(1))) is supported by gcc and clang. Originally the + * documentation claimed that it only increased the alignment, but actually it + * can decrease it on gcc, clang, and icc: + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502, + * https://gcc.godbolt.org/z/xYez1j67Y. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; } __attribute__((packed)) unalign; +#endif +static xxh_u32 XXH_read32(const void* ptr) +{ + typedef __attribute__((aligned(1))) xxh_u32 xxh_unalign32; + return *((const xxh_unalign32*)ptr); +} - There are alternatives possible, for example using two or more buffers of size `windowSize` each, though they consume more memory. +#else - Finally, if you control the compression process, you can also ignore all buffer size rules, - as long as the encoder and decoder progress in "lock-step", - aka use exactly the same buffer sizes, break contiguity at the same place, etc. +/* + * Portable and safe solution. Generally efficient. + * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + */ +static xxh_u32 XXH_read32(const void* memPtr) +{ + xxh_u32 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; +} - Once buffers are setup, start decompression, with ZSTD_decompressBegin(). - If decompression requires a dictionary, use ZSTD_decompressBegin_usingDict() or ZSTD_decompressBegin_usingDDict(). +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ - Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively. - ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue(). - ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail. - @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). - It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item. - It can also be an error code, which can be tested with ZSTD_isError(). +/* *** Endianness *** */ - A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. - Context can then be reset to start a new decompression. +/*! + * @ingroup tuning + * @def XXH_CPU_LITTLE_ENDIAN + * @brief Whether the target is little endian. + * + * Defined to 1 if the target is little endian, or 0 if it is big endian. + * It can be defined externally, for example on the compiler command line. + * + * If it is not defined, + * a runtime check (which is usually constant folded) is used instead. + * + * @note + * This is not necessarily defined to an integer constant. + * + * @see XXH_isLittleEndian() for the runtime check. + */ +#ifndef XXH_CPU_LITTLE_ENDIAN +/* + * Try to detect endianness automatically, to avoid the nonstandard behavior + * in `XXH_isLittleEndian()` + */ +# if defined(_WIN32) /* Windows is always little endian */ \ + || defined(__LITTLE_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 1 +# elif defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 0 +# else +/*! + * @internal + * @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN. + * + * Most compilers will constant fold this. + */ +static int XXH_isLittleEndian(void) +{ + /* + * Portable and well-defined behavior. + * Don't use static: it is detrimental to performance. + */ + const union { xxh_u32 u; xxh_u8 c[4]; } one = { 1 }; + return one.c[0]; +} +# define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian() +# endif +#endif - Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType(). - This information is not required to properly decode a frame. - == Special case : skippable frames == - Skippable frames allow integration of user-defined data into a flow of concatenated frames. - Skippable frames will be ignored (skipped) by decompressor. - The format of skippable frames is as follows : - a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F - b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits - c) Frame Content - any content (User Data) of length equal to Frame Size - For skippable frames ZSTD_getFrameHeader() returns zfhPtr->frameType==ZSTD_skippableFrame. - For skippable frames ZSTD_decompressContinue() always returns 0 : it only skips the content. -*/ -/*===== Buffer-less streaming decompression functions =====*/ -typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e; -typedef struct { - unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */ - unsigned long long windowSize; /* can be very large, up to <= frameContentSize */ - unsigned blockSizeMax; - ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */ - unsigned headerSize; - unsigned dictID; - unsigned checksumFlag; -} ZSTD_frameHeader; +/* **************************************** +* Compiler-specific Functions and Macros +******************************************/ +#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -/*! ZSTD_getFrameHeader() : - * decode Frame Header, or requires larger `srcSize`. - * @return : 0, `zfhPtr` is correctly filled, - * >0, `srcSize` is too small, value is wanted `srcSize` amount, - * or an error code, which can be tested using ZSTD_isError() */ -ZSTDLIB_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */ -/*! ZSTD_getFrameHeader_advanced() : - * same as ZSTD_getFrameHeader(), - * with added capability to select a format (like ZSTD_f_zstd1_magicless) */ -ZSTDLIB_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format); -ZSTDLIB_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */ +#ifdef __has_builtin +# define XXH_HAS_BUILTIN(x) __has_builtin(x) +#else +# define XXH_HAS_BUILTIN(x) 0 +#endif -ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); -ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); -ZSTDLIB_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); -ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); -ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -/* misc */ -ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); -typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; -ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); +/* + * C23 and future versions have standard "unreachable()". + * Once it has been implemented reliably we can add it as an + * additional case: + * + * ``` + * #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN) + * # include + * # ifdef unreachable + * # define XXH_UNREACHABLE() unreachable() + * # endif + * #endif + * ``` + * + * Note C++23 also has std::unreachable() which can be detected + * as follows: + * ``` + * #if defined(__cpp_lib_unreachable) && (__cpp_lib_unreachable >= 202202L) + * # include + * # define XXH_UNREACHABLE() std::unreachable() + * #endif + * ``` + * NB: `__cpp_lib_unreachable` is defined in the `` header. + * We don't use that as including `` in `extern "C"` blocks + * doesn't work on GCC12 + */ +#if XXH_HAS_BUILTIN(__builtin_unreachable) +# define XXH_UNREACHABLE() __builtin_unreachable() +#elif defined(_MSC_VER) +# define XXH_UNREACHABLE() __assume(0) +#else +# define XXH_UNREACHABLE() +#endif -/* ============================ */ -/** Block level API */ -/* ============================ */ +#if XXH_HAS_BUILTIN(__builtin_assume) +# define XXH_ASSUME(c) __builtin_assume(c) +#else +# define XXH_ASSUME(c) if (!(c)) { XXH_UNREACHABLE(); } +#endif /*! - Block functions produce and decode raw zstd blocks, without frame metadata. - Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes). - But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes. - - A few rules to respect : - - Compressing and decompressing require a context structure - + Use ZSTD_createCCtx() and ZSTD_createDCtx() - - It is necessary to init context before starting - + compression : any ZSTD_compressBegin*() variant, including with dictionary - + decompression : any ZSTD_decompressBegin*() variant, including with dictionary - + copyCCtx() and copyDCtx() can be used too - - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB - + If input is larger than a block size, it's necessary to split input data into multiple blocks - + For inputs larger than a single block, consider using regular ZSTD_compress() instead. - Frame metadata is not that costly, and quickly becomes negligible as source size grows larger than a block. - - When a block is considered not compressible enough, ZSTD_compressBlock() result will be 0 (zero) ! - ===> In which case, nothing is produced into `dst` ! - + User __must__ test for such outcome and deal directly with uncompressed data - + A block cannot be declared incompressible if ZSTD_compressBlock() return value was != 0. - Doing so would mess up with statistics history, leading to potential data corruption. - + ZSTD_decompressBlock() _doesn't accept uncompressed data as input_ !! - + In case of multiple successive blocks, should some of them be uncompressed, - decoder must be informed of their existence in order to follow proper history. - Use ZSTD_insertBlock() for such a case. -*/ - -/*===== Raw zstd block functions =====*/ -ZSTDLIB_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx); -ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -ZSTDLIB_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */ - - -#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */ - -#if defined (__cplusplus) -} -#endif -/**** ended inlining ../zstd.h ****/ -#define FSE_STATIC_LINKING_ONLY -/**** skipping file: fse.h ****/ -#define HUF_STATIC_LINKING_ONLY -/**** skipping file: huf.h ****/ -#ifndef XXH_STATIC_LINKING_ONLY -# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ -#endif -/**** start inlining xxhash.h ****/ -/* - * xxHash - Extremely Fast Hash algorithm - * Header File - * Copyright (c) 2012-2021, Yann Collet, Facebook, Inc. - * - * You can contact the author at : - * - xxHash source repository : https://github.com/Cyan4973/xxHash + * @internal + * @def XXH_rotl32(x,r) + * @brief 32-bit rotate left. * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. -*/ - -/* Notice extracted from xxHash homepage : - -xxHash is an extremely fast Hash algorithm, running at RAM speed limits. -It also successfully passes all tests from the SMHasher suite. - -Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) - -Name Speed Q.Score Author -xxHash 5.4 GB/s 10 -CrapWow 3.2 GB/s 2 Andrew -MumurHash 3a 2.7 GB/s 10 Austin Appleby -SpookyHash 2.0 GB/s 10 Bob Jenkins -SBox 1.4 GB/s 9 Bret Mulvey -Lookup3 1.2 GB/s 9 Bob Jenkins -SuperFastHash 1.2 GB/s 1 Paul Hsieh -CityHash64 1.05 GB/s 10 Pike & Alakuijala -FNV 0.55 GB/s 5 Fowler, Noll, Vo -CRC32 0.43 GB/s 9 -MD5-32 0.33 GB/s 10 Ronald L. Rivest -SHA1-32 0.28 GB/s 10 - -Q.Score is a measure of quality of the hash function. -It depends on successfully passing SMHasher test set. -10 is a perfect score. - -A 64-bits version, named XXH64, is available since r35. -It offers much better speed, but for 64-bits applications only. -Name Speed on 64 bits Speed on 32 bits -XXH64 13.8 GB/s 1.9 GB/s -XXH32 6.8 GB/s 6.0 GB/s -*/ + * @param x The 32-bit integer to be rotated. + * @param r The number of bits to rotate. + * @pre + * @p r > 0 && @p r < 32 + * @note + * @p x and @p r may be evaluated multiple times. + * @return The rotated result. + */ +#if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) \ + && XXH_HAS_BUILTIN(__builtin_rotateleft64) +# define XXH_rotl32 __builtin_rotateleft32 +# define XXH_rotl64 __builtin_rotateleft64 +/* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */ +#elif defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +# define XXH_rotl64(x,r) _rotl64(x,r) +#else +# define XXH_rotl32(x,r) (((x) << (r)) | ((x) >> (32 - (r)))) +# define XXH_rotl64(x,r) (((x) << (r)) | ((x) >> (64 - (r)))) +#endif -#if defined (__cplusplus) -extern "C" { +/*! + * @internal + * @fn xxh_u32 XXH_swap32(xxh_u32 x) + * @brief A 32-bit byteswap. + * + * @param x The 32-bit integer to byteswap. + * @return @p x, byteswapped. + */ +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap32 _byteswap_ulong +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +#else +static xxh_u32 XXH_swap32 (xxh_u32 x) +{ + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); +} #endif -#ifndef XXHASH_H_5627135585666179 -#define XXHASH_H_5627135585666179 1 +/* *************************** +* Memory reads +*****************************/ -/* **************************** -* Definitions -******************************/ -/**** skipping file: zstd_deps.h ****/ -typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; - +/*! + * @internal + * @brief Enum to indicate whether a pointer is aligned. + */ +typedef enum { + XXH_aligned, /*!< Aligned */ + XXH_unaligned /*!< Possibly unaligned */ +} XXH_alignment; -/* **************************** -* API modifier -******************************/ -/** XXH_PRIVATE_API -* This is useful if you want to include xxhash functions in `static` mode -* in order to inline them, and remove their symbol from the public list. -* Methodology : -* #define XXH_PRIVATE_API -* #include "xxhash.h" -* `xxhash.c` is automatically included. -* It's not useful to compile and link it as a separate module anymore. -*/ -#ifdef XXH_PRIVATE_API -# ifndef XXH_STATIC_LINKING_ONLY -# define XXH_STATIC_LINKING_ONLY -# endif -# if defined(__GNUC__) -# define XXH_PUBLIC_API static __inline __attribute__((unused)) -# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define XXH_PUBLIC_API static inline -# elif defined(_MSC_VER) -# define XXH_PUBLIC_API static __inline -# else -# define XXH_PUBLIC_API static /* this version may generate warnings for unused static functions; disable the relevant warning */ -# endif -#else -# define XXH_PUBLIC_API /* do nothing */ -#endif /* XXH_PRIVATE_API */ +/* + * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. + * + * This is ideal for older compilers which don't inline memcpy. + */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) -/*!XXH_NAMESPACE, aka Namespace Emulation : +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u32)bytePtr[1] << 8) + | ((xxh_u32)bytePtr[2] << 16) + | ((xxh_u32)bytePtr[3] << 24); +} -If you want to include _and expose_ xxHash functions from within your own library, -but also want to avoid symbol collisions with another library which also includes xxHash, +XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[3] + | ((xxh_u32)bytePtr[2] << 8) + | ((xxh_u32)bytePtr[1] << 16) + | ((xxh_u32)bytePtr[0] << 24); +} -you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library -with the value of XXH_NAMESPACE (so avoid to keep it NULL and avoid numeric values). +#else +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); +} -Note that no change is required within the calling program as long as it includes `xxhash.h` : -regular symbol name will be automatically translated by this header. -*/ -#ifdef XXH_NAMESPACE -# define XXH_CAT(A,B) A##B -# define XXH_NAME2(A,B) XXH_CAT(A,B) -# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) -# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) -# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) -# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) -# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) -# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) -# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) -# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) -# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) -# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) -# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) -# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) -# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) -# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) -# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) -# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) -# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) -# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) -# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) +static xxh_u32 XXH_readBE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); +} #endif +XXH_FORCE_INLINE xxh_u32 +XXH_readLE32_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) { + return XXH_readLE32(ptr); + } else { + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32*)ptr : XXH_swap32(*(const xxh_u32*)ptr); + } +} + /* ************************************* -* Version +* Misc ***************************************/ -#define XXH_VERSION_MAJOR 0 -#define XXH_VERSION_MINOR 6 -#define XXH_VERSION_RELEASE 2 -#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) -XXH_PUBLIC_API unsigned XXH_versionNumber (void); - +/*! @ingroup public */ +XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } -/* **************************** -* Simple Hash Functions -******************************/ -typedef unsigned int XXH32_hash_t; -typedef unsigned long long XXH64_hash_t; - -XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed); -XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed); - -/*! -XXH32() : - Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input". - The memory between input & input+length must be valid (allocated and read-accessible). - "seed" can be used to alter the result predictably. - Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s -XXH64() : - Calculate the 64-bits hash of sequence of length "len" stored at memory address "input". - "seed" can be used to alter the result predictably. - This function runs 2x faster on 64-bits systems, but slower on 32-bits systems (see benchmark). -*/ +/* ******************************************************************* +* 32-bit hash functions +*********************************************************************/ +/*! + * @} + * @defgroup XXH32_impl XXH32 implementation + * @ingroup impl + * + * Details on the XXH32 implementation. + * @{ + */ + /* #define instead of static const, to be used as initializers */ +#define XXH_PRIME32_1 0x9E3779B1U /*!< 0b10011110001101110111100110110001 */ +#define XXH_PRIME32_2 0x85EBCA77U /*!< 0b10000101111010111100101001110111 */ +#define XXH_PRIME32_3 0xC2B2AE3DU /*!< 0b11000010101100101010111000111101 */ +#define XXH_PRIME32_4 0x27D4EB2FU /*!< 0b00100111110101001110101100101111 */ +#define XXH_PRIME32_5 0x165667B1U /*!< 0b00010110010101100110011110110001 */ + +#ifdef XXH_OLD_NAMES +# define PRIME32_1 XXH_PRIME32_1 +# define PRIME32_2 XXH_PRIME32_2 +# define PRIME32_3 XXH_PRIME32_3 +# define PRIME32_4 XXH_PRIME32_4 +# define PRIME32_5 XXH_PRIME32_5 +#endif -/* **************************** -* Streaming Hash Functions -******************************/ -typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */ -typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ +/*! + * @internal + * @brief Normal stripe processing routine. + * + * This shuffles the bits so that any bit from @p input impacts several bits in + * @p acc. + * + * @param acc The accumulator lane. + * @param input The stripe of input to mix. + * @return The mixed accumulator lane. + */ +static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input) +{ + acc += input * XXH_PRIME32_2; + acc = XXH_rotl32(acc, 13); + acc *= XXH_PRIME32_1; +#if (defined(__SSE4_1__) || defined(__aarch64__) || defined(__wasm_simd128__)) && !defined(XXH_ENABLE_AUTOVECTORIZE) + /* + * UGLY HACK: + * A compiler fence is the only thing that prevents GCC and Clang from + * autovectorizing the XXH32 loop (pragmas and attributes don't work for some + * reason) without globally disabling SSE4.1. + * + * The reason we want to avoid vectorization is because despite working on + * 4 integers at a time, there are multiple factors slowing XXH32 down on + * SSE4: + * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on + * newer chips!) making it slightly slower to multiply four integers at + * once compared to four integers independently. Even when pmulld was + * fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE + * just to multiply unless doing a long operation. + * + * - Four instructions are required to rotate, + * movqda tmp, v // not required with VEX encoding + * pslld tmp, 13 // tmp <<= 13 + * psrld v, 19 // x >>= 19 + * por v, tmp // x |= tmp + * compared to one for scalar: + * roll v, 13 // reliably fast across the board + * shldl v, v, 13 // Sandy Bridge and later prefer this for some reason + * + * - Instruction level parallelism is actually more beneficial here because + * the SIMD actually serializes this operation: While v1 is rotating, v2 + * can load data, while v3 can multiply. SSE forces them to operate + * together. + * + * This is also enabled on AArch64, as Clang is *very aggressive* in vectorizing + * the loop. NEON is only faster on the A53, and with the newer cores, it is less + * than half the speed. + * + * Additionally, this is used on WASM SIMD128 because it JITs to the same + * SIMD instructions and has the same issue. + */ + XXH_COMPILER_GUARD(acc); +#endif + return acc; +} -/*! State allocation, compatible with dynamic libraries */ +/*! + * @internal + * @brief Mixes all bits to finalize the hash. + * + * The final mix ensures that all input bits have a chance to impact any bit in + * the output digest, resulting in an unbiased distribution. + * + * @param hash The hash to avalanche. + * @return The avalanched hash. + */ +static xxh_u32 XXH32_avalanche(xxh_u32 hash) +{ + hash ^= hash >> 15; + hash *= XXH_PRIME32_2; + hash ^= hash >> 13; + hash *= XXH_PRIME32_3; + hash ^= hash >> 16; + return hash; +} -XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); -XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); +#define XXH_get32bits(p) XXH_readLE32_align(p, align) -XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void); -XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); +/*! + * @internal + * @brief Processes the last 0-15 bytes of @p ptr. + * + * There may be up to 15 bytes remaining to consume from the input. + * This final stage will digest them to ensure that all input bytes are present + * in the final mix. + * + * @param hash The hash to finalize. + * @param ptr The pointer to the remaining input. + * @param len The remaining length, modulo 16. + * @param align Whether @p ptr is aligned. + * @return The finalized hash. + * @see XXH64_finalize(). + */ +static XXH_PUREF xxh_u32 +XXH32_finalize(xxh_u32 hash, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ +#define XXH_PROCESS1 do { \ + hash += (*ptr++) * XXH_PRIME32_5; \ + hash = XXH_rotl32(hash, 11) * XXH_PRIME32_1; \ +} while (0) + +#define XXH_PROCESS4 do { \ + hash += XXH_get32bits(ptr) * XXH_PRIME32_3; \ + ptr += 4; \ + hash = XXH_rotl32(hash, 17) * XXH_PRIME32_4; \ +} while (0) + + if (ptr==NULL) XXH_ASSERT(len == 0); + + /* Compact rerolled version; generally faster */ + if (!XXH32_ENDJMP) { + len &= 15; + while (len >= 4) { + XXH_PROCESS4; + len -= 4; + } + while (len > 0) { + XXH_PROCESS1; + --len; + } + return XXH32_avalanche(hash); + } else { + switch(len&15) /* or switch(bEnd - p) */ { + case 12: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 8: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 4: XXH_PROCESS4; + return XXH32_avalanche(hash); + + case 13: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 9: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 5: XXH_PROCESS4; + XXH_PROCESS1; + return XXH32_avalanche(hash); + + case 14: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 10: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 6: XXH_PROCESS4; + XXH_PROCESS1; + XXH_PROCESS1; + return XXH32_avalanche(hash); + + case 15: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 11: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 7: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 3: XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 2: XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 1: XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 0: return XXH32_avalanche(hash); + } + XXH_ASSERT(0); + return hash; /* reaching this point is deemed impossible */ + } +} +#ifdef XXH_OLD_NAMES +# define PROCESS1 XXH_PROCESS1 +# define PROCESS4 XXH_PROCESS4 +#else +# undef XXH_PROCESS1 +# undef XXH_PROCESS4 +#endif -/* hash streaming */ +/*! + * @internal + * @brief The implementation for @ref XXH32(). + * + * @param input , len , seed Directly passed from @ref XXH32(). + * @param align Whether @p input is aligned. + * @return The calculated hash. + */ +XXH_FORCE_INLINE XXH_PUREF xxh_u32 +XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align) +{ + xxh_u32 h32; -XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned int seed); -XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); -XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); + if (input==NULL) XXH_ASSERT(len == 0); -XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed); -XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); -XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr); + if (len>=16) { + const xxh_u8* const bEnd = input + len; + const xxh_u8* const limit = bEnd - 15; + xxh_u32 v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + xxh_u32 v2 = seed + XXH_PRIME32_2; + xxh_u32 v3 = seed + 0; + xxh_u32 v4 = seed - XXH_PRIME32_1; -/* -These functions generate the xxHash of an input provided in multiple segments. -Note that, for small input, they are slower than single-call functions, due to state management. -For small input, prefer `XXH32()` and `XXH64()` . + do { + v1 = XXH32_round(v1, XXH_get32bits(input)); input += 4; + v2 = XXH32_round(v2, XXH_get32bits(input)); input += 4; + v3 = XXH32_round(v3, XXH_get32bits(input)); input += 4; + v4 = XXH32_round(v4, XXH_get32bits(input)); input += 4; + } while (input < limit); + + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + } else { + h32 = seed + XXH_PRIME32_5; + } -XXH state must first be allocated, using XXH*_createState() . + h32 += (xxh_u32)len; -Start a new hash by initializing state with a seed, using XXH*_reset(). + return XXH32_finalize(h32, input, len&15, align); +} -Then, feed the hash state by calling XXH*_update() as many times as necessary. -Obviously, input must be allocated and read accessible. -The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed) +{ +#if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH32_state_t state; + XXH32_reset(&state, seed); + XXH32_update(&state, (const xxh_u8*)input, len); + return XXH32_digest(&state); +#else + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); + } } -Finally, a hash value can be produced anytime, by using XXH*_digest(). -This function returns the nn-bits hash as an int or long long. + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); +#endif +} -It's still possible to continue inserting input into the hash state after a digest, -and generate some new hashes later on, by calling again XXH*_digest(). -When done, free XXH state space if it was allocated dynamically. -*/ +/******* Hash streaming *******/ +#ifndef XXH_NO_STREAM +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) +{ + return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); +} +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} -/* ************************** -* Utils -****************************/ -#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* ! C99 */ -# define restrict /* disable restrict */ -#endif +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState) +{ + XXH_memcpy(dstState, srcState, sizeof(*dstState)); +} -XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dst_state, const XXH32_state_t* restrict src_state); -XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dst_state, const XXH64_state_t* restrict src_state); +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed) +{ + XXH_ASSERT(statePtr != NULL); + memset(statePtr, 0, sizeof(*statePtr)); + statePtr->v[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + statePtr->v[1] = seed + XXH_PRIME32_2; + statePtr->v[2] = seed + 0; + statePtr->v[3] = seed - XXH_PRIME32_1; + return XXH_OK; +} -/* ************************** -* Canonical representation -****************************/ -/* Default result type for XXH functions are primitive unsigned 32 and 64 bits. -* The canonical representation uses human-readable write convention, aka big-endian (large digits first). -* These functions allow transformation of hash result into and from its canonical format. -* This way, hash values can be written into a file / memory, and remain comparable on different systems and programs. -*/ -typedef struct { unsigned char digest[4]; } XXH32_canonical_t; -typedef struct { unsigned char digest[8]; } XXH64_canonical_t; +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode +XXH32_update(XXH32_state_t* state, const void* input, size_t len) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } -XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); -XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash); + { const xxh_u8* p = (const xxh_u8*)input; + const xxh_u8* const bEnd = p + len; -XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); -XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); + state->total_len_32 += (XXH32_hash_t)len; + state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16)); -#endif /* XXHASH_H_5627135585666179 */ + if (state->memsize + len < 16) { /* fill in tmp buffer */ + XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, len); + state->memsize += (XXH32_hash_t)len; + return XXH_OK; + } + if (state->memsize) { /* some data left from previous update */ + XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, 16-state->memsize); + { const xxh_u32* p32 = state->mem32; + state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p32)); p32++; + state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p32)); p32++; + state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p32)); p32++; + state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p32)); + } + p += 16-state->memsize; + state->memsize = 0; + } + if (p <= bEnd-16) { + const xxh_u8* const limit = bEnd - 16; -/* ================================================================================================ - This section contains definitions which are not guaranteed to remain stable. - They may change in future versions, becoming incompatible with a different version of the library. - They shall only be used with static linking. - Never use these definitions in association with dynamic linking ! -=================================================================================================== */ -#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXH_STATIC_H_3543687687345) -#define XXH_STATIC_H_3543687687345 - -/* These definitions are only meant to allow allocation of XXH state - statically, on stack, or in a struct for example. - Do not use members directly. */ - - struct XXH32_state_s { - unsigned total_len_32; - unsigned large_len; - unsigned v1; - unsigned v2; - unsigned v3; - unsigned v4; - unsigned mem32[4]; /* buffer defined as U32 for alignment */ - unsigned memsize; - unsigned reserved; /* never read nor write, will be removed in a future version */ - }; /* typedef'd to XXH32_state_t */ - - struct XXH64_state_s { - unsigned long long total_len; - unsigned long long v1; - unsigned long long v2; - unsigned long long v3; - unsigned long long v4; - unsigned long long mem64[4]; /* buffer defined as U64 for alignment */ - unsigned memsize; - unsigned reserved[2]; /* never read nor write, will be removed in a future version */ - }; /* typedef'd to XXH64_state_t */ - - -# ifdef XXH_PRIVATE_API -/**** start inlining xxhash.c ****/ -/* - * xxHash - Fast Hash algorithm - * Copyright (c) 2012-2021, Yann Collet, Facebook, Inc. - * - * You can contact the author at : - * - xxHash homepage: http://www.xxhash.com - * - xxHash source repository : https://github.com/Cyan4973/xxHash - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. -*/ + do { + state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p)); p+=4; + state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p)); p+=4; + state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p)); p+=4; + state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p)); p+=4; + } while (p<=limit); + } -/* ************************************* -* Tuning parameters -***************************************/ -/*!XXH_FORCE_MEMORY_ACCESS : - * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. - * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. - * The below switch allow to select different access method for improved performance. - * Method 0 (default) : use `memcpy()`. Safe and portable. - * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). - * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. - * Method 2 : direct access. This method doesn't depend on compiler but violate C standard. - * It can generate buggy code on targets which do not support unaligned memory accesses. - * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) - * See http://stackoverflow.com/a/32095106/646947 for details. - * Prefer these methods in priority order (0 > 1 > 2) - */ -#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ -# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) -# define XXH_FORCE_MEMORY_ACCESS 2 -# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \ - (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) || \ - defined(__ICCARM__) -# define XXH_FORCE_MEMORY_ACCESS 1 -# endif -#endif + if (p < bEnd) { + XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + } -/*!XXH_ACCEPT_NULL_INPUT_POINTER : - * If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. - * When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. - * By default, this option is disabled. To enable it, uncomment below define : - */ -/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */ + return XXH_OK; +} -/*!XXH_FORCE_NATIVE_FORMAT : - * By default, xxHash library provides endian-independent Hash values, based on little-endian convention. - * Results are therefore identical for little-endian and big-endian CPU. - * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. - * Should endian-independence be of no importance for your application, you may set the #define below to 1, - * to improve speed for Big-endian CPU. - * This option has no impact on Little_Endian CPU. - */ -#ifndef XXH_FORCE_NATIVE_FORMAT /* can be defined externally */ -# define XXH_FORCE_NATIVE_FORMAT 0 -#endif -/*!XXH_FORCE_ALIGN_CHECK : - * This is a minor performance trick, only useful with lots of very small keys. - * It means : check for aligned/unaligned input. - * The check costs one initial branch per hash; set to 0 when the input data - * is guaranteed to be aligned. - */ -#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ -# if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) -# define XXH_FORCE_ALIGN_CHECK 0 -# else -# define XXH_FORCE_ALIGN_CHECK 1 -# endif -#endif +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t* state) +{ + xxh_u32 h32; + if (state->large_len) { + h32 = XXH_rotl32(state->v[0], 1) + + XXH_rotl32(state->v[1], 7) + + XXH_rotl32(state->v[2], 12) + + XXH_rotl32(state->v[3], 18); + } else { + h32 = state->v[2] /* == seed */ + XXH_PRIME32_5; + } -/* ************************************* -* Includes & Memory related functions -***************************************/ -/* Modify the local functions below should you wish to use some other memory routines */ -/* for ZSTD_malloc(), ZSTD_free() */ -#define ZSTD_DEPS_NEED_MALLOC -/**** skipping file: zstd_deps.h ****/ -static void* XXH_malloc(size_t s) { return ZSTD_malloc(s); } -static void XXH_free (void* p) { ZSTD_free(p); } -static void* XXH_memcpy(void* dest, const void* src, size_t size) { return ZSTD_memcpy(dest,src,size); } + h32 += state->total_len_32; -#ifndef XXH_STATIC_LINKING_ONLY -# define XXH_STATIC_LINKING_ONLY -#endif -/**** skipping file: xxhash.h ****/ + return XXH32_finalize(h32, (const xxh_u8*)state->mem32, state->memsize, XXH_aligned); +} +#endif /* !XXH_NO_STREAM */ +/******* Canonical representation *******/ -/* ************************************* -* Compiler Specific Options -***************************************/ -/**** skipping file: compiler.h ****/ +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); +} +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) +{ + return XXH_readBE32(src); +} -/* ************************************* -* Basic Types -***************************************/ -/**** skipping file: mem.h ****/ +#ifndef XXH_NO_LONG_LONG -#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) +/* ******************************************************************* +* 64-bit hash functions +*********************************************************************/ +/*! + * @} + * @ingroup impl + * @{ + */ +/******* Memory access *******/ -/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ -static U32 XXH_read32(const void* memPtr) { return *(const U32*) memPtr; } -static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; } +typedef XXH64_hash_t xxh_u64; -#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) +#ifdef XXH_OLD_NAMES +# define U64 xxh_u64 +#endif -/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ -/* currently only defined for gcc and icc */ -typedef union { U32 u32; U64 u64; } __attribute__((packed)) unalign; +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE64 and XXH_readBE64. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) -static U32 XXH_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } -static U64 XXH_read64(const void* ptr) { return ((const unalign*)ptr)->u64; } +/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + return *(const xxh_u64*) memPtr; +} -#else +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) -/* portable and safe solution. Generally efficient. - * see : http://stackoverflow.com/a/32095106/646947 +/* + * __attribute__((aligned(1))) is supported by gcc and clang. Originally the + * documentation claimed that it only increased the alignment, but actually it + * can decrease it on gcc, clang, and icc: + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502, + * https://gcc.godbolt.org/z/xYez1j67Y. */ - -static U32 XXH_read32(const void* memPtr) +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) unalign64; +#endif +static xxh_u64 XXH_read64(const void* ptr) { - U32 val; - ZSTD_memcpy(&val, memPtr, sizeof(val)); - return val; + typedef __attribute__((aligned(1))) xxh_u64 xxh_unalign64; + return *((const xxh_unalign64*)ptr); } -static U64 XXH_read64(const void* memPtr) +#else + +/* + * Portable and safe solution. Generally efficient. + * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + */ +static xxh_u64 XXH_read64(const void* memPtr) { - U64 val; - ZSTD_memcpy(&val, memPtr, sizeof(val)); + xxh_u64 val; + XXH_memcpy(&val, memPtr, sizeof(val)); return val; } #endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ - -/* **************************************** -* Compiler-specific Functions and Macros -******************************************/ -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) - -/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */ -#if defined(_MSC_VER) -# define XXH_rotl32(x,r) _rotl(x,r) -# define XXH_rotl64(x,r) _rotl64(x,r) -#else -#if defined(__ICCARM__) -# include -# define XXH_rotl32(x,r) __ROR(x,(32 - r)) -#else -# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) -#endif -# define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r))) -#endif - #if defined(_MSC_VER) /* Visual Studio */ -# define XXH_swap32 _byteswap_ulong # define XXH_swap64 _byteswap_uint64 -#elif GCC_VERSION >= 403 -# define XXH_swap32 __builtin_bswap32 +#elif XXH_GCC_VERSION >= 403 # define XXH_swap64 __builtin_bswap64 #else -static U32 XXH_swap32 (U32 x) -{ - return ((x << 24) & 0xff000000 ) | - ((x << 8) & 0x00ff0000 ) | - ((x >> 8) & 0x0000ff00 ) | - ((x >> 24) & 0x000000ff ); -} -static U64 XXH_swap64 (U64 x) +static xxh_u64 XXH_swap64(xxh_u64 x) { return ((x << 56) & 0xff00000000000000ULL) | ((x << 40) & 0x00ff000000000000ULL) | @@ -6518,237 +10733,208 @@ static U64 XXH_swap64 (U64 x) #endif -/* ************************************* -* Architecture Macros -***************************************/ -typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; - -/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */ -#ifndef XXH_CPU_LITTLE_ENDIAN - static const int g_one = 1; -# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&g_one)) -#endif - - -/* *************************** -* Memory reads -*****************************/ -typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; - -FORCE_INLINE_TEMPLATE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) -{ - if (align==XXH_unaligned) - return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); - else - return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr); -} +/* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) -FORCE_INLINE_TEMPLATE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* memPtr) { - return XXH_readLE32_align(ptr, endian, XXH_unaligned); + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u64)bytePtr[1] << 8) + | ((xxh_u64)bytePtr[2] << 16) + | ((xxh_u64)bytePtr[3] << 24) + | ((xxh_u64)bytePtr[4] << 32) + | ((xxh_u64)bytePtr[5] << 40) + | ((xxh_u64)bytePtr[6] << 48) + | ((xxh_u64)bytePtr[7] << 56); } -static U32 XXH_readBE32(const void* ptr) +XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void* memPtr) { - return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[7] + | ((xxh_u64)bytePtr[6] << 8) + | ((xxh_u64)bytePtr[5] << 16) + | ((xxh_u64)bytePtr[4] << 24) + | ((xxh_u64)bytePtr[3] << 32) + | ((xxh_u64)bytePtr[2] << 40) + | ((xxh_u64)bytePtr[1] << 48) + | ((xxh_u64)bytePtr[0] << 56); } -FORCE_INLINE_TEMPLATE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) +#else +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* ptr) { - if (align==XXH_unaligned) - return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); - else - return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr); + return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); } -FORCE_INLINE_TEMPLATE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) +static xxh_u64 XXH_readBE64(const void* ptr) { - return XXH_readLE64_align(ptr, endian, XXH_unaligned); + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); } +#endif -static U64 XXH_readBE64(const void* ptr) +XXH_FORCE_INLINE xxh_u64 +XXH_readLE64_align(const void* ptr, XXH_alignment align) { - return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); + if (align==XXH_unaligned) + return XXH_readLE64(ptr); + else + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64*)ptr : XXH_swap64(*(const xxh_u64*)ptr); } -/* ************************************* -* Macros -***************************************/ -#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ - - -/* ************************************* -* Constants -***************************************/ -static const U32 PRIME32_1 = 2654435761U; -static const U32 PRIME32_2 = 2246822519U; -static const U32 PRIME32_3 = 3266489917U; -static const U32 PRIME32_4 = 668265263U; -static const U32 PRIME32_5 = 374761393U; - -static const U64 PRIME64_1 = 11400714785074694791ULL; -static const U64 PRIME64_2 = 14029467366897019727ULL; -static const U64 PRIME64_3 = 1609587929392839161ULL; -static const U64 PRIME64_4 = 9650029242287828579ULL; -static const U64 PRIME64_5 = 2870177450012600261ULL; - -XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } - +/******* xxh64 *******/ +/*! + * @} + * @defgroup XXH64_impl XXH64 implementation + * @ingroup impl + * + * Details on the XXH64 implementation. + * @{ + */ +/* #define rather that static const, to be used as initializers */ +#define XXH_PRIME64_1 0x9E3779B185EBCA87ULL /*!< 0b1001111000110111011110011011000110000101111010111100101010000111 */ +#define XXH_PRIME64_2 0xC2B2AE3D27D4EB4FULL /*!< 0b1100001010110010101011100011110100100111110101001110101101001111 */ +#define XXH_PRIME64_3 0x165667B19E3779F9ULL /*!< 0b0001011001010110011001111011000110011110001101110111100111111001 */ +#define XXH_PRIME64_4 0x85EBCA77C2B2AE63ULL /*!< 0b1000010111101011110010100111011111000010101100101010111001100011 */ +#define XXH_PRIME64_5 0x27D4EB2F165667C5ULL /*!< 0b0010011111010100111010110010111100010110010101100110011111000101 */ + +#ifdef XXH_OLD_NAMES +# define PRIME64_1 XXH_PRIME64_1 +# define PRIME64_2 XXH_PRIME64_2 +# define PRIME64_3 XXH_PRIME64_3 +# define PRIME64_4 XXH_PRIME64_4 +# define PRIME64_5 XXH_PRIME64_5 +#endif -/* ************************** -* Utils -****************************/ -XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dstState, const XXH32_state_t* restrict srcState) +/*! @copydoc XXH32_round */ +static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input) { - ZSTD_memcpy(dstState, srcState, sizeof(*dstState)); + acc += input * XXH_PRIME64_2; + acc = XXH_rotl64(acc, 31); + acc *= XXH_PRIME64_1; +#if (defined(__AVX512F__)) && !defined(XXH_ENABLE_AUTOVECTORIZE) + /* + * DISABLE AUTOVECTORIZATION: + * A compiler fence is used to prevent GCC and Clang from + * autovectorizing the XXH64 loop (pragmas and attributes don't work for some + * reason) without globally disabling AVX512. + * + * Autovectorization of XXH64 tends to be detrimental, + * though the exact outcome may change depending on exact cpu and compiler version. + * For information, it has been reported as detrimental for Skylake-X, + * but possibly beneficial for Zen4. + * + * The default is to disable auto-vectorization, + * but you can select to enable it instead using `XXH_ENABLE_AUTOVECTORIZE` build variable. + */ + XXH_COMPILER_GUARD(acc); +#endif + return acc; } -XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dstState, const XXH64_state_t* restrict srcState) +static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val) { - ZSTD_memcpy(dstState, srcState, sizeof(*dstState)); + val = XXH64_round(0, val); + acc ^= val; + acc = acc * XXH_PRIME64_1 + XXH_PRIME64_4; + return acc; } - -/* *************************** -* Simple Hash Functions -*****************************/ - -static U32 XXH32_round(U32 seed, U32 input) +/*! @copydoc XXH32_avalanche */ +static xxh_u64 XXH64_avalanche(xxh_u64 hash) { - seed += input * PRIME32_2; - seed = XXH_rotl32(seed, 13); - seed *= PRIME32_1; - return seed; + hash ^= hash >> 33; + hash *= XXH_PRIME64_2; + hash ^= hash >> 29; + hash *= XXH_PRIME64_3; + hash ^= hash >> 32; + return hash; } -FORCE_INLINE_TEMPLATE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* bEnd = p + len; - U32 h32; -#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align) - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (p==NULL) { - len=0; - bEnd=p=(const BYTE*)(size_t)16; - } -#endif - - if (len>=16) { - const BYTE* const limit = bEnd - 16; - U32 v1 = seed + PRIME32_1 + PRIME32_2; - U32 v2 = seed + PRIME32_2; - U32 v3 = seed + 0; - U32 v4 = seed - PRIME32_1; - do { - v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4; - v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4; - v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4; - v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4; - } while (p<=limit); +#define XXH_get64bits(p) XXH_readLE64_align(p, align) - h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); - } else { - h32 = seed + PRIME32_5; +/*! + * @internal + * @brief Processes the last 0-31 bytes of @p ptr. + * + * There may be up to 31 bytes remaining to consume from the input. + * This final stage will digest them to ensure that all input bytes are present + * in the final mix. + * + * @param hash The hash to finalize. + * @param ptr The pointer to the remaining input. + * @param len The remaining length, modulo 32. + * @param align Whether @p ptr is aligned. + * @return The finalized hash + * @see XXH32_finalize(). + */ +static XXH_PUREF xxh_u64 +XXH64_finalize(xxh_u64 hash, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ + if (ptr==NULL) XXH_ASSERT(len == 0); + len &= 31; + while (len >= 8) { + xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr)); + ptr += 8; + hash ^= k1; + hash = XXH_rotl64(hash,27) * XXH_PRIME64_1 + XXH_PRIME64_4; + len -= 8; } - - h32 += (U32) len; - - while (p+4<=bEnd) { - h32 += XXH_get32bits(p) * PRIME32_3; - h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; - p+=4; + if (len >= 4) { + hash ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1; + ptr += 4; + hash = XXH_rotl64(hash, 23) * XXH_PRIME64_2 + XXH_PRIME64_3; + len -= 4; } - - while (p 0) { + hash ^= (*ptr++) * XXH_PRIME64_5; + hash = XXH_rotl64(hash, 11) * XXH_PRIME64_1; + --len; } - - h32 ^= h32 >> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; - - return h32; + return XXH64_avalanche(hash); } - -XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed) -{ -#if 0 - /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ - XXH32_CREATESTATE_STATIC(state); - XXH32_reset(state, seed); - XXH32_update(state, input, len); - return XXH32_digest(state); +#ifdef XXH_OLD_NAMES +# define PROCESS1_64 XXH_PROCESS1_64 +# define PROCESS4_64 XXH_PROCESS4_64 +# define PROCESS8_64 XXH_PROCESS8_64 #else - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if (XXH_FORCE_ALIGN_CHECK) { - if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); - else - return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); - } } - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); - else - return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); +# undef XXH_PROCESS1_64 +# undef XXH_PROCESS4_64 +# undef XXH_PROCESS8_64 #endif -} - - -static U64 XXH64_round(U64 acc, U64 input) -{ - acc += input * PRIME64_2; - acc = XXH_rotl64(acc, 31); - acc *= PRIME64_1; - return acc; -} - -static U64 XXH64_mergeRound(U64 acc, U64 val) -{ - val = XXH64_round(0, val); - acc ^= val; - acc = acc * PRIME64_1 + PRIME64_4; - return acc; -} -FORCE_INLINE_TEMPLATE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align) +/*! + * @internal + * @brief The implementation for @ref XXH64(). + * + * @param input , len , seed Directly passed from @ref XXH64(). + * @param align Whether @p input is aligned. + * @return The calculated hash. + */ +XXH_FORCE_INLINE XXH_PUREF xxh_u64 +XXH64_endian_align(const xxh_u8* input, size_t len, xxh_u64 seed, XXH_alignment align) { - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - U64 h64; -#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align) - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (p==NULL) { - len=0; - bEnd=p=(const BYTE*)(size_t)32; - } -#endif + xxh_u64 h64; + if (input==NULL) XXH_ASSERT(len == 0); if (len>=32) { - const BYTE* const limit = bEnd - 32; - U64 v1 = seed + PRIME64_1 + PRIME64_2; - U64 v2 = seed + PRIME64_2; - U64 v3 = seed + 0; - U64 v4 = seed - PRIME64_1; + const xxh_u8* const bEnd = input + len; + const xxh_u8* const limit = bEnd - 31; + xxh_u64 v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + xxh_u64 v2 = seed + XXH_PRIME64_2; + xxh_u64 v3 = seed + 0; + xxh_u64 v4 = seed - XXH_PRIME64_1; do { - v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8; - v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8; - v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8; - v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8; - } while (p<=limit); + v1 = XXH64_round(v1, XXH_get64bits(input)); input+=8; + v2 = XXH64_round(v2, XXH_get64bits(input)); input+=8; + v3 = XXH64_round(v3, XXH_get64bits(input)); input+=8; + v4 = XXH64_round(v4, XXH_get64bits(input)); input+=8; + } while (input> 33; - h64 *= PRIME64_2; - h64 ^= h64 >> 29; - h64 *= PRIME64_3; - h64 ^= h64 >> 32; + h64 += (xxh_u64) len; - return h64; + return XXH64_finalize(h64, input, len, align); } -XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed) +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64 (XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) { -#if 0 +#if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2 /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ - XXH64_CREATESTATE_STATIC(state); - XXH64_reset(state, seed); - XXH64_update(state, input, len); - return XXH64_digest(state); + XXH64_state_t state; + XXH64_reset(&state, seed); + XXH64_update(&state, (const xxh_u8*)input, len); + return XXH64_digest(&state); #else - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - if (XXH_FORCE_ALIGN_CHECK) { if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); - else - return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); + return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); } } - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); - else - return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); -#endif -} - - -/* ************************************************** -* Advanced Hash Functions -****************************************************/ + return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); -XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) -{ - return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); -} -XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) -{ - XXH_free(statePtr); - return XXH_OK; +#endif } +/******* Hash Streaming *******/ +#ifndef XXH_NO_STREAM +/*! @ingroup XXH64_family*/ XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) { return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); } +/*! @ingroup XXH64_family */ XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) { XXH_free(statePtr); return XXH_OK; } - -/*** Hash feed ***/ - -XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed) +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dstState, const XXH64_state_t* srcState) { - XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ - ZSTD_memset(&state, 0, sizeof(state)-4); /* do not write into reserved, for future removal */ - state.v1 = seed + PRIME32_1 + PRIME32_2; - state.v2 = seed + PRIME32_2; - state.v3 = seed + 0; - state.v4 = seed - PRIME32_1; - ZSTD_memcpy(statePtr, &state, sizeof(state)); - return XXH_OK; + XXH_memcpy(dstState, srcState, sizeof(*dstState)); } - -XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed) +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed) { - XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ - ZSTD_memset(&state, 0, sizeof(state)-8); /* do not write into reserved, for future removal */ - state.v1 = seed + PRIME64_1 + PRIME64_2; - state.v2 = seed + PRIME64_2; - state.v3 = seed + 0; - state.v4 = seed - PRIME64_1; - ZSTD_memcpy(statePtr, &state, sizeof(state)); + XXH_ASSERT(statePtr != NULL); + memset(statePtr, 0, sizeof(*statePtr)); + statePtr->v[0] = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + statePtr->v[1] = seed + XXH_PRIME64_2; + statePtr->v[2] = seed + 0; + statePtr->v[3] = seed - XXH_PRIME64_1; return XXH_OK; } - -FORCE_INLINE_TEMPLATE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian) +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode +XXH64_update (XXH_NOESCAPE XXH64_state_t* state, XXH_NOESCAPE const void* input, size_t len) { - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (input==NULL) return XXH_ERROR; -#endif + { const xxh_u8* p = (const xxh_u8*)input; + const xxh_u8* const bEnd = p + len; - state->total_len_32 += (unsigned)len; - state->large_len |= (len>=16) | (state->total_len_32>=16); + state->total_len += len; - if (state->memsize + len < 16) { /* fill in tmp buffer */ - XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); - state->memsize += (unsigned)len; - return XXH_OK; - } + if (state->memsize + len < 32) { /* fill in tmp buffer */ + XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, len); + state->memsize += (xxh_u32)len; + return XXH_OK; + } - if (state->memsize) { /* some data left from previous update */ - XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize); - { const U32* p32 = state->mem32; - state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++; - state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++; - state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++; - state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); p32++; + if (state->memsize) { /* tmp buffer is full */ + XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, 32-state->memsize); + state->v[0] = XXH64_round(state->v[0], XXH_readLE64(state->mem64+0)); + state->v[1] = XXH64_round(state->v[1], XXH_readLE64(state->mem64+1)); + state->v[2] = XXH64_round(state->v[2], XXH_readLE64(state->mem64+2)); + state->v[3] = XXH64_round(state->v[3], XXH_readLE64(state->mem64+3)); + p += 32 - state->memsize; + state->memsize = 0; } - p += 16-state->memsize; - state->memsize = 0; - } - if (p <= bEnd-16) { - const BYTE* const limit = bEnd - 16; - U32 v1 = state->v1; - U32 v2 = state->v2; - U32 v3 = state->v3; - U32 v4 = state->v4; + if (p+32 <= bEnd) { + const xxh_u8* const limit = bEnd - 32; - do { - v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4; - v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4; - v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4; - v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4; - } while (p<=limit); + do { + state->v[0] = XXH64_round(state->v[0], XXH_readLE64(p)); p+=8; + state->v[1] = XXH64_round(state->v[1], XXH_readLE64(p)); p+=8; + state->v[2] = XXH64_round(state->v[2], XXH_readLE64(p)); p+=8; + state->v[3] = XXH64_round(state->v[3], XXH_readLE64(p)); p+=8; + } while (p<=limit); - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; - } + } - if (p < bEnd) { - XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); - state->memsize = (unsigned)(bEnd-p); + if (p < bEnd) { + XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } } return XXH_OK; } -XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_update_endian(state_in, input, len, XXH_littleEndian); - else - return XXH32_update_endian(state_in, input, len, XXH_bigEndian); -} - - -FORCE_INLINE_TEMPLATE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian) +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_digest(XXH_NOESCAPE const XXH64_state_t* state) { - const BYTE * p = (const BYTE*)state->mem32; - const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize; - U32 h32; + xxh_u64 h64; - if (state->large_len) { - h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); + if (state->total_len >= 32) { + h64 = XXH_rotl64(state->v[0], 1) + XXH_rotl64(state->v[1], 7) + XXH_rotl64(state->v[2], 12) + XXH_rotl64(state->v[3], 18); + h64 = XXH64_mergeRound(h64, state->v[0]); + h64 = XXH64_mergeRound(h64, state->v[1]); + h64 = XXH64_mergeRound(h64, state->v[2]); + h64 = XXH64_mergeRound(h64, state->v[3]); } else { - h32 = state->v3 /* == seed */ + PRIME32_5; - } - - h32 += state->total_len_32; - - while (p+4<=bEnd) { - h32 += XXH_readLE32(p, endian) * PRIME32_3; - h32 = XXH_rotl32(h32, 17) * PRIME32_4; - p+=4; - } - - while (pv[2] /*seed*/ + XXH_PRIME64_5; } - h32 ^= h32 >> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; + h64 += (xxh_u64) state->total_len; - return h32; + return XXH64_finalize(h64, (const xxh_u8*)state->mem64, (size_t)state->total_len, XXH_aligned); } +#endif /* !XXH_NO_STREAM */ +/******* Canonical representation *******/ -XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in) +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash) { - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_digest_endian(state_in, XXH_littleEndian); - else - return XXH32_digest_endian(state_in, XXH_bigEndian); + XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); } - - -/* **** XXH64 **** */ - -FORCE_INLINE_TEMPLATE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian) +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src) { - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (input==NULL) return XXH_ERROR; -#endif - - state->total_len += len; - - if (state->memsize + len < 32) { /* fill in tmp buffer */ - if (input != NULL) { - XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); - } - state->memsize += (U32)len; - return XXH_OK; - } - - if (state->memsize) { /* tmp buffer is full */ - XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize); - state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian)); - state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian)); - state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian)); - state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian)); - p += 32-state->memsize; - state->memsize = 0; - } - - if (p+32 <= bEnd) { - const BYTE* const limit = bEnd - 32; - U64 v1 = state->v1; - U64 v2 = state->v2; - U64 v3 = state->v3; - U64 v4 = state->v4; - - do { - v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8; - v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8; - v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8; - v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8; - } while (p<=limit); - - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; - } - - if (p < bEnd) { - XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); - state->memsize = (unsigned)(bEnd-p); - } - - return XXH_OK; + return XXH_readBE64(src); } -XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_update_endian(state_in, input, len, XXH_littleEndian); - else - return XXH64_update_endian(state_in, input, len, XXH_bigEndian); +#if defined (__cplusplus) } +#endif +#ifndef XXH_NO_XXH3 +/* ********************************************************************* +* XXH3 +* New generation hash designed for speed on small keys and vectorization +************************************************************************ */ +/*! + * @} + * @defgroup XXH3_impl XXH3 implementation + * @ingroup impl + * @{ + */ -FORCE_INLINE_TEMPLATE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian) -{ - const BYTE * p = (const BYTE*)state->mem64; - const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize; - U64 h64; - - if (state->total_len >= 32) { - U64 const v1 = state->v1; - U64 const v2 = state->v2; - U64 const v3 = state->v3; - U64 const v4 = state->v4; - - h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); - h64 = XXH64_mergeRound(h64, v1); - h64 = XXH64_mergeRound(h64, v2); - h64 = XXH64_mergeRound(h64, v3); - h64 = XXH64_mergeRound(h64, v4); - } else { - h64 = state->v3 + PRIME64_5; - } - - h64 += (U64) state->total_len; - - while (p+8<=bEnd) { - U64 const k1 = XXH64_round(0, XXH_readLE64(p, endian)); - h64 ^= k1; - h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; - p+=8; - } - - if (p+4<=bEnd) { - h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1; - h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; - p+=4; - } - - while (p> 33; - h64 *= PRIME64_2; - h64 ^= h64 >> 29; - h64 *= PRIME64_3; - h64 ^= h64 >> 32; +#if ((defined(sun) || defined(__sun)) && __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested with GCC 5.5 */ +# define XXH_RESTRICT /* disable */ +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */ +# define XXH_RESTRICT restrict +#elif (defined (__GNUC__) && ((__GNUC__ > 3) || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1))) \ + || (defined (__clang__)) \ + || (defined (_MSC_VER) && (_MSC_VER >= 1400)) \ + || (defined (__INTEL_COMPILER) && (__INTEL_COMPILER >= 1300)) +/* + * There are a LOT more compilers that recognize __restrict but this + * covers the major ones. + */ +# define XXH_RESTRICT __restrict +#else +# define XXH_RESTRICT /* disable */ +#endif - return h64; -} +#if (defined(__GNUC__) && (__GNUC__ >= 3)) \ + || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) \ + || defined(__clang__) +# define XXH_likely(x) __builtin_expect(x, 1) +# define XXH_unlikely(x) __builtin_expect(x, 0) +#else +# define XXH_likely(x) (x) +# define XXH_unlikely(x) (x) +#endif +#ifndef XXH_HAS_INCLUDE +# ifdef __has_include +/* + * Not defined as XXH_HAS_INCLUDE(x) (function-like) because + * this causes segfaults in Apple Clang 4.2 (on Mac OS X 10.7 Lion) + */ +# define XXH_HAS_INCLUDE __has_include +# else +# define XXH_HAS_INCLUDE(x) 0 +# endif +#endif -XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; +#if defined(__GNUC__) || defined(__clang__) +# if defined(__ARM_FEATURE_SVE) +# include +# endif +# if defined(__ARM_NEON__) || defined(__ARM_NEON) \ + || (defined(_M_ARM) && _M_ARM >= 7) \ + || defined(_M_ARM64) || defined(_M_ARM64EC) \ + || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE()) /* WASM SIMD128 via SIMDe */ +# define inline __inline__ /* circumvent a clang bug */ +# include +# undef inline +# elif defined(__AVX2__) +# include +# elif defined(__SSE2__) +# include +# endif +#endif - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_digest_endian(state_in, XXH_littleEndian); - else - return XXH64_digest_endian(state_in, XXH_bigEndian); -} +#if defined(_MSC_VER) +# include +#endif +/* + * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while + * remaining a true 64-bit/128-bit hash function. + * + * This is done by prioritizing a subset of 64-bit operations that can be + * emulated without too many steps on the average 32-bit machine. + * + * For example, these two lines seem similar, and run equally fast on 64-bit: + * + * xxh_u64 x; + * x ^= (x >> 47); // good + * x ^= (x >> 13); // bad + * + * However, to a 32-bit machine, there is a major difference. + * + * x ^= (x >> 47) looks like this: + * + * x.lo ^= (x.hi >> (47 - 32)); + * + * while x ^= (x >> 13) looks like this: + * + * // note: funnel shifts are not usually cheap. + * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13)); + * x.hi ^= (x.hi >> 13); + * + * The first one is significantly faster than the second, simply because the + * shift is larger than 32. This means: + * - All the bits we need are in the upper 32 bits, so we can ignore the lower + * 32 bits in the shift. + * - The shift result will always fit in the lower 32 bits, and therefore, + * we can ignore the upper 32 bits in the xor. + * + * Thanks to this optimization, XXH3 only requires these features to be efficient: + * + * - Usable unaligned access + * - A 32-bit or 64-bit ALU + * - If 32-bit, a decent ADC instruction + * - A 32 or 64-bit multiply with a 64-bit result + * - For the 128-bit variant, a decent byteswap helps short inputs. + * + * The first two are already required by XXH32, and almost all 32-bit and 64-bit + * platforms which can run XXH32 can run XXH3 efficiently. + * + * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one + * notable exception. + * + * First of all, Thumb-1 lacks support for the UMULL instruction which + * performs the important long multiply. This means numerous __aeabi_lmul + * calls. + * + * Second of all, the 8 functional registers are just not enough. + * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need + * Lo registers, and this shuffling results in thousands more MOVs than A32. + * + * A32 and T32 don't have this limitation. They can access all 14 registers, + * do a 32->64 multiply with UMULL, and the flexible operand allowing free + * shifts is helpful, too. + * + * Therefore, we do a quick sanity check. + * + * If compiling Thumb-1 for a target which supports ARM instructions, we will + * emit a warning, as it is not a "sane" platform to compile for. + * + * Usually, if this happens, it is because of an accident and you probably need + * to specify -march, as you likely meant to compile for a newer architecture. + * + * Credit: large sections of the vectorial and asm source code paths + * have been contributed by @easyaspi314 + */ +#if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM) +# warning "XXH3 is highly inefficient without ARM or Thumb-2." +#endif -/* ************************** -* Canonical representation -****************************/ +/* ========================================== + * Vectorization detection + * ========================================== */ -/*! Default XXH result types are basic unsigned 32 and 64 bits. -* The canonical representation follows human-readable write convention, aka big-endian (large digits first). -* These functions allow transformation of hash result into and from its canonical format. -* This way, hash values can be written into a file or buffer, and remain comparable across different systems and programs. -*/ +#ifdef XXH_DOXYGEN +/*! + * @ingroup tuning + * @brief Overrides the vectorization implementation chosen for XXH3. + * + * Can be defined to 0 to disable SIMD or any of the values mentioned in + * @ref XXH_VECTOR_TYPE. + * + * If this is not defined, it uses predefined macros to determine the best + * implementation. + */ +# define XXH_VECTOR XXH_SCALAR +/*! + * @ingroup tuning + * @brief Possible values for @ref XXH_VECTOR. + * + * Note that these are actually implemented as macros. + * + * If this is not defined, it is detected automatically. + * internal macro XXH_X86DISPATCH overrides this. + */ +enum XXH_VECTOR_TYPE /* fake enum */ { + XXH_SCALAR = 0, /*!< Portable scalar version */ + XXH_SSE2 = 1, /*!< + * SSE2 for Pentium 4, Opteron, all x86_64. + * + * @note SSE2 is also guaranteed on Windows 10, macOS, and + * Android x86. + */ + XXH_AVX2 = 2, /*!< AVX2 for Haswell and Bulldozer */ + XXH_AVX512 = 3, /*!< AVX512 for Skylake and Icelake */ + XXH_NEON = 4, /*!< + * NEON for most ARMv7-A, all AArch64, and WASM SIMD128 + * via the SIMDeverywhere polyfill provided with the + * Emscripten SDK. + */ + XXH_VSX = 5, /*!< VSX and ZVector for POWER8/z13 (64-bit) */ + XXH_SVE = 6, /*!< SVE for some ARMv8-A and ARMv9-A */ +}; +/*! + * @ingroup tuning + * @brief Selects the minimum alignment for XXH3's accumulators. + * + * When using SIMD, this should match the alignment required for said vector + * type, so, for example, 32 for AVX2. + * + * Default: Auto detected. + */ +# define XXH_ACC_ALIGN 8 +#endif -XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) -{ - XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); - if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); - ZSTD_memcpy(dst, &hash, sizeof(*dst)); -} +/* Actual definition */ +#ifndef XXH_DOXYGEN +# define XXH_SCALAR 0 +# define XXH_SSE2 1 +# define XXH_AVX2 2 +# define XXH_AVX512 3 +# define XXH_NEON 4 +# define XXH_VSX 5 +# define XXH_SVE 6 +#endif -XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash) -{ - XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); - if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); - ZSTD_memcpy(dst, &hash, sizeof(*dst)); -} +#ifndef XXH_VECTOR /* can be defined on command line */ +# if defined(__ARM_FEATURE_SVE) +# define XXH_VECTOR XXH_SVE +# elif ( \ + defined(__ARM_NEON__) || defined(__ARM_NEON) /* gcc */ \ + || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC) /* msvc */ \ + || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE()) /* wasm simd128 via SIMDe */ \ + ) && ( \ + defined(_WIN32) || defined(__LITTLE_ENDIAN__) /* little endian only */ \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) \ + ) +# define XXH_VECTOR XXH_NEON +# elif defined(__AVX512F__) +# define XXH_VECTOR XXH_AVX512 +# elif defined(__AVX2__) +# define XXH_VECTOR XXH_AVX2 +# elif defined(__SSE2__) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2)) +# define XXH_VECTOR XXH_SSE2 +# elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \ + || (defined(__s390x__) && defined(__VEC__)) \ + && defined(__GNUC__) /* TODO: IBM XL */ +# define XXH_VECTOR XXH_VSX +# else +# define XXH_VECTOR XXH_SCALAR +# endif +#endif -XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) -{ - return XXH_readBE32(src); -} +/* __ARM_FEATURE_SVE is only supported by GCC & Clang. */ +#if (XXH_VECTOR == XXH_SVE) && !defined(__ARM_FEATURE_SVE) +# ifdef _MSC_VER +# pragma warning(once : 4606) +# else +# warning "__ARM_FEATURE_SVE isn't supported. Use SCALAR instead." +# endif +# undef XXH_VECTOR +# define XXH_VECTOR XXH_SCALAR +#endif -XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src) -{ - return XXH_readBE64(src); -} -/**** ended inlining xxhash.c ****/ +/* + * Controls the alignment of the accumulator, + * for compatibility with aligned vector loads, which are usually faster. + */ +#ifndef XXH_ACC_ALIGN +# if defined(XXH_X86DISPATCH) +# define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */ +# elif XXH_VECTOR == XXH_SCALAR /* scalar */ +# define XXH_ACC_ALIGN 8 +# elif XXH_VECTOR == XXH_SSE2 /* sse2 */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX2 /* avx2 */ +# define XXH_ACC_ALIGN 32 +# elif XXH_VECTOR == XXH_NEON /* neon */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_VSX /* vsx */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX512 /* avx512 */ +# define XXH_ACC_ALIGN 64 +# elif XXH_VECTOR == XXH_SVE /* sve */ +# define XXH_ACC_ALIGN 64 # endif +#endif -#endif /* XXH_STATIC_LINKING_ONLY && XXH_STATIC_H_3543687687345 */ +#if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 \ + || XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512 +# define XXH_SEC_ALIGN XXH_ACC_ALIGN +#elif XXH_VECTOR == XXH_SVE +# define XXH_SEC_ALIGN XXH_ACC_ALIGN +#else +# define XXH_SEC_ALIGN 8 +#endif +#if defined(__GNUC__) || defined(__clang__) +# define XXH_ALIASING __attribute__((may_alias)) +#else +# define XXH_ALIASING /* nothing */ +#endif -#if defined (__cplusplus) -} +/* + * UGLY HACK: + * GCC usually generates the best code with -O3 for xxHash. + * + * However, when targeting AVX2, it is overzealous in its unrolling resulting + * in code roughly 3/4 the speed of Clang. + * + * There are other issues, such as GCC splitting _mm256_loadu_si256 into + * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which + * only applies to Sandy and Ivy Bridge... which don't even support AVX2. + * + * That is why when compiling the AVX2 version, it is recommended to use either + * -O2 -mavx2 -march=haswell + * or + * -O2 -mavx2 -mno-avx256-split-unaligned-load + * for decent performance, or to use Clang instead. + * + * Fortunately, we can control the first one with a pragma that forces GCC into + * -O2, but the other one we can't control without "failed to inline always + * inline function due to target mismatch" warnings. + */ +#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ + && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__OPTIMIZE__) && XXH_SIZE_OPT <= 0 /* respect -O0 and -Os */ +# pragma GCC push_options +# pragma GCC optimize("-O2") #endif -/**** ended inlining xxhash.h ****/ #if defined (__cplusplus) extern "C" { #endif -/* ---- static assert (debug) --- */ -#define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) -#define ZSTD_isError ERR_isError /* for inlining */ -#define FSE_isError ERR_isError -#define HUF_isError ERR_isError - - -/*-************************************* -* shared macros -***************************************/ -#undef MIN -#undef MAX -#define MIN(a,b) ((a)<(b) ? (a) : (b)) -#define MAX(a,b) ((a)>(b) ? (a) : (b)) +#if XXH_VECTOR == XXH_NEON -/** - * Ignore: this is an internal helper. +/* + * UGLY HACK: While AArch64 GCC on Linux does not seem to care, on macOS, GCC -O3 + * optimizes out the entire hashLong loop because of the aliasing violation. * - * This is a helper function to help force C99-correctness during compilation. - * Under strict compilation modes, variadic macro arguments can't be empty. - * However, variadic function arguments can be. Using a function therefore lets - * us statically check that at least one (string) argument was passed, - * independent of the compilation flags. + * However, GCC is also inefficient at load-store optimization with vld1q/vst1q, + * so the only option is to mark it as aliasing. */ -static INLINE_KEYWORD UNUSED_ATTR -void _force_has_format_string(const char *format, ...) { - (void)format; -} +typedef uint64x2_t xxh_aliasing_uint64x2_t XXH_ALIASING; -/** - * Ignore: this is an internal helper. +/*! + * @internal + * @brief `vld1q_u64` but faster and alignment-safe. * - * We want to force this function invocation to be syntactically correct, but - * we don't want to force runtime evaluation of its arguments. - */ -#define _FORCE_HAS_FORMAT_STRING(...) \ - if (0) { \ - _force_has_format_string(__VA_ARGS__); \ - } - -/** - * Return the specified error if the condition evaluates to true. + * On AArch64, unaligned access is always safe, but on ARMv7-a, it is only + * *conditionally* safe (`vld1` has an alignment bit like `movdq[ua]` in x86). * - * In debug modes, prints additional information. - * In order to do that (particularly, printing the conditional that failed), - * this can't just wrap RETURN_ERROR(). + * GCC for AArch64 sees `vld1q_u8` as an intrinsic instead of a load, so it + * prohibits load-store optimizations. Therefore, a direct dereference is used. + * + * Otherwise, `vld1q_u8` is used with `vreinterpretq_u8_u64` to do a safe + * unaligned load. */ -#define RETURN_ERROR_IF(cond, err, ...) \ - if (cond) { \ - RAWLOG(3, "%s:%d: ERROR!: check %s failed, returning %s", \ - __FILE__, __LINE__, ZSTD_QUOTE(cond), ZSTD_QUOTE(ERROR(err))); \ - _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ - RAWLOG(3, ": " __VA_ARGS__); \ - RAWLOG(3, "\n"); \ - return ERROR(err); \ - } +#if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) +XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr) /* silence -Wcast-align */ +{ + return *(xxh_aliasing_uint64x2_t const *)ptr; +} +#else +XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr) +{ + return vreinterpretq_u64_u8(vld1q_u8((uint8_t const*)ptr)); +} +#endif -/** - * Unconditionally return the specified error. +/*! + * @internal + * @brief `vmlal_u32` on low and high halves of a vector. * - * In debug modes, prints additional information. + * This is a workaround for AArch64 GCC < 11 which implemented arm_neon.h with + * inline assembly and were therefore incapable of merging the `vget_{low, high}_u32` + * with `vmlal_u32`. */ -#define RETURN_ERROR(err, ...) \ - do { \ - RAWLOG(3, "%s:%d: ERROR!: unconditional check failed, returning %s", \ - __FILE__, __LINE__, ZSTD_QUOTE(ERROR(err))); \ - _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ - RAWLOG(3, ": " __VA_ARGS__); \ - RAWLOG(3, "\n"); \ - return ERROR(err); \ - } while(0); +#if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 11 +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + /* Inline assembly is the only way */ + __asm__("umlal %0.2d, %1.2s, %2.2s" : "+w" (acc) : "w" (lhs), "w" (rhs)); + return acc; +} +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + /* This intrinsic works as expected */ + return vmlal_high_u32(acc, lhs, rhs); +} +#else +/* Portable intrinsic versions */ +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + return vmlal_u32(acc, vget_low_u32(lhs), vget_low_u32(rhs)); +} +/*! @copydoc XXH_vmlal_low_u32 + * Assume the compiler converts this to vmlal_high_u32 on aarch64 */ +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + return vmlal_u32(acc, vget_high_u32(lhs), vget_high_u32(rhs)); +} +#endif -/** - * If the provided expression evaluates to an error code, returns that error code. +/*! + * @ingroup tuning + * @brief Controls the NEON to scalar ratio for XXH3 * - * In debug modes, prints additional information. + * This can be set to 2, 4, 6, or 8. + * + * ARM Cortex CPUs are _very_ sensitive to how their pipelines are used. + * + * For example, the Cortex-A73 can dispatch 3 micro-ops per cycle, but only 2 of those + * can be NEON. If you are only using NEON instructions, you are only using 2/3 of the CPU + * bandwidth. + * + * This is even more noticeable on the more advanced cores like the Cortex-A76 which + * can dispatch 8 micro-ops per cycle, but still only 2 NEON micro-ops at once. + * + * Therefore, to make the most out of the pipeline, it is beneficial to run 6 NEON lanes + * and 2 scalar lanes, which is chosen by default. + * + * This does not apply to Apple processors or 32-bit processors, which run better with + * full NEON. These will default to 8. Additionally, size-optimized builds run 8 lanes. + * + * This change benefits CPUs with large micro-op buffers without negatively affecting + * most other CPUs: + * + * | Chipset | Dispatch type | NEON only | 6:2 hybrid | Diff. | + * |:----------------------|:--------------------|----------:|-----------:|------:| + * | Snapdragon 730 (A76) | 2 NEON/8 micro-ops | 8.8 GB/s | 10.1 GB/s | ~16% | + * | Snapdragon 835 (A73) | 2 NEON/3 micro-ops | 5.1 GB/s | 5.3 GB/s | ~5% | + * | Marvell PXA1928 (A53) | In-order dual-issue | 1.9 GB/s | 1.9 GB/s | 0% | + * | Apple M1 | 4 NEON/8 micro-ops | 37.3 GB/s | 36.1 GB/s | ~-3% | + * + * It also seems to fix some bad codegen on GCC, making it almost as fast as clang. + * + * When using WASM SIMD128, if this is 2 or 6, SIMDe will scalarize 2 of the lanes meaning + * it effectively becomes worse 4. + * + * @see XXH3_accumulate_512_neon() */ -#define FORWARD_IF_ERROR(err, ...) \ - do { \ - size_t const err_code = (err); \ - if (ERR_isError(err_code)) { \ - RAWLOG(3, "%s:%d: ERROR!: forwarding error in %s: %s", \ - __FILE__, __LINE__, ZSTD_QUOTE(err), ERR_getErrorName(err_code)); \ - _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ - RAWLOG(3, ": " __VA_ARGS__); \ - RAWLOG(3, "\n"); \ - return err_code; \ - } \ - } while(0); - - -/*-************************************* -* Common constants -***************************************/ -#define ZSTD_OPT_NUM (1<<12) - -#define ZSTD_REP_NUM 3 /* number of repcodes */ -#define ZSTD_REP_MOVE (ZSTD_REP_NUM-1) -static UNUSED_ATTR const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 }; +# ifndef XXH3_NEON_LANES +# if (defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) || defined(_M_ARM64EC)) \ + && !defined(__APPLE__) && XXH_SIZE_OPT <= 0 +# define XXH3_NEON_LANES 6 +# else +# define XXH3_NEON_LANES XXH_ACC_NB +# endif +# endif +#endif /* XXH_VECTOR == XXH_NEON */ -#define KB *(1 <<10) -#define MB *(1 <<20) -#define GB *(1U<<30) +#if defined (__cplusplus) +} /* extern "C" */ +#endif -#define BIT7 128 -#define BIT6 64 -#define BIT5 32 -#define BIT4 16 -#define BIT1 2 -#define BIT0 1 +/* + * VSX and Z Vector helpers. + * + * This is very messy, and any pull requests to clean this up are welcome. + * + * There are a lot of problems with supporting VSX and s390x, due to + * inconsistent intrinsics, spotty coverage, and multiple endiannesses. + */ +#if XXH_VECTOR == XXH_VSX +/* Annoyingly, these headers _may_ define three macros: `bool`, `vector`, + * and `pixel`. This is a problem for obvious reasons. + * + * These keywords are unnecessary; the spec literally says they are + * equivalent to `__bool`, `__vector`, and `__pixel` and may be undef'd + * after including the header. + * + * We use pragma push_macro/pop_macro to keep the namespace clean. */ +# pragma push_macro("bool") +# pragma push_macro("vector") +# pragma push_macro("pixel") +/* silence potential macro redefined warnings */ +# undef bool +# undef vector +# undef pixel + +# if defined(__s390x__) +# include +# else +# include +# endif -#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10 -static UNUSED_ATTR const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 }; -static UNUSED_ATTR const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 }; +/* Restore the original macro values, if applicable. */ +# pragma pop_macro("pixel") +# pragma pop_macro("vector") +# pragma pop_macro("bool") -#define ZSTD_FRAMEIDSIZE 4 /* magic number size */ +typedef __vector unsigned long long xxh_u64x2; +typedef __vector unsigned char xxh_u8x16; +typedef __vector unsigned xxh_u32x4; -#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ -static UNUSED_ATTR const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; -typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e; +/* + * UGLY HACK: Similar to aarch64 macOS GCC, s390x GCC has the same aliasing issue. + */ +typedef xxh_u64x2 xxh_aliasing_u64x2 XXH_ALIASING; + +# ifndef XXH_VSX_BE +# if defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_VSX_BE 1 +# elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__ +# warning "-maltivec=be is not recommended. Please use native endianness." +# define XXH_VSX_BE 1 +# else +# define XXH_VSX_BE 0 +# endif +# endif /* !defined(XXH_VSX_BE) */ -#define ZSTD_FRAMECHECKSUMSIZE 4 +# if XXH_VSX_BE +# if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__)) +# define XXH_vec_revb vec_revb +# else +#if defined (__cplusplus) +extern "C" { +#endif +/*! + * A polyfill for POWER9's vec_revb(). + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val) +{ + xxh_u8x16 const vByteSwap = { 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00, + 0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 }; + return vec_perm(val, val, vByteSwap); +} +#if defined (__cplusplus) +} /* extern "C" */ +#endif +# endif +# endif /* XXH_VSX_BE */ -#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */ -#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */ +#if defined (__cplusplus) +extern "C" { +#endif +/*! + * Performs an unaligned vector load and byte swaps it on big endian. + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr) +{ + xxh_u64x2 ret; + XXH_memcpy(&ret, ptr, sizeof(xxh_u64x2)); +# if XXH_VSX_BE + ret = XXH_vec_revb(ret); +# endif + return ret; +} -#define HufLog 12 -typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e; +/* + * vec_mulo and vec_mule are very problematic intrinsics on PowerPC + * + * These intrinsics weren't added until GCC 8, despite existing for a while, + * and they are endian dependent. Also, their meaning swap depending on version. + * */ +# if defined(__s390x__) + /* s390x is always big endian, no issue on this platform */ +# define XXH_vec_mulo vec_mulo +# define XXH_vec_mule vec_mule +# elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) && !defined(__ibmxl__) +/* Clang has a better way to control this, we can just use the builtin which doesn't swap. */ + /* The IBM XL Compiler (which defined __clang__) only implements the vec_* operations */ +# define XXH_vec_mulo __builtin_altivec_vmulouw +# define XXH_vec_mule __builtin_altivec_vmuleuw +# else +/* gcc needs inline assembly */ +/* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +# endif /* XXH_vec_mulo, XXH_vec_mule */ -#define LONGNBSEQ 0x7F00 +#if defined (__cplusplus) +} /* extern "C" */ +#endif -#define MINMATCH 3 +#endif /* XXH_VECTOR == XXH_VSX */ + +#if XXH_VECTOR == XXH_SVE +#define ACCRND(acc, offset) \ +do { \ + svuint64_t input_vec = svld1_u64(mask, xinput + offset); \ + svuint64_t secret_vec = svld1_u64(mask, xsecret + offset); \ + svuint64_t mixed = sveor_u64_x(mask, secret_vec, input_vec); \ + svuint64_t swapped = svtbl_u64(input_vec, kSwap); \ + svuint64_t mixed_lo = svextw_u64_x(mask, mixed); \ + svuint64_t mixed_hi = svlsr_n_u64_x(mask, mixed, 32); \ + svuint64_t mul = svmad_u64_x(mask, mixed_lo, mixed_hi, swapped); \ + acc = svadd_u64_x(mask, acc, mul); \ +} while (0) +#endif /* XXH_VECTOR == XXH_SVE */ -#define Litbits 8 -#define MaxLit ((1<= 1 +# define XXH_PREFETCH(ptr) (void)(ptr) +# elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */ +# include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ +# define XXH_PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) +# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) +# define XXH_PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) +# else +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +# endif +#endif /* XXH_NO_PREFETCH */ -#define ZSTD_MAX_HUF_HEADER_SIZE 128 /* header + <= 127 byte tree description */ -/* Each table cannot take more than #symbols * FSELog bits */ -#define ZSTD_MAX_FSE_HEADERS_SIZE (((MaxML + 1) * MLFSELog + (MaxLL + 1) * LLFSELog + (MaxOff + 1) * OffFSELog + 7) / 8) +#if defined (__cplusplus) +extern "C" { +#endif +/* ========================================== + * XXH3 default settings + * ========================================== */ -static UNUSED_ATTR const U32 LL_bits[MaxLL+1] = { - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 1, 1, 1, 1, 2, 2, 3, 3, - 4, 6, 7, 8, 9,10,11,12, - 13,14,15,16 -}; -static UNUSED_ATTR const S16 LL_defaultNorm[MaxLL+1] = { - 4, 3, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 1, 1, 1, - 2, 2, 2, 2, 2, 2, 2, 2, - 2, 3, 2, 1, 1, 1, 1, 1, - -1,-1,-1,-1 -}; -#define LL_DEFAULTNORMLOG 6 /* for static allocation */ -static UNUSED_ATTR const U32 LL_defaultNormLog = LL_DEFAULTNORMLOG; +#define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */ -static UNUSED_ATTR const U32 ML_bits[MaxML+1] = { - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 1, 1, 1, 1, 2, 2, 3, 3, - 4, 4, 5, 7, 8, 9,10,11, - 12,13,14,15,16 -}; -static UNUSED_ATTR const S16 ML_defaultNorm[MaxML+1] = { - 1, 4, 3, 2, 2, 2, 2, 2, - 2, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1,-1,-1, - -1,-1,-1,-1,-1 -}; -#define ML_DEFAULTNORMLOG 6 /* for static allocation */ -static UNUSED_ATTR const U32 ML_defaultNormLog = ML_DEFAULTNORMLOG; +#if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN) +# error "default keyset is not large enough" +#endif -static UNUSED_ATTR const S16 OF_defaultNorm[DefaultMaxOff+1] = { - 1, 1, 1, 1, 1, 1, 2, 2, - 2, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - -1,-1,-1,-1,-1 +/*! Pseudorandom secret taken directly from FARSH. */ +XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = { + 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c, + 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f, + 0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21, + 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c, + 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3, + 0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8, + 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d, + 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64, + 0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb, + 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e, + 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce, + 0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e, }; -#define OF_DEFAULTNORMLOG 5 /* for static allocation */ -static UNUSED_ATTR const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG; +static const xxh_u64 PRIME_MX1 = 0x165667919E3779F9ULL; /*!< 0b0001011001010110011001111001000110011110001101110111100111111001 */ +static const xxh_u64 PRIME_MX2 = 0x9FB21C651E98DF25ULL; /*!< 0b1001111110110010000111000110010100011110100110001101111100100101 */ -/*-******************************************* -* Shared functions to include for inlining -*********************************************/ -static void ZSTD_copy8(void* dst, const void* src) { -#if !defined(ZSTD_NO_INTRINSICS) && defined(__ARM_NEON) - vst1_u8((uint8_t*)dst, vld1_u8((const uint8_t*)src)); -#else - ZSTD_memcpy(dst, src, 8); +#ifdef XXH_OLD_NAMES +# define kSecret XXH3_kSecret #endif -} -#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; } -static void ZSTD_copy16(void* dst, const void* src) { -#if !defined(ZSTD_NO_INTRINSICS) && defined(__ARM_NEON) - vst1q_u8((uint8_t*)dst, vld1q_u8((const uint8_t*)src)); +#ifdef XXH_DOXYGEN +/*! + * @brief Calculates a 32-bit to 64-bit long multiply. + * + * Implemented as a macro. + * + * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it doesn't + * need to (but it shouldn't need to anyways, it is about 7 instructions to do + * a 64x64 multiply...). Since we know that this will _always_ emit `MULL`, we + * use that instead of the normal method. + * + * If you are compiling for platforms like Thumb-1 and don't have a better option, + * you may also want to write your own long multiply routine here. + * + * @param x, y Numbers to be multiplied + * @return 64-bit product of the low 32 bits of @p x and @p y. + */ +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64(xxh_u64 x, xxh_u64 y) +{ + return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF); +} +#elif defined(_MSC_VER) && defined(_M_IX86) +# define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y)) #else - ZSTD_memcpy(dst, src, 16); +/* + * Downcast + upcast is usually better than masking on older compilers like + * GCC 4.2 (especially 32-bit ones), all without affecting newer compilers. + * + * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both operands + * and perform a full 64x64 multiply -- entirely redundant on 32-bit. + */ +# define XXH_mult32to64(x, y) ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y)) #endif -} -#define COPY16(d,s) { ZSTD_copy16(d,s); d+=16; s+=16; } - -#define WILDCOPY_OVERLENGTH 32 -#define WILDCOPY_VECLEN 16 - -typedef enum { - ZSTD_no_overlap, - ZSTD_overlap_src_before_dst - /* ZSTD_overlap_dst_before_src, */ -} ZSTD_overlap_e; -/*! ZSTD_wildcopy() : - * Custom version of ZSTD_memcpy(), can over read/write up to WILDCOPY_OVERLENGTH bytes (if length==0) - * @param ovtype controls the overlap detection - * - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart. - * - ZSTD_overlap_src_before_dst: The src and dst may overlap, but they MUST be at least 8 bytes apart. - * The src buffer must be before the dst buffer. +/*! + * @brief Calculates a 64->128-bit long multiply. + * + * Uses `__uint128_t` and `_umul128` if available, otherwise uses a scalar + * version. + * + * @param lhs , rhs The 64-bit integers to be multiplied + * @return The 128-bit result represented in an @ref XXH128_hash_t. */ -MEM_STATIC FORCE_INLINE_ATTR -void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e const ovtype) +static XXH128_hash_t +XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) { - ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src; - const BYTE* ip = (const BYTE*)src; - BYTE* op = (BYTE*)dst; - BYTE* const oend = op + length; + /* + * GCC/Clang __uint128_t method. + * + * On most 64-bit targets, GCC and Clang define a __uint128_t type. + * This is usually the best way as it usually uses a native long 64-bit + * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64. + * + * Usually. + * + * Despite being a 32-bit platform, Clang (and emscripten) define this type + * despite not having the arithmetic for it. This results in a laggy + * compiler builtin call which calculates a full 128-bit multiply. + * In that case it is best to use the portable one. + * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677 + */ +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__wasm__) \ + && defined(__SIZEOF_INT128__) \ + || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + + __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs; + XXH128_hash_t r128; + r128.low64 = (xxh_u64)(product); + r128.high64 = (xxh_u64)(product >> 64); + return r128; + + /* + * MSVC for x64's _umul128 method. + * + * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct); + * + * This compiles to single operand MUL on x64. + */ +#elif (defined(_M_X64) || defined(_M_IA64)) && !defined(_M_ARM64EC) + +#ifndef _MSC_VER +# pragma intrinsic(_umul128) +#endif + xxh_u64 product_high; + xxh_u64 const product_low = _umul128(lhs, rhs, &product_high); + XXH128_hash_t r128; + r128.low64 = product_low; + r128.high64 = product_high; + return r128; + + /* + * MSVC for ARM64's __umulh method. + * + * This compiles to the same MUL + UMULH as GCC/Clang's __uint128_t method. + */ +#elif defined(_M_ARM64) || defined(_M_ARM64EC) - assert(diff >= 8 || (ovtype == ZSTD_no_overlap && diff <= -WILDCOPY_VECLEN)); +#ifndef _MSC_VER +# pragma intrinsic(__umulh) +#endif + XXH128_hash_t r128; + r128.low64 = lhs * rhs; + r128.high64 = __umulh(lhs, rhs); + return r128; - if (ovtype == ZSTD_overlap_src_before_dst && diff < WILDCOPY_VECLEN) { - /* Handle short offset copies. */ - do { - COPY8(op, ip) - } while (op < oend); - } else { - assert(diff >= WILDCOPY_VECLEN || diff <= -WILDCOPY_VECLEN); - /* Separate out the first COPY16() call because the copy length is - * almost certain to be short, so the branches have different - * probabilities. Since it is almost certain to be short, only do - * one COPY16() in the first call. Then, do two calls per loop since - * at that point it is more likely to have a high trip count. - */ -#ifdef __aarch64__ - do { - COPY16(op, ip); - } - while (op < oend); #else - ZSTD_copy16(op, ip); - if (16 >= length) return; - op += 16; - ip += 16; - do { - COPY16(op, ip); - COPY16(op, ip); - } - while (op < oend); + /* + * Portable scalar method. Optimized for 32-bit and 64-bit ALUs. + * + * This is a fast and simple grade school multiply, which is shown below + * with base 10 arithmetic instead of base 0x100000000. + * + * 9 3 // D2 lhs = 93 + * x 7 5 // D2 rhs = 75 + * ---------- + * 1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15 + * 4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45 + * 2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21 + * + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63 + * --------- + * 2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27 + * + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67 + * --------- + * 6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975 + * + * The reasons for adding the products like this are: + * 1. It avoids manual carry tracking. Just like how + * (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX. + * This avoids a lot of complexity. + * + * 2. It hints for, and on Clang, compiles to, the powerful UMAAL + * instruction available in ARM's Digital Signal Processing extension + * in 32-bit ARMv6 and later, which is shown below: + * + * void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm) + * { + * xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm; + * *RdLo = (xxh_u32)(product & 0xFFFFFFFF); + * *RdHi = (xxh_u32)(product >> 32); + * } + * + * This instruction was designed for efficient long multiplication, and + * allows this to be calculated in only 4 instructions at speeds + * comparable to some 64-bit ALUs. + * + * 3. It isn't terrible on other platforms. Usually this will be a couple + * of 32-bit ADD/ADCs. + */ + + /* First calculate all of the cross products. */ + xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF); + xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF); + xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32); + xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32, rhs >> 32); + + /* Now add the products together. These will never overflow. */ + xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi; + xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32) + hi_hi; + xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF); + + XXH128_hash_t r128; + r128.low64 = lower; + r128.high64 = upper; + return r128; #endif - } } -MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +/*! + * @brief Calculates a 64-bit to 128-bit multiply, then XOR folds it. + * + * The reason for the separate function is to prevent passing too many structs + * around by value. This will hopefully inline the multiply, but we don't force it. + * + * @param lhs , rhs The 64-bit integers to multiply + * @return The low 64 bits of the product XOR'd by the high 64 bits. + * @see XXH_mult64to128() + */ +static xxh_u64 +XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs) { - size_t const length = MIN(dstCapacity, srcSize); - if (length > 0) { - ZSTD_memcpy(dst, src, length); - } - return length; + XXH128_hash_t product = XXH_mult64to128(lhs, rhs); + return product.low64 ^ product.high64; } -/* define "workspace is too large" as this number of times larger than needed */ -#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 +/*! Seems to produce slightly better code on GCC for some reason. */ +XXH_FORCE_INLINE XXH_CONSTF xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift) +{ + XXH_ASSERT(0 <= shift && shift < 64); + return v64 ^ (v64 >> shift); +} -/* when workspace is continuously too large - * during at least this number of times, - * context's memory usage is considered wasteful, - * because it's sized to handle a worst case scenario which rarely happens. - * In which case, resize it down to free some memory */ -#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 +/* + * This is a fast avalanche stage, + * suitable when input bits are already partially mixed + */ +static XXH64_hash_t XXH3_avalanche(xxh_u64 h64) +{ + h64 = XXH_xorshift64(h64, 37); + h64 *= PRIME_MX1; + h64 = XXH_xorshift64(h64, 32); + return h64; +} -/* Controls whether the input/output buffer is buffered or stable. */ -typedef enum { - ZSTD_bm_buffered = 0, /* Buffer the input/output */ - ZSTD_bm_stable = 1 /* ZSTD_inBuffer/ZSTD_outBuffer is stable */ -} ZSTD_bufferMode_e; +/* + * This is a stronger avalanche, + * inspired by Pelle Evensen's rrmxmx + * preferable when input has not been previously mixed + */ +static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len) +{ + /* this mix is inspired by Pelle Evensen's rrmxmx */ + h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24); + h64 *= PRIME_MX2; + h64 ^= (h64 >> 35) + len ; + h64 *= PRIME_MX2; + return XXH_xorshift64(h64, 28); +} -/*-******************************************* -* Private declarations -*********************************************/ -typedef struct seqDef_s { - U32 offset; /* Offset code of the sequence */ - U16 litLength; - U16 matchLength; -} seqDef; +/* ========================================== + * Short keys + * ========================================== + * One of the shortcomings of XXH32 and XXH64 was that their performance was + * sub-optimal on short lengths. It used an iterative algorithm which strongly + * favored lengths that were a multiple of 4 or 8. + * + * Instead of iterating over individual inputs, we use a set of single shot + * functions which piece together a range of lengths and operate in constant time. + * + * Additionally, the number of multiplies has been significantly reduced. This + * reduces latency, especially when emulating 64-bit multiplies on 32-bit. + * + * Depending on the platform, this may or may not be faster than XXH32, but it + * is almost guaranteed to be faster than XXH64. + */ -typedef struct { - seqDef* sequencesStart; - seqDef* sequences; /* ptr to end of sequences */ - BYTE* litStart; - BYTE* lit; /* ptr to end of literals */ - BYTE* llCode; - BYTE* mlCode; - BYTE* ofCode; - size_t maxNbSeq; - size_t maxNbLit; - - /* longLengthPos and longLengthID to allow us to represent either a single litLength or matchLength - * in the seqStore that has a value larger than U16 (if it exists). To do so, we increment - * the existing value of the litLength or matchLength by 0x10000. +/* + * At very short lengths, there isn't enough input to fully hide secrets, or use + * the entire secret. + * + * There is also only a limited amount of mixing we can do before significantly + * impacting performance. + * + * Therefore, we use different sections of the secret and always mix two secret + * samples with an XOR. This should have no effect on performance on the + * seedless or withSeed variants because everything _should_ be constant folded + * by modern compilers. + * + * The XOR mixing hides individual parts of the secret and increases entropy. + * + * This adds an extra layer of strength for custom secrets. + */ +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combined = { input[0], 0x01, input[0], input[0] } + * len = 2: combined = { input[1], 0x02, input[0], input[1] } + * len = 3: combined = { input[2], 0x03, input[0], input[1] } */ - U32 longLengthID; /* 0 == no longLength; 1 == Represent the long literal; 2 == Represent the long match; */ - U32 longLengthPos; /* Index of the sequence to apply long length modification to */ -} seqStore_t; + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u64 const bitflip = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const keyed = (xxh_u64)combined ^ bitflip; + return XXH64_avalanche(keyed); + } +} -typedef struct { - U32 litLength; - U32 matchLength; -} ZSTD_sequenceLength; +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input1 = XXH_readLE32(input); + xxh_u32 const input2 = XXH_readLE32(input + len - 4); + xxh_u64 const bitflip = (XXH_readLE64(secret+8) ^ XXH_readLE64(secret+16)) - seed; + xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32); + xxh_u64 const keyed = input64 ^ bitflip; + return XXH3_rrmxmx(keyed, len); + } +} -/** - * Returns the ZSTD_sequenceLength for the given sequences. It handles the decoding of long sequences - * indicated by longLengthPos and longLengthID, and adds MINMATCH back to matchLength. +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflip1 = (XXH_readLE64(secret+24) ^ XXH_readLE64(secret+32)) + seed; + xxh_u64 const bitflip2 = (XXH_readLE64(secret+40) ^ XXH_readLE64(secret+48)) - seed; + xxh_u64 const input_lo = XXH_readLE64(input) ^ bitflip1; + xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2; + xxh_u64 const acc = len + + XXH_swap64(input_lo) + input_hi + + XXH3_mul128_fold64(input_lo, input_hi); + return XXH3_avalanche(acc); + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (XXH_likely(len > 8)) return XXH3_len_9to16_64b(input, len, secret, seed); + if (XXH_likely(len >= 4)) return XXH3_len_4to8_64b(input, len, secret, seed); + if (len) return XXH3_len_1to3_64b(input, len, secret, seed); + return XXH64_avalanche(seed ^ (XXH_readLE64(secret+56) ^ XXH_readLE64(secret+64))); + } +} + +/* + * DISCLAIMER: There are known *seed-dependent* multicollisions here due to + * multiplication by zero, affecting hashes of lengths 17 to 240. + * + * However, they are very unlikely. + * + * Keep this in mind when using the unseeded XXH3_64bits() variant: As with all + * unseeded non-cryptographic hashes, it does not attempt to defend itself + * against specially crafted inputs, only random inputs. + * + * Compared to classic UMAC where a 1 in 2^31 chance of 4 consecutive bytes + * cancelling out the secret is taken an arbitrary number of times (addressed + * in XXH3_accumulate_512), this collision is very unlikely with random inputs + * and/or proper seeding: + * + * This only has a 1 in 2^63 chance of 8 consecutive bytes cancelling out, in a + * function that is only called up to 16 times per hash with up to 240 bytes of + * input. + * + * This is not too bad for a non-cryptographic hash function, especially with + * only 64 bit outputs. + * + * The 128-bit variant (which trades some speed for strength) is NOT affected + * by this, although it is always a good idea to use a proper seed if you care + * about strength. */ -MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore, seqDef const* seq) +XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64) +{ +#if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like XXH32 hack */ + /* + * UGLY HACK: + * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in + * slower code. + * + * By forcing seed64 into a register, we disrupt the cost model and + * cause it to scalarize. See `XXH32_round()` + * + * FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600, + * XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on + * GCC 9.2, despite both emitting scalar code. + * + * GCC generates much better scalar code than Clang for the rest of XXH3, + * which is why finding a more optimal codepath is an interest. + */ + XXH_COMPILER_GUARD(seed64); +#endif + { xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 const input_hi = XXH_readLE64(input+8); + return XXH3_mul128_fold64( + input_lo ^ (XXH_readLE64(secret) + seed64), + input_hi ^ (XXH_readLE64(secret+8) - seed64) + ); + } +} + +/* For mid range keys, XXH3 uses a Mum-hash variant. */ +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_17to128_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) { - ZSTD_sequenceLength seqLen; - seqLen.litLength = seq->litLength; - seqLen.matchLength = seq->matchLength + MINMATCH; - if (seqStore->longLengthPos == (U32)(seq - seqStore->sequencesStart)) { - if (seqStore->longLengthID == 1) { - seqLen.litLength += 0xFFFF; - } - if (seqStore->longLengthID == 2) { - seqLen.matchLength += 0xFFFF; + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { xxh_u64 acc = len * XXH_PRIME64_1; +#if XXH_SIZE_OPT >= 1 + /* Smaller and cleaner, but slightly slower. */ + unsigned int i = (unsigned int)(len - 1) / 32; + do { + acc += XXH3_mix16B(input+16 * i, secret+32*i, seed); + acc += XXH3_mix16B(input+len-16*(i+1), secret+32*i+16, seed); + } while (i-- != 0); +#else + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc += XXH3_mix16B(input+48, secret+96, seed); + acc += XXH3_mix16B(input+len-64, secret+112, seed); + } + acc += XXH3_mix16B(input+32, secret+64, seed); + acc += XXH3_mix16B(input+len-48, secret+80, seed); + } + acc += XXH3_mix16B(input+16, secret+32, seed); + acc += XXH3_mix16B(input+len-32, secret+48, seed); } + acc += XXH3_mix16B(input+0, secret+0, seed); + acc += XXH3_mix16B(input+len-16, secret+16, seed); +#endif + return XXH3_avalanche(acc); } - return seqLen; } -/** - * Contains the compressed frame size and an upper-bound for the decompressed frame size. - * Note: before using `compressedSize`, check for errors using ZSTD_isError(). - * similarly, before using `decompressedBound`, check for errors using: - * `decompressedBound != ZSTD_CONTENTSIZE_ERROR` +/*! + * @brief Maximum size of "short" key in bytes. */ -typedef struct { - size_t compressedSize; - unsigned long long decompressedBound; -} ZSTD_frameSizeInfo; /* decompress & legacy */ +#define XXH3_MIDSIZE_MAX 240 + +XXH_NO_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + #define XXH3_MIDSIZE_STARTOFFSET 3 + #define XXH3_MIDSIZE_LASTOFFSET 17 + + { xxh_u64 acc = len * XXH_PRIME64_1; + xxh_u64 acc_end; + unsigned int const nbRounds = (unsigned int)len / 16; + unsigned int i; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + for (i=0; i<8; i++) { + acc += XXH3_mix16B(input+(16*i), secret+(16*i), seed); + } + /* last bytes */ + acc_end = XXH3_mix16B(input + len - 16, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed); + XXH_ASSERT(nbRounds >= 8); + acc = XXH3_avalanche(acc); +#if defined(__clang__) /* Clang */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86. + * In everywhere else, it uses scalar code. + * + * For 64->128-bit multiplies, even if the NEON was 100% optimal, it + * would still be slower than UMAAL (see XXH_mult64to128). + * + * Unfortunately, Clang doesn't handle the long multiplies properly and + * converts them to the nonexistent "vmulq_u64" intrinsic, which is then + * scalarized into an ugly mess of VMOV.32 instructions. + * + * This mess is difficult to avoid without turning autovectorization + * off completely, but they are usually relatively minor and/or not + * worth it to fix. + * + * This loop is the easiest to fix, as unlike XXH32, this pragma + * _actually works_ because it is a loop vectorization instead of an + * SLP vectorization. + */ + #pragma clang loop vectorize(disable) +#endif + for (i=8 ; i < nbRounds; i++) { + /* + * Prevents clang for unrolling the acc loop and interleaving with this one. + */ + XXH_COMPILER_GUARD(acc); + acc_end += XXH3_mix16B(input+(16*i), secret+(16*(i-8)) + XXH3_MIDSIZE_STARTOFFSET, seed); + } + return XXH3_avalanche(acc + acc_end); + } +} -const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBuilder */ -void ZSTD_seqToCodes(const seqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */ -/* custom memory allocation functions */ -void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem); -void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem); -void ZSTD_customFree(void* ptr, ZSTD_customMem customMem); +/* ======= Long Keys ======= */ +#define XXH_STRIPE_LEN 64 +#define XXH_SECRET_CONSUME_RATE 8 /* nb of secret bytes consumed at each accumulation */ +#define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64)) -MEM_STATIC U32 ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus */ -{ - assert(val != 0); - { -# if defined(_MSC_VER) /* Visual */ -# if STATIC_BMI2 == 1 - return _lzcnt_u32(val)^31; -# else - unsigned long r=0; - return _BitScanReverse(&r, val) ? (unsigned)r : 0; -# endif -# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */ - return __builtin_clz (val) ^ 31; -# elif defined(__ICCARM__) /* IAR Intrinsic */ - return 31 - __CLZ(val); -# else /* Software version */ - static const U32 DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; - U32 v = val; - v |= v >> 1; - v |= v >> 2; - v |= v >> 4; - v |= v >> 8; - v |= v >> 16; - return DeBruijnClz[(v * 0x07C4ACDDU) >> 27]; -# endif - } -} +#ifdef XXH_OLD_NAMES +# define STRIPE_LEN XXH_STRIPE_LEN +# define ACC_NB XXH_ACC_NB +#endif +#ifndef XXH_PREFETCH_DIST +# ifdef __clang__ +# define XXH_PREFETCH_DIST 320 +# else +# if (XXH_VECTOR == XXH_AVX512) +# define XXH_PREFETCH_DIST 512 +# else +# define XXH_PREFETCH_DIST 384 +# endif +# endif /* __clang__ */ +#endif /* XXH_PREFETCH_DIST */ -/* ZSTD_invalidateRepCodes() : - * ensures next compression will not use repcodes from previous block. - * Note : only works with regular variant; - * do not use with extDict variant ! */ -void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx); /* zstdmt, adaptive_compression (shouldn't get this definition from here) */ +/* + * These macros are to generate an XXH3_accumulate() function. + * The two arguments select the name suffix and target attribute. + * + * The name of this symbol is XXH3_accumulate_() and it calls + * XXH3_accumulate_512_(). + * + * It may be useful to hand implement this function if the compiler fails to + * optimize the inline function. + */ +#define XXH3_ACCUMULATE_TEMPLATE(name) \ +void \ +XXH3_accumulate_##name(xxh_u64* XXH_RESTRICT acc, \ + const xxh_u8* XXH_RESTRICT input, \ + const xxh_u8* XXH_RESTRICT secret, \ + size_t nbStripes) \ +{ \ + size_t n; \ + for (n = 0; n < nbStripes; n++ ) { \ + const xxh_u8* const in = input + n*XXH_STRIPE_LEN; \ + XXH_PREFETCH(in + XXH_PREFETCH_DIST); \ + XXH3_accumulate_512_##name( \ + acc, \ + in, \ + secret + n*XXH_SECRET_CONSUME_RATE); \ + } \ +} + + +XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64) +{ + if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64); + XXH_memcpy(dst, &v64, sizeof(v64)); +} + +/* Several intrinsic functions below are supposed to accept __int64 as argument, + * as documented in https://software.intel.com/sites/landingpage/IntrinsicsGuide/ . + * However, several environments do not define __int64 type, + * requiring a workaround. + */ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) + typedef int64_t xxh_i64; +#else + /* the following type must have a width of 64-bit */ + typedef long long xxh_i64; +#endif -typedef struct { - blockType_e blockType; - U32 lastBlock; - U32 origSize; -} blockProperties_t; /* declared here for decompress and fullbench */ +/* + * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the most optimized. + * + * It is a hardened version of UMAC, based off of FARSH's implementation. + * + * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD + * implementations, and it is ridiculously fast. + * + * We harden it by mixing the original input to the accumulators as well as the product. + * + * This means that in the (relatively likely) case of a multiply by zero, the + * original input is preserved. + * + * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve + * cross-pollination, as otherwise the upper and lower halves would be + * essentially independent. + * + * This doesn't matter on 64-bit hashes since they all get merged together in + * the end, so we skip the extra step. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ -/*! ZSTD_getcBlockSize() : - * Provides the size of compressed block from block header `src` */ -/* Used by: decompress, fullbench (does not get its definition from here) */ -size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, - blockProperties_t* bpPtr); +#if (XXH_VECTOR == XXH_AVX512) \ + || (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0) -/*! ZSTD_decodeSeqHeaders() : - * decode sequence header from src */ -/* Used by: decompress, fullbench (does not get its definition from here) */ -size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, - const void* src, size_t srcSize); +#ifndef XXH_TARGET_AVX512 +# define XXH_TARGET_AVX512 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_accumulate_512_avx512(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + __m512i* const xacc = (__m512i *) acc; + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + + { + /* data_vec = input[0]; */ + __m512i const data_vec = _mm512_loadu_si512 (input); + /* key_vec = secret[0]; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + /* data_key = data_vec ^ key_vec; */ + __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m512i const data_key_lo = _mm512_srli_epi64 (data_key, 32); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m512i const product = _mm512_mul_epu32 (data_key, data_key_lo); + /* xacc[0] += swap(data_vec); */ + __m512i const data_swap = _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2)); + __m512i const sum = _mm512_add_epi64(*xacc, data_swap); + /* xacc[0] += product; */ + *xacc = _mm512_add_epi64(product, sum); + } +} +XXH_FORCE_INLINE XXH_TARGET_AVX512 XXH3_ACCUMULATE_TEMPLATE(avx512) +/* + * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing. + * + * Multiplication isn't perfect, as explained by Google in HighwayHash: + * + * // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to + * // varying degrees. In descending order of goodness, bytes + * // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32. + * // As expected, the upper and lower bytes are much worse. + * + * Source: https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291 + * + * Since our algorithm uses a pseudorandom secret to add some variance into the + * mix, we don't need to (or want to) mix as often or as much as HighwayHash does. + * + * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid + * extraction. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ -#if defined (__cplusplus) +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_scrambleAcc_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + { __m512i* const xacc = (__m512i*) acc; + const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1); + + /* xacc[0] ^= (xacc[0] >> 47) */ + __m512i const acc_vec = *xacc; + __m512i const shifted = _mm512_srli_epi64 (acc_vec, 47); + /* xacc[0] ^= secret; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + __m512i const data_key = _mm512_ternarylogic_epi32(key_vec, acc_vec, shifted, 0x96 /* key_vec ^ acc_vec ^ shifted */); + + /* xacc[0] *= XXH_PRIME32_1; */ + __m512i const data_key_hi = _mm512_srli_epi64 (data_key, 32); + __m512i const prod_lo = _mm512_mul_epu32 (data_key, prime32); + __m512i const prod_hi = _mm512_mul_epu32 (data_key_hi, prime32); + *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32)); + } } -#endif -#endif /* ZSTD_CCOMMON_H_MODULE */ -/**** ended inlining zstd_internal.h ****/ +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_initCustomSecret_avx512(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64); + XXH_ASSERT(((size_t)customSecret & 63) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i); + __m512i const seed_pos = _mm512_set1_epi64((xxh_i64)seed64); + __m512i const seed = _mm512_mask_sub_epi64(seed_pos, 0xAA, _mm512_set1_epi8(0), seed_pos); + + const __m512i* const src = (const __m512i*) ((const void*) XXH3_kSecret); + __m512i* const dest = ( __m512i*) customSecret; + int i; + XXH_ASSERT(((size_t)src & 63) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 63) == 0); + for (i=0; i < nbRounds; ++i) { + dest[i] = _mm512_add_epi64(_mm512_load_si512(src + i), seed); + } } +} +#endif -/*-**************************************** -* Version -******************************************/ -unsigned ZSTD_versionNumber(void) { return ZSTD_VERSION_NUMBER; } +#if (XXH_VECTOR == XXH_AVX2) \ + || (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0) -const char* ZSTD_versionString(void) { return ZSTD_VERSION_STRING; } +#ifndef XXH_TARGET_AVX2 +# define XXH_TARGET_AVX2 /* disable attribute target */ +#endif +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_accumulate_512_avx2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { __m256i* const xacc = (__m256i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xinput = (const __m256i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* data_vec = xinput[i]; */ + __m256i const data_vec = _mm256_loadu_si256 (xinput+i); + /* key_vec = xsecret[i]; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m256i const data_key_lo = _mm256_srli_epi64 (data_key, 32); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m256i const product = _mm256_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2)); + __m256i const sum = _mm256_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm256_add_epi64(product, sum); + } } +} +XXH_FORCE_INLINE XXH_TARGET_AVX2 XXH3_ACCUMULATE_TEMPLATE(avx2) + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_scrambleAcc_avx2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { __m256i* const xacc = (__m256i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m256i const acc_vec = xacc[i]; + __m256i const shifted = _mm256_srli_epi64 (acc_vec, 47); + __m256i const data_vec = _mm256_xor_si256 (acc_vec, shifted); + /* xacc[i] ^= xsecret; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m256i const data_key_hi = _mm256_srli_epi64 (data_key, 32); + __m256i const prod_lo = _mm256_mul_epu32 (data_key, prime32); + __m256i const prod_hi = _mm256_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32)); + } + } +} -/*-**************************************** -* ZSTD Error Management -******************************************/ -#undef ZSTD_isError /* defined within zstd_internal.h */ -/*! ZSTD_isError() : - * tells if a return value is an error code - * symbol is required for external callers */ -unsigned ZSTD_isError(size_t code) { return ERR_isError(code); } +XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0); + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64); + (void)(&XXH_writeLE64); + XXH_PREFETCH(customSecret); + { __m256i const seed = _mm256_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64, (xxh_i64)(0U - seed64), (xxh_i64)seed64); -/*! ZSTD_getErrorName() : - * provides error code string from function result (useful for debugging) */ -const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); } + const __m256i* const src = (const __m256i*) ((const void*) XXH3_kSecret); + __m256i* dest = ( __m256i*) customSecret; -/*! ZSTD_getError() : - * convert a `size_t` function result into a proper ZSTD_errorCode enum */ -ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); } +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dest); +# endif + XXH_ASSERT(((size_t)src & 31) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 31) == 0); + + /* GCC -O2 need unroll loop manually */ + dest[0] = _mm256_add_epi64(_mm256_load_si256(src+0), seed); + dest[1] = _mm256_add_epi64(_mm256_load_si256(src+1), seed); + dest[2] = _mm256_add_epi64(_mm256_load_si256(src+2), seed); + dest[3] = _mm256_add_epi64(_mm256_load_si256(src+3), seed); + dest[4] = _mm256_add_epi64(_mm256_load_si256(src+4), seed); + dest[5] = _mm256_add_epi64(_mm256_load_si256(src+5), seed); + } +} -/*! ZSTD_getErrorString() : - * provides error code string from enum */ -const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); } +#endif +/* x86dispatch always generates SSE2 */ +#if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH) +#ifndef XXH_TARGET_SSE2 +# define XXH_TARGET_SSE2 /* disable attribute target */ +#endif -/*=************************************************************** -* Custom allocator -****************************************************************/ -void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem) -{ - if (customMem.customAlloc) - return customMem.customAlloc(customMem.opaque, size); - return ZSTD_malloc(size); +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_accumulate_512_sse2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* SSE2 is just a half-scale version of the AVX2 version. */ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { __m128i* const xacc = (__m128i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xinput = (const __m128i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* data_vec = xinput[i]; */ + __m128i const data_vec = _mm_loadu_si128 (xinput+i); + /* key_vec = xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m128i const data_key_lo = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m128i const product = _mm_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2)); + __m128i const sum = _mm_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm_add_epi64(product, sum); + } } } - -void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem) -{ - if (customMem.customAlloc) { - /* calloc implemented as malloc+memset; - * not as efficient as calloc, but next best guess for custom malloc */ - void* const ptr = customMem.customAlloc(customMem.opaque, size); - ZSTD_memset(ptr, 0, size); - return ptr; +XXH_FORCE_INLINE XXH_TARGET_SSE2 XXH3_ACCUMULATE_TEMPLATE(sse2) + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_scrambleAcc_sse2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { __m128i* const xacc = (__m128i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m128i const acc_vec = xacc[i]; + __m128i const shifted = _mm_srli_epi64 (acc_vec, 47); + __m128i const data_vec = _mm_xor_si128 (acc_vec, shifted); + /* xacc[i] ^= xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m128i const data_key_hi = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + __m128i const prod_lo = _mm_mul_epu32 (data_key, prime32); + __m128i const prod_hi = _mm_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32)); + } } - return ZSTD_calloc(1, size); } -void ZSTD_customFree(void* ptr, ZSTD_customMem customMem) +XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) { - if (ptr!=NULL) { - if (customMem.customFree) - customMem.customFree(customMem.opaque, ptr); - else - ZSTD_free(ptr); - } + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i); + +# if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900 + /* MSVC 32bit mode does not support _mm_set_epi64x before 2015 */ + XXH_ALIGN(16) const xxh_i64 seed64x2[2] = { (xxh_i64)seed64, (xxh_i64)(0U - seed64) }; + __m128i const seed = _mm_load_si128((__m128i const*)seed64x2); +# else + __m128i const seed = _mm_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64); +# endif + int i; + + const void* const src16 = XXH3_kSecret; + __m128i* dst16 = (__m128i*) customSecret; +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dst16); +# endif + XXH_ASSERT(((size_t)src16 & 15) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dst16 & 15) == 0); + + for (i=0; i < nbRounds; ++i) { + dst16[i] = _mm_add_epi64(_mm_load_si128((const __m128i *)src16+i), seed); + } } } -/**** ended inlining common/zstd_common.c ****/ -/**** start inlining decompress/huf_decompress.c ****/ -/* ****************************************************************** - * huff0 huffman decoder, - * part of Finite State Entropy library - * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc. - * - * You can contact the author at : - * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. -****************************************************************** */ +#endif -/* ************************************************************** -* Dependencies -****************************************************************/ -/**** skipping file: ../common/zstd_deps.h ****/ -/**** skipping file: ../common/compiler.h ****/ -/**** skipping file: ../common/bitstream.h ****/ -/**** skipping file: ../common/fse.h ****/ -#define HUF_STATIC_LINKING_ONLY -/**** skipping file: ../common/huf.h ****/ -/**** skipping file: ../common/error_private.h ****/ +#if (XXH_VECTOR == XXH_NEON) -/* ************************************************************** -* Macros -****************************************************************/ +/* forward declarations for the scalar routines */ +XXH_FORCE_INLINE void +XXH3_scalarRound(void* XXH_RESTRICT acc, void const* XXH_RESTRICT input, + void const* XXH_RESTRICT secret, size_t lane); -/* These two optional macros force the use one way or another of the two - * Huffman decompression implementations. You can't force in both directions - * at the same time. +XXH_FORCE_INLINE void +XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT secret, size_t lane); + +/*! + * @internal + * @brief The bulk processing loop for NEON and WASM SIMD128. + * + * The NEON code path is actually partially scalar when running on AArch64. This + * is to optimize the pipelining and can have up to 15% speedup depending on the + * CPU, and it also mitigates some GCC codegen issues. + * + * @see XXH3_NEON_LANES for configuring this and details about this optimization. + * + * NEON's 32-bit to 64-bit long multiply takes a half vector of 32-bit + * integers instead of the other platforms which mask full 64-bit vectors, + * so the setup is more complicated than just shifting right. + * + * Additionally, there is an optimization for 4 lanes at once noted below. + * + * Since, as stated, the most optimal amount of lanes for Cortexes is 6, + * there needs to be *three* versions of the accumulate operation used + * for the remaining 2 lanes. + * + * WASM's SIMD128 uses SIMDe's arm_neon.h polyfill because the intrinsics overlap + * nearly perfectly. */ -#if defined(HUF_FORCE_DECOMPRESS_X1) && \ - defined(HUF_FORCE_DECOMPRESS_X2) -#error "Cannot force the use of the X1 and X2 decoders at the same time!" + +XXH_FORCE_INLINE void +XXH3_accumulate_512_neon( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + XXH_STATIC_ASSERT(XXH3_NEON_LANES > 0 && XXH3_NEON_LANES <= XXH_ACC_NB && XXH3_NEON_LANES % 2 == 0); + { /* GCC for darwin arm64 does not like aliasing here */ + xxh_aliasing_uint64x2_t* const xacc = (xxh_aliasing_uint64x2_t*) acc; + /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */ + uint8_t const* xinput = (const uint8_t *) input; + uint8_t const* xsecret = (const uint8_t *) secret; + + size_t i; +#ifdef __wasm_simd128__ + /* + * On WASM SIMD128, Clang emits direct address loads when XXH3_kSecret + * is constant propagated, which results in it converting it to this + * inside the loop: + * + * a = v128.load(XXH3_kSecret + 0 + $secret_offset, offset = 0) + * b = v128.load(XXH3_kSecret + 16 + $secret_offset, offset = 0) + * ... + * + * This requires a full 32-bit address immediate (and therefore a 6 byte + * instruction) as well as an add for each offset. + * + * Putting an asm guard prevents it from folding (at the cost of losing + * the alignment hint), and uses the free offset in `v128.load` instead + * of adding secret_offset each time which overall reduces code size by + * about a kilobyte and improves performance. + */ + XXH_COMPILER_GUARD(xsecret); #endif + /* Scalar lanes use the normal scalarRound routine */ + for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { + XXH3_scalarRound(acc, input, secret, i); + } + i = 0; + /* 4 NEON lanes at a time. */ + for (; i+1 < XXH3_NEON_LANES / 2; i+=2) { + /* data_vec = xinput[i]; */ + uint64x2_t data_vec_1 = XXH_vld1q_u64(xinput + (i * 16)); + uint64x2_t data_vec_2 = XXH_vld1q_u64(xinput + ((i+1) * 16)); + /* key_vec = xsecret[i]; */ + uint64x2_t key_vec_1 = XXH_vld1q_u64(xsecret + (i * 16)); + uint64x2_t key_vec_2 = XXH_vld1q_u64(xsecret + ((i+1) * 16)); + /* data_swap = swap(data_vec) */ + uint64x2_t data_swap_1 = vextq_u64(data_vec_1, data_vec_1, 1); + uint64x2_t data_swap_2 = vextq_u64(data_vec_2, data_vec_2, 1); + /* data_key = data_vec ^ key_vec; */ + uint64x2_t data_key_1 = veorq_u64(data_vec_1, key_vec_1); + uint64x2_t data_key_2 = veorq_u64(data_vec_2, key_vec_2); + + /* + * If we reinterpret the 64x2 vectors as 32x4 vectors, we can use a + * de-interleave operation for 4 lanes in 1 step with `vuzpq_u32` to + * get one vector with the low 32 bits of each lane, and one vector + * with the high 32 bits of each lane. + * + * The intrinsic returns a double vector because the original ARMv7-a + * instruction modified both arguments in place. AArch64 and SIMD128 emit + * two instructions from this intrinsic. + * + * [ dk11L | dk11H | dk12L | dk12H ] -> [ dk11L | dk12L | dk21L | dk22L ] + * [ dk21L | dk21H | dk22L | dk22H ] -> [ dk11H | dk12H | dk21H | dk22H ] + */ + uint32x4x2_t unzipped = vuzpq_u32( + vreinterpretq_u32_u64(data_key_1), + vreinterpretq_u32_u64(data_key_2) + ); + /* data_key_lo = data_key & 0xFFFFFFFF */ + uint32x4_t data_key_lo = unzipped.val[0]; + /* data_key_hi = data_key >> 32 */ + uint32x4_t data_key_hi = unzipped.val[1]; + /* + * Then, we can split the vectors horizontally and multiply which, as for most + * widening intrinsics, have a variant that works on both high half vectors + * for free on AArch64. A similar instruction is available on SIMD128. + * + * sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi + */ + uint64x2_t sum_1 = XXH_vmlal_low_u32(data_swap_1, data_key_lo, data_key_hi); + uint64x2_t sum_2 = XXH_vmlal_high_u32(data_swap_2, data_key_lo, data_key_hi); + /* + * Clang reorders + * a += b * c; // umlal swap.2d, dkl.2s, dkh.2s + * c += a; // add acc.2d, acc.2d, swap.2d + * to + * c += a; // add acc.2d, acc.2d, swap.2d + * c += b * c; // umlal acc.2d, dkl.2s, dkh.2s + * + * While it would make sense in theory since the addition is faster, + * for reasons likely related to umlal being limited to certain NEON + * pipelines, this is worse. A compiler guard fixes this. + */ + XXH_COMPILER_GUARD_CLANG_NEON(sum_1); + XXH_COMPILER_GUARD_CLANG_NEON(sum_2); + /* xacc[i] = acc_vec + sum; */ + xacc[i] = vaddq_u64(xacc[i], sum_1); + xacc[i+1] = vaddq_u64(xacc[i+1], sum_2); + } + /* Operate on the remaining NEON lanes 2 at a time. */ + for (; i < XXH3_NEON_LANES / 2; i++) { + /* data_vec = xinput[i]; */ + uint64x2_t data_vec = XXH_vld1q_u64(xinput + (i * 16)); + /* key_vec = xsecret[i]; */ + uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16)); + /* acc_vec_2 = swap(data_vec) */ + uint64x2_t data_swap = vextq_u64(data_vec, data_vec, 1); + /* data_key = data_vec ^ key_vec; */ + uint64x2_t data_key = veorq_u64(data_vec, key_vec); + /* For two lanes, just use VMOVN and VSHRN. */ + /* data_key_lo = data_key & 0xFFFFFFFF; */ + uint32x2_t data_key_lo = vmovn_u64(data_key); + /* data_key_hi = data_key >> 32; */ + uint32x2_t data_key_hi = vshrn_n_u64(data_key, 32); + /* sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi; */ + uint64x2_t sum = vmlal_u32(data_swap, data_key_lo, data_key_hi); + /* Same Clang workaround as before */ + XXH_COMPILER_GUARD_CLANG_NEON(sum); + /* xacc[i] = acc_vec + sum; */ + xacc[i] = vaddq_u64 (xacc[i], sum); + } + } +} +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(neon) +XXH_FORCE_INLINE void +XXH3_scrambleAcc_neon(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); -/* ************************************************************** -* Error Management -****************************************************************/ -#define HUF_isError ERR_isError + { xxh_aliasing_uint64x2_t* xacc = (xxh_aliasing_uint64x2_t*) acc; + uint8_t const* xsecret = (uint8_t const*) secret; + size_t i; + /* WASM uses operator overloads and doesn't need these. */ +#ifndef __wasm_simd128__ + /* { prime32_1, prime32_1 } */ + uint32x2_t const kPrimeLo = vdup_n_u32(XXH_PRIME32_1); + /* { 0, prime32_1, 0, prime32_1 } */ + uint32x4_t const kPrimeHi = vreinterpretq_u32_u64(vdupq_n_u64((xxh_u64)XXH_PRIME32_1 << 32)); +#endif -/* ************************************************************** -* Byte alignment for workSpace management -****************************************************************/ -#define HUF_ALIGN(x, a) HUF_ALIGN_MASK((x), (a) - 1) -#define HUF_ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask)) + /* AArch64 uses both scalar and neon at the same time */ + for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { + XXH3_scalarScrambleRound(acc, secret, i); + } + for (i=0; i < XXH3_NEON_LANES / 2; i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + uint64x2_t acc_vec = xacc[i]; + uint64x2_t shifted = vshrq_n_u64(acc_vec, 47); + uint64x2_t data_vec = veorq_u64(acc_vec, shifted); + + /* xacc[i] ^= xsecret[i]; */ + uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16)); + uint64x2_t data_key = veorq_u64(data_vec, key_vec); + /* xacc[i] *= XXH_PRIME32_1 */ +#ifdef __wasm_simd128__ + /* SIMD128 has multiply by u64x2, use it instead of expanding and scalarizing */ + xacc[i] = data_key * XXH_PRIME32_1; +#else + /* + * Expanded version with portable NEON intrinsics + * + * lo(x) * lo(y) + (hi(x) * lo(y) << 32) + * + * prod_hi = hi(data_key) * lo(prime) << 32 + * + * Since we only need 32 bits of this multiply a trick can be used, reinterpreting the vector + * as a uint32x4_t and multiplying by { 0, prime, 0, prime } to cancel out the unwanted bits + * and avoid the shift. + */ + uint32x4_t prod_hi = vmulq_u32 (vreinterpretq_u32_u64(data_key), kPrimeHi); + /* Extract low bits for vmlal_u32 */ + uint32x2_t data_key_lo = vmovn_u64(data_key); + /* xacc[i] = prod_hi + lo(data_key) * XXH_PRIME32_1; */ + xacc[i] = vmlal_u32(vreinterpretq_u64_u32(prod_hi), data_key_lo, kPrimeLo); +#endif + } + } +} +#endif +#if (XXH_VECTOR == XXH_VSX) -/* ************************************************************** -* BMI2 Variant Wrappers -****************************************************************/ -#if DYNAMIC_BMI2 +XXH_FORCE_INLINE void +XXH3_accumulate_512_vsx( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* presumed aligned */ + xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc; + xxh_u8 const* const xinput = (xxh_u8 const*) input; /* no alignment restriction */ + xxh_u8 const* const xsecret = (xxh_u8 const*) secret; /* no alignment restriction */ + xxh_u64x2 const v32 = { 32, 32 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* data_vec = xinput[i]; */ + xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + 16*i); + /* key_vec = xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16*i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + /* shuffled = (data_key << 32) | (data_key >> 32); */ + xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32); + /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & 0xFFFFFFFF); */ + xxh_u64x2 const product = XXH_vec_mulo((xxh_u32x4)data_key, shuffled); + /* acc_vec = xacc[i]; */ + xxh_u64x2 acc_vec = xacc[i]; + acc_vec += product; + + /* swap high and low halves */ +#ifdef __s390x__ + acc_vec += vec_permi(data_vec, data_vec, 2); +#else + acc_vec += vec_xxpermdi(data_vec, data_vec, 2); +#endif + xacc[i] = acc_vec; + } +} +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(vsx) + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_vsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc; + const xxh_u8* const xsecret = (const xxh_u8*) secret; + /* constants */ + xxh_u64x2 const v32 = { 32, 32 }; + xxh_u64x2 const v47 = { 47, 47 }; + xxh_u32x4 const prime = { XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + xxh_u64x2 const acc_vec = xacc[i]; + xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47); + + /* xacc[i] ^= xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16*i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + + /* xacc[i] *= XXH_PRIME32_1 */ + /* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & 0xFFFFFFFF); */ + xxh_u64x2 const prod_even = XXH_vec_mule((xxh_u32x4)data_key, prime); + /* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32); */ + xxh_u64x2 const prod_odd = XXH_vec_mulo((xxh_u32x4)data_key, prime); + xacc[i] = prod_odd + (prod_even << v32); + } } +} -#define HUF_DGEN(fn) \ - \ - static size_t fn##_default( \ - void* dst, size_t dstSize, \ - const void* cSrc, size_t cSrcSize, \ - const HUF_DTable* DTable) \ - { \ - return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ - } \ - \ - static TARGET_ATTRIBUTE("bmi2") size_t fn##_bmi2( \ - void* dst, size_t dstSize, \ - const void* cSrc, size_t cSrcSize, \ - const HUF_DTable* DTable) \ - { \ - return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ - } \ - \ - static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ - size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \ - { \ - if (bmi2) { \ - return fn##_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); \ - } \ - return fn##_default(dst, dstSize, cSrc, cSrcSize, DTable); \ +#endif + +#if (XXH_VECTOR == XXH_SVE) + +XXH_FORCE_INLINE void +XXH3_accumulate_512_sve( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + uint64_t *xacc = (uint64_t *)acc; + const uint64_t *xinput = (const uint64_t *)(const void *)input; + const uint64_t *xsecret = (const uint64_t *)(const void *)secret; + svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1); + uint64_t element_count = svcntd(); + if (element_count >= 8) { + svbool_t mask = svptrue_pat_b64(SV_VL8); + svuint64_t vacc = svld1_u64(mask, xacc); + ACCRND(vacc, 0); + svst1_u64(mask, xacc, vacc); + } else if (element_count == 2) { /* sve128 */ + svbool_t mask = svptrue_pat_b64(SV_VL2); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 2); + svuint64_t acc2 = svld1_u64(mask, xacc + 4); + svuint64_t acc3 = svld1_u64(mask, xacc + 6); + ACCRND(acc0, 0); + ACCRND(acc1, 2); + ACCRND(acc2, 4); + ACCRND(acc3, 6); + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 2, acc1); + svst1_u64(mask, xacc + 4, acc2); + svst1_u64(mask, xacc + 6, acc3); + } else { + svbool_t mask = svptrue_pat_b64(SV_VL4); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 4); + ACCRND(acc0, 0); + ACCRND(acc1, 4); + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 4, acc1); + } +} + +XXH_FORCE_INLINE void +XXH3_accumulate_sve(xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, + size_t nbStripes) +{ + if (nbStripes != 0) { + uint64_t *xacc = (uint64_t *)acc; + const uint64_t *xinput = (const uint64_t *)(const void *)input; + const uint64_t *xsecret = (const uint64_t *)(const void *)secret; + svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1); + uint64_t element_count = svcntd(); + if (element_count >= 8) { + svbool_t mask = svptrue_pat_b64(SV_VL8); + svuint64_t vacc = svld1_u64(mask, xacc + 0); + do { + /* svprfd(svbool_t, void *, enum svfprop); */ + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(vacc, 0); + xinput += 8; + xsecret += 1; + nbStripes--; + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, vacc); + } else if (element_count == 2) { /* sve128 */ + svbool_t mask = svptrue_pat_b64(SV_VL2); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 2); + svuint64_t acc2 = svld1_u64(mask, xacc + 4); + svuint64_t acc3 = svld1_u64(mask, xacc + 6); + do { + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(acc0, 0); + ACCRND(acc1, 2); + ACCRND(acc2, 4); + ACCRND(acc3, 6); + xinput += 8; + xsecret += 1; + nbStripes--; + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 2, acc1); + svst1_u64(mask, xacc + 4, acc2); + svst1_u64(mask, xacc + 6, acc3); + } else { + svbool_t mask = svptrue_pat_b64(SV_VL4); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 4); + do { + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(acc0, 0); + ACCRND(acc1, 4); + xinput += 8; + xsecret += 1; + nbStripes--; + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 4, acc1); + } } +} + +#endif +/* scalar variants - universal */ + +#if defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__)) +/* + * In XXH3_scalarRound(), GCC and Clang have a similar codegen issue, where they + * emit an excess mask and a full 64-bit multiply-add (MADD X-form). + * + * While this might not seem like much, as AArch64 is a 64-bit architecture, only + * big Cortex designs have a full 64-bit multiplier. + * + * On the little cores, the smaller 32-bit multiplier is used, and full 64-bit + * multiplies expand to 2-3 multiplies in microcode. This has a major penalty + * of up to 4 latency cycles and 2 stall cycles in the multiply pipeline. + * + * Thankfully, AArch64 still provides the 32-bit long multiply-add (UMADDL) which does + * not have this penalty and does the mask automatically. + */ +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc) +{ + xxh_u64 ret; + /* note: %x = 64-bit register, %w = 32-bit register */ + __asm__("umaddl %x0, %w1, %w2, %x3" : "=r" (ret) : "r" (lhs), "r" (rhs), "r" (acc)); + return ret; +} #else +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc) +{ + return XXH_mult32to64((xxh_u32)lhs, (xxh_u32)rhs) + acc; +} +#endif -#define HUF_DGEN(fn) \ - static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ - size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \ - { \ - (void)bmi2; \ - return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ +/*! + * @internal + * @brief Scalar round for @ref XXH3_accumulate_512_scalar(). + * + * This is extracted to its own function because the NEON path uses a combination + * of NEON and scalar. + */ +XXH_FORCE_INLINE void +XXH3_scalarRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT input, + void const* XXH_RESTRICT secret, + size_t lane) +{ + xxh_u64* xacc = (xxh_u64*) acc; + xxh_u8 const* xinput = (xxh_u8 const*) input; + xxh_u8 const* xsecret = (xxh_u8 const*) secret; + XXH_ASSERT(lane < XXH_ACC_NB); + XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN-1)) == 0); + { + xxh_u64 const data_val = XXH_readLE64(xinput + lane * 8); + xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + lane * 8); + xacc[lane ^ 1] += data_val; /* swap adjacent lanes */ + xacc[lane] = XXH_mult32to64_add64(data_key /* & 0xFFFFFFFF */, data_key >> 32, xacc[lane]); } +} +/*! + * @internal + * @brief Processes a 64 byte block of data using the scalar path. + */ +XXH_FORCE_INLINE void +XXH3_accumulate_512_scalar(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + size_t i; + /* ARM GCC refuses to unroll this loop, resulting in a 24% slowdown on ARMv6. */ +#if defined(__GNUC__) && !defined(__clang__) \ + && (defined(__arm__) || defined(__thumb2__)) \ + && defined(__ARM_FEATURE_UNALIGNED) /* no unaligned access just wastes bytes */ \ + && XXH_SIZE_OPT <= 0 +# pragma GCC unroll 8 #endif + for (i=0; i < XXH_ACC_NB; i++) { + XXH3_scalarRound(acc, input, secret, i); + } +} +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(scalar) +/*! + * @internal + * @brief Scalar scramble step for @ref XXH3_scrambleAcc_scalar(). + * + * This is extracted to its own function because the NEON path uses a combination + * of NEON and scalar. + */ +XXH_FORCE_INLINE void +XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT secret, + size_t lane) +{ + xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */ + const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */ + XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN-1)) == 0); + XXH_ASSERT(lane < XXH_ACC_NB); + { + xxh_u64 const key64 = XXH_readLE64(xsecret + lane * 8); + xxh_u64 acc64 = xacc[lane]; + acc64 = XXH_xorshift64(acc64, 47); + acc64 ^= key64; + acc64 *= XXH_PRIME32_1; + xacc[lane] = acc64; + } +} -/*-***************************/ -/* generic DTableDesc */ -/*-***************************/ -typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc; +/*! + * @internal + * @brief Scrambles the accumulators after a large chunk has been read + */ +XXH_FORCE_INLINE void +XXH3_scrambleAcc_scalar(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + size_t i; + for (i=0; i < XXH_ACC_NB; i++) { + XXH3_scalarScrambleRound(acc, secret, i); + } +} -static DTableDesc HUF_getDTableDesc(const HUF_DTable* table) +XXH_FORCE_INLINE void +XXH3_initCustomSecret_scalar(void* XXH_RESTRICT customSecret, xxh_u64 seed64) { - DTableDesc dtd; - ZSTD_memcpy(&dtd, table, sizeof(dtd)); - return dtd; + /* + * We need a separate pointer for the hack below, + * which requires a non-const pointer. + * Any decent compiler will optimize this out otherwise. + */ + const xxh_u8* kSecretPtr = XXH3_kSecret; + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + +#if defined(__GNUC__) && defined(__aarch64__) + /* + * UGLY HACK: + * GCC and Clang generate a bunch of MOV/MOVK pairs for aarch64, and they are + * placed sequentially, in order, at the top of the unrolled loop. + * + * While MOVK is great for generating constants (2 cycles for a 64-bit + * constant compared to 4 cycles for LDR), it fights for bandwidth with + * the arithmetic instructions. + * + * I L S + * MOVK + * MOVK + * MOVK + * MOVK + * ADD + * SUB STR + * STR + * By forcing loads from memory (as the asm line causes the compiler to assume + * that XXH3_kSecretPtr has been changed), the pipelines are used more + * efficiently: + * I L S + * LDR + * ADD LDR + * SUB STR + * STR + * + * See XXH3_NEON_LANES for details on the pipsline. + * + * XXH3_64bits_withSeed, len == 256, Snapdragon 835 + * without hack: 2654.4 MB/s + * with hack: 3202.9 MB/s + */ + XXH_COMPILER_GUARD(kSecretPtr); +#endif + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16; + int i; + for (i=0; i < nbRounds; i++) { + /* + * The asm hack causes the compiler to assume that kSecretPtr aliases with + * customSecret, and on aarch64, this prevented LDP from merging two + * loads together for free. Putting the loads together before the stores + * properly generates LDP. + */ + xxh_u64 lo = XXH_readLE64(kSecretPtr + 16*i) + seed64; + xxh_u64 hi = XXH_readLE64(kSecretPtr + 16*i + 8) - seed64; + XXH_writeLE64((xxh_u8*)customSecret + 16*i, lo); + XXH_writeLE64((xxh_u8*)customSecret + 16*i + 8, hi); + } } } -#ifndef HUF_FORCE_DECOMPRESS_X2 +typedef void (*XXH3_f_accumulate)(xxh_u64* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, size_t); +typedef void (*XXH3_f_scrambleAcc)(void* XXH_RESTRICT, const void*); +typedef void (*XXH3_f_initCustomSecret)(void* XXH_RESTRICT, xxh_u64); -/*-***************************/ -/* single-symbol decoding */ -/*-***************************/ -typedef struct { BYTE byte; BYTE nbBits; } HUF_DEltX1; /* single-symbol decoding */ -/** - * Packs 4 HUF_DEltX1 structs into a U64. This is used to lay down 4 entries at - * a time. - */ -static U64 HUF_DEltX1_set4(BYTE symbol, BYTE nbBits) { - U64 D4; - if (MEM_isLittleEndian()) { - D4 = symbol + (nbBits << 8); - } else { - D4 = (symbol << 8) + nbBits; - } - D4 *= 0x0001000100010001ULL; - return D4; -} +#if (XXH_VECTOR == XXH_AVX512) -typedef struct { - U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; - U32 rankStart[HUF_TABLELOG_ABSOLUTEMAX + 1]; - U32 statsWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; - BYTE symbols[HUF_SYMBOLVALUE_MAX + 1]; - BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; -} HUF_ReadDTableX1_Workspace; +#define XXH3_accumulate_512 XXH3_accumulate_512_avx512 +#define XXH3_accumulate XXH3_accumulate_avx512 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx512 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx512 +#elif (XXH_VECTOR == XXH_AVX2) -size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize) -{ - return HUF_readDTableX1_wksp_bmi2(DTable, src, srcSize, workSpace, wkspSize, /* bmi2 */ 0); -} +#define XXH3_accumulate_512 XXH3_accumulate_512_avx2 +#define XXH3_accumulate XXH3_accumulate_avx2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx2 -size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2) -{ - U32 tableLog = 0; - U32 nbSymbols = 0; - size_t iSize; - void* const dtPtr = DTable + 1; - HUF_DEltX1* const dt = (HUF_DEltX1*)dtPtr; - HUF_ReadDTableX1_Workspace* wksp = (HUF_ReadDTableX1_Workspace*)workSpace; +#elif (XXH_VECTOR == XXH_SSE2) - DEBUG_STATIC_ASSERT(HUF_DECOMPRESS_WORKSPACE_SIZE >= sizeof(*wksp)); - if (sizeof(*wksp) > wkspSize) return ERROR(tableLog_tooLarge); +#define XXH3_accumulate_512 XXH3_accumulate_512_sse2 +#define XXH3_accumulate XXH3_accumulate_sse2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_sse2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_sse2 - DEBUG_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable)); - /* ZSTD_memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */ +#elif (XXH_VECTOR == XXH_NEON) - iSize = HUF_readStats_wksp(wksp->huffWeight, HUF_SYMBOLVALUE_MAX + 1, wksp->rankVal, &nbSymbols, &tableLog, src, srcSize, wksp->statsWksp, sizeof(wksp->statsWksp), bmi2); - if (HUF_isError(iSize)) return iSize; +#define XXH3_accumulate_512 XXH3_accumulate_512_neon +#define XXH3_accumulate XXH3_accumulate_neon +#define XXH3_scrambleAcc XXH3_scrambleAcc_neon +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar - /* Table header */ - { DTableDesc dtd = HUF_getDTableDesc(DTable); - if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, Huffman tree cannot fit in */ - dtd.tableType = 0; - dtd.tableLog = (BYTE)tableLog; - ZSTD_memcpy(DTable, &dtd, sizeof(dtd)); - } +#elif (XXH_VECTOR == XXH_VSX) - /* Compute symbols and rankStart given rankVal: - * - * rankVal already contains the number of values of each weight. - * - * symbols contains the symbols ordered by weight. First are the rankVal[0] - * weight 0 symbols, followed by the rankVal[1] weight 1 symbols, and so on. - * symbols[0] is filled (but unused) to avoid a branch. - * - * rankStart contains the offset where each rank belongs in the DTable. - * rankStart[0] is not filled because there are no entries in the table for - * weight 0. - */ - { - int n; - int nextRankStart = 0; - int const unroll = 4; - int const nLimit = (int)nbSymbols - unroll + 1; - for (n=0; n<(int)tableLog+1; n++) { - U32 const curr = nextRankStart; - nextRankStart += wksp->rankVal[n]; - wksp->rankStart[n] = curr; - } - for (n=0; n < nLimit; n += unroll) { - int u; - for (u=0; u < unroll; ++u) { - size_t const w = wksp->huffWeight[n+u]; - wksp->symbols[wksp->rankStart[w]++] = (BYTE)(n+u); - } - } - for (; n < (int)nbSymbols; ++n) { - size_t const w = wksp->huffWeight[n]; - wksp->symbols[wksp->rankStart[w]++] = (BYTE)n; - } - } +#define XXH3_accumulate_512 XXH3_accumulate_512_vsx +#define XXH3_accumulate XXH3_accumulate_vsx +#define XXH3_scrambleAcc XXH3_scrambleAcc_vsx +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar - /* fill DTable - * We fill all entries of each weight in order. - * That way length is a constant for each iteration of the outter loop. - * We can switch based on the length to a different inner loop which is - * optimized for that particular case. - */ - { - U32 w; - int symbol=wksp->rankVal[0]; - int rankStart=0; - for (w=1; wrankVal[w]; - int const length = (1 << w) >> 1; - int uStart = rankStart; - BYTE const nbBits = (BYTE)(tableLog + 1 - w); - int s; - int u; - switch (length) { - case 1: - for (s=0; ssymbols[symbol + s]; - D.nbBits = nbBits; - dt[uStart] = D; - uStart += 1; - } - break; - case 2: - for (s=0; ssymbols[symbol + s]; - D.nbBits = nbBits; - dt[uStart+0] = D; - dt[uStart+1] = D; - uStart += 2; - } - break; - case 4: - for (s=0; ssymbols[symbol + s], nbBits); - MEM_write64(dt + uStart, D4); - uStart += 4; - } - break; - case 8: - for (s=0; ssymbols[symbol + s], nbBits); - MEM_write64(dt + uStart, D4); - MEM_write64(dt + uStart + 4, D4); - uStart += 8; - } - break; - default: - for (s=0; ssymbols[symbol + s], nbBits); - for (u=0; u < length; u += 16) { - MEM_write64(dt + uStart + u + 0, D4); - MEM_write64(dt + uStart + u + 4, D4); - MEM_write64(dt + uStart + u + 8, D4); - MEM_write64(dt + uStart + u + 12, D4); - } - assert(u == length); - uStart += length; - } - break; - } - symbol += symbolCount; - rankStart += symbolCount * length; - } - } - return iSize; -} +#elif (XXH_VECTOR == XXH_SVE) +#define XXH3_accumulate_512 XXH3_accumulate_512_sve +#define XXH3_accumulate XXH3_accumulate_sve +#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar -FORCE_INLINE_TEMPLATE BYTE -HUF_decodeSymbolX1(BIT_DStream_t* Dstream, const HUF_DEltX1* dt, const U32 dtLog) -{ - size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */ - BYTE const c = dt[val].byte; - BIT_skipBits(Dstream, dt[val].nbBits); - return c; -} +#else /* scalar */ -#define HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) \ - *ptr++ = HUF_decodeSymbolX1(DStreamPtr, dt, dtLog) +#define XXH3_accumulate_512 XXH3_accumulate_512_scalar +#define XXH3_accumulate XXH3_accumulate_scalar +#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar -#define HUF_DECODE_SYMBOLX1_1(ptr, DStreamPtr) \ - if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ - HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) +#endif -#define HUF_DECODE_SYMBOLX1_2(ptr, DStreamPtr) \ - if (MEM_64bits()) \ - HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) +#if XXH_SIZE_OPT >= 1 /* don't do SIMD for initialization */ +# undef XXH3_initCustomSecret +# define XXH3_initCustomSecret XXH3_initCustomSecret_scalar +#endif -HINT_INLINE size_t -HUF_decodeStreamX1(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX1* const dt, const U32 dtLog) +XXH_FORCE_INLINE void +XXH3_hashLong_internal_loop(xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) { - BYTE* const pStart = p; + size_t const nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE; + size_t const block_len = XXH_STRIPE_LEN * nbStripesPerBlock; + size_t const nb_blocks = (len - 1) / block_len; - /* up to 4 symbols at a time */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-3)) { - HUF_DECODE_SYMBOLX1_2(p, bitDPtr); - HUF_DECODE_SYMBOLX1_1(p, bitDPtr); - HUF_DECODE_SYMBOLX1_2(p, bitDPtr); - HUF_DECODE_SYMBOLX1_0(p, bitDPtr); - } + size_t n; - /* [0-3] symbols remaining */ - if (MEM_32bits()) - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd)) - HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); - /* no more data to retrieve from bitstream, no need to reload */ - while (p < pEnd) - HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + for (n = 0; n < nb_blocks; n++) { + f_acc(acc, input + n*block_len, secret, nbStripesPerBlock); + f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN); + } + + /* last partial block */ + XXH_ASSERT(len > XXH_STRIPE_LEN); + { size_t const nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN; + XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE)); + f_acc(acc, input + nb_blocks*block_len, secret, nbStripes); - return pEnd-pStart; + /* last stripe */ + { const xxh_u8* const p = input + len - XXH_STRIPE_LEN; +#define XXH_SECRET_LASTACC_START 7 /* not aligned on 8, last secret is different from acc & scrambler */ + XXH3_accumulate_512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START); + } } } -FORCE_INLINE_TEMPLATE size_t -HUF_decompress1X1_usingDTable_internal_body( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) +XXH_FORCE_INLINE xxh_u64 +XXH3_mix2Accs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret) { - BYTE* op = (BYTE*)dst; - BYTE* const oend = op + dstSize; - const void* dtPtr = DTable + 1; - const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr; - BIT_DStream_t bitD; - DTableDesc const dtd = HUF_getDTableDesc(DTable); - U32 const dtLog = dtd.tableLog; - - CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) ); - - HUF_decodeStreamX1(op, &bitD, oend, dt, dtLog); - - if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); - - return dstSize; + return XXH3_mul128_fold64( + acc[0] ^ XXH_readLE64(secret), + acc[1] ^ XXH_readLE64(secret+8) ); } -FORCE_INLINE_TEMPLATE size_t -HUF_decompress4X1_usingDTable_internal_body( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) +static XXH64_hash_t +XXH3_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start) { - /* Check */ - if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + xxh_u64 result64 = start; + size_t i = 0; - { const BYTE* const istart = (const BYTE*) cSrc; - BYTE* const ostart = (BYTE*) dst; - BYTE* const oend = ostart + dstSize; - BYTE* const olimit = oend - 3; - const void* const dtPtr = DTable + 1; - const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr; + for (i = 0; i < 4; i++) { + result64 += XXH3_mix2Accs(acc+2*i, secret + 16*i); +#if defined(__clang__) /* Clang */ \ + && (defined(__arm__) || defined(__thumb__)) /* ARMv7 */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Prevent autovectorization on Clang ARMv7-a. Exact same problem as + * the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b. + * XXH3_64bits, len == 256, Snapdragon 835: + * without hack: 2063.7 MB/s + * with hack: 2560.7 MB/s + */ + XXH_COMPILER_GUARD(result64); +#endif + } - /* Init */ - BIT_DStream_t bitD1; - BIT_DStream_t bitD2; - BIT_DStream_t bitD3; - BIT_DStream_t bitD4; - size_t const length1 = MEM_readLE16(istart); - size_t const length2 = MEM_readLE16(istart+2); - size_t const length3 = MEM_readLE16(istart+4); - size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); - const BYTE* const istart1 = istart + 6; /* jumpTable */ - const BYTE* const istart2 = istart1 + length1; - const BYTE* const istart3 = istart2 + length2; - const BYTE* const istart4 = istart3 + length3; - const size_t segmentSize = (dstSize+3) / 4; - BYTE* const opStart2 = ostart + segmentSize; - BYTE* const opStart3 = opStart2 + segmentSize; - BYTE* const opStart4 = opStart3 + segmentSize; - BYTE* op1 = ostart; - BYTE* op2 = opStart2; - BYTE* op3 = opStart3; - BYTE* op4 = opStart4; - DTableDesc const dtd = HUF_getDTableDesc(DTable); - U32 const dtLog = dtd.tableLog; - U32 endSignal = 1; + return XXH3_avalanche(result64); +} - if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ - CHECK_F( BIT_initDStream(&bitD1, istart1, length1) ); - CHECK_F( BIT_initDStream(&bitD2, istart2, length2) ); - CHECK_F( BIT_initDStream(&bitD3, istart3, length3) ); - CHECK_F( BIT_initDStream(&bitD4, istart4, length4) ); +#define XXH3_INIT_ACC { XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \ + XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 } - /* up to 16 symbols per loop (4 symbols per stream) in 64-bit mode */ - for ( ; (endSignal) & (op4 < olimit) ; ) { - HUF_DECODE_SYMBOLX1_2(op1, &bitD1); - HUF_DECODE_SYMBOLX1_2(op2, &bitD2); - HUF_DECODE_SYMBOLX1_2(op3, &bitD3); - HUF_DECODE_SYMBOLX1_2(op4, &bitD4); - HUF_DECODE_SYMBOLX1_1(op1, &bitD1); - HUF_DECODE_SYMBOLX1_1(op2, &bitD2); - HUF_DECODE_SYMBOLX1_1(op3, &bitD3); - HUF_DECODE_SYMBOLX1_1(op4, &bitD4); - HUF_DECODE_SYMBOLX1_2(op1, &bitD1); - HUF_DECODE_SYMBOLX1_2(op2, &bitD2); - HUF_DECODE_SYMBOLX1_2(op3, &bitD3); - HUF_DECODE_SYMBOLX1_2(op4, &bitD4); - HUF_DECODE_SYMBOLX1_0(op1, &bitD1); - HUF_DECODE_SYMBOLX1_0(op2, &bitD2); - HUF_DECODE_SYMBOLX1_0(op3, &bitD3); - HUF_DECODE_SYMBOLX1_0(op4, &bitD4); - endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished; - endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished; - endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished; - endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished; - } +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_internal(const void* XXH_RESTRICT input, size_t len, + const void* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; - /* check corruption */ - /* note : should not be necessary : op# advance in lock step, and we control op4. - * but curiously, binary generated by gcc 7.2 & 7.3 with -mbmi2 runs faster when >=1 test is present */ - if (op1 > opStart2) return ERROR(corruption_detected); - if (op2 > opStart3) return ERROR(corruption_detected); - if (op3 > opStart4) return ERROR(corruption_detected); - /* note : op4 supposed already verified within main loop */ + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, f_acc, f_scramble); - /* finish bitStreams one by one */ - HUF_decodeStreamX1(op1, &bitD1, opStart2, dt, dtLog); - HUF_decodeStreamX1(op2, &bitD2, opStart3, dt, dtLog); - HUF_decodeStreamX1(op3, &bitD3, opStart4, dt, dtLog); - HUF_decodeStreamX1(op4, &bitD4, oend, dt, dtLog); + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + /* do not align on 8, so that the secret is different from the accumulator */ +#define XXH_SECRET_MERGEACCS_START 11 + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + return XXH3_mergeAccs(acc, (const xxh_u8*)secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)len * XXH_PRIME64_1); +} + +/* + * It's important for performance to transmit secret's size (when it's static) + * so that the compiler can properly optimize the vectorized loop. + * This makes a big performance difference for "medium" keys (<1 KB) when using AVX instruction set. + * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE + * breaks -Og, this is XXH_NO_INLINE. + */ +XXH3_WITH_SECRET_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_64b_internal(input, len, secret, secretLen, XXH3_accumulate, XXH3_scrambleAcc); +} - /* check */ - { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); - if (!endCheck) return ERROR(corruption_detected); } +/* + * It's preferable for performance that XXH3_hashLong is not inlined, + * as it results in a smaller function for small data, easier to the instruction cache. + * Note that inside this no_inline function, we do inline the internal loop, + * and provide a statically defined secret size to allow optimization of vector loop. + */ +XXH_NO_INLINE XXH_PUREF XXH64_hash_t +XXH3_hashLong_64b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_accumulate, XXH3_scrambleAcc); +} - /* decoded size */ - return dstSize; +/* + * XXH3_hashLong_64b_withSeed(): + * Generate a custom key based on alteration of default XXH3_kSecret with the seed, + * and then use this key for long mode hashing. + * + * This operation is decently fast but nonetheless costs a little bit of time. + * Try to avoid it whenever possible (typically when seed==0). + * + * It's important for performance that XXH3_hashLong is not inlined. Not sure + * why (uop cache maybe?), but the difference is large and easily measurable. + */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed_internal(const void* input, size_t len, + XXH64_hash_t seed, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ +#if XXH_SIZE_OPT <= 0 + if (seed == 0) + return XXH3_hashLong_64b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc, f_scramble); +#endif + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed); + return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret), + f_acc, f_scramble); } } +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_64b_withSeed_internal(input, len, seed, + XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret); +} + -typedef size_t (*HUF_decompress_usingDTable_t)(void *dst, size_t dstSize, - const void *cSrc, - size_t cSrcSize, - const HUF_DTable *DTable); +typedef XXH64_hash_t (*XXH3_hashLong64_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const xxh_u8* XXH_RESTRICT, size_t); -HUF_DGEN(HUF_decompress1X1_usingDTable_internal) -HUF_DGEN(HUF_decompress4X1_usingDTable_internal) +XXH_FORCE_INLINE XXH64_hash_t +XXH3_64bits_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong64_f f_hashLong) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secretLen` condition is not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + * Also, note that function signature doesn't offer room to return an error. + */ + if (len <= 16) + return XXH3_len_0to16_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hashLong(input, len, seed64, (const xxh_u8*)secret, secretLen); +} +/* === Public entry point === */ -size_t HUF_decompress1X1_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length) { - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (dtd.tableType != 0) return ERROR(GENERIC); - return HUF_decompress1X1_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); + return XXH3_64bits_internal(input, length, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_default); } -size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize) +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecret(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize) { - const BYTE* ip = (const BYTE*) cSrc; + return XXH3_64bits_internal(input, length, 0, secret, secretSize, XXH3_hashLong_64b_withSecret); +} - size_t const hSize = HUF_readDTableX1_wksp(DCtx, cSrc, cSrcSize, workSpace, wkspSize); - if (HUF_isError(hSize)) return hSize; - if (hSize >= cSrcSize) return ERROR(srcSize_wrong); - ip += hSize; cSrcSize -= hSize; +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed) +{ + return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed); +} - return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, /* bmi2 */ 0); +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + if (length <= XXH3_MIDSIZE_MAX) + return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_64b_withSecret(input, length, seed, (const xxh_u8*)secret, secretSize); } -size_t HUF_decompress4X1_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) +/* === XXH3 streaming === */ +#ifndef XXH_NO_STREAM +/* + * Malloc's a pointer that is always aligned to align. + * + * This must be freed with `XXH_alignedFree()`. + * + * malloc typically guarantees 16 byte alignment on 64-bit systems and 8 byte + * alignment on 32-bit. This isn't enough for the 32 byte aligned loads in AVX2 + * or on 32-bit, the 16 byte aligned loads in SSE2 and NEON. + * + * This underalignment previously caused a rather obvious crash which went + * completely unnoticed due to XXH3_createState() not actually being tested. + * Credit to RedSpah for noticing this bug. + * + * The alignment is done manually: Functions like posix_memalign or _mm_malloc + * are avoided: To maintain portability, we would have to write a fallback + * like this anyways, and besides, testing for the existence of library + * functions without relying on external build tools is impossible. + * + * The method is simple: Overallocate, manually align, and store the offset + * to the original behind the returned pointer. + * + * Align must be a power of 2 and 8 <= align <= 128. + */ +static XXH_MALLOCF void* XXH_alignedMalloc(size_t s, size_t align) +{ + XXH_ASSERT(align <= 128 && align >= 8); /* range check */ + XXH_ASSERT((align & (align-1)) == 0); /* power of 2 */ + XXH_ASSERT(s != 0 && s < (s + align)); /* empty/overflow */ + { /* Overallocate to make room for manual realignment and an offset byte */ + xxh_u8* base = (xxh_u8*)XXH_malloc(s + align); + if (base != NULL) { + /* + * Get the offset needed to align this pointer. + * + * Even if the returned pointer is aligned, there will always be + * at least one byte to store the offset to the original pointer. + */ + size_t offset = align - ((size_t)base & (align - 1)); /* base % align */ + /* Add the offset for the now-aligned pointer */ + xxh_u8* ptr = base + offset; + + XXH_ASSERT((size_t)ptr % align == 0); + + /* Store the offset immediately before the returned pointer. */ + ptr[-1] = (xxh_u8)offset; + return ptr; + } + return NULL; + } +} +/* + * Frees an aligned pointer allocated by XXH_alignedMalloc(). Don't pass + * normal malloc'd pointers, XXH_alignedMalloc has a specific data layout. + */ +static void XXH_alignedFree(void* p) +{ + if (p != NULL) { + xxh_u8* ptr = (xxh_u8*)p; + /* Get the offset byte we added in XXH_malloc. */ + xxh_u8 offset = ptr[-1]; + /* Free the original malloc'd pointer */ + xxh_u8* base = ptr - offset; + XXH_free(base); + } +} +/*! @ingroup XXH3_family */ +/*! + * @brief Allocate an @ref XXH3_state_t. + * + * @return An allocated pointer of @ref XXH3_state_t on success. + * @return `NULL` on failure. + * + * @note Must be freed with XXH3_freeState(). + */ +XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void) { - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (dtd.tableType != 0) return ERROR(GENERIC); - return HUF_decompress4X1_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); + XXH3_state_t* const state = (XXH3_state_t*)XXH_alignedMalloc(sizeof(XXH3_state_t), 64); + if (state==NULL) return NULL; + XXH3_INITSTATE(state); + return state; } -static size_t HUF_decompress4X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize, int bmi2) +/*! @ingroup XXH3_family */ +/*! + * @brief Frees an @ref XXH3_state_t. + * + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). + * + * @return @ref XXH_OK. + * + * @note Must be allocated with XXH3_createState(). + */ +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr) { - const BYTE* ip = (const BYTE*) cSrc; + XXH_alignedFree(statePtr); + return XXH_OK; +} - size_t const hSize = HUF_readDTableX1_wksp_bmi2(dctx, cSrc, cSrcSize, workSpace, wkspSize, bmi2); - if (HUF_isError(hSize)) return hSize; - if (hSize >= cSrcSize) return ERROR(srcSize_wrong); - ip += hSize; cSrcSize -= hSize; +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API void +XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state) +{ + XXH_memcpy(dst_state, src_state, sizeof(*dst_state)); +} - return HUF_decompress4X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2); +static void +XXH3_reset_internal(XXH3_state_t* statePtr, + XXH64_hash_t seed, + const void* secret, size_t secretSize) +{ + size_t const initStart = offsetof(XXH3_state_t, bufferedSize); + size_t const initLength = offsetof(XXH3_state_t, nbStripesPerBlock) - initStart; + XXH_ASSERT(offsetof(XXH3_state_t, nbStripesPerBlock) > initStart); + XXH_ASSERT(statePtr != NULL); + /* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */ + memset((char*)statePtr + initStart, 0, initLength); + statePtr->acc[0] = XXH_PRIME32_3; + statePtr->acc[1] = XXH_PRIME64_1; + statePtr->acc[2] = XXH_PRIME64_2; + statePtr->acc[3] = XXH_PRIME64_3; + statePtr->acc[4] = XXH_PRIME64_4; + statePtr->acc[5] = XXH_PRIME32_2; + statePtr->acc[6] = XXH_PRIME64_5; + statePtr->acc[7] = XXH_PRIME32_1; + statePtr->seed = seed; + statePtr->useSeed = (seed != 0); + statePtr->extSecret = (const unsigned char*)secret; + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + statePtr->secretLimit = secretSize - XXH_STRIPE_LEN; + statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; } -size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize) +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize) { - return HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, 0); + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, secret, secretSize); + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + return XXH_OK; } +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed) +{ + if (statePtr == NULL) return XXH_ERROR; + if (seed==0) return XXH3_64bits_reset(statePtr); + if ((seed != statePtr->seed) || (statePtr->extSecret != NULL)) + XXH3_initCustomSecret(statePtr->customSecret, seed); + XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} -#endif /* HUF_FORCE_DECOMPRESS_X2 */ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed64) +{ + if (statePtr == NULL) return XXH_ERROR; + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + XXH3_reset_internal(statePtr, seed64, secret, secretSize); + statePtr->useSeed = 1; /* always, even if seed64==0 */ + return XXH_OK; +} +/*! + * @internal + * @brief Processes a large input for XXH3_update() and XXH3_digest_long(). + * + * Unlike XXH3_hashLong_internal_loop(), this can process data that overlaps a block. + * + * @param acc Pointer to the 8 accumulator lanes + * @param nbStripesSoFarPtr In/out pointer to the number of leftover stripes in the block* + * @param nbStripesPerBlock Number of stripes in a block + * @param input Input pointer + * @param nbStripes Number of stripes to process + * @param secret Secret pointer + * @param secretLimit Offset of the last block in @p secret + * @param f_acc Pointer to an XXH3_accumulate implementation + * @param f_scramble Pointer to an XXH3_scrambleAcc implementation + * @return Pointer past the end of @p input after processing + */ +XXH_FORCE_INLINE const xxh_u8 * +XXH3_consumeStripes(xxh_u64* XXH_RESTRICT acc, + size_t* XXH_RESTRICT nbStripesSoFarPtr, size_t nbStripesPerBlock, + const xxh_u8* XXH_RESTRICT input, size_t nbStripes, + const xxh_u8* XXH_RESTRICT secret, size_t secretLimit, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + const xxh_u8* initialSecret = secret + *nbStripesSoFarPtr * XXH_SECRET_CONSUME_RATE; + /* Process full blocks */ + if (nbStripes >= (nbStripesPerBlock - *nbStripesSoFarPtr)) { + /* Process the initial partial block... */ + size_t nbStripesThisIter = nbStripesPerBlock - *nbStripesSoFarPtr; -#ifndef HUF_FORCE_DECOMPRESS_X1 + do { + /* Accumulate and scramble */ + f_acc(acc, input, initialSecret, nbStripesThisIter); + f_scramble(acc, secret + secretLimit); + input += nbStripesThisIter * XXH_STRIPE_LEN; + nbStripes -= nbStripesThisIter; + /* Then continue the loop with the full block size */ + nbStripesThisIter = nbStripesPerBlock; + initialSecret = secret; + } while (nbStripes >= nbStripesPerBlock); + *nbStripesSoFarPtr = 0; + } + /* Process a partial block */ + if (nbStripes > 0) { + f_acc(acc, input, initialSecret, nbStripes); + input += nbStripes * XXH_STRIPE_LEN; + *nbStripesSoFarPtr += nbStripes; + } + /* Return end pointer */ + return input; +} -/* *************************/ -/* double-symbols decoding */ -/* *************************/ +#ifndef XXH3_STREAM_USE_STACK +# if XXH_SIZE_OPT <= 0 && !defined(__clang__) /* clang doesn't need additional stack space */ +# define XXH3_STREAM_USE_STACK 1 +# endif +#endif +/* + * Both XXH3_64bits_update and XXH3_128bits_update use this routine. + */ +XXH_FORCE_INLINE XXH_errorcode +XXH3_update(XXH3_state_t* XXH_RESTRICT const state, + const xxh_u8* XXH_RESTRICT input, size_t len, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } -typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX2; /* double-symbols decoding */ -typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t; -typedef U32 rankValCol_t[HUF_TABLELOG_MAX + 1]; -typedef rankValCol_t rankVal_t[HUF_TABLELOG_MAX]; + XXH_ASSERT(state != NULL); + { const xxh_u8* const bEnd = input + len; + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; +#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* For some reason, gcc and MSVC seem to suffer greatly + * when operating accumulators directly into state. + * Operating into stack space seems to enable proper optimization. + * clang, on the other hand, doesn't seem to need this trick */ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[8]; + XXH_memcpy(acc, state->acc, sizeof(acc)); +#else + xxh_u64* XXH_RESTRICT const acc = state->acc; +#endif + state->totalLen += len; + XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE); + + /* small input : just fill in tmp buffer */ + if (len <= XXH3_INTERNALBUFFER_SIZE - state->bufferedSize) { + XXH_memcpy(state->buffer + state->bufferedSize, input, len); + state->bufferedSize += (XXH32_hash_t)len; + return XXH_OK; + } + /* total input is now > XXH3_INTERNALBUFFER_SIZE */ + #define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN) + XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0); /* clean multiple */ -/* HUF_fillDTableX2Level2() : - * `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */ -static void HUF_fillDTableX2Level2(HUF_DEltX2* DTable, U32 sizeLog, const U32 consumed, - const U32* rankValOrigin, const int minWeight, - const sortedSymbol_t* sortedSymbols, const U32 sortedListSize, - U32 nbBitsBaseline, U16 baseSeq) -{ - HUF_DEltX2 DElt; - U32 rankVal[HUF_TABLELOG_MAX + 1]; + /* + * Internal buffer is partially filled (always, except at beginning) + * Complete it, then consume it. + */ + if (state->bufferedSize) { + size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize; + XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize); + input += loadSize; + XXH3_consumeStripes(acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, XXH3_INTERNALBUFFER_STRIPES, + secret, state->secretLimit, + f_acc, f_scramble); + state->bufferedSize = 0; + } + XXH_ASSERT(input < bEnd); + if (bEnd - input > XXH3_INTERNALBUFFER_SIZE) { + size_t nbStripes = (size_t)(bEnd - 1 - input) / XXH_STRIPE_LEN; + input = XXH3_consumeStripes(acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + input, nbStripes, + secret, state->secretLimit, + f_acc, f_scramble); + XXH_memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN); - /* get pre-calculated rankVal */ - ZSTD_memcpy(rankVal, rankValOrigin, sizeof(rankVal)); + } + /* Some remaining input (always) : buffer it */ + XXH_ASSERT(input < bEnd); + XXH_ASSERT(bEnd - input <= XXH3_INTERNALBUFFER_SIZE); + XXH_ASSERT(state->bufferedSize == 0); + XXH_memcpy(state->buffer, input, (size_t)(bEnd-input)); + state->bufferedSize = (XXH32_hash_t)(bEnd-input); +#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* save stack accumulators into state */ + XXH_memcpy(state->acc, acc, sizeof(acc)); +#endif + } - /* fill skipped values */ - if (minWeight>1) { - U32 i, skipSize = rankVal[minWeight]; - MEM_writeLE16(&(DElt.sequence), baseSeq); - DElt.nbBits = (BYTE)(consumed); - DElt.length = 1; - for (i = 0; i < skipSize; i++) - DTable[i] = DElt; - } - - /* fill DTable */ - { U32 s; for (s=0; s= 1 */ - - rankVal[weight] += length; - } } + return XXH_OK; } +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len) +{ + return XXH3_update(state, (const xxh_u8*)input, len, + XXH3_accumulate, XXH3_scrambleAcc); +} -static void HUF_fillDTableX2(HUF_DEltX2* DTable, const U32 targetLog, - const sortedSymbol_t* sortedList, const U32 sortedListSize, - const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight, - const U32 nbBitsBaseline) + +XXH_FORCE_INLINE void +XXH3_digest_long (XXH64_hash_t* acc, + const XXH3_state_t* state, + const unsigned char* secret) { - U32 rankVal[HUF_TABLELOG_MAX + 1]; - const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */ - const U32 minBits = nbBitsBaseline - maxWeight; - U32 s; + xxh_u8 lastStripe[XXH_STRIPE_LEN]; + const xxh_u8* lastStripePtr; - ZSTD_memcpy(rankVal, rankValOrigin, sizeof(rankVal)); + /* + * Digest on a local copy. This way, the state remains unaltered, and it can + * continue ingesting more input afterwards. + */ + XXH_memcpy(acc, state->acc, sizeof(state->acc)); + if (state->bufferedSize >= XXH_STRIPE_LEN) { + /* Consume remaining stripes then point to remaining data in buffer */ + size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN; + size_t nbStripesSoFar = state->nbStripesSoFar; + XXH3_consumeStripes(acc, + &nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, nbStripes, + secret, state->secretLimit, + XXH3_accumulate, XXH3_scrambleAcc); + lastStripePtr = state->buffer + state->bufferedSize - XXH_STRIPE_LEN; + } else { /* bufferedSize < XXH_STRIPE_LEN */ + /* Copy to temp buffer */ + size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize; + XXH_ASSERT(state->bufferedSize > 0); /* there is always some input buffered */ + XXH_memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, catchupSize); + XXH_memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize); + lastStripePtr = lastStripe; + } + /* Last stripe */ + XXH3_accumulate_512(acc, + lastStripePtr, + secret + state->secretLimit - XXH_SECRET_LASTACC_START); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + return XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + } + /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */ + if (state->useSeed) + return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} +#endif /* !XXH_NO_STREAM */ - /* fill DTable */ - for (s=0; s= minBits) { /* enough room for a second symbol */ - U32 sortedRank; - int minWeight = nbBits + scaleLog; - if (minWeight < 1) minWeight = 1; - sortedRank = rankStart[minWeight]; - HUF_fillDTableX2Level2(DTable+start, targetLog-nbBits, nbBits, - rankValOrigin[nbBits], minWeight, - sortedList+sortedRank, sortedListSize-sortedRank, - nbBitsBaseline, symbol); - } else { - HUF_DEltX2 DElt; - MEM_writeLE16(&(DElt.sequence), symbol); - DElt.nbBits = (BYTE)(nbBits); - DElt.length = 1; - { U32 const end = start + length; - U32 u; - for (u = start; u < end; u++) DTable[u] = DElt; - } } - rankVal[weight] += length; +/* ========================================== + * XXH3 128 bits (a.k.a XXH128) + * ========================================== + * XXH3's 128-bit variant has better mixing and strength than the 64-bit variant, + * even without counting the significantly larger output size. + * + * For example, extra steps are taken to avoid the seed-dependent collisions + * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B). + * + * This strength naturally comes at the cost of some speed, especially on short + * lengths. Note that longer hashes are about as fast as the 64-bit version + * due to it using only a slight modification of the 64-bit loop. + * + * XXH128 is also more oriented towards 64-bit machines. It is still extremely + * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64). + */ + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_1to3_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + /* A doubled version of 1to3_64b with different constants. */ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combinedl = { input[0], 0x01, input[0], input[0] } + * len = 2: combinedl = { input[1], 0x02, input[0], input[1] } + * len = 3: combinedl = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combinedl = ((xxh_u32)c1 <<16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13); + xxh_u64 const bitflipl = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const bitfliph = (XXH_readLE32(secret+8) ^ XXH_readLE32(secret+12)) - seed; + xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl; + xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph; + XXH128_hash_t h128; + h128.low64 = XXH64_avalanche(keyed_lo); + h128.high64 = XXH64_avalanche(keyed_hi); + return h128; } } -size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, - const void* src, size_t srcSize, - void* workSpace, size_t wkspSize) +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_4to8_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) { - U32 tableLog, maxW, sizeOfSort, nbSymbols; - DTableDesc dtd = HUF_getDTableDesc(DTable); - U32 const maxTableLog = dtd.maxTableLog; - size_t iSize; - void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */ - HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr; - U32 *rankStart; + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input_lo = XXH_readLE32(input); + xxh_u32 const input_hi = XXH_readLE32(input + len - 4); + xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32); + xxh_u64 const bitflip = (XXH_readLE64(secret+16) ^ XXH_readLE64(secret+24)) + seed; + xxh_u64 const keyed = input_64 ^ bitflip; - rankValCol_t* rankVal; - U32* rankStats; - U32* rankStart0; - sortedSymbol_t* sortedSymbol; - BYTE* weightList; - size_t spaceUsed32 = 0; - - rankVal = (rankValCol_t *)((U32 *)workSpace + spaceUsed32); - spaceUsed32 += (sizeof(rankValCol_t) * HUF_TABLELOG_MAX) >> 2; - rankStats = (U32 *)workSpace + spaceUsed32; - spaceUsed32 += HUF_TABLELOG_MAX + 1; - rankStart0 = (U32 *)workSpace + spaceUsed32; - spaceUsed32 += HUF_TABLELOG_MAX + 2; - sortedSymbol = (sortedSymbol_t *)workSpace + (spaceUsed32 * sizeof(U32)) / sizeof(sortedSymbol_t); - spaceUsed32 += HUF_ALIGN(sizeof(sortedSymbol_t) * (HUF_SYMBOLVALUE_MAX + 1), sizeof(U32)) >> 2; - weightList = (BYTE *)((U32 *)workSpace + spaceUsed32); - spaceUsed32 += HUF_ALIGN(HUF_SYMBOLVALUE_MAX + 1, sizeof(U32)) >> 2; - - if ((spaceUsed32 << 2) > wkspSize) return ERROR(tableLog_tooLarge); - - rankStart = rankStart0 + 1; - ZSTD_memset(rankStats, 0, sizeof(U32) * (2 * HUF_TABLELOG_MAX + 2 + 1)); + /* Shift len to the left to ensure it is even, this avoids even multiplies. */ + XXH128_hash_t m128 = XXH_mult64to128(keyed, XXH_PRIME64_1 + (len << 2)); - DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(HUF_DTable)); /* if compiler fails here, assertion is wrong */ - if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); - /* ZSTD_memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */ + m128.high64 += (m128.low64 << 1); + m128.low64 ^= (m128.high64 >> 3); - iSize = HUF_readStats(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize); - if (HUF_isError(iSize)) return iSize; + m128.low64 = XXH_xorshift64(m128.low64, 35); + m128.low64 *= PRIME_MX2; + m128.low64 = XXH_xorshift64(m128.low64, 28); + m128.high64 = XXH3_avalanche(m128.high64); + return m128; + } +} - /* check result */ - if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_9to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflipl = (XXH_readLE64(secret+32) ^ XXH_readLE64(secret+40)) - seed; + xxh_u64 const bitfliph = (XXH_readLE64(secret+48) ^ XXH_readLE64(secret+56)) + seed; + xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 input_hi = XXH_readLE64(input + len - 8); + XXH128_hash_t m128 = XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1); + /* + * Put len in the middle of m128 to ensure that the length gets mixed to + * both the low and high bits in the 128x64 multiply below. + */ + m128.low64 += (xxh_u64)(len - 1) << 54; + input_hi ^= bitfliph; + /* + * Add the high 32 bits of input_hi to the high 32 bits of m128, then + * add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to + * the high 64 bits of m128. + * + * The best approach to this operation is different on 32-bit and 64-bit. + */ + if (sizeof(void *) < sizeof(xxh_u64)) { /* 32-bit */ + /* + * 32-bit optimized version, which is more readable. + * + * On 32-bit, it removes an ADC and delays a dependency between the two + * halves of m128.high64, but it generates an extra mask on 64-bit. + */ + m128.high64 += (input_hi & 0xFFFFFFFF00000000ULL) + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2); + } else { + /* + * 64-bit optimized (albeit more confusing) version. + * + * Uses some properties of addition and multiplication to remove the mask: + * + * Let: + * a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF) + * b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000) + * c = XXH_PRIME32_2 + * + * a + (b * c) + * Inverse Property: x + y - x == y + * a + (b * (1 + c - 1)) + * Distributive Property: x * (y + z) == (x * y) + (x * z) + * a + (b * 1) + (b * (c - 1)) + * Identity Property: x * 1 == x + * a + b + (b * (c - 1)) + * + * Substitute a, b, and c: + * input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + * + * Since input_hi.hi + input_hi.lo == input_hi, we get this: + * input_hi + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + */ + m128.high64 += input_hi + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2 - 1); + } + /* m128 ^= XXH_swap64(m128 >> 64); */ + m128.low64 ^= XXH_swap64(m128.high64); - /* find maxWeight */ - for (maxW = tableLog; rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */ + { /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */ + XXH128_hash_t h128 = XXH_mult64to128(m128.low64, XXH_PRIME64_2); + h128.high64 += m128.high64 * XXH_PRIME64_2; - /* Get start index of each weight */ - { U32 w, nextRankStart = 0; - for (w=1; w= XXH3_SECRET_SIZE_MIN + */ +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_0to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed); + if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed); + if (len) return XXH3_len_1to3_128b(input, len, secret, seed); + { XXH128_hash_t h128; + xxh_u64 const bitflipl = XXH_readLE64(secret+64) ^ XXH_readLE64(secret+72); + xxh_u64 const bitfliph = XXH_readLE64(secret+80) ^ XXH_readLE64(secret+88); + h128.low64 = XXH64_avalanche(seed ^ bitflipl); + h128.high64 = XXH64_avalanche( seed ^ bitfliph); + return h128; + } } +} + +/* + * A bit slower than XXH3_mix16B, but handles multiply by zero better. + */ +XXH_FORCE_INLINE XXH128_hash_t +XXH128_mix32B(XXH128_hash_t acc, const xxh_u8* input_1, const xxh_u8* input_2, + const xxh_u8* secret, XXH64_hash_t seed) +{ + acc.low64 += XXH3_mix16B (input_1, secret+0, seed); + acc.low64 ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8); + acc.high64 += XXH3_mix16B (input_2, secret+16, seed); + acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8); + return acc; +} + + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_17to128_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { XXH128_hash_t acc; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + +#if XXH_SIZE_OPT >= 1 + { + /* Smaller, but slightly slower. */ + unsigned int i = (unsigned int)(len - 1) / 32; + do { + acc = XXH128_mix32B(acc, input+16*i, input+len-16*(i+1), secret+32*i, seed); + } while (i-- != 0); + } +#else + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc = XXH128_mix32B(acc, input+48, input+len-64, secret+96, seed); + } + acc = XXH128_mix32B(acc, input+32, input+len-48, secret+64, seed); + } + acc = XXH128_mix32B(acc, input+16, input+len-32, secret+32, seed); + } + acc = XXH128_mix32B(acc, input, input+len-16, secret, seed); +#endif + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; } - rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/ - sizeOfSort = nextRankStart; } +} - /* sort symbols by weight */ - { U32 s; - for (s=0; s= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + { XXH128_hash_t acc; + unsigned i; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + /* + * We set as `i` as offset + 32. We do this so that unchanged + * `len` can be used as upper bound. This reaches a sweet spot + * where both x86 and aarch64 get simple agen and good codegen + * for the loop. + */ + for (i = 32; i < 160; i += 32) { + acc = XXH128_mix32B(acc, + input + i - 32, + input + i - 16, + secret + i - 32, + seed); + } + acc.low64 = XXH3_avalanche(acc.low64); + acc.high64 = XXH3_avalanche(acc.high64); + /* + * NB: `i <= len` will duplicate the last 32-bytes if + * len % 32 was zero. This is an unfortunate necessity to keep + * the hash result stable. + */ + for (i=160; i <= len; i += 32) { + acc = XXH128_mix32B(acc, + input + i - 32, + input + i - 16, + secret + XXH3_MIDSIZE_STARTOFFSET + i - 160, + seed); + } + /* last bytes */ + acc = XXH128_mix32B(acc, + input + len - 16, + input + len - 32, + secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16, + (XXH64_hash_t)0 - seed); + + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; } - rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */ } +} - /* Build rankVal */ - { U32* const rankVal0 = rankVal[0]; - { int const rescale = (maxTableLog-tableLog) - 1; /* tableLog <= maxTableLog */ - U32 nextRankVal = 0; - U32 w; - for (w=1; w> consumed; - } } } } +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_internal(const void* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, secret, secretSize, f_acc, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)len * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs(acc, + secret + secretSize + - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)len * XXH_PRIME64_2)); + return h128; + } +} - HUF_fillDTableX2(dt, maxTableLog, - sortedSymbol, sizeOfSort, - rankStart0, rankVal, maxW, - tableLog+1); +/* + * It's important for performance that XXH3_hashLong() is not inlined. + */ +XXH_NO_INLINE XXH_PUREF XXH128_hash_t +XXH3_hashLong_128b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_accumulate, XXH3_scrambleAcc); +} + +/* + * It's important for performance to pass @p secretLen (when it's static) + * to the compiler, so that it can properly optimize the vectorized loop. + * + * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE + * breaks -Og, this is XXH_NO_INLINE. + */ +XXH3_WITH_SECRET_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, secretLen, + XXH3_accumulate, XXH3_scrambleAcc); +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ + if (seed64 == 0) + return XXH3_hashLong_128b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc, f_scramble); + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed64); + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, sizeof(secret), + f_acc, f_scramble); + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_128b_withSeed_internal(input, len, seed64, + XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret); +} + +typedef XXH128_hash_t (*XXH3_hashLong128_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const void* XXH_RESTRICT, size_t); + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_128bits_internal(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong128_f f_hl128) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secret` conditions are not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + */ + if (len <= 16) + return XXH3_len_0to16_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hl128(input, len, seed64, secret, secretLen); +} + + +/* === Public XXH128 API === */ + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* input, size_t len) +{ + return XXH3_128bits_internal(input, len, 0, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_default); +} - dtd.tableLog = (BYTE)maxTableLog; - dtd.tableType = 1; - ZSTD_memcpy(DTable, &dtd, sizeof(dtd)); - return iSize; +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecret(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + return XXH3_128bits_internal(input, len, 0, + (const xxh_u8*)secret, secretSize, + XXH3_hashLong_128b_withSecret); } +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSeed(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_internal(input, len, seed, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_withSeed); +} -FORCE_INLINE_TEMPLATE U32 -HUF_decodeSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog) +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) { - size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ - ZSTD_memcpy(op, dt+val, 2); - BIT_skipBits(DStream, dt[val].nbBits); - return dt[val].length; + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_128b_withSecret(input, len, seed, secret, secretSize); } -FORCE_INLINE_TEMPLATE U32 -HUF_decodeLastSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog) +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) { - size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ - ZSTD_memcpy(op, dt+val, 1); - if (dt[val].length==1) BIT_skipBits(DStream, dt[val].nbBits); - else { - if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) { - BIT_skipBits(DStream, dt[val].nbBits); - if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8)) - /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ - DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); - } } - return 1; + return XXH3_128bits_withSeed(input, len, seed); } -#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \ - ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog) -#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ - if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ - ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog) +/* === XXH3 128-bit streaming === */ +#ifndef XXH_NO_STREAM +/* + * All initialization and update functions are identical to 64-bit streaming variant. + * The only difference is the finalization routine. + */ -#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ - if (MEM_64bits()) \ - ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog) +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr) +{ + return XXH3_64bits_reset(statePtr); +} -HINT_INLINE size_t -HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, - const HUF_DEltX2* const dt, const U32 dtLog) +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize) { - BYTE* const pStart = p; + return XXH3_64bits_reset_withSecret(statePtr, secret, secretSize); +} - /* up to 8 symbols at a time */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-(sizeof(bitDPtr->bitContainer)-1))) { - HUF_DECODE_SYMBOLX2_2(p, bitDPtr); - HUF_DECODE_SYMBOLX2_1(p, bitDPtr); - HUF_DECODE_SYMBOLX2_2(p, bitDPtr); - HUF_DECODE_SYMBOLX2_0(p, bitDPtr); +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed) +{ + return XXH3_64bits_reset_withSeed(statePtr, seed); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + return XXH3_64bits_reset_withSecretandSeed(statePtr, secret, secretSize, seed); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len) +{ + return XXH3_64bits_update(state, input, len); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs(acc, + secret + state->secretLimit + XXH_STRIPE_LEN + - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)state->totalLen * XXH_PRIME64_2)); + return h128; + } } + /* len <= XXH3_MIDSIZE_MAX : short code */ + if (state->seed) + return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} +#endif /* !XXH_NO_STREAM */ +/* 128-bit utility functions */ - /* closer to end : up to 2 symbols at a time */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd-2)) - HUF_DECODE_SYMBOLX2_0(p, bitDPtr); +/* return : 1 is equal, 0 if different */ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2) +{ + /* note : XXH128_hash_t is compact, it has no padding byte */ + return !(memcmp(&h1, &h2, sizeof(h1))); +} - while (p <= pEnd-2) - HUF_DECODE_SYMBOLX2_0(p, bitDPtr); /* no need to reload : reached the end of DStream */ +/* This prototype is compatible with stdlib's qsort(). + * @return : >0 if *h128_1 > *h128_2 + * <0 if *h128_1 < *h128_2 + * =0 if *h128_1 == *h128_2 */ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2) +{ + XXH128_hash_t const h1 = *(const XXH128_hash_t*)h128_1; + XXH128_hash_t const h2 = *(const XXH128_hash_t*)h128_2; + int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64); + /* note : bets that, in most cases, hash values are different */ + if (hcmp) return hcmp; + return (h1.low64 > h2.low64) - (h2.low64 > h1.low64); +} - if (p < pEnd) - p += HUF_decodeLastSymbolX2(p, bitDPtr, dt, dtLog); - return p-pStart; +/*====== Canonical representation ======*/ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API void +XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) { + hash.high64 = XXH_swap64(hash.high64); + hash.low64 = XXH_swap64(hash.low64); + } + XXH_memcpy(dst, &hash.high64, sizeof(hash.high64)); + XXH_memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64)); } -FORCE_INLINE_TEMPLATE size_t -HUF_decompress1X2_usingDTable_internal_body( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src) { - BIT_DStream_t bitD; + XXH128_hash_t h; + h.high64 = XXH_readBE64(src); + h.low64 = XXH_readBE64(src->digest + 8); + return h; +} - /* Init */ - CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) ); - /* decode */ - { BYTE* const ostart = (BYTE*) dst; - BYTE* const oend = ostart + dstSize; - const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */ - const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; - DTableDesc const dtd = HUF_getDTableDesc(DTable); - HUF_decodeStreamX2(ostart, &bitD, oend, dt, dtd.tableLog); + +/* ========================================== + * Secret generators + * ========================================== + */ +#define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x)) + +XXH_FORCE_INLINE void XXH3_combine16(void* dst, XXH128_hash_t h128) +{ + XXH_writeLE64( dst, XXH_readLE64(dst) ^ h128.low64 ); + XXH_writeLE64( (char*)dst+8, XXH_readLE64((char*)dst+8) ^ h128.high64 ); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize) +{ +#if (XXH_DEBUGLEVEL >= 1) + XXH_ASSERT(secretBuffer != NULL); + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); +#else + /* production mode, assert() are disabled */ + if (secretBuffer == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; +#endif + + if (customSeedSize == 0) { + customSeed = XXH3_kSecret; + customSeedSize = XXH_SECRET_DEFAULT_SIZE; } +#if (XXH_DEBUGLEVEL >= 1) + XXH_ASSERT(customSeed != NULL); +#else + if (customSeed == NULL) return XXH_ERROR; +#endif - /* check */ - if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); + /* Fill secretBuffer with a copy of customSeed - repeat as needed */ + { size_t pos = 0; + while (pos < secretSize) { + size_t const toCopy = XXH_MIN((secretSize - pos), customSeedSize); + memcpy((char*)secretBuffer + pos, customSeed, toCopy); + pos += toCopy; + } } - /* decoded size */ - return dstSize; + { size_t const nbSeg16 = secretSize / 16; + size_t n; + XXH128_canonical_t scrambler; + XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0)); + for (n=0; n cSrcSize) return ERROR(corruption_detected); /* overflow */ - CHECK_F( BIT_initDStream(&bitD1, istart1, length1) ); - CHECK_F( BIT_initDStream(&bitD2, istart2, length2) ); - CHECK_F( BIT_initDStream(&bitD3, istart3, length3) ); - CHECK_F( BIT_initDStream(&bitD4, istart4, length4) ); +/* Pop our optimization override from above */ +#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ + && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__OPTIMIZE__) && XXH_SIZE_OPT <= 0 /* respect -O0 and -Os */ +# pragma GCC pop_options +#endif - /* 16-32 symbols per loop (4-8 symbols per stream) */ - for ( ; (endSignal) & (op4 < olimit); ) { -#if defined(__clang__) && (defined(__x86_64__) || defined(__i386__)) - HUF_DECODE_SYMBOLX2_2(op1, &bitD1); - HUF_DECODE_SYMBOLX2_1(op1, &bitD1); - HUF_DECODE_SYMBOLX2_2(op1, &bitD1); - HUF_DECODE_SYMBOLX2_0(op1, &bitD1); - HUF_DECODE_SYMBOLX2_2(op2, &bitD2); - HUF_DECODE_SYMBOLX2_1(op2, &bitD2); - HUF_DECODE_SYMBOLX2_2(op2, &bitD2); - HUF_DECODE_SYMBOLX2_0(op2, &bitD2); - endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished; - endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished; - HUF_DECODE_SYMBOLX2_2(op3, &bitD3); - HUF_DECODE_SYMBOLX2_1(op3, &bitD3); - HUF_DECODE_SYMBOLX2_2(op3, &bitD3); - HUF_DECODE_SYMBOLX2_0(op3, &bitD3); - HUF_DECODE_SYMBOLX2_2(op4, &bitD4); - HUF_DECODE_SYMBOLX2_1(op4, &bitD4); - HUF_DECODE_SYMBOLX2_2(op4, &bitD4); - HUF_DECODE_SYMBOLX2_0(op4, &bitD4); - endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished; - endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished; + +#if defined (__cplusplus) +} /* extern "C" */ +#endif + +#endif /* XXH_NO_LONG_LONG */ +#endif /* XXH_NO_XXH3 */ + +/*! + * @} + */ +#endif /* XXH_IMPLEMENTATION */ +/**** ended inlining xxhash.h ****/ +#ifndef ZSTD_NO_TRACE +/**** start inlining zstd_trace.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_TRACE_H +#define ZSTD_TRACE_H + +#include + +/* weak symbol support + * For now, enable conservatively: + * - Only GNUC + * - Only ELF + * - Only x86-64, i386, aarch64 and risc-v. + * Also, explicitly disable on platforms known not to work so they aren't + * forgotten in the future. + */ +#if !defined(ZSTD_HAVE_WEAK_SYMBOLS) && \ + defined(__GNUC__) && defined(__ELF__) && \ + (defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || \ + defined(_M_IX86) || defined(__aarch64__) || defined(__riscv)) && \ + !defined(__APPLE__) && !defined(_WIN32) && !defined(__MINGW32__) && \ + !defined(__CYGWIN__) && !defined(_AIX) +# define ZSTD_HAVE_WEAK_SYMBOLS 1 #else - HUF_DECODE_SYMBOLX2_2(op1, &bitD1); - HUF_DECODE_SYMBOLX2_2(op2, &bitD2); - HUF_DECODE_SYMBOLX2_2(op3, &bitD3); - HUF_DECODE_SYMBOLX2_2(op4, &bitD4); - HUF_DECODE_SYMBOLX2_1(op1, &bitD1); - HUF_DECODE_SYMBOLX2_1(op2, &bitD2); - HUF_DECODE_SYMBOLX2_1(op3, &bitD3); - HUF_DECODE_SYMBOLX2_1(op4, &bitD4); - HUF_DECODE_SYMBOLX2_2(op1, &bitD1); - HUF_DECODE_SYMBOLX2_2(op2, &bitD2); - HUF_DECODE_SYMBOLX2_2(op3, &bitD3); - HUF_DECODE_SYMBOLX2_2(op4, &bitD4); - HUF_DECODE_SYMBOLX2_0(op1, &bitD1); - HUF_DECODE_SYMBOLX2_0(op2, &bitD2); - HUF_DECODE_SYMBOLX2_0(op3, &bitD3); - HUF_DECODE_SYMBOLX2_0(op4, &bitD4); - endSignal = (U32)LIKELY( - (BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished) - & (BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished) - & (BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished) - & (BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished)); +# define ZSTD_HAVE_WEAK_SYMBOLS 0 +#endif +#if ZSTD_HAVE_WEAK_SYMBOLS +# define ZSTD_WEAK_ATTR __attribute__((__weak__)) +#else +# define ZSTD_WEAK_ATTR #endif - } - /* check corruption */ - if (op1 > opStart2) return ERROR(corruption_detected); - if (op2 > opStart3) return ERROR(corruption_detected); - if (op3 > opStart4) return ERROR(corruption_detected); - /* note : op4 already verified within main loop */ +/* Only enable tracing when weak symbols are available. */ +#ifndef ZSTD_TRACE +# define ZSTD_TRACE ZSTD_HAVE_WEAK_SYMBOLS +#endif - /* finish bitStreams one by one */ - HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog); - HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog); - HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog); - HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog); +#if ZSTD_TRACE - /* check */ - { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); - if (!endCheck) return ERROR(corruption_detected); } +struct ZSTD_CCtx_s; +struct ZSTD_DCtx_s; +struct ZSTD_CCtx_params_s; + +typedef struct { + /** + * ZSTD_VERSION_NUMBER + * + * This is guaranteed to be the first member of ZSTD_trace. + * Otherwise, this struct is not stable between versions. If + * the version number does not match your expectation, you + * should not interpret the rest of the struct. + */ + unsigned version; + /** + * Non-zero if streaming (de)compression is used. + */ + int streaming; + /** + * The dictionary ID. + */ + unsigned dictionaryID; + /** + * Is the dictionary cold? + * Only set on decompression. + */ + int dictionaryIsCold; + /** + * The dictionary size or zero if no dictionary. + */ + size_t dictionarySize; + /** + * The uncompressed size of the data. + */ + size_t uncompressedSize; + /** + * The compressed size of the data. + */ + size_t compressedSize; + /** + * The fully resolved CCtx parameters (NULL on decompression). + */ + struct ZSTD_CCtx_params_s const* params; + /** + * The ZSTD_CCtx pointer (NULL on decompression). + */ + struct ZSTD_CCtx_s const* cctx; + /** + * The ZSTD_DCtx pointer (NULL on compression). + */ + struct ZSTD_DCtx_s const* dctx; +} ZSTD_Trace; + +/** + * A tracing context. It must be 0 when tracing is disabled. + * Otherwise, any non-zero value returned by a tracing begin() + * function is presented to any subsequent calls to end(). + * + * Any non-zero value is treated as tracing is enabled and not + * interpreted by the library. + * + * Two possible uses are: + * * A timestamp for when the begin() function was called. + * * A unique key identifying the (de)compression, like the + * address of the [dc]ctx pointer if you need to track + * more information than just a timestamp. + */ +typedef unsigned long long ZSTD_TraceCtx; + +/** + * Trace the beginning of a compression call. + * @param cctx The dctx pointer for the compression. + * It can be used as a key to map begin() to end(). + * @returns Non-zero if tracing is enabled. The return value is + * passed to ZSTD_trace_compress_end(). + */ +ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_compress_begin( + struct ZSTD_CCtx_s const* cctx); - /* decoded size */ - return dstSize; - } -} +/** + * Trace the end of a compression call. + * @param ctx The return value of ZSTD_trace_compress_begin(). + * @param trace The zstd tracing info. + */ +ZSTD_WEAK_ATTR void ZSTD_trace_compress_end( + ZSTD_TraceCtx ctx, + ZSTD_Trace const* trace); -HUF_DGEN(HUF_decompress1X2_usingDTable_internal) -HUF_DGEN(HUF_decompress4X2_usingDTable_internal) +/** + * Trace the beginning of a decompression call. + * @param dctx The dctx pointer for the decompression. + * It can be used as a key to map begin() to end(). + * @returns Non-zero if tracing is enabled. The return value is + * passed to ZSTD_trace_compress_end(). + */ +ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_decompress_begin( + struct ZSTD_DCtx_s const* dctx); -size_t HUF_decompress1X2_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (dtd.tableType != 1) return ERROR(GENERIC); - return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -} +/** + * Trace the end of a decompression call. + * @param ctx The return value of ZSTD_trace_decompress_begin(). + * @param trace The zstd tracing info. + */ +ZSTD_WEAK_ATTR void ZSTD_trace_decompress_end( + ZSTD_TraceCtx ctx, + ZSTD_Trace const* trace); -size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize) -{ - const BYTE* ip = (const BYTE*) cSrc; +#endif /* ZSTD_TRACE */ - size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize, - workSpace, wkspSize); - if (HUF_isError(hSize)) return hSize; - if (hSize >= cSrcSize) return ERROR(srcSize_wrong); - ip += hSize; cSrcSize -= hSize; +#endif /* ZSTD_TRACE_H */ +/**** ended inlining zstd_trace.h ****/ +#else +# define ZSTD_TRACE 0 +#endif - return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, /* bmi2 */ 0); -} +/* ---- static assert (debug) --- */ +#define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) +#define ZSTD_isError ERR_isError /* for inlining */ +#define FSE_isError ERR_isError +#define HUF_isError ERR_isError -size_t HUF_decompress4X2_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (dtd.tableType != 1) return ERROR(GENERIC); - return HUF_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -} +/*-************************************* +* shared macros +***************************************/ +#undef MIN +#undef MAX +#define MIN(a,b) ((a)<(b) ? (a) : (b)) +#define MAX(a,b) ((a)>(b) ? (a) : (b)) +#define BOUNDED(min,val,max) (MAX(min,MIN(val,max))) -static size_t HUF_decompress4X2_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize, int bmi2) -{ - const BYTE* ip = (const BYTE*) cSrc; - size_t hSize = HUF_readDTableX2_wksp(dctx, cSrc, cSrcSize, - workSpace, wkspSize); - if (HUF_isError(hSize)) return hSize; - if (hSize >= cSrcSize) return ERROR(srcSize_wrong); - ip += hSize; cSrcSize -= hSize; +/*-************************************* +* Common constants +***************************************/ +#define ZSTD_OPT_NUM (1<<12) - return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2); -} +#define ZSTD_REP_NUM 3 /* number of repcodes */ +static UNUSED_ATTR const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 }; -size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize) -{ - return HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, /* bmi2 */ 0); -} +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) +#define BIT7 128 +#define BIT6 64 +#define BIT5 32 +#define BIT4 16 +#define BIT1 2 +#define BIT0 1 -#endif /* HUF_FORCE_DECOMPRESS_X1 */ +#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10 +static UNUSED_ATTR const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 }; +static UNUSED_ATTR const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 }; +#define ZSTD_FRAMEIDSIZE 4 /* magic number size */ -/* ***********************************/ -/* Universal decompression selectors */ -/* ***********************************/ +#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ +static UNUSED_ATTR const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; +typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e; -size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc const dtd = HUF_getDTableDesc(DTable); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)dtd; - assert(dtd.tableType == 0); - return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)dtd; - assert(dtd.tableType == 1); - return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -#else - return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) : - HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -#endif -} +#define ZSTD_FRAMECHECKSUMSIZE 4 -size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc const dtd = HUF_getDTableDesc(DTable); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)dtd; - assert(dtd.tableType == 0); - return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)dtd; - assert(dtd.tableType == 1); - return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -#else - return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) : - HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -#endif -} +#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */ +#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */) /* for a non-null block */ +#define MIN_LITERALS_FOR_4_STREAMS 6 +typedef enum { set_basic, set_rle, set_compressed, set_repeat } SymbolEncodingType_e; -#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2) -typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t; -static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] = -{ - /* single, double, quad */ - {{0,0}, {1,1}, {2,2}}, /* Q==0 : impossible */ - {{0,0}, {1,1}, {2,2}}, /* Q==1 : impossible */ - {{ 38,130}, {1313, 74}, {2151, 38}}, /* Q == 2 : 12-18% */ - {{ 448,128}, {1353, 74}, {2238, 41}}, /* Q == 3 : 18-25% */ - {{ 556,128}, {1353, 74}, {2238, 47}}, /* Q == 4 : 25-32% */ - {{ 714,128}, {1418, 74}, {2436, 53}}, /* Q == 5 : 32-38% */ - {{ 883,128}, {1437, 74}, {2464, 61}}, /* Q == 6 : 38-44% */ - {{ 897,128}, {1515, 75}, {2622, 68}}, /* Q == 7 : 44-50% */ - {{ 926,128}, {1613, 75}, {2730, 75}}, /* Q == 8 : 50-56% */ - {{ 947,128}, {1729, 77}, {3359, 77}}, /* Q == 9 : 56-62% */ - {{1107,128}, {2083, 81}, {4006, 84}}, /* Q ==10 : 62-69% */ - {{1177,128}, {2379, 87}, {4785, 88}}, /* Q ==11 : 69-75% */ - {{1242,128}, {2415, 93}, {5155, 84}}, /* Q ==12 : 75-81% */ - {{1349,128}, {2644,106}, {5260,106}}, /* Q ==13 : 81-87% */ - {{1455,128}, {2422,124}, {4174,124}}, /* Q ==14 : 87-93% */ - {{ 722,128}, {1891,145}, {1936,146}}, /* Q ==15 : 93-99% */ -}; -#endif +#define LONGNBSEQ 0x7F00 -/** HUF_selectDecoder() : - * Tells which decoder is likely to decode faster, - * based on a set of pre-computed metrics. - * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 . - * Assumption : 0 < dstSize <= 128 KB */ -U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize) -{ - assert(dstSize > 0); - assert(dstSize <= 128*1024); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)dstSize; - (void)cSrcSize; - return 0; -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)dstSize; - (void)cSrcSize; - return 1; -#else - /* decoder timing evaluation */ - { U32 const Q = (cSrcSize >= dstSize) ? 15 : (U32)(cSrcSize * 16 / dstSize); /* Q < 16 */ - U32 const D256 = (U32)(dstSize >> 8); - U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256); - U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256); - DTime1 += DTime1 >> 3; /* advantage to algorithm using less memory, to reduce cache eviction */ - return DTime1 < DTime0; - } -#endif -} +#define MINMATCH 3 +#define Litbits 8 +#define LitHufLog 11 +#define MaxLit ((1< dstSize) return ERROR(corruption_detected); /* invalid */ - if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ - if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ +static UNUSED_ATTR const U8 ML_bits[MaxML+1] = { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 2, 2, 3, 3, + 4, 4, 5, 7, 8, 9,10,11, + 12,13,14,15,16 +}; +static UNUSED_ATTR const S16 ML_defaultNorm[MaxML+1] = { + 1, 4, 3, 2, 2, 2, 2, 2, + 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1,-1,-1, + -1,-1,-1,-1,-1 +}; +#define ML_DEFAULTNORMLOG 6 /* for static allocation */ +static UNUSED_ATTR const U32 ML_defaultNormLog = ML_DEFAULTNORMLOG; - { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)algoNb; - assert(algoNb == 0); - return HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, - cSrcSize, workSpace, wkspSize); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)algoNb; - assert(algoNb == 1); - return HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, - cSrcSize, workSpace, wkspSize); -#else - return algoNb ? HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, - cSrcSize, workSpace, wkspSize): - HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, - cSrcSize, workSpace, wkspSize); -#endif - } -} +static UNUSED_ATTR const S16 OF_defaultNorm[DefaultMaxOff+1] = { + 1, 1, 1, 1, 1, 1, 2, 2, + 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + -1,-1,-1,-1,-1 +}; +#define OF_DEFAULTNORMLOG 5 /* for static allocation */ +static UNUSED_ATTR const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG; -size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2) -{ - DTableDesc const dtd = HUF_getDTableDesc(DTable); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)dtd; - assert(dtd.tableType == 0); - return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)dtd; - assert(dtd.tableType == 1); - return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); +/*-******************************************* +* Shared functions to include for inlining +*********************************************/ +static void ZSTD_copy8(void* dst, const void* src) { +#if defined(ZSTD_ARCH_ARM_NEON) + vst1_u8((uint8_t*)dst, vld1_u8((const uint8_t*)src)); #else - return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) : - HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); + ZSTD_memcpy(dst, src, 8); #endif } +#define COPY8(d,s) do { ZSTD_copy8(d,s); d+=8; s+=8; } while (0) -#ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2) -{ - const BYTE* ip = (const BYTE*) cSrc; - - size_t const hSize = HUF_readDTableX1_wksp_bmi2(dctx, cSrc, cSrcSize, workSpace, wkspSize, bmi2); - if (HUF_isError(hSize)) return hSize; - if (hSize >= cSrcSize) return ERROR(srcSize_wrong); - ip += hSize; cSrcSize -= hSize; - - return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2); -} -#endif - -size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2) -{ - DTableDesc const dtd = HUF_getDTableDesc(DTable); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)dtd; - assert(dtd.tableType == 0); - return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)dtd; - assert(dtd.tableType == 1); - return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); +/* Need to use memmove here since the literal buffer can now be located within + the dst buffer. In circumstances where the op "catches up" to where the + literal buffer is, there can be partial overlaps in this call on the final + copy if the literal is being shifted by less than 16 bytes. */ +static void ZSTD_copy16(void* dst, const void* src) { +#if defined(ZSTD_ARCH_ARM_NEON) + vst1q_u8((uint8_t*)dst, vld1q_u8((const uint8_t*)src)); +#elif defined(ZSTD_ARCH_X86_SSE2) + _mm_storeu_si128((__m128i*)dst, _mm_loadu_si128((const __m128i*)src)); +#elif defined(__clang__) + ZSTD_memmove(dst, src, 16); #else - return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) : - HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); + /* ZSTD_memmove is not inlined properly by gcc */ + BYTE copy16_buf[16]; + ZSTD_memcpy(copy16_buf, src, 16); + ZSTD_memcpy(dst, copy16_buf, 16); #endif } +#define COPY16(d,s) do { ZSTD_copy16(d,s); d+=16; s+=16; } while (0) + +#define WILDCOPY_OVERLENGTH 32 +#define WILDCOPY_VECLEN 16 -size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2) +typedef enum { + ZSTD_no_overlap, + ZSTD_overlap_src_before_dst + /* ZSTD_overlap_dst_before_src, */ +} ZSTD_overlap_e; + +/*! ZSTD_wildcopy() : + * Custom version of ZSTD_memcpy(), can over read/write up to WILDCOPY_OVERLENGTH bytes (if length==0) + * @param ovtype controls the overlap detection + * - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart. + * - ZSTD_overlap_src_before_dst: The src and dst may overlap, but they MUST be at least 8 bytes apart. + * The src buffer must be before the dst buffer. + */ +MEM_STATIC FORCE_INLINE_ATTR +void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e const ovtype) { - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize == 0) return ERROR(corruption_detected); + ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src; + const BYTE* ip = (const BYTE*)src; + BYTE* op = (BYTE*)dst; + BYTE* const oend = op + length; - { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)algoNb; - assert(algoNb == 0); - return HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)algoNb; - assert(algoNb == 1); - return HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2); -#else - return algoNb ? HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2) : - HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2); -#endif + if (ovtype == ZSTD_overlap_src_before_dst && diff < WILDCOPY_VECLEN) { + /* Handle short offset copies. */ + do { + COPY8(op, ip); + } while (op < oend); + } else { + assert(diff >= WILDCOPY_VECLEN || diff <= -WILDCOPY_VECLEN); + /* Separate out the first COPY16() call because the copy length is + * almost certain to be short, so the branches have different + * probabilities. Since it is almost certain to be short, only do + * one COPY16() in the first call. Then, do two calls per loop since + * at that point it is more likely to have a high trip count. + */ + ZSTD_copy16(op, ip); + if (16 >= length) return; + op += 16; + ip += 16; + do { + COPY16(op, ip); + COPY16(op, ip); + } + while (op < oend); } } -#ifndef ZSTD_NO_UNUSED_FUNCTIONS -#ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_readDTableX1(HUF_DTable* DTable, const void* src, size_t srcSize) +MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) { - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_readDTableX1_wksp(DTable, src, srcSize, - workSpace, sizeof(workSpace)); + size_t const length = MIN(dstCapacity, srcSize); + if (length > 0) { + ZSTD_memcpy(dst, src, length); + } + return length; } -size_t HUF_decompress1X1_DCtx(HUF_DTable* DCtx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_decompress1X1_DCtx_wksp(DCtx, dst, dstSize, cSrc, cSrcSize, - workSpace, sizeof(workSpace)); -} +/* define "workspace is too large" as this number of times larger than needed */ +#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 -size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUF_CREATE_STATIC_DTABLEX1(DTable, HUF_TABLELOG_MAX); - return HUF_decompress1X1_DCtx (DTable, dst, dstSize, cSrc, cSrcSize); -} -#endif +/* when workspace is continuously too large + * during at least this number of times, + * context's memory usage is considered wasteful, + * because it's sized to handle a worst case scenario which rarely happens. + * In which case, resize it down to free some memory */ +#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_readDTableX2(HUF_DTable* DTable, const void* src, size_t srcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_readDTableX2_wksp(DTable, src, srcSize, - workSpace, sizeof(workSpace)); -} +/* Controls whether the input/output buffer is buffered or stable. */ +typedef enum { + ZSTD_bm_buffered = 0, /* Buffer the input/output */ + ZSTD_bm_stable = 1 /* ZSTD_inBuffer/ZSTD_outBuffer is stable */ +} ZSTD_bufferMode_e; -size_t HUF_decompress1X2_DCtx(HUF_DTable* DCtx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_decompress1X2_DCtx_wksp(DCtx, dst, dstSize, cSrc, cSrcSize, - workSpace, sizeof(workSpace)); -} -size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX); - return HUF_decompress1X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); -} -#endif +/*-******************************************* +* Private declarations +*********************************************/ -#ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_decompress4X1_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, - workSpace, sizeof(workSpace)); -} -size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUF_CREATE_STATIC_DTABLEX1(DTable, HUF_TABLELOG_MAX); - return HUF_decompress4X1_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); -} -#endif +/** + * Contains the compressed frame size and an upper-bound for the decompressed frame size. + * Note: before using `compressedSize`, check for errors using ZSTD_isError(). + * similarly, before using `decompressedBound`, check for errors using: + * `decompressedBound != ZSTD_CONTENTSIZE_ERROR` + */ +typedef struct { + size_t nbBlocks; + size_t compressedSize; + unsigned long long decompressedBound; +} ZSTD_frameSizeInfo; /* decompress & legacy */ -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, - workSpace, sizeof(workSpace)); -} +/* ZSTD_invalidateRepCodes() : + * ensures next compression will not use repcodes from previous block. + * Note : only works with regular variant; + * do not use with extDict variant ! */ +void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx); /* zstdmt, adaptive_compression (shouldn't get this definition from here) */ -size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX); - return HUF_decompress4X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); -} -#endif -typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); +typedef struct { + blockType_e blockType; + U32 lastBlock; + U32 origSize; +} blockProperties_t; /* declared here for decompress and fullbench */ -size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ -#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2) - static const decompressionAlgo decompress[2] = { HUF_decompress4X1, HUF_decompress4X2 }; -#endif +/*! ZSTD_getcBlockSize() : + * Provides the size of compressed block from block header `src` */ +/* Used by: decompress, fullbench */ +size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, + blockProperties_t* bpPtr); - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ - if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ - if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ +/*! ZSTD_decodeSeqHeaders() : + * decode sequence header from src */ +/* Used by: zstd_decompress_block, fullbench */ +size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, + const void* src, size_t srcSize); - { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)algoNb; - assert(algoNb == 0); - return HUF_decompress4X1(dst, dstSize, cSrc, cSrcSize); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)algoNb; - assert(algoNb == 1); - return HUF_decompress4X2(dst, dstSize, cSrc, cSrcSize); -#else - return decompress[algoNb](dst, dstSize, cSrc, cSrcSize); -#endif - } +/** + * @returns true iff the CPU supports dynamic BMI2 dispatch. + */ +MEM_STATIC int ZSTD_cpuSupportsBmi2(void) +{ + ZSTD_cpuid_t cpuid = ZSTD_cpuid(); + return ZSTD_cpuid_bmi1(cpuid) && ZSTD_cpuid_bmi2(cpuid); } -size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ - if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ - if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ +#endif /* ZSTD_CCOMMON_H_MODULE */ +/**** ended inlining zstd_internal.h ****/ - { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)algoNb; - assert(algoNb == 0); - return HUF_decompress4X1_DCtx(dctx, dst, dstSize, cSrc, cSrcSize); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)algoNb; - assert(algoNb == 1); - return HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize); -#else - return algoNb ? HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : - HUF_decompress4X1_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; -#endif - } -} -size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_decompress4X_hufOnly_wksp(dctx, dst, dstSize, cSrc, cSrcSize, - workSpace, sizeof(workSpace)); -} +/*-**************************************** +* Version +******************************************/ +unsigned ZSTD_versionNumber(void) { return ZSTD_VERSION_NUMBER; } -size_t HUF_decompress1X_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_decompress1X_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, - workSpace, sizeof(workSpace)); -} -#endif -/**** ended inlining decompress/huf_decompress.c ****/ -/**** start inlining decompress/zstd_ddict.c ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ +const char* ZSTD_versionString(void) { return ZSTD_VERSION_STRING; } -/* zstd_ddict.c : - * concentrates all logic that needs to know the internals of ZSTD_DDict object */ -/*-******************************************************* -* Dependencies -*********************************************************/ -/**** skipping file: ../common/zstd_deps.h ****/ -/**** start inlining ../common/cpu.h ****/ -/* - * Copyright (c) 2018-2021, Facebook, Inc. - * All rights reserved. +/*-**************************************** +* ZSTD Error Management +******************************************/ +#undef ZSTD_isError /* defined within zstd_internal.h */ +/*! ZSTD_isError() : + * tells if a return value is an error code + * symbol is required for external callers */ +unsigned ZSTD_isError(size_t code) { return ERR_isError(code); } + +/*! ZSTD_getErrorName() : + * provides error code string from function result (useful for debugging) */ +const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); } + +/*! ZSTD_getError() : + * convert a `size_t` function result into a proper ZSTD_errorCode enum */ +ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); } + +/*! ZSTD_getErrorString() : + * provides error code string from enum */ +const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); } +/**** ended inlining common/zstd_common.c ****/ + +/**** start inlining decompress/huf_decompress.c ****/ +/* ****************************************************************** + * huff0 huffman decoder, + * part of Finite State Entropy library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy * * This source code is licensed under both the BSD-style license (found in the * LICENSE file in the root directory of this source tree) and the GPLv2 (found * in the COPYING file in the root directory of this source tree). * You may select, at your option, one of the above-listed licenses. - */ +****************************************************************** */ -#ifndef ZSTD_COMMON_CPU_H -#define ZSTD_COMMON_CPU_H +/* ************************************************************** +* Dependencies +****************************************************************/ +/**** skipping file: ../common/zstd_deps.h ****/ +/**** skipping file: ../common/compiler.h ****/ +/**** skipping file: ../common/bitstream.h ****/ +/**** skipping file: ../common/fse.h ****/ +/**** skipping file: ../common/huf.h ****/ +/**** skipping file: ../common/error_private.h ****/ +/**** skipping file: ../common/zstd_internal.h ****/ +/**** skipping file: ../common/bits.h ****/ -/** - * Implementation taken from folly/CpuId.h - * https://github.com/facebook/folly/blob/master/folly/CpuId.h - */ +/* ************************************************************** +* Constants +****************************************************************/ -/**** skipping file: mem.h ****/ +#define HUF_DECODER_FAST_TABLELOG 11 -#ifdef _MSC_VER -#include +/* ************************************************************** +* Macros +****************************************************************/ + +#ifdef HUF_DISABLE_FAST_DECODE +# define HUF_ENABLE_FAST_DECODE 0 +#else +# define HUF_ENABLE_FAST_DECODE 1 #endif -typedef struct { - U32 f1c; - U32 f1d; - U32 f7b; - U32 f7c; -} ZSTD_cpuid_t; +/* These two optional macros force the use one way or another of the two + * Huffman decompression implementations. You can't force in both directions + * at the same time. + */ +#if defined(HUF_FORCE_DECOMPRESS_X1) && \ + defined(HUF_FORCE_DECOMPRESS_X2) +#error "Cannot force the use of the X1 and X2 decoders at the same time!" +#endif -MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) { - U32 f1c = 0; - U32 f1d = 0; - U32 f7b = 0; - U32 f7c = 0; -#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) - int reg[4]; - __cpuid((int*)reg, 0); - { - int const n = reg[0]; - if (n >= 1) { - __cpuid((int*)reg, 1); - f1c = (U32)reg[2]; - f1d = (U32)reg[3]; - } - if (n >= 7) { - __cpuidex((int*)reg, 7, 0); - f7b = (U32)reg[1]; - f7c = (U32)reg[2]; - } - } -#elif defined(__i386__) && defined(__PIC__) && !defined(__clang__) && defined(__GNUC__) - /* The following block like the normal cpuid branch below, but gcc - * reserves ebx for use of its pic register so we must specially - * handle the save and restore to avoid clobbering the register - */ - U32 n; - __asm__( - "pushl %%ebx\n\t" - "cpuid\n\t" - "popl %%ebx\n\t" - : "=a"(n) - : "a"(0) - : "ecx", "edx"); - if (n >= 1) { - U32 f1a; - __asm__( - "pushl %%ebx\n\t" - "cpuid\n\t" - "popl %%ebx\n\t" - : "=a"(f1a), "=c"(f1c), "=d"(f1d) - : "a"(1)); - } - if (n >= 7) { - __asm__( - "pushl %%ebx\n\t" - "cpuid\n\t" - "movl %%ebx, %%eax\n\t" - "popl %%ebx" - : "=a"(f7b), "=c"(f7c) - : "a"(7), "c"(0) - : "edx"); - } -#elif defined(__x86_64__) || defined(_M_X64) || defined(__i386__) - U32 n; - __asm__("cpuid" : "=a"(n) : "a"(0) : "ebx", "ecx", "edx"); - if (n >= 1) { - U32 f1a; - __asm__("cpuid" : "=a"(f1a), "=c"(f1c), "=d"(f1d) : "a"(1) : "ebx"); - } - if (n >= 7) { - U32 f7a; - __asm__("cpuid" - : "=a"(f7a), "=b"(f7b), "=c"(f7c) - : "a"(7), "c"(0) - : "edx"); - } +/* When DYNAMIC_BMI2 is enabled, fast decoders are only called when bmi2 is + * supported at runtime, so we can add the BMI2 target attribute. + * When it is disabled, we will still get BMI2 if it is enabled statically. + */ +#if DYNAMIC_BMI2 +# define HUF_FAST_BMI2_ATTRS BMI2_TARGET_ATTRIBUTE +#else +# define HUF_FAST_BMI2_ATTRS #endif - { - ZSTD_cpuid_t cpuid; - cpuid.f1c = f1c; - cpuid.f1d = f1d; - cpuid.f7b = f7b; - cpuid.f7c = f7c; - return cpuid; - } -} -#define X(name, r, bit) \ - MEM_STATIC int ZSTD_cpuid_##name(ZSTD_cpuid_t const cpuid) { \ - return ((cpuid.r) & (1U << bit)) != 0; \ - } +#ifdef __cplusplus +# define HUF_EXTERN_C extern "C" +#else +# define HUF_EXTERN_C +#endif +#define HUF_ASM_DECL HUF_EXTERN_C -/* cpuid(1): Processor Info and Feature Bits. */ -#define C(name, bit) X(name, f1c, bit) - C(sse3, 0) - C(pclmuldq, 1) - C(dtes64, 2) - C(monitor, 3) - C(dscpl, 4) - C(vmx, 5) - C(smx, 6) - C(eist, 7) - C(tm2, 8) - C(ssse3, 9) - C(cnxtid, 10) - C(fma, 12) - C(cx16, 13) - C(xtpr, 14) - C(pdcm, 15) - C(pcid, 17) - C(dca, 18) - C(sse41, 19) - C(sse42, 20) - C(x2apic, 21) - C(movbe, 22) - C(popcnt, 23) - C(tscdeadline, 24) - C(aes, 25) - C(xsave, 26) - C(osxsave, 27) - C(avx, 28) - C(f16c, 29) - C(rdrand, 30) -#undef C -#define D(name, bit) X(name, f1d, bit) - D(fpu, 0) - D(vme, 1) - D(de, 2) - D(pse, 3) - D(tsc, 4) - D(msr, 5) - D(pae, 6) - D(mce, 7) - D(cx8, 8) - D(apic, 9) - D(sep, 11) - D(mtrr, 12) - D(pge, 13) - D(mca, 14) - D(cmov, 15) - D(pat, 16) - D(pse36, 17) - D(psn, 18) - D(clfsh, 19) - D(ds, 21) - D(acpi, 22) - D(mmx, 23) - D(fxsr, 24) - D(sse, 25) - D(sse2, 26) - D(ss, 27) - D(htt, 28) - D(tm, 29) - D(pbe, 31) -#undef D +#if DYNAMIC_BMI2 +# define HUF_NEED_BMI2_FUNCTION 1 +#else +# define HUF_NEED_BMI2_FUNCTION 0 +#endif -/* cpuid(7): Extended Features. */ -#define B(name, bit) X(name, f7b, bit) - B(bmi1, 3) - B(hle, 4) - B(avx2, 5) - B(smep, 7) - B(bmi2, 8) - B(erms, 9) - B(invpcid, 10) - B(rtm, 11) - B(mpx, 14) - B(avx512f, 16) - B(avx512dq, 17) - B(rdseed, 18) - B(adx, 19) - B(smap, 20) - B(avx512ifma, 21) - B(pcommit, 22) - B(clflushopt, 23) - B(clwb, 24) - B(avx512pf, 26) - B(avx512er, 27) - B(avx512cd, 28) - B(sha, 29) - B(avx512bw, 30) - B(avx512vl, 31) -#undef B -#define C(name, bit) X(name, f7c, bit) - C(prefetchwt1, 0) - C(avx512vbmi, 1) -#undef C +/* ************************************************************** +* Error Management +****************************************************************/ +#define HUF_isError ERR_isError + + +/* ************************************************************** +* Byte alignment for workSpace management +****************************************************************/ +#define HUF_ALIGN(x, a) HUF_ALIGN_MASK((x), (a) - 1) +#define HUF_ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask)) + + +/* ************************************************************** +* BMI2 Variant Wrappers +****************************************************************/ +typedef size_t (*HUF_DecompressUsingDTableFn)(void *dst, size_t dstSize, + const void *cSrc, + size_t cSrcSize, + const HUF_DTable *DTable); + +#if DYNAMIC_BMI2 + +#define HUF_DGEN(fn) \ + \ + static size_t fn##_default( \ + void* dst, size_t dstSize, \ + const void* cSrc, size_t cSrcSize, \ + const HUF_DTable* DTable) \ + { \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + \ + static BMI2_TARGET_ATTRIBUTE size_t fn##_bmi2( \ + void* dst, size_t dstSize, \ + const void* cSrc, size_t cSrcSize, \ + const HUF_DTable* DTable) \ + { \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + \ + static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ + size_t cSrcSize, HUF_DTable const* DTable, int flags) \ + { \ + if (flags & HUF_flags_bmi2) { \ + return fn##_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + return fn##_default(dst, dstSize, cSrc, cSrcSize, DTable); \ + } -#undef X +#else -#endif /* ZSTD_COMMON_CPU_H */ -/**** ended inlining ../common/cpu.h ****/ -/**** skipping file: ../common/mem.h ****/ -#define FSE_STATIC_LINKING_ONLY -/**** skipping file: ../common/fse.h ****/ -#define HUF_STATIC_LINKING_ONLY -/**** skipping file: ../common/huf.h ****/ -/**** start inlining zstd_decompress_internal.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ +#define HUF_DGEN(fn) \ + static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ + size_t cSrcSize, HUF_DTable const* DTable, int flags) \ + { \ + (void)flags; \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } +#endif -/* zstd_decompress_internal: - * objects and definitions shared within lib/decompress modules */ - #ifndef ZSTD_DECOMPRESS_INTERNAL_H - #define ZSTD_DECOMPRESS_INTERNAL_H +/*-***************************/ +/* generic DTableDesc */ +/*-***************************/ +typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc; +static DTableDesc HUF_getDTableDesc(const HUF_DTable* table) +{ + DTableDesc dtd; + ZSTD_memcpy(&dtd, table, sizeof(dtd)); + return dtd; +} -/*-******************************************************* - * Dependencies - *********************************************************/ -/**** skipping file: ../common/mem.h ****/ -/**** skipping file: ../common/zstd_internal.h ****/ -/**** start inlining ../common/zstd_trace.h ****/ -/* - * Copyright (c) 2016-2021, Facebook, Inc. - * All rights reserved. +static size_t HUF_initFastDStream(BYTE const* ip) { + BYTE const lastByte = ip[7]; + size_t const bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0; + size_t const value = MEM_readLEST(ip) | 1; + assert(bitsConsumed <= 8); + assert(sizeof(size_t) == 8); + return value << bitsConsumed; +} + + +/** + * The input/output arguments to the Huffman fast decoding loop: * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. + * ip [in/out] - The input pointers, must be updated to reflect what is consumed. + * op [in/out] - The output pointers, must be updated to reflect what is written. + * bits [in/out] - The bitstream containers, must be updated to reflect the current state. + * dt [in] - The decoding table. + * ilowest [in] - The beginning of the valid range of the input. Decoders may read + * down to this pointer. It may be below iend[0]. + * oend [in] - The end of the output stream. op[3] must not cross oend. + * iend [in] - The end of each input stream. ip[i] may cross iend[i], + * as long as it is above ilowest, but that indicates corruption. */ +typedef struct { + BYTE const* ip[4]; + BYTE* op[4]; + U64 bits[4]; + void const* dt; + BYTE const* ilowest; + BYTE* oend; + BYTE const* iend[4]; +} HUF_DecompressFastArgs; -#ifndef ZSTD_TRACE_H -#define ZSTD_TRACE_H +typedef void (*HUF_DecompressFastLoopFn)(HUF_DecompressFastArgs*); -#if defined (__cplusplus) -extern "C" { -#endif +/** + * Initializes args for the fast decoding loop. + * @returns 1 on success + * 0 if the fallback implementation should be used. + * Or an error code on failure. + */ +static size_t HUF_DecompressFastArgs_init(HUF_DecompressFastArgs* args, void* dst, size_t dstSize, void const* src, size_t srcSize, const HUF_DTable* DTable) +{ + void const* dt = DTable + 1; + U32 const dtLog = HUF_getDTableDesc(DTable).tableLog; -#include + const BYTE* const istart = (const BYTE*)src; -/* weak symbol support */ -#if !defined(ZSTD_HAVE_WEAK_SYMBOLS) && defined(__GNUC__) && \ - !defined(__APPLE__) && !defined(_WIN32) && !defined(__MINGW32__) && \ - !defined(__CYGWIN__) -# define ZSTD_HAVE_WEAK_SYMBOLS 1 -#else -# define ZSTD_HAVE_WEAK_SYMBOLS 0 -#endif -#if ZSTD_HAVE_WEAK_SYMBOLS -# define ZSTD_WEAK_ATTR __attribute__((__weak__)) -#else -# define ZSTD_WEAK_ATTR -#endif + BYTE* const oend = ZSTD_maybeNullPtrAdd((BYTE*)dst, dstSize); -/* Only enable tracing when weak symbols are available. */ -#ifndef ZSTD_TRACE -# define ZSTD_TRACE ZSTD_HAVE_WEAK_SYMBOLS -#endif + /* The fast decoding loop assumes 64-bit little-endian. + * This condition is false on x32. + */ + if (!MEM_isLittleEndian() || MEM_32bits()) + return 0; -#if ZSTD_TRACE + /* Avoid nullptr addition */ + if (dstSize == 0) + return 0; + assert(dst != NULL); -struct ZSTD_CCtx_s; -struct ZSTD_DCtx_s; -struct ZSTD_CCtx_params_s; + /* strict minimum : jump table + 1 byte per stream */ + if (srcSize < 10) + return ERROR(corruption_detected); -typedef struct { - /** - * ZSTD_VERSION_NUMBER - * - * This is guaranteed to be the first member of ZSTD_trace. - * Otherwise, this struct is not stable between versions. If - * the version number does not match your expectation, you - * should not interpret the rest of the struct. - */ - unsigned version; - /** - * Non-zero if streaming (de)compression is used. - */ - unsigned streaming; - /** - * The dictionary ID. - */ - unsigned dictionaryID; - /** - * Is the dictionary cold? - * Only set on decompression. - */ - unsigned dictionaryIsCold; - /** - * The dictionary size or zero if no dictionary. - */ - size_t dictionarySize; - /** - * The uncompressed size of the data. - */ - size_t uncompressedSize; - /** - * The compressed size of the data. - */ - size_t compressedSize; - /** - * The fully resolved CCtx parameters (NULL on decompression). - */ - struct ZSTD_CCtx_params_s const* params; - /** - * The ZSTD_CCtx pointer (NULL on decompression). + /* Must have at least 8 bytes per stream because we don't handle initializing smaller bit containers. + * If table log is not correct at this point, fallback to the old decoder. + * On small inputs we don't have enough data to trigger the fast loop, so use the old decoder. */ - struct ZSTD_CCtx_s const* cctx; - /** - * The ZSTD_DCtx pointer (NULL on compression). + if (dtLog != HUF_DECODER_FAST_TABLELOG) + return 0; + + /* Read the jump table. */ + { + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = srcSize - (length1 + length2 + length3 + 6); + args->iend[0] = istart + 6; /* jumpTable */ + args->iend[1] = args->iend[0] + length1; + args->iend[2] = args->iend[1] + length2; + args->iend[3] = args->iend[2] + length3; + + /* HUF_initFastDStream() requires this, and this small of an input + * won't benefit from the ASM loop anyways. + */ + if (length1 < 8 || length2 < 8 || length3 < 8 || length4 < 8) + return 0; + if (length4 > srcSize) return ERROR(corruption_detected); /* overflow */ + } + /* ip[] contains the position that is currently loaded into bits[]. */ + args->ip[0] = args->iend[1] - sizeof(U64); + args->ip[1] = args->iend[2] - sizeof(U64); + args->ip[2] = args->iend[3] - sizeof(U64); + args->ip[3] = (BYTE const*)src + srcSize - sizeof(U64); + + /* op[] contains the output pointers. */ + args->op[0] = (BYTE*)dst; + args->op[1] = args->op[0] + (dstSize+3)/4; + args->op[2] = args->op[1] + (dstSize+3)/4; + args->op[3] = args->op[2] + (dstSize+3)/4; + + /* No point to call the ASM loop for tiny outputs. */ + if (args->op[3] >= oend) + return 0; + + /* bits[] is the bit container. + * It is read from the MSB down to the LSB. + * It is shifted left as it is read, and zeros are + * shifted in. After the lowest valid bit a 1 is + * set, so that CountTrailingZeros(bits[]) can be used + * to count how many bits we've consumed. + */ + args->bits[0] = HUF_initFastDStream(args->ip[0]); + args->bits[1] = HUF_initFastDStream(args->ip[1]); + args->bits[2] = HUF_initFastDStream(args->ip[2]); + args->bits[3] = HUF_initFastDStream(args->ip[3]); + + /* The decoders must be sure to never read beyond ilowest. + * This is lower than iend[0], but allowing decoders to read + * down to ilowest can allow an extra iteration or two in the + * fast loop. */ - struct ZSTD_DCtx_s const* dctx; -} ZSTD_Trace; + args->ilowest = istart; -/** - * A tracing context. It must be 0 when tracing is disabled. - * Otherwise, any non-zero value returned by a tracing begin() - * function is presented to any subsequent calls to end(). - * - * Any non-zero value is treated as tracing is enabled and not - * interpreted by the library. - * - * Two possible uses are: - * * A timestamp for when the begin() function was called. - * * A unique key identifying the (de)compression, like the - * address of the [dc]ctx pointer if you need to track - * more information than just a timestamp. - */ -typedef unsigned long long ZSTD_TraceCtx; + args->oend = oend; + args->dt = dt; -/** - * Trace the beginning of a compression call. - * @param cctx The dctx pointer for the compression. - * It can be used as a key to map begin() to end(). - * @returns Non-zero if tracing is enabled. The return value is - * passed to ZSTD_trace_compress_end(). - */ -ZSTD_TraceCtx ZSTD_trace_compress_begin(struct ZSTD_CCtx_s const* cctx); + return 1; +} + +static size_t HUF_initRemainingDStream(BIT_DStream_t* bit, HUF_DecompressFastArgs const* args, int stream, BYTE* segmentEnd) +{ + /* Validate that we haven't overwritten. */ + if (args->op[stream] > segmentEnd) + return ERROR(corruption_detected); + /* Validate that we haven't read beyond iend[]. + * Note that ip[] may be < iend[] because the MSB is + * the next bit to read, and we may have consumed 100% + * of the stream, so down to iend[i] - 8 is valid. + */ + if (args->ip[stream] < args->iend[stream] - 8) + return ERROR(corruption_detected); + + /* Construct the BIT_DStream_t. */ + assert(sizeof(size_t) == 8); + bit->bitContainer = MEM_readLEST(args->ip[stream]); + bit->bitsConsumed = ZSTD_countTrailingZeros64(args->bits[stream]); + bit->start = (const char*)args->ilowest; + bit->limitPtr = bit->start + sizeof(size_t); + bit->ptr = (const char*)args->ip[stream]; + + return 0; +} + +/* Calls X(N) for each stream 0, 1, 2, 3. */ +#define HUF_4X_FOR_EACH_STREAM(X) \ + do { \ + X(0); \ + X(1); \ + X(2); \ + X(3); \ + } while (0) + +/* Calls X(N, var) for each stream 0, 1, 2, 3. */ +#define HUF_4X_FOR_EACH_STREAM_WITH_VAR(X, var) \ + do { \ + X(0, (var)); \ + X(1, (var)); \ + X(2, (var)); \ + X(3, (var)); \ + } while (0) + + +#ifndef HUF_FORCE_DECOMPRESS_X2 + +/*-***************************/ +/* single-symbol decoding */ +/*-***************************/ +typedef struct { BYTE nbBits; BYTE byte; } HUF_DEltX1; /* single-symbol decoding */ /** - * Trace the end of a compression call. - * @param ctx The return value of ZSTD_trace_compress_begin(). - * @param trace The zstd tracing info. + * Packs 4 HUF_DEltX1 structs into a U64. This is used to lay down 4 entries at + * a time. */ -void ZSTD_trace_compress_end( - ZSTD_TraceCtx ctx, - ZSTD_Trace const* trace); +static U64 HUF_DEltX1_set4(BYTE symbol, BYTE nbBits) { + U64 D4; + if (MEM_isLittleEndian()) { + D4 = (U64)((symbol << 8) + nbBits); + } else { + D4 = (U64)(symbol + (nbBits << 8)); + } + assert(D4 < (1U << 16)); + D4 *= 0x0001000100010001ULL; + return D4; +} /** - * Trace the beginning of a decompression call. - * @param dctx The dctx pointer for the decompression. - * It can be used as a key to map begin() to end(). - * @returns Non-zero if tracing is enabled. The return value is - * passed to ZSTD_trace_compress_end(). + * Increase the tableLog to targetTableLog and rescales the stats. + * If tableLog > targetTableLog this is a no-op. + * @returns New tableLog */ -ZSTD_TraceCtx ZSTD_trace_decompress_begin(struct ZSTD_DCtx_s const* dctx); +static U32 HUF_rescaleStats(BYTE* huffWeight, U32* rankVal, U32 nbSymbols, U32 tableLog, U32 targetTableLog) +{ + if (tableLog > targetTableLog) + return tableLog; + if (tableLog < targetTableLog) { + U32 const scale = targetTableLog - tableLog; + U32 s; + /* Increase the weight for all non-zero probability symbols by scale. */ + for (s = 0; s < nbSymbols; ++s) { + huffWeight[s] += (BYTE)((huffWeight[s] == 0) ? 0 : scale); + } + /* Update rankVal to reflect the new weights. + * All weights except 0 get moved to weight + scale. + * Weights [1, scale] are empty. + */ + for (s = targetTableLog; s > scale; --s) { + rankVal[s] = rankVal[s - scale]; + } + for (s = scale; s > 0; --s) { + rankVal[s] = 0; + } + } + return targetTableLog; +} + +typedef struct { + U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; + U32 rankStart[HUF_TABLELOG_ABSOLUTEMAX + 1]; + U32 statsWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; + BYTE symbols[HUF_SYMBOLVALUE_MAX + 1]; + BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; +} HUF_ReadDTableX1_Workspace; + +size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags) +{ + U32 tableLog = 0; + U32 nbSymbols = 0; + size_t iSize; + void* const dtPtr = DTable + 1; + HUF_DEltX1* const dt = (HUF_DEltX1*)dtPtr; + HUF_ReadDTableX1_Workspace* wksp = (HUF_ReadDTableX1_Workspace*)workSpace; + + DEBUG_STATIC_ASSERT(HUF_DECOMPRESS_WORKSPACE_SIZE >= sizeof(*wksp)); + if (sizeof(*wksp) > wkspSize) return ERROR(tableLog_tooLarge); + + DEBUG_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable)); + /* ZSTD_memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */ + + iSize = HUF_readStats_wksp(wksp->huffWeight, HUF_SYMBOLVALUE_MAX + 1, wksp->rankVal, &nbSymbols, &tableLog, src, srcSize, wksp->statsWksp, sizeof(wksp->statsWksp), flags); + if (HUF_isError(iSize)) return iSize; + + + /* Table header */ + { DTableDesc dtd = HUF_getDTableDesc(DTable); + U32 const maxTableLog = dtd.maxTableLog + 1; + U32 const targetTableLog = MIN(maxTableLog, HUF_DECODER_FAST_TABLELOG); + tableLog = HUF_rescaleStats(wksp->huffWeight, wksp->rankVal, nbSymbols, tableLog, targetTableLog); + if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, Huffman tree cannot fit in */ + dtd.tableType = 0; + dtd.tableLog = (BYTE)tableLog; + ZSTD_memcpy(DTable, &dtd, sizeof(dtd)); + } -/** - * Trace the end of a decompression call. - * @param ctx The return value of ZSTD_trace_decompress_begin(). - * @param trace The zstd tracing info. - */ -void ZSTD_trace_decompress_end( - ZSTD_TraceCtx ctx, - ZSTD_Trace const* trace); + /* Compute symbols and rankStart given rankVal: + * + * rankVal already contains the number of values of each weight. + * + * symbols contains the symbols ordered by weight. First are the rankVal[0] + * weight 0 symbols, followed by the rankVal[1] weight 1 symbols, and so on. + * symbols[0] is filled (but unused) to avoid a branch. + * + * rankStart contains the offset where each rank belongs in the DTable. + * rankStart[0] is not filled because there are no entries in the table for + * weight 0. + */ + { int n; + U32 nextRankStart = 0; + int const unroll = 4; + int const nLimit = (int)nbSymbols - unroll + 1; + for (n=0; n<(int)tableLog+1; n++) { + U32 const curr = nextRankStart; + nextRankStart += wksp->rankVal[n]; + wksp->rankStart[n] = curr; + } + for (n=0; n < nLimit; n += unroll) { + int u; + for (u=0; u < unroll; ++u) { + size_t const w = wksp->huffWeight[n+u]; + wksp->symbols[wksp->rankStart[w]++] = (BYTE)(n+u); + } + } + for (; n < (int)nbSymbols; ++n) { + size_t const w = wksp->huffWeight[n]; + wksp->symbols[wksp->rankStart[w]++] = (BYTE)n; + } + } -#endif /* ZSTD_TRACE */ + /* fill DTable + * We fill all entries of each weight in order. + * That way length is a constant for each iteration of the outer loop. + * We can switch based on the length to a different inner loop which is + * optimized for that particular case. + */ + { U32 w; + int symbol = wksp->rankVal[0]; + int rankStart = 0; + for (w=1; wrankVal[w]; + int const length = (1 << w) >> 1; + int uStart = rankStart; + BYTE const nbBits = (BYTE)(tableLog + 1 - w); + int s; + int u; + switch (length) { + case 1: + for (s=0; ssymbols[symbol + s]; + D.nbBits = nbBits; + dt[uStart] = D; + uStart += 1; + } + break; + case 2: + for (s=0; ssymbols[symbol + s]; + D.nbBits = nbBits; + dt[uStart+0] = D; + dt[uStart+1] = D; + uStart += 2; + } + break; + case 4: + for (s=0; ssymbols[symbol + s], nbBits); + MEM_write64(dt + uStart, D4); + uStart += 4; + } + break; + case 8: + for (s=0; ssymbols[symbol + s], nbBits); + MEM_write64(dt + uStart, D4); + MEM_write64(dt + uStart + 4, D4); + uStart += 8; + } + break; + default: + for (s=0; ssymbols[symbol + s], nbBits); + for (u=0; u < length; u += 16) { + MEM_write64(dt + uStart + u + 0, D4); + MEM_write64(dt + uStart + u + 4, D4); + MEM_write64(dt + uStart + u + 8, D4); + MEM_write64(dt + uStart + u + 12, D4); + } + assert(u == length); + uStart += length; + } + break; + } + symbol += symbolCount; + rankStart += symbolCount * length; + } + } + return iSize; +} -#if defined (__cplusplus) +FORCE_INLINE_TEMPLATE BYTE +HUF_decodeSymbolX1(BIT_DStream_t* Dstream, const HUF_DEltX1* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */ + BYTE const c = dt[val].byte; + BIT_skipBits(Dstream, dt[val].nbBits); + return c; } -#endif -#endif /* ZSTD_TRACE_H */ -/**** ended inlining ../common/zstd_trace.h ****/ +#define HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) \ + do { *ptr++ = HUF_decodeSymbolX1(DStreamPtr, dt, dtLog); } while (0) +#define HUF_DECODE_SYMBOLX1_1(ptr, DStreamPtr) \ + do { \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ + HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr); \ + } while (0) +#define HUF_DECODE_SYMBOLX1_2(ptr, DStreamPtr) \ + do { \ + if (MEM_64bits()) \ + HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr); \ + } while (0) -/*-******************************************************* - * Constants - *********************************************************/ -static UNUSED_ATTR const U32 LL_base[MaxLL+1] = { - 0, 1, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, - 16, 18, 20, 22, 24, 28, 32, 40, - 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, - 0x2000, 0x4000, 0x8000, 0x10000 }; +HINT_INLINE size_t +HUF_decodeStreamX1(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX1* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; -static UNUSED_ATTR const U32 OF_base[MaxOff+1] = { - 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, - 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, - 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, - 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD }; + /* up to 4 symbols at a time */ + if ((pEnd - p) > 3) { + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-3)) { + HUF_DECODE_SYMBOLX1_2(p, bitDPtr); + HUF_DECODE_SYMBOLX1_1(p, bitDPtr); + HUF_DECODE_SYMBOLX1_2(p, bitDPtr); + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + } + } else { + BIT_reloadDStream(bitDPtr); + } -static UNUSED_ATTR const U32 OF_bits[MaxOff+1] = { - 0, 1, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 21, 22, 23, - 24, 25, 26, 27, 28, 29, 30, 31 }; + /* [0-3] symbols remaining */ + if (MEM_32bits()) + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd)) + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); -static UNUSED_ATTR const U32 ML_base[MaxML+1] = { - 3, 4, 5, 6, 7, 8, 9, 10, - 11, 12, 13, 14, 15, 16, 17, 18, - 19, 20, 21, 22, 23, 24, 25, 26, - 27, 28, 29, 30, 31, 32, 33, 34, - 35, 37, 39, 41, 43, 47, 51, 59, - 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, - 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 }; + /* no more data to retrieve from bitstream, no need to reload */ + while (p < pEnd) + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + return (size_t)(pEnd-pStart); +} -/*-******************************************************* - * Decompression types - *********************************************************/ - typedef struct { - U32 fastMode; - U32 tableLog; - } ZSTD_seqSymbol_header; +FORCE_INLINE_TEMPLATE size_t +HUF_decompress1X1_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + BYTE* op = (BYTE*)dst; + BYTE* const oend = ZSTD_maybeNullPtrAdd(op, dstSize); + const void* dtPtr = DTable + 1; + const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr; + BIT_DStream_t bitD; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; - typedef struct { - U16 nextState; - BYTE nbAdditionalBits; - BYTE nbBits; - U32 baseValue; - } ZSTD_seqSymbol; + CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) ); - #define SEQSYMBOL_TABLE_SIZE(log) (1 + (1 << (log))) + HUF_decodeStreamX1(op, &bitD, oend, dt, dtLog); -#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE (sizeof(S16) * (MaxSeq + 1) + (1u << MaxFSELog) + sizeof(U64)) -#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32 ((ZSTD_BUILD_FSE_TABLE_WKSP_SIZE + sizeof(U32) - 1) / sizeof(U32)) + if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); -typedef struct { - ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]; /* Note : Space reserved for FSE Tables */ - ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]; /* is also used as temporary workspace while building hufTable during DDict creation */ - ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]; /* and therefore must be at least HUF_DECOMPRESS_WORKSPACE_SIZE large */ - HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */ - U32 rep[ZSTD_REP_NUM]; - U32 workspace[ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32]; -} ZSTD_entropyDTables_t; + return dstSize; +} -typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, - ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock, - ZSTDds_decompressLastBlock, ZSTDds_checkChecksum, - ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage; +/* HUF_decompress4X1_usingDTable_internal_body(): + * Conditions : + * @dstSize >= 6 + */ +FORCE_INLINE_TEMPLATE size_t +HUF_decompress4X1_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + /* Check */ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + if (dstSize < 6) return ERROR(corruption_detected); /* stream 4-split doesn't work */ -typedef enum { zdss_init=0, zdss_loadHeader, - zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage; + { const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + BYTE* const olimit = oend - 3; + const void* const dtPtr = DTable + 1; + const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr; -typedef enum { - ZSTD_use_indefinitely = -1, /* Use the dictionary indefinitely */ - ZSTD_dont_use = 0, /* Do not use the dictionary (if one exists free it) */ - ZSTD_use_once = 1 /* Use the dictionary once and set to ZSTD_dont_use */ -} ZSTD_dictUses_e; + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + const size_t segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + U32 endSignal = 1; -/* Hashset for storing references to multiple ZSTD_DDict within ZSTD_DCtx */ -typedef struct { - const ZSTD_DDict** ddictPtrTable; - size_t ddictPtrTableSize; - size_t ddictPtrCount; -} ZSTD_DDictHashSet; + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + if (opStart4 > oend) return ERROR(corruption_detected); /* overflow */ + assert(dstSize >= 6); /* validated above */ + CHECK_F( BIT_initDStream(&bitD1, istart1, length1) ); + CHECK_F( BIT_initDStream(&bitD2, istart2, length2) ); + CHECK_F( BIT_initDStream(&bitD3, istart3, length3) ); + CHECK_F( BIT_initDStream(&bitD4, istart4, length4) ); -struct ZSTD_DCtx_s -{ - const ZSTD_seqSymbol* LLTptr; - const ZSTD_seqSymbol* MLTptr; - const ZSTD_seqSymbol* OFTptr; - const HUF_DTable* HUFptr; - ZSTD_entropyDTables_t entropy; - U32 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; /* space needed when building huffman tables */ - const void* previousDstEnd; /* detect continuity */ - const void* prefixStart; /* start of current segment */ - const void* virtualStart; /* virtual start of previous segment if it was just before current one */ - const void* dictEnd; /* end of previous segment */ - size_t expected; - ZSTD_frameHeader fParams; - U64 processedCSize; - U64 decodedSize; - blockType_e bType; /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block decompression stages */ - ZSTD_dStage stage; - U32 litEntropy; - U32 fseEntropy; - XXH64_state_t xxhState; - size_t headerSize; - ZSTD_format_e format; - ZSTD_forceIgnoreChecksum_e forceIgnoreChecksum; /* User specified: if == 1, will ignore checksums in compressed frame. Default == 0 */ - U32 validateChecksum; /* if == 1, will validate checksum. Is == 1 if (fParams.checksumFlag == 1) and (forceIgnoreChecksum == 0). */ - const BYTE* litPtr; - ZSTD_customMem customMem; - size_t litSize; - size_t rleSize; - size_t staticSize; - int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */ + /* up to 16 symbols per loop (4 symbols per stream) in 64-bit mode */ + if ((size_t)(oend - op4) >= sizeof(size_t)) { + for ( ; (endSignal) & (op4 < olimit) ; ) { + HUF_DECODE_SYMBOLX1_2(op1, &bitD1); + HUF_DECODE_SYMBOLX1_2(op2, &bitD2); + HUF_DECODE_SYMBOLX1_2(op3, &bitD3); + HUF_DECODE_SYMBOLX1_2(op4, &bitD4); + HUF_DECODE_SYMBOLX1_1(op1, &bitD1); + HUF_DECODE_SYMBOLX1_1(op2, &bitD2); + HUF_DECODE_SYMBOLX1_1(op3, &bitD3); + HUF_DECODE_SYMBOLX1_1(op4, &bitD4); + HUF_DECODE_SYMBOLX1_2(op1, &bitD1); + HUF_DECODE_SYMBOLX1_2(op2, &bitD2); + HUF_DECODE_SYMBOLX1_2(op3, &bitD3); + HUF_DECODE_SYMBOLX1_2(op4, &bitD4); + HUF_DECODE_SYMBOLX1_0(op1, &bitD1); + HUF_DECODE_SYMBOLX1_0(op2, &bitD2); + HUF_DECODE_SYMBOLX1_0(op3, &bitD3); + HUF_DECODE_SYMBOLX1_0(op4, &bitD4); + endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished; + } + } - /* dictionary */ - ZSTD_DDict* ddictLocal; - const ZSTD_DDict* ddict; /* set by ZSTD_initDStream_usingDDict(), or ZSTD_DCtx_refDDict() */ - U32 dictID; - int ddictIsCold; /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */ - ZSTD_dictUses_e dictUses; - ZSTD_DDictHashSet* ddictSet; /* Hash set for multiple ddicts */ - ZSTD_refMultipleDDicts_e refMultipleDDicts; /* User specified: if == 1, will allow references to multiple DDicts. Default == 0 (disabled) */ + /* check corruption */ + /* note : should not be necessary : op# advance in lock step, and we control op4. + * but curiously, binary generated by gcc 7.2 & 7.3 with -mbmi2 runs faster when >=1 test is present */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 supposed already verified within main loop */ - /* streaming */ - ZSTD_dStreamStage streamStage; - char* inBuff; - size_t inBuffSize; - size_t inPos; - size_t maxWindowSize; - char* outBuff; - size_t outBuffSize; - size_t outStart; - size_t outEnd; - size_t lhSize; - void* legacyContext; - U32 previousLegacyVersion; - U32 legacyVersion; - U32 hostageByte; - int noForwardProgress; - ZSTD_bufferMode_e outBufferMode; - ZSTD_outBuffer expectedOutBuffer; + /* finish bitStreams one by one */ + HUF_decodeStreamX1(op1, &bitD1, opStart2, dt, dtLog); + HUF_decodeStreamX1(op2, &bitD2, opStart3, dt, dtLog); + HUF_decodeStreamX1(op3, &bitD3, opStart4, dt, dtLog); + HUF_decodeStreamX1(op4, &bitD4, oend, dt, dtLog); - /* workspace */ - BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH]; - BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; + /* check */ + { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endCheck) return ERROR(corruption_detected); } - size_t oversizedDuration; + /* decoded size */ + return dstSize; + } +} -#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION - void const* dictContentBeginForFuzzing; - void const* dictContentEndForFuzzing; +#if HUF_NEED_BMI2_FUNCTION +static BMI2_TARGET_ATTRIBUTE +size_t HUF_decompress4X1_usingDTable_internal_bmi2(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} #endif - /* Tracing */ -#if ZSTD_TRACE - ZSTD_TraceCtx traceCtx; -#endif -}; /* typedef'd to ZSTD_DCtx within "zstd.h" */ +static +size_t HUF_decompress4X1_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} +#if ZSTD_ENABLE_ASM_X86_64_BMI2 -/*-******************************************************* - * Shared internal functions - *********************************************************/ +HUF_ASM_DECL void HUF_decompress4X1_usingDTable_internal_fast_asm_loop(HUF_DecompressFastArgs* args) ZSTDLIB_HIDDEN; -/*! ZSTD_loadDEntropy() : - * dict : must point at beginning of a valid zstd dictionary. - * @return : size of dictionary header (size of magic number + dict ID + entropy tables) */ -size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, - const void* const dict, size_t const dictSize); +#endif -/*! ZSTD_checkContinuity() : - * check if next `dst` follows previous position, where decompression ended. - * If yes, do nothing (continue on current segment). - * If not, classify previous segment as "external dictionary", and start a new segment. - * This function cannot fail. */ -void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize); +static HUF_FAST_BMI2_ATTRS +void HUF_decompress4X1_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs* args) +{ + U64 bits[4]; + BYTE const* ip[4]; + BYTE* op[4]; + U16 const* const dtable = (U16 const*)args->dt; + BYTE* const oend = args->oend; + BYTE const* const ilowest = args->ilowest; + /* Copy the arguments to local variables */ + ZSTD_memcpy(&bits, &args->bits, sizeof(bits)); + ZSTD_memcpy((void*)(&ip), &args->ip, sizeof(ip)); + ZSTD_memcpy(&op, &args->op, sizeof(op)); -#endif /* ZSTD_DECOMPRESS_INTERNAL_H */ -/**** ended inlining zstd_decompress_internal.h ****/ -/**** start inlining zstd_ddict.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ + assert(MEM_isLittleEndian()); + assert(!MEM_32bits()); + + for (;;) { + BYTE* olimit; + int stream; + + /* Assert loop preconditions */ +#ifndef NDEBUG + for (stream = 0; stream < 4; ++stream) { + assert(op[stream] <= (stream == 3 ? oend : op[stream + 1])); + assert(ip[stream] >= ilowest); + } +#endif + /* Compute olimit */ + { + /* Each iteration produces 5 output symbols per stream */ + size_t const oiters = (size_t)(oend - op[3]) / 5; + /* Each iteration consumes up to 11 bits * 5 = 55 bits < 7 bytes + * per stream. + */ + size_t const iiters = (size_t)(ip[0] - ilowest) / 7; + /* We can safely run iters iterations before running bounds checks */ + size_t const iters = MIN(oiters, iiters); + size_t const symbols = iters * 5; + + /* We can simply check that op[3] < olimit, instead of checking all + * of our bounds, since we can't hit the other bounds until we've run + * iters iterations, which only happens when op[3] == olimit. + */ + olimit = op[3] + symbols; + /* Exit fast decoding loop once we reach the end. */ + if (op[3] == olimit) + break; -#ifndef ZSTD_DDICT_H -#define ZSTD_DDICT_H + /* Exit the decoding loop if any input pointer has crossed the + * previous one. This indicates corruption, and a precondition + * to our loop is that ip[i] >= ip[0]. + */ + for (stream = 1; stream < 4; ++stream) { + if (ip[stream] < ip[stream - 1]) + goto _out; + } + } -/*-******************************************************* - * Dependencies - *********************************************************/ -/**** skipping file: ../common/zstd_deps.h ****/ -/**** skipping file: ../zstd.h ****/ +#ifndef NDEBUG + for (stream = 1; stream < 4; ++stream) { + assert(ip[stream] >= ip[stream - 1]); + } +#endif +#define HUF_4X1_DECODE_SYMBOL(_stream, _symbol) \ + do { \ + int const index = (int)(bits[(_stream)] >> 53); \ + int const entry = (int)dtable[index]; \ + bits[(_stream)] <<= (entry & 0x3F); \ + op[(_stream)][(_symbol)] = (BYTE)((entry >> 8) & 0xFF); \ + } while (0) + +#define HUF_4X1_RELOAD_STREAM(_stream) \ + do { \ + int const ctz = ZSTD_countTrailingZeros64(bits[(_stream)]); \ + int const nbBits = ctz & 7; \ + int const nbBytes = ctz >> 3; \ + op[(_stream)] += 5; \ + ip[(_stream)] -= nbBytes; \ + bits[(_stream)] = MEM_read64(ip[(_stream)]) | 1; \ + bits[(_stream)] <<= nbBits; \ + } while (0) + + /* Manually unroll the loop because compilers don't consistently + * unroll the inner loops, which destroys performance. + */ + do { + /* Decode 5 symbols in each of the 4 streams */ + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 0); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 1); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 2); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 3); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 4); + + /* Reload each of the 4 the bitstreams */ + HUF_4X_FOR_EACH_STREAM(HUF_4X1_RELOAD_STREAM); + } while (op[3] < olimit); + +#undef HUF_4X1_DECODE_SYMBOL +#undef HUF_4X1_RELOAD_STREAM + } -/*-******************************************************* - * Interface - *********************************************************/ +_out: -/* note: several prototypes are already published in `zstd.h` : - * ZSTD_createDDict() - * ZSTD_createDDict_byReference() - * ZSTD_createDDict_advanced() - * ZSTD_freeDDict() - * ZSTD_initStaticDDict() - * ZSTD_sizeof_DDict() - * ZSTD_estimateDDictSize() - * ZSTD_getDictID_fromDict() + /* Save the final values of each of the state variables back to args. */ + ZSTD_memcpy(&args->bits, &bits, sizeof(bits)); + ZSTD_memcpy((void*)(&args->ip), &ip, sizeof(ip)); + ZSTD_memcpy(&args->op, &op, sizeof(op)); +} + +/** + * @returns @p dstSize on success (>= 6) + * 0 if the fallback implementation should be used + * An error if an error occurred */ +static HUF_FAST_BMI2_ATTRS +size_t +HUF_decompress4X1_usingDTable_internal_fast( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable, + HUF_DecompressFastLoopFn loopFn) +{ + void const* dt = DTable + 1; + BYTE const* const ilowest = (BYTE const*)cSrc; + BYTE* const oend = ZSTD_maybeNullPtrAdd((BYTE*)dst, dstSize); + HUF_DecompressFastArgs args; + { size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable); + FORWARD_IF_ERROR(ret, "Failed to init fast loop args"); + if (ret == 0) + return 0; + } -const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict); -size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict); + assert(args.ip[0] >= args.ilowest); + loopFn(&args); + + /* Our loop guarantees that ip[] >= ilowest and that we haven't + * overwritten any op[]. + */ + assert(args.ip[0] >= ilowest); + assert(args.ip[0] >= ilowest); + assert(args.ip[1] >= ilowest); + assert(args.ip[2] >= ilowest); + assert(args.ip[3] >= ilowest); + assert(args.op[3] <= oend); + + assert(ilowest == args.ilowest); + assert(ilowest + 6 == args.iend[0]); + (void)ilowest; + + /* finish bit streams one by one. */ + { size_t const segmentSize = (dstSize+3) / 4; + BYTE* segmentEnd = (BYTE*)dst; + int i; + for (i = 0; i < 4; ++i) { + BIT_DStream_t bit; + if (segmentSize <= (size_t)(oend - segmentEnd)) + segmentEnd += segmentSize; + else + segmentEnd = oend; + FORWARD_IF_ERROR(HUF_initRemainingDStream(&bit, &args, i, segmentEnd), "corruption"); + /* Decompress and validate that we've produced exactly the expected length. */ + args.op[i] += HUF_decodeStreamX1(args.op[i], &bit, segmentEnd, (HUF_DEltX1 const*)dt, HUF_DECODER_FAST_TABLELOG); + if (args.op[i] != segmentEnd) return ERROR(corruption_detected); + } + } -void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + /* decoded size */ + assert(dstSize != 0); + return dstSize; +} +HUF_DGEN(HUF_decompress1X1_usingDTable_internal) +static size_t HUF_decompress4X1_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable, int flags) +{ + HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X1_usingDTable_internal_default; + HUF_DecompressFastLoopFn loopFn = HUF_decompress4X1_usingDTable_internal_fast_c_loop; -#endif /* ZSTD_DDICT_H */ -/**** ended inlining zstd_ddict.h ****/ +#if DYNAMIC_BMI2 + if (flags & HUF_flags_bmi2) { + fallbackFn = HUF_decompress4X1_usingDTable_internal_bmi2; +# if ZSTD_ENABLE_ASM_X86_64_BMI2 + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop; + } +# endif + } else { + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); + } +#endif -#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) -/**** start inlining ../legacy/zstd_legacy.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ +#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__) + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop; + } +#endif -#ifndef ZSTD_LEGACY_H -#define ZSTD_LEGACY_H + if (HUF_ENABLE_FAST_DECODE && !(flags & HUF_flags_disableFast)) { + size_t const ret = HUF_decompress4X1_usingDTable_internal_fast(dst, dstSize, cSrc, cSrcSize, DTable, loopFn); + if (ret != 0) + return ret; + } + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); +} -#if defined (__cplusplus) -extern "C" { -#endif +static size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int flags) +{ + const BYTE* ip = (const BYTE*) cSrc; -/* ************************************* -* Includes -***************************************/ -/**** skipping file: ../common/mem.h ****/ -/**** skipping file: ../common/error_private.h ****/ -/**** skipping file: ../common/zstd_internal.h ****/ + size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize, flags); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; -#if !defined (ZSTD_LEGACY_SUPPORT) || (ZSTD_LEGACY_SUPPORT == 0) -# undef ZSTD_LEGACY_SUPPORT -# define ZSTD_LEGACY_SUPPORT 8 -#endif + return HUF_decompress4X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags); +} -#if (ZSTD_LEGACY_SUPPORT <= 1) -/**** start inlining zstd_v01.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ +#endif /* HUF_FORCE_DECOMPRESS_X2 */ -#ifndef ZSTD_V01_H_28739879432 -#define ZSTD_V01_H_28739879432 -#if defined (__cplusplus) -extern "C" { -#endif +#ifndef HUF_FORCE_DECOMPRESS_X1 -/* ************************************* -* Includes -***************************************/ -#include /* size_t */ +/* *************************/ +/* double-symbols decoding */ +/* *************************/ +typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX2; /* double-symbols decoding */ +typedef struct { BYTE symbol; } sortedSymbol_t; +typedef U32 rankValCol_t[HUF_TABLELOG_MAX + 1]; +typedef rankValCol_t rankVal_t[HUF_TABLELOG_MAX]; -/* ************************************* -* Simple one-step function -***************************************/ /** -ZSTDv01_decompress() : decompress ZSTD frames compliant with v0.1.x format - compressedSize : is the exact source size - maxOriginalSize : is the size of the 'dst' buffer, which must be already allocated. - It must be equal or larger than originalSize, otherwise decompression will fail. - return : the number of bytes decompressed into destination buffer (originalSize) - or an errorCode if it fails (which can be tested using ZSTDv01_isError()) -*/ -size_t ZSTDv01_decompress( void* dst, size_t maxOriginalSize, - const void* src, size_t compressedSize); + * Constructs a HUF_DEltX2 in a U32. + */ +static U32 HUF_buildDEltX2U32(U32 symbol, U32 nbBits, U32 baseSeq, int level) +{ + U32 seq; + DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, sequence) == 0); + DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, nbBits) == 2); + DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, length) == 3); + DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(U32)); + if (MEM_isLittleEndian()) { + seq = level == 1 ? symbol : (baseSeq + (symbol << 8)); + return seq + (nbBits << 16) + ((U32)level << 24); + } else { + seq = level == 1 ? (symbol << 8) : ((baseSeq << 8) + symbol); + return (seq << 16) + (nbBits << 8) + (U32)level; + } +} + +/** + * Constructs a HUF_DEltX2. + */ +static HUF_DEltX2 HUF_buildDEltX2(U32 symbol, U32 nbBits, U32 baseSeq, int level) +{ + HUF_DEltX2 DElt; + U32 const val = HUF_buildDEltX2U32(symbol, nbBits, baseSeq, level); + DEBUG_STATIC_ASSERT(sizeof(DElt) == sizeof(val)); + ZSTD_memcpy(&DElt, &val, sizeof(val)); + return DElt; +} - /** - ZSTDv01_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.1.x format - srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' - cSize (output parameter) : the number of bytes that would be read to decompress this frame - or an error code if it fails (which can be tested using ZSTDv01_isError()) - dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame - or ZSTD_CONTENTSIZE_ERROR if an error occurs +/** + * Constructs 2 HUF_DEltX2s and packs them into a U64. + */ +static U64 HUF_buildDEltX2U64(U32 symbol, U32 nbBits, U16 baseSeq, int level) +{ + U32 DElt = HUF_buildDEltX2U32(symbol, nbBits, baseSeq, level); + return (U64)DElt + ((U64)DElt << 32); +} - note : assumes `cSize` and `dBound` are _not_ NULL. +/** + * Fills the DTable rank with all the symbols from [begin, end) that are each + * nbBits long. + * + * @param DTableRank The start of the rank in the DTable. + * @param begin The first symbol to fill (inclusive). + * @param end The last symbol to fill (exclusive). + * @param nbBits Each symbol is nbBits long. + * @param tableLog The table log. + * @param baseSeq If level == 1 { 0 } else { the first level symbol } + * @param level The level in the table. Must be 1 or 2. */ -void ZSTDv01_findFrameSizeInfoLegacy(const void *src, size_t srcSize, - size_t* cSize, unsigned long long* dBound); +static void HUF_fillDTableX2ForWeight( + HUF_DEltX2* DTableRank, + sortedSymbol_t const* begin, sortedSymbol_t const* end, + U32 nbBits, U32 tableLog, + U16 baseSeq, int const level) +{ + U32 const length = 1U << ((tableLog - nbBits) & 0x1F /* quiet static-analyzer */); + const sortedSymbol_t* ptr; + assert(level >= 1 && level <= 2); + switch (length) { + case 1: + for (ptr = begin; ptr != end; ++ptr) { + HUF_DEltX2 const DElt = HUF_buildDEltX2(ptr->symbol, nbBits, baseSeq, level); + *DTableRank++ = DElt; + } + break; + case 2: + for (ptr = begin; ptr != end; ++ptr) { + HUF_DEltX2 const DElt = HUF_buildDEltX2(ptr->symbol, nbBits, baseSeq, level); + DTableRank[0] = DElt; + DTableRank[1] = DElt; + DTableRank += 2; + } + break; + case 4: + for (ptr = begin; ptr != end; ++ptr) { + U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level); + ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2)); + DTableRank += 4; + } + break; + case 8: + for (ptr = begin; ptr != end; ++ptr) { + U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level); + ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 4, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 6, &DEltX2, sizeof(DEltX2)); + DTableRank += 8; + } + break; + default: + for (ptr = begin; ptr != end; ++ptr) { + U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level); + HUF_DEltX2* const DTableRankEnd = DTableRank + length; + for (; DTableRank != DTableRankEnd; DTableRank += 8) { + ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 4, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 6, &DEltX2, sizeof(DEltX2)); + } + } + break; + } +} -/** -ZSTDv01_isError() : tells if the result of ZSTDv01_decompress() is an error -*/ -unsigned ZSTDv01_isError(size_t code); +/* HUF_fillDTableX2Level2() : + * `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */ +static void HUF_fillDTableX2Level2(HUF_DEltX2* DTable, U32 targetLog, const U32 consumedBits, + const U32* rankVal, const int minWeight, const int maxWeight1, + const sortedSymbol_t* sortedSymbols, U32 const* rankStart, + U32 nbBitsBaseline, U16 baseSeq) +{ + /* Fill skipped values (all positions up to rankVal[minWeight]). + * These are positions only get a single symbol because the combined weight + * is too large. + */ + if (minWeight>1) { + U32 const length = 1U << ((targetLog - consumedBits) & 0x1F /* quiet static-analyzer */); + U64 const DEltX2 = HUF_buildDEltX2U64(baseSeq, consumedBits, /* baseSeq */ 0, /* level */ 1); + int const skipSize = rankVal[minWeight]; + assert(length > 1); + assert((U32)skipSize < length); + switch (length) { + case 2: + assert(skipSize == 1); + ZSTD_memcpy(DTable, &DEltX2, sizeof(DEltX2)); + break; + case 4: + assert(skipSize <= 4); + ZSTD_memcpy(DTable + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + 2, &DEltX2, sizeof(DEltX2)); + break; + default: + { + int i; + for (i = 0; i < skipSize; i += 8) { + ZSTD_memcpy(DTable + i + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + i + 2, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + i + 4, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + i + 6, &DEltX2, sizeof(DEltX2)); + } + } + } + } + /* Fill each of the second level symbols by weight. */ + { + int w; + for (w = minWeight; w < maxWeight1; ++w) { + int const begin = rankStart[w]; + int const end = rankStart[w+1]; + U32 const nbBits = nbBitsBaseline - w; + U32 const totalBits = nbBits + consumedBits; + HUF_fillDTableX2ForWeight( + DTable + rankVal[w], + sortedSymbols + begin, sortedSymbols + end, + totalBits, targetLog, + baseSeq, /* level */ 2); + } + } +} -/* ************************************* -* Advanced functions -***************************************/ -typedef struct ZSTDv01_Dctx_s ZSTDv01_Dctx; -ZSTDv01_Dctx* ZSTDv01_createDCtx(void); -size_t ZSTDv01_freeDCtx(ZSTDv01_Dctx* dctx); +static void HUF_fillDTableX2(HUF_DEltX2* DTable, const U32 targetLog, + const sortedSymbol_t* sortedList, + const U32* rankStart, rankValCol_t* rankValOrigin, const U32 maxWeight, + const U32 nbBitsBaseline) +{ + U32* const rankVal = rankValOrigin[0]; + const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */ + const U32 minBits = nbBitsBaseline - maxWeight; + int w; + int const wEnd = (int)maxWeight + 1; + + /* Fill DTable in order of weight. */ + for (w = 1; w < wEnd; ++w) { + int const begin = (int)rankStart[w]; + int const end = (int)rankStart[w+1]; + U32 const nbBits = nbBitsBaseline - w; + + if (targetLog-nbBits >= minBits) { + /* Enough room for a second symbol. */ + int start = rankVal[w]; + U32 const length = 1U << ((targetLog - nbBits) & 0x1F /* quiet static-analyzer */); + int minWeight = nbBits + scaleLog; + int s; + if (minWeight < 1) minWeight = 1; + /* Fill the DTable for every symbol of weight w. + * These symbols get at least 1 second symbol. + */ + for (s = begin; s != end; ++s) { + HUF_fillDTableX2Level2( + DTable + start, targetLog, nbBits, + rankValOrigin[nbBits], minWeight, wEnd, + sortedList, rankStart, + nbBitsBaseline, sortedList[s].symbol); + start += length; + } + } else { + /* Only a single symbol. */ + HUF_fillDTableX2ForWeight( + DTable + rankVal[w], + sortedList + begin, sortedList + end, + nbBits, targetLog, + /* baseSeq */ 0, /* level */ 1); + } + } +} -size_t ZSTDv01_decompressDCtx(void* ctx, - void* dst, size_t maxOriginalSize, - const void* src, size_t compressedSize); +typedef struct { + rankValCol_t rankVal[HUF_TABLELOG_MAX]; + U32 rankStats[HUF_TABLELOG_MAX + 1]; + U32 rankStart0[HUF_TABLELOG_MAX + 3]; + sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1]; + BYTE weightList[HUF_SYMBOLVALUE_MAX + 1]; + U32 calleeWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; +} HUF_ReadDTableX2_Workspace; -/* ************************************* -* Streaming functions -***************************************/ -size_t ZSTDv01_resetDCtx(ZSTDv01_Dctx* dctx); +size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize, int flags) +{ + U32 tableLog, maxW, nbSymbols; + DTableDesc dtd = HUF_getDTableDesc(DTable); + U32 maxTableLog = dtd.maxTableLog; + size_t iSize; + void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */ + HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr; + U32 *rankStart; -size_t ZSTDv01_nextSrcSizeToDecompress(ZSTDv01_Dctx* dctx); -size_t ZSTDv01_decompressContinue(ZSTDv01_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); -/** - Use above functions alternatively. - ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue(). - ZSTD_decompressContinue() will use previous data blocks to improve compression if they are located prior to current block. - Result is the number of bytes regenerated within 'dst'. - It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header. -*/ + HUF_ReadDTableX2_Workspace* const wksp = (HUF_ReadDTableX2_Workspace*)workSpace; -/* ************************************* -* Prefix - version detection -***************************************/ -#define ZSTDv01_magicNumber 0xFD2FB51E /* Big Endian version */ -#define ZSTDv01_magicNumberLE 0x1EB52FFD /* Little Endian version */ + if (sizeof(*wksp) > wkspSize) return ERROR(GENERIC); + rankStart = wksp->rankStart0 + 1; + ZSTD_memset(wksp->rankStats, 0, sizeof(wksp->rankStats)); + ZSTD_memset(wksp->rankStart0, 0, sizeof(wksp->rankStart0)); -#if defined (__cplusplus) -} -#endif + DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(HUF_DTable)); /* if compiler fails here, assertion is wrong */ + if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); + /* ZSTD_memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */ -#endif /* ZSTD_V01_H_28739879432 */ -/**** ended inlining zstd_v01.h ****/ -#endif -#if (ZSTD_LEGACY_SUPPORT <= 2) -/**** start inlining zstd_v02.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ + iSize = HUF_readStats_wksp(wksp->weightList, HUF_SYMBOLVALUE_MAX + 1, wksp->rankStats, &nbSymbols, &tableLog, src, srcSize, wksp->calleeWksp, sizeof(wksp->calleeWksp), flags); + if (HUF_isError(iSize)) return iSize; -#ifndef ZSTD_V02_H_4174539423 -#define ZSTD_V02_H_4174539423 + /* check result */ + if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ + if (tableLog <= HUF_DECODER_FAST_TABLELOG && maxTableLog > HUF_DECODER_FAST_TABLELOG) maxTableLog = HUF_DECODER_FAST_TABLELOG; -#if defined (__cplusplus) -extern "C" { -#endif + /* find maxWeight */ + for (maxW = tableLog; wksp->rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */ -/* ************************************* -* Includes -***************************************/ -#include /* size_t */ + /* Get start index of each weight */ + { U32 w, nextRankStart = 0; + for (w=1; wrankStats[w]; + rankStart[w] = curr; + } + rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/ + rankStart[maxW+1] = nextRankStart; + } + /* sort symbols by weight */ + { U32 s; + for (s=0; sweightList[s]; + U32 const r = rankStart[w]++; + wksp->sortedSymbol[r].symbol = (BYTE)s; + } + rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */ + } -/* ************************************* -* Simple one-step function -***************************************/ -/** -ZSTDv02_decompress() : decompress ZSTD frames compliant with v0.2.x format - compressedSize : is the exact source size - maxOriginalSize : is the size of the 'dst' buffer, which must be already allocated. - It must be equal or larger than originalSize, otherwise decompression will fail. - return : the number of bytes decompressed into destination buffer (originalSize) - or an errorCode if it fails (which can be tested using ZSTDv01_isError()) -*/ -size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize, - const void* src, size_t compressedSize); + /* Build rankVal */ + { U32* const rankVal0 = wksp->rankVal[0]; + { int const rescale = (maxTableLog-tableLog) - 1; /* tableLog <= maxTableLog */ + U32 nextRankVal = 0; + U32 w; + for (w=1; wrankStats[w] << (w+rescale); + rankVal0[w] = curr; + } } + { U32 const minBits = tableLog+1 - maxW; + U32 consumed; + for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) { + U32* const rankValPtr = wksp->rankVal[consumed]; + U32 w; + for (w = 1; w < maxW+1; w++) { + rankValPtr[w] = rankVal0[w] >> consumed; + } } } } - /** - ZSTDv02_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.2.x format - srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' - cSize (output parameter) : the number of bytes that would be read to decompress this frame - or an error code if it fails (which can be tested using ZSTDv01_isError()) - dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame - or ZSTD_CONTENTSIZE_ERROR if an error occurs + HUF_fillDTableX2(dt, maxTableLog, + wksp->sortedSymbol, + wksp->rankStart0, wksp->rankVal, maxW, + tableLog+1); - note : assumes `cSize` and `dBound` are _not_ NULL. - */ -void ZSTDv02_findFrameSizeInfoLegacy(const void *src, size_t srcSize, - size_t* cSize, unsigned long long* dBound); + dtd.tableLog = (BYTE)maxTableLog; + dtd.tableType = 1; + ZSTD_memcpy(DTable, &dtd, sizeof(dtd)); + return iSize; +} -/** -ZSTDv02_isError() : tells if the result of ZSTDv02_decompress() is an error -*/ -unsigned ZSTDv02_isError(size_t code); +FORCE_INLINE_TEMPLATE U32 +HUF_decodeSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + ZSTD_memcpy(op, &dt[val].sequence, 2); + BIT_skipBits(DStream, dt[val].nbBits); + return dt[val].length; +} -/* ************************************* -* Advanced functions -***************************************/ -typedef struct ZSTDv02_Dctx_s ZSTDv02_Dctx; -ZSTDv02_Dctx* ZSTDv02_createDCtx(void); -size_t ZSTDv02_freeDCtx(ZSTDv02_Dctx* dctx); +FORCE_INLINE_TEMPLATE U32 +HUF_decodeLastSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + ZSTD_memcpy(op, &dt[val].sequence, 1); + if (dt[val].length==1) { + BIT_skipBits(DStream, dt[val].nbBits); + } else { + if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) { + BIT_skipBits(DStream, dt[val].nbBits); + if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8)) + /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ + DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); + } + } + return 1; +} -size_t ZSTDv02_decompressDCtx(void* ctx, - void* dst, size_t maxOriginalSize, - const void* src, size_t compressedSize); +#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \ + do { ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog); } while (0) -/* ************************************* -* Streaming functions -***************************************/ -size_t ZSTDv02_resetDCtx(ZSTDv02_Dctx* dctx); +#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ + do { \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ + ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog); \ + } while (0) -size_t ZSTDv02_nextSrcSizeToDecompress(ZSTDv02_Dctx* dctx); -size_t ZSTDv02_decompressContinue(ZSTDv02_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); -/** - Use above functions alternatively. - ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue(). - ZSTD_decompressContinue() will use previous data blocks to improve compression if they are located prior to current block. - Result is the number of bytes regenerated within 'dst'. - It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header. -*/ +#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ + do { \ + if (MEM_64bits()) \ + ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog); \ + } while (0) -/* ************************************* -* Prefix - version detection -***************************************/ -#define ZSTDv02_magicNumber 0xFD2FB522 /* v0.2 */ +HINT_INLINE size_t +HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, + const HUF_DEltX2* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 8 symbols at a time */ + if ((size_t)(pEnd - p) >= sizeof(bitDPtr->bitContainer)) { + if (dtLog <= 11 && MEM_64bits()) { + /* up to 10 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-9)) { + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + } + } else { + /* up to 8 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-(sizeof(bitDPtr->bitContainer)-1))) { + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_1(p, bitDPtr); + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + } + } + } else { + BIT_reloadDStream(bitDPtr); + } + /* closer to end : up to 2 symbols at a time */ + if ((size_t)(pEnd - p) >= 2) { + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd-2)) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); -#if defined (__cplusplus) -} -#endif + while (p <= pEnd-2) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); /* no need to reload : reached the end of DStream */ + } -#endif /* ZSTD_V02_H_4174539423 */ -/**** ended inlining zstd_v02.h ****/ -#endif -#if (ZSTD_LEGACY_SUPPORT <= 3) -/**** start inlining zstd_v03.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ + if (p < pEnd) + p += HUF_decodeLastSymbolX2(p, bitDPtr, dt, dtLog); -#ifndef ZSTD_V03_H_298734209782 -#define ZSTD_V03_H_298734209782 + return p-pStart; +} -#if defined (__cplusplus) -extern "C" { -#endif +FORCE_INLINE_TEMPLATE size_t +HUF_decompress1X2_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + BIT_DStream_t bitD; -/* ************************************* -* Includes -***************************************/ -#include /* size_t */ + /* Init */ + CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) ); + /* decode */ + { BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ZSTD_maybeNullPtrAdd(ostart, dstSize); + const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */ + const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + HUF_decodeStreamX2(ostart, &bitD, oend, dt, dtd.tableLog); + } -/* ************************************* -* Simple one-step function -***************************************/ -/** -ZSTDv03_decompress() : decompress ZSTD frames compliant with v0.3.x format - compressedSize : is the exact source size - maxOriginalSize : is the size of the 'dst' buffer, which must be already allocated. - It must be equal or larger than originalSize, otherwise decompression will fail. - return : the number of bytes decompressed into destination buffer (originalSize) - or an errorCode if it fails (which can be tested using ZSTDv01_isError()) -*/ -size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize, - const void* src, size_t compressedSize); + /* check */ + if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); - /** - ZSTDv03_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.3.x format - srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' - cSize (output parameter) : the number of bytes that would be read to decompress this frame - or an error code if it fails (which can be tested using ZSTDv01_isError()) - dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame - or ZSTD_CONTENTSIZE_ERROR if an error occurs + /* decoded size */ + return dstSize; +} - note : assumes `cSize` and `dBound` are _not_ NULL. +/* HUF_decompress4X2_usingDTable_internal_body(): + * Conditions: + * @dstSize >= 6 */ - void ZSTDv03_findFrameSizeInfoLegacy(const void *src, size_t srcSize, - size_t* cSize, unsigned long long* dBound); - - /** -ZSTDv03_isError() : tells if the result of ZSTDv03_decompress() is an error -*/ -unsigned ZSTDv03_isError(size_t code); +FORCE_INLINE_TEMPLATE size_t +HUF_decompress4X2_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + if (dstSize < 6) return ERROR(corruption_detected); /* stream 4-split doesn't work */ + { const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + BYTE* const olimit = oend - (sizeof(size_t)-1); + const void* const dtPtr = DTable+1; + const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; -/* ************************************* -* Advanced functions -***************************************/ -typedef struct ZSTDv03_Dctx_s ZSTDv03_Dctx; -ZSTDv03_Dctx* ZSTDv03_createDCtx(void); -size_t ZSTDv03_freeDCtx(ZSTDv03_Dctx* dctx); + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + size_t const segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + U32 endSignal = 1; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; -size_t ZSTDv03_decompressDCtx(void* ctx, - void* dst, size_t maxOriginalSize, - const void* src, size_t compressedSize); + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + if (opStart4 > oend) return ERROR(corruption_detected); /* overflow */ + assert(dstSize >= 6 /* validated above */); + CHECK_F( BIT_initDStream(&bitD1, istart1, length1) ); + CHECK_F( BIT_initDStream(&bitD2, istart2, length2) ); + CHECK_F( BIT_initDStream(&bitD3, istart3, length3) ); + CHECK_F( BIT_initDStream(&bitD4, istart4, length4) ); -/* ************************************* -* Streaming functions -***************************************/ -size_t ZSTDv03_resetDCtx(ZSTDv03_Dctx* dctx); + /* 16-32 symbols per loop (4-8 symbols per stream) */ + if ((size_t)(oend - op4) >= sizeof(size_t)) { + for ( ; (endSignal) & (op4 < olimit); ) { +#if defined(__clang__) && (defined(__x86_64__) || defined(__i386__)) + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_1(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_0(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_1(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_0(op2, &bitD2); + endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished; + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_1(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_0(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_1(op4, &bitD4); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_0(op4, &bitD4); + endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished; +#else + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_1(op1, &bitD1); + HUF_DECODE_SYMBOLX2_1(op2, &bitD2); + HUF_DECODE_SYMBOLX2_1(op3, &bitD3); + HUF_DECODE_SYMBOLX2_1(op4, &bitD4); + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_0(op1, &bitD1); + HUF_DECODE_SYMBOLX2_0(op2, &bitD2); + HUF_DECODE_SYMBOLX2_0(op3, &bitD3); + HUF_DECODE_SYMBOLX2_0(op4, &bitD4); + endSignal = (U32)LIKELY((U32) + (BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished)); +#endif + } + } -size_t ZSTDv03_nextSrcSizeToDecompress(ZSTDv03_Dctx* dctx); -size_t ZSTDv03_decompressContinue(ZSTDv03_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); -/** - Use above functions alternatively. - ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue(). - ZSTD_decompressContinue() will use previous data blocks to improve compression if they are located prior to current block. - Result is the number of bytes regenerated within 'dst'. - It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header. -*/ + /* check corruption */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 already verified within main loop */ -/* ************************************* -* Prefix - version detection -***************************************/ -#define ZSTDv03_magicNumber 0xFD2FB523 /* v0.3 */ + /* finish bitStreams one by one */ + HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog); + HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog); + HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog); + HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog); + /* check */ + { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endCheck) return ERROR(corruption_detected); } -#if defined (__cplusplus) + /* decoded size */ + return dstSize; + } } -#endif -#endif /* ZSTD_V03_H_298734209782 */ -/**** ended inlining zstd_v03.h ****/ +#if HUF_NEED_BMI2_FUNCTION +static BMI2_TARGET_ATTRIBUTE +size_t HUF_decompress4X2_usingDTable_internal_bmi2(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} #endif -#if (ZSTD_LEGACY_SUPPORT <= 4) -/**** start inlining zstd_v04.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ -#ifndef ZSTD_V04_H_91868324769238 -#define ZSTD_V04_H_91868324769238 - -#if defined (__cplusplus) -extern "C" { -#endif +static +size_t HUF_decompress4X2_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} -/* ************************************* -* Includes -***************************************/ -#include /* size_t */ +#if ZSTD_ENABLE_ASM_X86_64_BMI2 +HUF_ASM_DECL void HUF_decompress4X2_usingDTable_internal_fast_asm_loop(HUF_DecompressFastArgs* args) ZSTDLIB_HIDDEN; -/* ************************************* -* Simple one-step function -***************************************/ -/** -ZSTDv04_decompress() : decompress ZSTD frames compliant with v0.4.x format - compressedSize : is the exact source size - maxOriginalSize : is the size of the 'dst' buffer, which must be already allocated. - It must be equal or larger than originalSize, otherwise decompression will fail. - return : the number of bytes decompressed into destination buffer (originalSize) - or an errorCode if it fails (which can be tested using ZSTDv01_isError()) -*/ -size_t ZSTDv04_decompress( void* dst, size_t maxOriginalSize, - const void* src, size_t compressedSize); +#endif - /** - ZSTDv04_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.4.x format - srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' - cSize (output parameter) : the number of bytes that would be read to decompress this frame - or an error code if it fails (which can be tested using ZSTDv01_isError()) - dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame - or ZSTD_CONTENTSIZE_ERROR if an error occurs +static HUF_FAST_BMI2_ATTRS +void HUF_decompress4X2_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs* args) +{ + U64 bits[4]; + BYTE const* ip[4]; + BYTE* op[4]; + BYTE* oend[4]; + HUF_DEltX2 const* const dtable = (HUF_DEltX2 const*)args->dt; + BYTE const* const ilowest = args->ilowest; - note : assumes `cSize` and `dBound` are _not_ NULL. - */ - void ZSTDv04_findFrameSizeInfoLegacy(const void *src, size_t srcSize, - size_t* cSize, unsigned long long* dBound); + /* Copy the arguments to local registers. */ + ZSTD_memcpy(&bits, &args->bits, sizeof(bits)); + ZSTD_memcpy((void*)(&ip), &args->ip, sizeof(ip)); + ZSTD_memcpy(&op, &args->op, sizeof(op)); -/** -ZSTDv04_isError() : tells if the result of ZSTDv04_decompress() is an error -*/ -unsigned ZSTDv04_isError(size_t code); + oend[0] = op[1]; + oend[1] = op[2]; + oend[2] = op[3]; + oend[3] = args->oend; + assert(MEM_isLittleEndian()); + assert(!MEM_32bits()); -/* ************************************* -* Advanced functions -***************************************/ -typedef struct ZSTDv04_Dctx_s ZSTDv04_Dctx; -ZSTDv04_Dctx* ZSTDv04_createDCtx(void); -size_t ZSTDv04_freeDCtx(ZSTDv04_Dctx* dctx); + for (;;) { + BYTE* olimit; + int stream; + + /* Assert loop preconditions */ +#ifndef NDEBUG + for (stream = 0; stream < 4; ++stream) { + assert(op[stream] <= oend[stream]); + assert(ip[stream] >= ilowest); + } +#endif + /* Compute olimit */ + { + /* Each loop does 5 table lookups for each of the 4 streams. + * Each table lookup consumes up to 11 bits of input, and produces + * up to 2 bytes of output. + */ + /* We can consume up to 7 bytes of input per iteration per stream. + * We also know that each input pointer is >= ip[0]. So we can run + * iters loops before running out of input. + */ + size_t iters = (size_t)(ip[0] - ilowest) / 7; + /* Each iteration can produce up to 10 bytes of output per stream. + * Each output stream my advance at different rates. So take the + * minimum number of safe iterations among all the output streams. + */ + for (stream = 0; stream < 4; ++stream) { + size_t const oiters = (size_t)(oend[stream] - op[stream]) / 10; + iters = MIN(iters, oiters); + } -size_t ZSTDv04_decompressDCtx(ZSTDv04_Dctx* dctx, - void* dst, size_t maxOriginalSize, - const void* src, size_t compressedSize); + /* Each iteration produces at least 5 output symbols. So until + * op[3] crosses olimit, we know we haven't executed iters + * iterations yet. This saves us maintaining an iters counter, + * at the expense of computing the remaining # of iterations + * more frequently. + */ + olimit = op[3] + (iters * 5); + /* Exit the fast decoding loop once we reach the end. */ + if (op[3] == olimit) + break; -/* ************************************* -* Direct Streaming -***************************************/ -size_t ZSTDv04_resetDCtx(ZSTDv04_Dctx* dctx); + /* Exit the decoding loop if any input pointer has crossed the + * previous one. This indicates corruption, and a precondition + * to our loop is that ip[i] >= ip[0]. + */ + for (stream = 1; stream < 4; ++stream) { + if (ip[stream] < ip[stream - 1]) + goto _out; + } + } -size_t ZSTDv04_nextSrcSizeToDecompress(ZSTDv04_Dctx* dctx); -size_t ZSTDv04_decompressContinue(ZSTDv04_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); -/** - Use above functions alternatively. - ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue(). - ZSTD_decompressContinue() will use previous data blocks to improve compression if they are located prior to current block. - Result is the number of bytes regenerated within 'dst'. - It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header. -*/ +#ifndef NDEBUG + for (stream = 1; stream < 4; ++stream) { + assert(ip[stream] >= ip[stream - 1]); + } +#endif +#define HUF_4X2_DECODE_SYMBOL(_stream, _decode3) \ + do { \ + if ((_decode3) || (_stream) != 3) { \ + int const index = (int)(bits[(_stream)] >> 53); \ + HUF_DEltX2 const entry = dtable[index]; \ + MEM_write16(op[(_stream)], entry.sequence); \ + bits[(_stream)] <<= (entry.nbBits) & 0x3F; \ + op[(_stream)] += (entry.length); \ + } \ + } while (0) + +#define HUF_4X2_RELOAD_STREAM(_stream) \ + do { \ + HUF_4X2_DECODE_SYMBOL(3, 1); \ + { \ + int const ctz = ZSTD_countTrailingZeros64(bits[(_stream)]); \ + int const nbBits = ctz & 7; \ + int const nbBytes = ctz >> 3; \ + ip[(_stream)] -= nbBytes; \ + bits[(_stream)] = MEM_read64(ip[(_stream)]) | 1; \ + bits[(_stream)] <<= nbBits; \ + } \ + } while (0) + + /* Manually unroll the loop because compilers don't consistently + * unroll the inner loops, which destroys performance. + */ + do { + /* Decode 5 symbols from each of the first 3 streams. + * The final stream will be decoded during the reload phase + * to reduce register pressure. + */ + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); + + /* Decode one symbol from the final stream */ + HUF_4X2_DECODE_SYMBOL(3, 1); + + /* Decode 4 symbols from the final stream & reload bitstreams. + * The final stream is reloaded last, meaning that all 5 symbols + * are decoded from the final stream before it is reloaded. + */ + HUF_4X_FOR_EACH_STREAM(HUF_4X2_RELOAD_STREAM); + } while (op[3] < olimit); + } -/* ************************************* -* Buffered Streaming -***************************************/ -typedef struct ZBUFFv04_DCtx_s ZBUFFv04_DCtx; -ZBUFFv04_DCtx* ZBUFFv04_createDCtx(void); -size_t ZBUFFv04_freeDCtx(ZBUFFv04_DCtx* dctx); +#undef HUF_4X2_DECODE_SYMBOL +#undef HUF_4X2_RELOAD_STREAM -size_t ZBUFFv04_decompressInit(ZBUFFv04_DCtx* dctx); -size_t ZBUFFv04_decompressWithDictionary(ZBUFFv04_DCtx* dctx, const void* dict, size_t dictSize); +_out: -size_t ZBUFFv04_decompressContinue(ZBUFFv04_DCtx* dctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr); + /* Save the final values of each of the state variables back to args. */ + ZSTD_memcpy(&args->bits, &bits, sizeof(bits)); + ZSTD_memcpy((void*)(&args->ip), &ip, sizeof(ip)); + ZSTD_memcpy(&args->op, &op, sizeof(op)); +} -/** ************************************************ -* Streaming decompression -* -* A ZBUFF_DCtx object is required to track streaming operation. -* Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources. -* Use ZBUFF_decompressInit() to start a new decompression operation. -* ZBUFF_DCtx objects can be reused multiple times. -* -* Optionally, a reference to a static dictionary can be set, using ZBUFF_decompressWithDictionary() -* It must be the same content as the one set during compression phase. -* Dictionary content must remain accessible during the decompression process. -* -* Use ZBUFF_decompressContinue() repetitively to consume your input. -* *srcSizePtr and *maxDstSizePtr can be any size. -* The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr. -* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. -* The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst. -* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency) -* or 0 when a frame is completely decoded -* or an error code, which can be tested using ZBUFF_isError(). -* -* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize / ZBUFF_recommendedDOutSize -* output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when it's decoded. -* input : ZBUFF_recommendedDInSize==128Kb+3; just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . -* **************************************************/ -unsigned ZBUFFv04_isError(size_t errorCode); -const char* ZBUFFv04_getErrorName(size_t errorCode); +static HUF_FAST_BMI2_ATTRS size_t +HUF_decompress4X2_usingDTable_internal_fast( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable, + HUF_DecompressFastLoopFn loopFn) { + void const* dt = DTable + 1; + const BYTE* const ilowest = (const BYTE*)cSrc; + BYTE* const oend = ZSTD_maybeNullPtrAdd((BYTE*)dst, dstSize); + HUF_DecompressFastArgs args; + { + size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable); + FORWARD_IF_ERROR(ret, "Failed to init asm args"); + if (ret == 0) + return 0; + } -/** The below functions provide recommended buffer sizes for Compression or Decompression operations. -* These sizes are not compulsory, they just tend to offer better latency */ -size_t ZBUFFv04_recommendedDInSize(void); -size_t ZBUFFv04_recommendedDOutSize(void); + assert(args.ip[0] >= args.ilowest); + loopFn(&args); + /* note : op4 already verified within main loop */ + assert(args.ip[0] >= ilowest); + assert(args.ip[1] >= ilowest); + assert(args.ip[2] >= ilowest); + assert(args.ip[3] >= ilowest); + assert(args.op[3] <= oend); -/* ************************************* -* Prefix - version detection -***************************************/ -#define ZSTDv04_magicNumber 0xFD2FB524 /* v0.4 */ + assert(ilowest == args.ilowest); + assert(ilowest + 6 == args.iend[0]); + (void)ilowest; + /* finish bitStreams one by one */ + { + size_t const segmentSize = (dstSize+3) / 4; + BYTE* segmentEnd = (BYTE*)dst; + int i; + for (i = 0; i < 4; ++i) { + BIT_DStream_t bit; + if (segmentSize <= (size_t)(oend - segmentEnd)) + segmentEnd += segmentSize; + else + segmentEnd = oend; + FORWARD_IF_ERROR(HUF_initRemainingDStream(&bit, &args, i, segmentEnd), "corruption"); + args.op[i] += HUF_decodeStreamX2(args.op[i], &bit, segmentEnd, (HUF_DEltX2 const*)dt, HUF_DECODER_FAST_TABLELOG); + if (args.op[i] != segmentEnd) + return ERROR(corruption_detected); + } + } -#if defined (__cplusplus) + /* decoded size */ + return dstSize; } -#endif - -#endif /* ZSTD_V04_H_91868324769238 */ -/**** ended inlining zstd_v04.h ****/ -#endif -#if (ZSTD_LEGACY_SUPPORT <= 5) -/**** start inlining zstd_v05.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ -#ifndef ZSTDv05_H -#define ZSTDv05_H +static size_t HUF_decompress4X2_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable, int flags) +{ + HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X2_usingDTable_internal_default; + HUF_DecompressFastLoopFn loopFn = HUF_decompress4X2_usingDTable_internal_fast_c_loop; -#if defined (__cplusplus) -extern "C" { +#if DYNAMIC_BMI2 + if (flags & HUF_flags_bmi2) { + fallbackFn = HUF_decompress4X2_usingDTable_internal_bmi2; +# if ZSTD_ENABLE_ASM_X86_64_BMI2 + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop; + } +# endif + } else { + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); + } #endif -/*-************************************* -* Dependencies -***************************************/ -#include /* size_t */ -/**** skipping file: ../common/mem.h ****/ - +#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__) + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop; + } +#endif -/* ************************************* -* Simple functions -***************************************/ -/*! ZSTDv05_decompress() : - `compressedSize` : is the _exact_ size of the compressed blob, otherwise decompression will fail. - `dstCapacity` must be large enough, equal or larger than originalSize. - @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), - or an errorCode if it fails (which can be tested using ZSTDv05_isError()) */ -size_t ZSTDv05_decompress( void* dst, size_t dstCapacity, - const void* src, size_t compressedSize); - - /** - ZSTDv05_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.5.x format - srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' - cSize (output parameter) : the number of bytes that would be read to decompress this frame - or an error code if it fails (which can be tested using ZSTDv01_isError()) - dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame - or ZSTD_CONTENTSIZE_ERROR if an error occurs - - note : assumes `cSize` and `dBound` are _not_ NULL. - */ -void ZSTDv05_findFrameSizeInfoLegacy(const void *src, size_t srcSize, - size_t* cSize, unsigned long long* dBound); + if (HUF_ENABLE_FAST_DECODE && !(flags & HUF_flags_disableFast)) { + size_t const ret = HUF_decompress4X2_usingDTable_internal_fast(dst, dstSize, cSrc, cSrcSize, DTable, loopFn); + if (ret != 0) + return ret; + } + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); +} -/* ************************************* -* Helper functions -***************************************/ -/* Error Management */ -unsigned ZSTDv05_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ -const char* ZSTDv05_getErrorName(size_t code); /*!< provides readable string for an error code */ +HUF_DGEN(HUF_decompress1X2_usingDTable_internal) +size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int flags) +{ + const BYTE* ip = (const BYTE*) cSrc; -/* ************************************* -* Explicit memory management -***************************************/ -/** Decompression context */ -typedef struct ZSTDv05_DCtx_s ZSTDv05_DCtx; -ZSTDv05_DCtx* ZSTDv05_createDCtx(void); -size_t ZSTDv05_freeDCtx(ZSTDv05_DCtx* dctx); /*!< @return : errorCode */ - -/** ZSTDv05_decompressDCtx() : -* Same as ZSTDv05_decompress(), but requires an already allocated ZSTDv05_DCtx (see ZSTDv05_createDCtx()) */ -size_t ZSTDv05_decompressDCtx(ZSTDv05_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); - - -/*-*********************** -* Simple Dictionary API -*************************/ -/*! ZSTDv05_decompress_usingDict() : -* Decompression using a pre-defined Dictionary content (see dictBuilder). -* Dictionary must be identical to the one used during compression, otherwise regenerated data will be corrupted. -* Note : dict can be NULL, in which case, it's equivalent to ZSTDv05_decompressDCtx() */ -size_t ZSTDv05_decompress_usingDict(ZSTDv05_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize); + size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize, + workSpace, wkspSize, flags); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; -/*-************************ -* Advanced Streaming API -***************************/ -typedef enum { ZSTDv05_fast, ZSTDv05_greedy, ZSTDv05_lazy, ZSTDv05_lazy2, ZSTDv05_btlazy2, ZSTDv05_opt, ZSTDv05_btopt } ZSTDv05_strategy; -typedef struct { - U64 srcSize; - U32 windowLog; /* the only useful information to retrieve */ - U32 contentLog; U32 hashLog; U32 searchLog; U32 searchLength; U32 targetLength; ZSTDv05_strategy strategy; -} ZSTDv05_parameters; -size_t ZSTDv05_getFrameParams(ZSTDv05_parameters* params, const void* src, size_t srcSize); + return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, flags); +} -size_t ZSTDv05_decompressBegin_usingDict(ZSTDv05_DCtx* dctx, const void* dict, size_t dictSize); -void ZSTDv05_copyDCtx(ZSTDv05_DCtx* dstDCtx, const ZSTDv05_DCtx* srcDCtx); -size_t ZSTDv05_nextSrcSizeToDecompress(ZSTDv05_DCtx* dctx); -size_t ZSTDv05_decompressContinue(ZSTDv05_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +static size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int flags) +{ + const BYTE* ip = (const BYTE*) cSrc; + size_t hSize = HUF_readDTableX2_wksp(dctx, cSrc, cSrcSize, + workSpace, wkspSize, flags); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; -/*-*********************** -* ZBUFF API -*************************/ -typedef struct ZBUFFv05_DCtx_s ZBUFFv05_DCtx; -ZBUFFv05_DCtx* ZBUFFv05_createDCtx(void); -size_t ZBUFFv05_freeDCtx(ZBUFFv05_DCtx* dctx); + return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags); +} -size_t ZBUFFv05_decompressInit(ZBUFFv05_DCtx* dctx); -size_t ZBUFFv05_decompressInitDictionary(ZBUFFv05_DCtx* dctx, const void* dict, size_t dictSize); +#endif /* HUF_FORCE_DECOMPRESS_X1 */ -size_t ZBUFFv05_decompressContinue(ZBUFFv05_DCtx* dctx, - void* dst, size_t* dstCapacityPtr, - const void* src, size_t* srcSizePtr); -/*-*************************************************************************** -* Streaming decompression -* -* A ZBUFFv05_DCtx object is required to track streaming operations. -* Use ZBUFFv05_createDCtx() and ZBUFFv05_freeDCtx() to create/release resources. -* Use ZBUFFv05_decompressInit() to start a new decompression operation, -* or ZBUFFv05_decompressInitDictionary() if decompression requires a dictionary. -* Note that ZBUFFv05_DCtx objects can be reused multiple times. -* -* Use ZBUFFv05_decompressContinue() repetitively to consume your input. -* *srcSizePtr and *dstCapacityPtr can be any size. -* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. -* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. -* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters or change @dst. -* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency) -* or 0 when a frame is completely decoded -* or an error code, which can be tested using ZBUFFv05_isError(). -* -* Hint : recommended buffer sizes (not compulsory) : ZBUFFv05_recommendedDInSize() / ZBUFFv05_recommendedDOutSize() -* output : ZBUFFv05_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. -* input : ZBUFFv05_recommendedDInSize==128Kb+3; just follow indications from ZBUFFv05_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . -* *******************************************************************************/ +/* ***********************************/ +/* Universal decompression selectors */ +/* ***********************************/ -/* ************************************* -* Tool functions -***************************************/ -unsigned ZBUFFv05_isError(size_t errorCode); -const char* ZBUFFv05_getErrorName(size_t errorCode); +#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2) +typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t; +static const algo_time_t algoTime[16 /* Quantization */][2 /* single, double */] = +{ + /* single, double, quad */ + {{0,0}, {1,1}}, /* Q==0 : impossible */ + {{0,0}, {1,1}}, /* Q==1 : impossible */ + {{ 150,216}, { 381,119}}, /* Q == 2 : 12-18% */ + {{ 170,205}, { 514,112}}, /* Q == 3 : 18-25% */ + {{ 177,199}, { 539,110}}, /* Q == 4 : 25-32% */ + {{ 197,194}, { 644,107}}, /* Q == 5 : 32-38% */ + {{ 221,192}, { 735,107}}, /* Q == 6 : 38-44% */ + {{ 256,189}, { 881,106}}, /* Q == 7 : 44-50% */ + {{ 359,188}, {1167,109}}, /* Q == 8 : 50-56% */ + {{ 582,187}, {1570,114}}, /* Q == 9 : 56-62% */ + {{ 688,187}, {1712,122}}, /* Q ==10 : 62-69% */ + {{ 825,186}, {1965,136}}, /* Q ==11 : 69-75% */ + {{ 976,185}, {2131,150}}, /* Q ==12 : 75-81% */ + {{1180,186}, {2070,175}}, /* Q ==13 : 81-87% */ + {{1377,185}, {1731,202}}, /* Q ==14 : 87-93% */ + {{1412,185}, {1695,202}}, /* Q ==15 : 93-99% */ +}; +#endif -/** Functions below provide recommended buffer sizes for Compression or Decompression operations. -* These sizes are just hints, and tend to offer better latency */ -size_t ZBUFFv05_recommendedDInSize(void); -size_t ZBUFFv05_recommendedDOutSize(void); +/** HUF_selectDecoder() : + * Tells which decoder is likely to decode faster, + * based on a set of pre-computed metrics. + * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 . + * Assumption : 0 < dstSize <= 128 KB */ +U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize) +{ + assert(dstSize > 0); + assert(dstSize <= 128*1024); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dstSize; + (void)cSrcSize; + return 0; +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dstSize; + (void)cSrcSize; + return 1; +#else + /* decoder timing evaluation */ + { U32 const Q = (cSrcSize >= dstSize) ? 15 : (U32)(cSrcSize * 16 / dstSize); /* Q < 16 */ + U32 const D256 = (U32)(dstSize >> 8); + U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256); + U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256); + DTime1 += DTime1 >> 5; /* small advantage to algorithm using less memory, to reduce cache eviction */ + return DTime1 < DTime0; + } +#endif +} +size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int flags) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ + if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize, flags); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize, flags); +#else + return algoNb ? HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize, flags): + HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize, flags); +#endif + } +} -/*-************************************* -* Constants -***************************************/ -#define ZSTDv05_MAGICNUMBER 0xFD2FB525 /* v0.5 */ +size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dtd; + assert(dtd.tableType == 0); + return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dtd; + assert(dtd.tableType == 1); + return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#else + return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags) : + HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#endif +} +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags) +{ + const BYTE* ip = (const BYTE*) cSrc; + size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize, flags); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; -#if defined (__cplusplus) + return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags); } #endif -#endif /* ZSTDv0505_H */ -/**** ended inlining zstd_v05.h ****/ +size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dtd; + assert(dtd.tableType == 0); + return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dtd; + assert(dtd.tableType == 1); + return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#else + return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags) : + HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#endif +} + +size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize == 0) return ERROR(corruption_detected); + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags); +#else + return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags) : + HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags); #endif -#if (ZSTD_LEGACY_SUPPORT <= 6) -/**** start inlining zstd_v06.h ****/ + } +} +/**** ended inlining decompress/huf_decompress.c ****/ +/**** start inlining decompress/zstd_ddict.c ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -10318,174 +16965,78 @@ size_t ZBUFFv05_recommendedDOutSize(void); * You may select, at your option, one of the above-listed licenses. */ -#ifndef ZSTDv06_H -#define ZSTDv06_H - -#if defined (__cplusplus) -extern "C" { -#endif - -/*====== Dependency ======*/ -#include /* size_t */ - - -/*====== Export for Windows ======*/ -/*! -* ZSTDv06_DLL_EXPORT : -* Enable exporting of functions when building a Windows DLL -*/ -#if defined(_WIN32) && defined(ZSTDv06_DLL_EXPORT) && (ZSTDv06_DLL_EXPORT==1) -# define ZSTDLIBv06_API __declspec(dllexport) -#else -# define ZSTDLIBv06_API -#endif - - -/* ************************************* -* Simple functions -***************************************/ -/*! ZSTDv06_decompress() : - `compressedSize` : is the _exact_ size of the compressed blob, otherwise decompression will fail. - `dstCapacity` must be large enough, equal or larger than originalSize. - @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), - or an errorCode if it fails (which can be tested using ZSTDv06_isError()) */ -ZSTDLIBv06_API size_t ZSTDv06_decompress( void* dst, size_t dstCapacity, - const void* src, size_t compressedSize); - -/** -ZSTDv06_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.6.x format - srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' - cSize (output parameter) : the number of bytes that would be read to decompress this frame - or an error code if it fails (which can be tested using ZSTDv01_isError()) - dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame - or ZSTD_CONTENTSIZE_ERROR if an error occurs - - note : assumes `cSize` and `dBound` are _not_ NULL. -*/ -void ZSTDv06_findFrameSizeInfoLegacy(const void *src, size_t srcSize, - size_t* cSize, unsigned long long* dBound); - -/* ************************************* -* Helper functions -***************************************/ -ZSTDLIBv06_API size_t ZSTDv06_compressBound(size_t srcSize); /*!< maximum compressed size (worst case scenario) */ - -/* Error Management */ -ZSTDLIBv06_API unsigned ZSTDv06_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ -ZSTDLIBv06_API const char* ZSTDv06_getErrorName(size_t code); /*!< provides readable string for an error code */ - - -/* ************************************* -* Explicit memory management -***************************************/ -/** Decompression context */ -typedef struct ZSTDv06_DCtx_s ZSTDv06_DCtx; -ZSTDLIBv06_API ZSTDv06_DCtx* ZSTDv06_createDCtx(void); -ZSTDLIBv06_API size_t ZSTDv06_freeDCtx(ZSTDv06_DCtx* dctx); /*!< @return : errorCode */ - -/** ZSTDv06_decompressDCtx() : -* Same as ZSTDv06_decompress(), but requires an already allocated ZSTDv06_DCtx (see ZSTDv06_createDCtx()) */ -ZSTDLIBv06_API size_t ZSTDv06_decompressDCtx(ZSTDv06_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); - - -/*-*********************** -* Dictionary API -*************************/ -/*! ZSTDv06_decompress_usingDict() : -* Decompression using a pre-defined Dictionary content (see dictBuilder). -* Dictionary must be identical to the one used during compression, otherwise regenerated data will be corrupted. -* Note : dict can be NULL, in which case, it's equivalent to ZSTDv06_decompressDCtx() */ -ZSTDLIBv06_API size_t ZSTDv06_decompress_usingDict(ZSTDv06_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize); - - -/*-************************ -* Advanced Streaming API -***************************/ -struct ZSTDv06_frameParams_s { unsigned long long frameContentSize; unsigned windowLog; }; -typedef struct ZSTDv06_frameParams_s ZSTDv06_frameParams; - -ZSTDLIBv06_API size_t ZSTDv06_getFrameParams(ZSTDv06_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */ -ZSTDLIBv06_API size_t ZSTDv06_decompressBegin_usingDict(ZSTDv06_DCtx* dctx, const void* dict, size_t dictSize); -ZSTDLIBv06_API void ZSTDv06_copyDCtx(ZSTDv06_DCtx* dctx, const ZSTDv06_DCtx* preparedDCtx); - -ZSTDLIBv06_API size_t ZSTDv06_nextSrcSizeToDecompress(ZSTDv06_DCtx* dctx); -ZSTDLIBv06_API size_t ZSTDv06_decompressContinue(ZSTDv06_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); - - - -/* ************************************* -* ZBUFF API -***************************************/ - -typedef struct ZBUFFv06_DCtx_s ZBUFFv06_DCtx; -ZSTDLIBv06_API ZBUFFv06_DCtx* ZBUFFv06_createDCtx(void); -ZSTDLIBv06_API size_t ZBUFFv06_freeDCtx(ZBUFFv06_DCtx* dctx); - -ZSTDLIBv06_API size_t ZBUFFv06_decompressInit(ZBUFFv06_DCtx* dctx); -ZSTDLIBv06_API size_t ZBUFFv06_decompressInitDictionary(ZBUFFv06_DCtx* dctx, const void* dict, size_t dictSize); - -ZSTDLIBv06_API size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* dctx, - void* dst, size_t* dstCapacityPtr, - const void* src, size_t* srcSizePtr); +/* zstd_ddict.c : + * concentrates all logic that needs to know the internals of ZSTD_DDict object */ -/*-*************************************************************************** -* Streaming decompression howto -* -* A ZBUFFv06_DCtx object is required to track streaming operations. -* Use ZBUFFv06_createDCtx() and ZBUFFv06_freeDCtx() to create/release resources. -* Use ZBUFFv06_decompressInit() to start a new decompression operation, -* or ZBUFFv06_decompressInitDictionary() if decompression requires a dictionary. -* Note that ZBUFFv06_DCtx objects can be re-init multiple times. -* -* Use ZBUFFv06_decompressContinue() repetitively to consume your input. -* *srcSizePtr and *dstCapacityPtr can be any size. -* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. -* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. -* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change `dst`. -* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency), -* or 0 when a frame is completely decoded, -* or an error code, which can be tested using ZBUFFv06_isError(). -* -* Hint : recommended buffer sizes (not compulsory) : ZBUFFv06_recommendedDInSize() and ZBUFFv06_recommendedDOutSize() -* output : ZBUFFv06_recommendedDOutSize== 128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. -* input : ZBUFFv06_recommendedDInSize == 128KB + 3; -* just follow indications from ZBUFFv06_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . -* *******************************************************************************/ +/*-******************************************************* +* Dependencies +*********************************************************/ +/**** start inlining ../common/allocations.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ +/* This file provides custom allocation primitives + */ -/* ************************************* -* Tool functions -***************************************/ -ZSTDLIBv06_API unsigned ZBUFFv06_isError(size_t errorCode); -ZSTDLIBv06_API const char* ZBUFFv06_getErrorName(size_t errorCode); +#define ZSTD_DEPS_NEED_MALLOC +/**** skipping file: zstd_deps.h ****/ -/** Functions below provide recommended buffer sizes for Compression or Decompression operations. -* These sizes are just hints, they tend to offer better latency */ -ZSTDLIBv06_API size_t ZBUFFv06_recommendedDInSize(void); -ZSTDLIBv06_API size_t ZBUFFv06_recommendedDOutSize(void); +/**** skipping file: compiler.h ****/ +#define ZSTD_STATIC_LINKING_ONLY +/**** skipping file: ../zstd.h ****/ +#ifndef ZSTD_ALLOCATIONS_H +#define ZSTD_ALLOCATIONS_H -/*-************************************* -* Constants -***************************************/ -#define ZSTDv06_MAGICNUMBER 0xFD2FB526 /* v0.6 */ +/* custom memory allocation functions */ +MEM_STATIC void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem) +{ + if (customMem.customAlloc) + return customMem.customAlloc(customMem.opaque, size); + return ZSTD_malloc(size); +} +MEM_STATIC void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem) +{ + if (customMem.customAlloc) { + /* calloc implemented as malloc+memset; + * not as efficient as calloc, but next best guess for custom malloc */ + void* const ptr = customMem.customAlloc(customMem.opaque, size); + ZSTD_memset(ptr, 0, size); + return ptr; + } + return ZSTD_calloc(1, size); +} -#if defined (__cplusplus) +MEM_STATIC void ZSTD_customFree(void* ptr, ZSTD_customMem customMem) +{ + if (ptr!=NULL) { + if (customMem.customFree) + customMem.customFree(customMem.opaque, ptr); + else + ZSTD_free(ptr); + } } -#endif -#endif /* ZSTDv06_BUFFERED_H */ -/**** ended inlining zstd_v06.h ****/ -#endif -#if (ZSTD_LEGACY_SUPPORT <= 7) -/**** start inlining zstd_v07.h ****/ +#endif /* ZSTD_ALLOCATIONS_H */ +/**** ended inlining ../common/allocations.h ****/ +/**** skipping file: ../common/zstd_deps.h ****/ +/**** skipping file: ../common/cpu.h ****/ +/**** skipping file: ../common/mem.h ****/ +#define FSE_STATIC_LINKING_ONLY +/**** skipping file: ../common/fse.h ****/ +/**** skipping file: ../common/huf.h ****/ +/**** start inlining zstd_decompress_internal.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -10494,551 +17045,286 @@ ZSTDLIBv06_API size_t ZBUFFv06_recommendedDOutSize(void); * You may select, at your option, one of the above-listed licenses. */ -#ifndef ZSTDv07_H_235446 -#define ZSTDv07_H_235446 -#if defined (__cplusplus) -extern "C" { -#endif +/* zstd_decompress_internal: + * objects and definitions shared within lib/decompress modules */ -/*====== Dependency ======*/ -#include /* size_t */ + #ifndef ZSTD_DECOMPRESS_INTERNAL_H + #define ZSTD_DECOMPRESS_INTERNAL_H -/*====== Export for Windows ======*/ -/*! -* ZSTDv07_DLL_EXPORT : -* Enable exporting of functions when building a Windows DLL -*/ -#if defined(_WIN32) && defined(ZSTDv07_DLL_EXPORT) && (ZSTDv07_DLL_EXPORT==1) -# define ZSTDLIBv07_API __declspec(dllexport) -#else -# define ZSTDLIBv07_API -#endif +/*-******************************************************* + * Dependencies + *********************************************************/ +/**** skipping file: ../common/mem.h ****/ +/**** skipping file: ../common/zstd_internal.h ****/ -/* ************************************* -* Simple API -***************************************/ -/*! ZSTDv07_getDecompressedSize() : -* @return : decompressed size if known, 0 otherwise. - note 1 : if `0`, follow up with ZSTDv07_getFrameParams() to know precise failure cause. - note 2 : decompressed size could be wrong or intentionally modified ! - always ensure results fit within application's authorized limits */ -unsigned long long ZSTDv07_getDecompressedSize(const void* src, size_t srcSize); - -/*! ZSTDv07_decompress() : - `compressedSize` : must be _exact_ size of compressed input, otherwise decompression will fail. - `dstCapacity` must be equal or larger than originalSize. - @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), - or an errorCode if it fails (which can be tested using ZSTDv07_isError()) */ -ZSTDLIBv07_API size_t ZSTDv07_decompress( void* dst, size_t dstCapacity, - const void* src, size_t compressedSize); -/** -ZSTDv07_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.7.x format - srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' - cSize (output parameter) : the number of bytes that would be read to decompress this frame - or an error code if it fails (which can be tested using ZSTDv01_isError()) - dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame - or ZSTD_CONTENTSIZE_ERROR if an error occurs - - note : assumes `cSize` and `dBound` are _not_ NULL. -*/ -void ZSTDv07_findFrameSizeInfoLegacy(const void *src, size_t srcSize, - size_t* cSize, unsigned long long* dBound); +/*-******************************************************* + * Constants + *********************************************************/ +static UNUSED_ATTR const U32 LL_base[MaxLL+1] = { + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 18, 20, 22, 24, 28, 32, 40, + 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, + 0x2000, 0x4000, 0x8000, 0x10000 }; -/*====== Helper functions ======*/ -ZSTDLIBv07_API unsigned ZSTDv07_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ -ZSTDLIBv07_API const char* ZSTDv07_getErrorName(size_t code); /*!< provides readable string from an error code */ +static UNUSED_ATTR const U32 OF_base[MaxOff+1] = { + 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, + 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, + 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, + 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD }; +static UNUSED_ATTR const U8 OF_bits[MaxOff+1] = { + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31 }; -/*-************************************* -* Explicit memory management -***************************************/ -/** Decompression context */ -typedef struct ZSTDv07_DCtx_s ZSTDv07_DCtx; -ZSTDLIBv07_API ZSTDv07_DCtx* ZSTDv07_createDCtx(void); -ZSTDLIBv07_API size_t ZSTDv07_freeDCtx(ZSTDv07_DCtx* dctx); /*!< @return : errorCode */ +static UNUSED_ATTR const U32 ML_base[MaxML+1] = { + 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, + 27, 28, 29, 30, 31, 32, 33, 34, + 35, 37, 39, 41, 43, 47, 51, 59, + 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, + 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 }; -/** ZSTDv07_decompressDCtx() : -* Same as ZSTDv07_decompress(), requires an allocated ZSTDv07_DCtx (see ZSTDv07_createDCtx()) */ -ZSTDLIBv07_API size_t ZSTDv07_decompressDCtx(ZSTDv07_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +/*-******************************************************* + * Decompression types + *********************************************************/ + typedef struct { + U32 fastMode; + U32 tableLog; + } ZSTD_seqSymbol_header; -/*-************************ -* Simple dictionary API -***************************/ -/*! ZSTDv07_decompress_usingDict() : -* Decompression using a pre-defined Dictionary content (see dictBuilder). -* Dictionary must be identical to the one used during compression. -* Note : This function load the dictionary, resulting in a significant startup time */ -ZSTDLIBv07_API size_t ZSTDv07_decompress_usingDict(ZSTDv07_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize); + typedef struct { + U16 nextState; + BYTE nbAdditionalBits; + BYTE nbBits; + U32 baseValue; + } ZSTD_seqSymbol; + #define SEQSYMBOL_TABLE_SIZE(log) (1 + (1 << (log))) -/*-************************** -* Advanced Dictionary API -****************************/ -/*! ZSTDv07_createDDict() : -* Create a digested dictionary, ready to start decompression operation without startup delay. -* `dict` can be released after creation */ -typedef struct ZSTDv07_DDict_s ZSTDv07_DDict; -ZSTDLIBv07_API ZSTDv07_DDict* ZSTDv07_createDDict(const void* dict, size_t dictSize); -ZSTDLIBv07_API size_t ZSTDv07_freeDDict(ZSTDv07_DDict* ddict); - -/*! ZSTDv07_decompress_usingDDict() : -* Decompression using a pre-digested Dictionary -* Faster startup than ZSTDv07_decompress_usingDict(), recommended when same dictionary is used multiple times. */ -ZSTDLIBv07_API size_t ZSTDv07_decompress_usingDDict(ZSTDv07_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTDv07_DDict* ddict); +#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE (sizeof(S16) * (MaxSeq + 1) + (1u << MaxFSELog) + sizeof(U64)) +#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32 ((ZSTD_BUILD_FSE_TABLE_WKSP_SIZE + sizeof(U32) - 1) / sizeof(U32)) +#define ZSTD_HUFFDTABLE_CAPACITY_LOG 12 typedef struct { - unsigned long long frameContentSize; - unsigned windowSize; - unsigned dictID; - unsigned checksumFlag; -} ZSTDv07_frameParams; - -ZSTDLIBv07_API size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */ - - - - -/* ************************************* -* Streaming functions -***************************************/ -typedef struct ZBUFFv07_DCtx_s ZBUFFv07_DCtx; -ZSTDLIBv07_API ZBUFFv07_DCtx* ZBUFFv07_createDCtx(void); -ZSTDLIBv07_API size_t ZBUFFv07_freeDCtx(ZBUFFv07_DCtx* dctx); - -ZSTDLIBv07_API size_t ZBUFFv07_decompressInit(ZBUFFv07_DCtx* dctx); -ZSTDLIBv07_API size_t ZBUFFv07_decompressInitDictionary(ZBUFFv07_DCtx* dctx, const void* dict, size_t dictSize); + ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]; /* Note : Space reserved for FSE Tables */ + ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]; /* is also used as temporary workspace while building hufTable during DDict creation */ + ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]; /* and therefore must be at least HUF_DECOMPRESS_WORKSPACE_SIZE large */ + HUF_DTable hufTable[HUF_DTABLE_SIZE(ZSTD_HUFFDTABLE_CAPACITY_LOG)]; /* can accommodate HUF_decompress4X */ + U32 rep[ZSTD_REP_NUM]; + U32 workspace[ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32]; +} ZSTD_entropyDTables_t; -ZSTDLIBv07_API size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* dctx, - void* dst, size_t* dstCapacityPtr, - const void* src, size_t* srcSizePtr); +typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, + ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock, + ZSTDds_decompressLastBlock, ZSTDds_checkChecksum, + ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage; -/*-*************************************************************************** -* Streaming decompression howto -* -* A ZBUFFv07_DCtx object is required to track streaming operations. -* Use ZBUFFv07_createDCtx() and ZBUFFv07_freeDCtx() to create/release resources. -* Use ZBUFFv07_decompressInit() to start a new decompression operation, -* or ZBUFFv07_decompressInitDictionary() if decompression requires a dictionary. -* Note that ZBUFFv07_DCtx objects can be re-init multiple times. -* -* Use ZBUFFv07_decompressContinue() repetitively to consume your input. -* *srcSizePtr and *dstCapacityPtr can be any size. -* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. -* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. -* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change `dst`. -* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency), -* or 0 when a frame is completely decoded, -* or an error code, which can be tested using ZBUFFv07_isError(). -* -* Hint : recommended buffer sizes (not compulsory) : ZBUFFv07_recommendedDInSize() and ZBUFFv07_recommendedDOutSize() -* output : ZBUFFv07_recommendedDOutSize== 128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. -* input : ZBUFFv07_recommendedDInSize == 128KB + 3; -* just follow indications from ZBUFFv07_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . -* *******************************************************************************/ +typedef enum { zdss_init=0, zdss_loadHeader, + zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage; +typedef enum { + ZSTD_use_indefinitely = -1, /* Use the dictionary indefinitely */ + ZSTD_dont_use = 0, /* Do not use the dictionary (if one exists free it) */ + ZSTD_use_once = 1 /* Use the dictionary once and set to ZSTD_dont_use */ +} ZSTD_dictUses_e; -/* ************************************* -* Tool functions -***************************************/ -ZSTDLIBv07_API unsigned ZBUFFv07_isError(size_t errorCode); -ZSTDLIBv07_API const char* ZBUFFv07_getErrorName(size_t errorCode); +/* Hashset for storing references to multiple ZSTD_DDict within ZSTD_DCtx */ +typedef struct { + const ZSTD_DDict** ddictPtrTable; + size_t ddictPtrTableSize; + size_t ddictPtrCount; +} ZSTD_DDictHashSet; -/** Functions below provide recommended buffer sizes for Compression or Decompression operations. -* These sizes are just hints, they tend to offer better latency */ -ZSTDLIBv07_API size_t ZBUFFv07_recommendedDInSize(void); -ZSTDLIBv07_API size_t ZBUFFv07_recommendedDOutSize(void); +#ifndef ZSTD_DECODER_INTERNAL_BUFFER +# define ZSTD_DECODER_INTERNAL_BUFFER (1 << 16) +#endif +#define ZSTD_LBMIN 64 +#define ZSTD_LBMAX (128 << 10) -/*-************************************* -* Constants -***************************************/ -#define ZSTDv07_MAGICNUMBER 0xFD2FB527 /* v0.7 */ +/* extra buffer, compensates when dst is not large enough to store litBuffer */ +#define ZSTD_LITBUFFEREXTRASIZE BOUNDED(ZSTD_LBMIN, ZSTD_DECODER_INTERNAL_BUFFER, ZSTD_LBMAX) +typedef enum { + ZSTD_not_in_dst = 0, /* Stored entirely within litExtraBuffer */ + ZSTD_in_dst = 1, /* Stored entirely within dst (in memory after current output write) */ + ZSTD_split = 2 /* Split between litExtraBuffer and dst */ +} ZSTD_litLocation_e; -#if defined (__cplusplus) -} +struct ZSTD_DCtx_s +{ + const ZSTD_seqSymbol* LLTptr; + const ZSTD_seqSymbol* MLTptr; + const ZSTD_seqSymbol* OFTptr; + const HUF_DTable* HUFptr; + ZSTD_entropyDTables_t entropy; + U32 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; /* space needed when building huffman tables */ + const void* previousDstEnd; /* detect continuity */ + const void* prefixStart; /* start of current segment */ + const void* virtualStart; /* virtual start of previous segment if it was just before current one */ + const void* dictEnd; /* end of previous segment */ + size_t expected; + ZSTD_FrameHeader fParams; + U64 processedCSize; + U64 decodedSize; + blockType_e bType; /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block decompression stages */ + ZSTD_dStage stage; + U32 litEntropy; + U32 fseEntropy; + XXH64_state_t xxhState; + size_t headerSize; + ZSTD_format_e format; + ZSTD_forceIgnoreChecksum_e forceIgnoreChecksum; /* User specified: if == 1, will ignore checksums in compressed frame. Default == 0 */ + U32 validateChecksum; /* if == 1, will validate checksum. Is == 1 if (fParams.checksumFlag == 1) and (forceIgnoreChecksum == 0). */ + const BYTE* litPtr; + ZSTD_customMem customMem; + size_t litSize; + size_t rleSize; + size_t staticSize; + int isFrameDecompression; +#if DYNAMIC_BMI2 + int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */ #endif -#endif /* ZSTDv07_H_235446 */ -/**** ended inlining zstd_v07.h ****/ -#endif + /* dictionary */ + ZSTD_DDict* ddictLocal; + const ZSTD_DDict* ddict; /* set by ZSTD_initDStream_usingDDict(), or ZSTD_DCtx_refDDict() */ + U32 dictID; + int ddictIsCold; /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */ + ZSTD_dictUses_e dictUses; + ZSTD_DDictHashSet* ddictSet; /* Hash set for multiple ddicts */ + ZSTD_refMultipleDDicts_e refMultipleDDicts; /* User specified: if == 1, will allow references to multiple DDicts. Default == 0 (disabled) */ + int disableHufAsm; + int maxBlockSizeParam; -/** ZSTD_isLegacy() : - @return : > 0 if supported by legacy decoder. 0 otherwise. - return value is the version. -*/ -MEM_STATIC unsigned ZSTD_isLegacy(const void* src, size_t srcSize) -{ - U32 magicNumberLE; - if (srcSize<4) return 0; - magicNumberLE = MEM_readLE32(src); - switch(magicNumberLE) - { -#if (ZSTD_LEGACY_SUPPORT <= 1) - case ZSTDv01_magicNumberLE:return 1; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 2) - case ZSTDv02_magicNumber : return 2; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 3) - case ZSTDv03_magicNumber : return 3; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 4) - case ZSTDv04_magicNumber : return 4; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 5) - case ZSTDv05_MAGICNUMBER : return 5; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 6) - case ZSTDv06_MAGICNUMBER : return 6; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 7) - case ZSTDv07_MAGICNUMBER : return 7; + /* streaming */ + ZSTD_dStreamStage streamStage; + char* inBuff; + size_t inBuffSize; + size_t inPos; + size_t maxWindowSize; + char* outBuff; + size_t outBuffSize; + size_t outStart; + size_t outEnd; + size_t lhSize; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + void* legacyContext; + U32 previousLegacyVersion; + U32 legacyVersion; #endif - default : return 0; - } -} + U32 hostageByte; + int noForwardProgress; + ZSTD_bufferMode_e outBufferMode; + ZSTD_outBuffer expectedOutBuffer; + + /* workspace */ + BYTE* litBuffer; + const BYTE* litBufferEnd; + ZSTD_litLocation_e litBufferLocation; + BYTE litExtraBuffer[ZSTD_LITBUFFEREXTRASIZE + WILDCOPY_OVERLENGTH]; /* literal buffer can be split between storage within dst and within this scratch buffer */ + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; + size_t oversizedDuration; -MEM_STATIC unsigned long long ZSTD_getDecompressedSize_legacy(const void* src, size_t srcSize) -{ - U32 const version = ZSTD_isLegacy(src, srcSize); - if (version < 5) return 0; /* no decompressed size in frame header, or not a legacy format */ -#if (ZSTD_LEGACY_SUPPORT <= 5) - if (version==5) { - ZSTDv05_parameters fParams; - size_t const frResult = ZSTDv05_getFrameParams(&fParams, src, srcSize); - if (frResult != 0) return 0; - return fParams.srcSize; - } +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + void const* dictContentBeginForFuzzing; + void const* dictContentEndForFuzzing; #endif -#if (ZSTD_LEGACY_SUPPORT <= 6) - if (version==6) { - ZSTDv06_frameParams fParams; - size_t const frResult = ZSTDv06_getFrameParams(&fParams, src, srcSize); - if (frResult != 0) return 0; - return fParams.frameContentSize; - } + + /* Tracing */ +#if ZSTD_TRACE + ZSTD_TraceCtx traceCtx; #endif -#if (ZSTD_LEGACY_SUPPORT <= 7) - if (version==7) { - ZSTDv07_frameParams fParams; - size_t const frResult = ZSTDv07_getFrameParams(&fParams, src, srcSize); - if (frResult != 0) return 0; - return fParams.frameContentSize; - } +}; /* typedef'd to ZSTD_DCtx within "zstd.h" */ + +MEM_STATIC int ZSTD_DCtx_get_bmi2(const struct ZSTD_DCtx_s *dctx) { +#if DYNAMIC_BMI2 + return dctx->bmi2; +#else + (void)dctx; + return 0; #endif - return 0; /* should not be possible */ } +/*-******************************************************* + * Shared internal functions + *********************************************************/ -MEM_STATIC size_t ZSTD_decompressLegacy( - void* dst, size_t dstCapacity, - const void* src, size_t compressedSize, - const void* dict,size_t dictSize) -{ - U32 const version = ZSTD_isLegacy(src, compressedSize); - (void)dst; (void)dstCapacity; (void)dict; (void)dictSize; /* unused when ZSTD_LEGACY_SUPPORT >= 8 */ - switch(version) - { -#if (ZSTD_LEGACY_SUPPORT <= 1) - case 1 : - return ZSTDv01_decompress(dst, dstCapacity, src, compressedSize); -#endif -#if (ZSTD_LEGACY_SUPPORT <= 2) - case 2 : - return ZSTDv02_decompress(dst, dstCapacity, src, compressedSize); -#endif -#if (ZSTD_LEGACY_SUPPORT <= 3) - case 3 : - return ZSTDv03_decompress(dst, dstCapacity, src, compressedSize); -#endif -#if (ZSTD_LEGACY_SUPPORT <= 4) - case 4 : - return ZSTDv04_decompress(dst, dstCapacity, src, compressedSize); -#endif -#if (ZSTD_LEGACY_SUPPORT <= 5) - case 5 : - { size_t result; - ZSTDv05_DCtx* const zd = ZSTDv05_createDCtx(); - if (zd==NULL) return ERROR(memory_allocation); - result = ZSTDv05_decompress_usingDict(zd, dst, dstCapacity, src, compressedSize, dict, dictSize); - ZSTDv05_freeDCtx(zd); - return result; - } -#endif -#if (ZSTD_LEGACY_SUPPORT <= 6) - case 6 : - { size_t result; - ZSTDv06_DCtx* const zd = ZSTDv06_createDCtx(); - if (zd==NULL) return ERROR(memory_allocation); - result = ZSTDv06_decompress_usingDict(zd, dst, dstCapacity, src, compressedSize, dict, dictSize); - ZSTDv06_freeDCtx(zd); - return result; - } -#endif -#if (ZSTD_LEGACY_SUPPORT <= 7) - case 7 : - { size_t result; - ZSTDv07_DCtx* const zd = ZSTDv07_createDCtx(); - if (zd==NULL) return ERROR(memory_allocation); - result = ZSTDv07_decompress_usingDict(zd, dst, dstCapacity, src, compressedSize, dict, dictSize); - ZSTDv07_freeDCtx(zd); - return result; - } -#endif - default : - return ERROR(prefix_unknown); - } -} +/*! ZSTD_loadDEntropy() : + * dict : must point at beginning of a valid zstd dictionary. + * @return : size of dictionary header (size of magic number + dict ID + entropy tables) */ +size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, + const void* const dict, size_t const dictSize); -MEM_STATIC ZSTD_frameSizeInfo ZSTD_findFrameSizeInfoLegacy(const void *src, size_t srcSize) -{ - ZSTD_frameSizeInfo frameSizeInfo; - U32 const version = ZSTD_isLegacy(src, srcSize); - switch(version) - { -#if (ZSTD_LEGACY_SUPPORT <= 1) - case 1 : - ZSTDv01_findFrameSizeInfoLegacy(src, srcSize, - &frameSizeInfo.compressedSize, - &frameSizeInfo.decompressedBound); - break; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 2) - case 2 : - ZSTDv02_findFrameSizeInfoLegacy(src, srcSize, - &frameSizeInfo.compressedSize, - &frameSizeInfo.decompressedBound); - break; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 3) - case 3 : - ZSTDv03_findFrameSizeInfoLegacy(src, srcSize, - &frameSizeInfo.compressedSize, - &frameSizeInfo.decompressedBound); - break; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 4) - case 4 : - ZSTDv04_findFrameSizeInfoLegacy(src, srcSize, - &frameSizeInfo.compressedSize, - &frameSizeInfo.decompressedBound); - break; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 5) - case 5 : - ZSTDv05_findFrameSizeInfoLegacy(src, srcSize, - &frameSizeInfo.compressedSize, - &frameSizeInfo.decompressedBound); - break; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 6) - case 6 : - ZSTDv06_findFrameSizeInfoLegacy(src, srcSize, - &frameSizeInfo.compressedSize, - &frameSizeInfo.decompressedBound); - break; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 7) - case 7 : - ZSTDv07_findFrameSizeInfoLegacy(src, srcSize, - &frameSizeInfo.compressedSize, - &frameSizeInfo.decompressedBound); - break; -#endif - default : - frameSizeInfo.compressedSize = ERROR(prefix_unknown); - frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR; - break; - } - if (!ZSTD_isError(frameSizeInfo.compressedSize) && frameSizeInfo.compressedSize > srcSize) { - frameSizeInfo.compressedSize = ERROR(srcSize_wrong); - frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR; - } - return frameSizeInfo; -} +/*! ZSTD_checkContinuity() : + * check if next `dst` follows previous position, where decompression ended. + * If yes, do nothing (continue on current segment). + * If not, classify previous segment as "external dictionary", and start a new segment. + * This function cannot fail. */ +void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize); -MEM_STATIC size_t ZSTD_findFrameCompressedSizeLegacy(const void *src, size_t srcSize) -{ - ZSTD_frameSizeInfo frameSizeInfo = ZSTD_findFrameSizeInfoLegacy(src, srcSize); - return frameSizeInfo.compressedSize; -} -MEM_STATIC size_t ZSTD_freeLegacyStreamContext(void* legacyContext, U32 version) -{ - switch(version) - { - default : - case 1 : - case 2 : - case 3 : - (void)legacyContext; - return ERROR(version_unsupported); -#if (ZSTD_LEGACY_SUPPORT <= 4) - case 4 : return ZBUFFv04_freeDCtx((ZBUFFv04_DCtx*)legacyContext); -#endif -#if (ZSTD_LEGACY_SUPPORT <= 5) - case 5 : return ZBUFFv05_freeDCtx((ZBUFFv05_DCtx*)legacyContext); -#endif -#if (ZSTD_LEGACY_SUPPORT <= 6) - case 6 : return ZBUFFv06_freeDCtx((ZBUFFv06_DCtx*)legacyContext); -#endif -#if (ZSTD_LEGACY_SUPPORT <= 7) - case 7 : return ZBUFFv07_freeDCtx((ZBUFFv07_DCtx*)legacyContext); -#endif - } -} +#endif /* ZSTD_DECOMPRESS_INTERNAL_H */ +/**** ended inlining zstd_decompress_internal.h ****/ +/**** start inlining zstd_ddict.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ -MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U32 newVersion, - const void* dict, size_t dictSize) -{ - DEBUGLOG(5, "ZSTD_initLegacyStream for v0.%u", newVersion); - if (prevVersion != newVersion) ZSTD_freeLegacyStreamContext(*legacyContext, prevVersion); - switch(newVersion) - { - default : - case 1 : - case 2 : - case 3 : - (void)dict; (void)dictSize; - return 0; -#if (ZSTD_LEGACY_SUPPORT <= 4) - case 4 : - { - ZBUFFv04_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv04_createDCtx() : (ZBUFFv04_DCtx*)*legacyContext; - if (dctx==NULL) return ERROR(memory_allocation); - ZBUFFv04_decompressInit(dctx); - ZBUFFv04_decompressWithDictionary(dctx, dict, dictSize); - *legacyContext = dctx; - return 0; - } -#endif -#if (ZSTD_LEGACY_SUPPORT <= 5) - case 5 : - { - ZBUFFv05_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv05_createDCtx() : (ZBUFFv05_DCtx*)*legacyContext; - if (dctx==NULL) return ERROR(memory_allocation); - ZBUFFv05_decompressInitDictionary(dctx, dict, dictSize); - *legacyContext = dctx; - return 0; - } -#endif -#if (ZSTD_LEGACY_SUPPORT <= 6) - case 6 : - { - ZBUFFv06_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv06_createDCtx() : (ZBUFFv06_DCtx*)*legacyContext; - if (dctx==NULL) return ERROR(memory_allocation); - ZBUFFv06_decompressInitDictionary(dctx, dict, dictSize); - *legacyContext = dctx; - return 0; - } -#endif -#if (ZSTD_LEGACY_SUPPORT <= 7) - case 7 : - { - ZBUFFv07_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv07_createDCtx() : (ZBUFFv07_DCtx*)*legacyContext; - if (dctx==NULL) return ERROR(memory_allocation); - ZBUFFv07_decompressInitDictionary(dctx, dict, dictSize); - *legacyContext = dctx; - return 0; - } -#endif - } -} +#ifndef ZSTD_DDICT_H +#define ZSTD_DDICT_H + +/*-******************************************************* + * Dependencies + *********************************************************/ +/**** skipping file: ../common/zstd_deps.h ****/ +/**** skipping file: ../zstd.h ****/ +/*-******************************************************* + * Interface + *********************************************************/ -MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version, - ZSTD_outBuffer* output, ZSTD_inBuffer* input) -{ - DEBUGLOG(5, "ZSTD_decompressLegacyStream for v0.%u", version); - switch(version) - { - default : - case 1 : - case 2 : - case 3 : - (void)legacyContext; (void)output; (void)input; - return ERROR(version_unsupported); -#if (ZSTD_LEGACY_SUPPORT <= 4) - case 4 : - { - ZBUFFv04_DCtx* dctx = (ZBUFFv04_DCtx*) legacyContext; - const void* src = (const char*)input->src + input->pos; - size_t readSize = input->size - input->pos; - void* dst = (char*)output->dst + output->pos; - size_t decodedSize = output->size - output->pos; - size_t const hintSize = ZBUFFv04_decompressContinue(dctx, dst, &decodedSize, src, &readSize); - output->pos += decodedSize; - input->pos += readSize; - return hintSize; - } -#endif -#if (ZSTD_LEGACY_SUPPORT <= 5) - case 5 : - { - ZBUFFv05_DCtx* dctx = (ZBUFFv05_DCtx*) legacyContext; - const void* src = (const char*)input->src + input->pos; - size_t readSize = input->size - input->pos; - void* dst = (char*)output->dst + output->pos; - size_t decodedSize = output->size - output->pos; - size_t const hintSize = ZBUFFv05_decompressContinue(dctx, dst, &decodedSize, src, &readSize); - output->pos += decodedSize; - input->pos += readSize; - return hintSize; - } -#endif -#if (ZSTD_LEGACY_SUPPORT <= 6) - case 6 : - { - ZBUFFv06_DCtx* dctx = (ZBUFFv06_DCtx*) legacyContext; - const void* src = (const char*)input->src + input->pos; - size_t readSize = input->size - input->pos; - void* dst = (char*)output->dst + output->pos; - size_t decodedSize = output->size - output->pos; - size_t const hintSize = ZBUFFv06_decompressContinue(dctx, dst, &decodedSize, src, &readSize); - output->pos += decodedSize; - input->pos += readSize; - return hintSize; - } -#endif -#if (ZSTD_LEGACY_SUPPORT <= 7) - case 7 : - { - ZBUFFv07_DCtx* dctx = (ZBUFFv07_DCtx*) legacyContext; - const void* src = (const char*)input->src + input->pos; - size_t readSize = input->size - input->pos; - void* dst = (char*)output->dst + output->pos; - size_t decodedSize = output->size - output->pos; - size_t const hintSize = ZBUFFv07_decompressContinue(dctx, dst, &decodedSize, src, &readSize); - output->pos += decodedSize; - input->pos += readSize; - return hintSize; - } -#endif - } -} +/* note: several prototypes are already published in `zstd.h` : + * ZSTD_createDDict() + * ZSTD_createDDict_byReference() + * ZSTD_createDDict_advanced() + * ZSTD_freeDDict() + * ZSTD_initStaticDDict() + * ZSTD_sizeof_DDict() + * ZSTD_estimateDDictSize() + * ZSTD_getDictID_fromDict() + */ +const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict); +size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict); + +void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); -#if defined (__cplusplus) -} -#endif -#endif /* ZSTD_LEGACY_H */ -/**** ended inlining ../legacy/zstd_legacy.h ****/ + +#endif /* ZSTD_DDICT_H */ +/**** ended inlining zstd_ddict.h ****/ + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) +#error Using excluded file: ../legacy/zstd_legacy.h (re-amalgamate source to fix) #endif @@ -11147,7 +17433,7 @@ static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, ZSTD_memcpy(internalBuffer, dict, dictSize); } ddict->dictSize = dictSize; - ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ + ddict->entropy.hufTable[0] = (HUF_DTable)((ZSTD_HUFFDTABLE_CAPACITY_LOG)*0x1000001); /* cover both little and big endian */ /* parse dictionary content */ FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) , ""); @@ -11253,12 +17539,12 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict) unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) { if (ddict==NULL) return 0; - return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize); + return ddict->dictID; } /**** ended inlining decompress/zstd_ddict.c ****/ /**** start inlining decompress/zstd_decompress.c ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -11315,20 +17601,20 @@ unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) * Dependencies *********************************************************/ /**** skipping file: ../common/zstd_deps.h ****/ -/**** skipping file: ../common/cpu.h ****/ +/**** skipping file: ../common/allocations.h ****/ +/**** skipping file: ../common/error_private.h ****/ +/**** skipping file: ../common/zstd_internal.h ****/ /**** skipping file: ../common/mem.h ****/ -/**** skipping file: ../common/zstd_trace.h ****/ +/**** skipping file: ../common/bits.h ****/ #define FSE_STATIC_LINKING_ONLY /**** skipping file: ../common/fse.h ****/ -#define HUF_STATIC_LINKING_ONLY /**** skipping file: ../common/huf.h ****/ /**** skipping file: ../common/xxhash.h ****/ -/**** skipping file: ../common/zstd_internal.h ****/ /**** skipping file: zstd_decompress_internal.h ****/ /**** skipping file: zstd_ddict.h ****/ /**** start inlining zstd_decompress_block.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -11362,6 +17648,12 @@ unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) */ + /* Streaming state is used to inform allocation of the literal buffer */ +typedef enum { + not_streaming = 0, + is_streaming = 1 +} streaming_operation; + /* ZSTD_decompressBlock_internal() : * decompress block, starting at `src`, * into destination buffer `dst`. @@ -11370,7 +17662,7 @@ unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) */ size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, - const void* src, size_t srcSize, const int frame); + const void* src, size_t srcSize, const streaming_operation streaming); /* ZSTD_buildFSETable() : * generate FSE decoding table for one symbol (ll, ml or off) @@ -11383,16 +17675,21 @@ size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, */ void ZSTD_buildFSETable(ZSTD_seqSymbol* dt, const short* normalizedCounter, unsigned maxSymbolValue, - const U32* baseValue, const U32* nbAdditionalBits, + const U32* baseValue, const U8* nbAdditionalBits, unsigned tableLog, void* wksp, size_t wkspSize, int bmi2); +/* Internal definition of ZSTD_decompressBlock() to avoid deprecation warnings. */ +size_t ZSTD_decompressBlock_deprecated(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + #endif /* ZSTD_DEC_BLOCK_H */ /**** ended inlining zstd_decompress_block.h ****/ #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) -/**** skipping file: ../legacy/zstd_legacy.h ****/ +#error Using excluded file: ../legacy/zstd_legacy.h (re-amalgamate source to fix) #endif @@ -11402,11 +17699,11 @@ void ZSTD_buildFSETable(ZSTD_seqSymbol* dt, *************************************/ #define DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT 4 -#define DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT 3 /* These two constants represent SIZE_MULT/COUNT_MULT load factor without using a float. - * Currently, that means a 0.75 load factor. - * So, if count * COUNT_MULT / size * SIZE_MULT != 0, then we've exceeded - * the load factor of the ddict hash set. - */ +#define DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT 3 /* These two constants represent SIZE_MULT/COUNT_MULT load factor without using a float. + * Currently, that means a 0.75 load factor. + * So, if count * COUNT_MULT / size * SIZE_MULT != 0, then we've exceeded + * the load factor of the ddict hash set. + */ #define DDICT_HASHSET_TABLE_BASE_SIZE 64 #define DDICT_HASHSET_RESIZE_FACTOR 2 @@ -11500,12 +17797,15 @@ static const ZSTD_DDict* ZSTD_DDictHashSet_getDDict(ZSTD_DDictHashSet* hashSet, static ZSTD_DDictHashSet* ZSTD_createDDictHashSet(ZSTD_customMem customMem) { ZSTD_DDictHashSet* ret = (ZSTD_DDictHashSet*)ZSTD_customMalloc(sizeof(ZSTD_DDictHashSet), customMem); DEBUGLOG(4, "Allocating new hash set"); + if (!ret) + return NULL; ret->ddictPtrTable = (const ZSTD_DDict**)ZSTD_customCalloc(DDICT_HASHSET_TABLE_BASE_SIZE * sizeof(ZSTD_DDict*), customMem); - ret->ddictPtrTableSize = DDICT_HASHSET_TABLE_BASE_SIZE; - ret->ddictPtrCount = 0; - if (!ret || !ret->ddictPtrTable) { + if (!ret->ddictPtrTable) { + ZSTD_customFree(ret, customMem); return NULL; } + ret->ddictPtrTableSize = DDICT_HASHSET_TABLE_BASE_SIZE; + ret->ddictPtrCount = 0; return ret; } @@ -11564,6 +17864,8 @@ static void ZSTD_DCtx_resetParameters(ZSTD_DCtx* dctx) dctx->outBufferMode = ZSTD_bm_buffered; dctx->forceIgnoreChecksum = ZSTD_d_validateChecksum; dctx->refMultipleDDicts = ZSTD_rmd_refSingleDDict; + dctx->disableHufAsm = 0; + dctx->maxBlockSizeParam = 0; } static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx) @@ -11578,11 +17880,16 @@ static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx) dctx->inBuffSize = 0; dctx->outBuffSize = 0; dctx->streamStage = zdss_init; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) dctx->legacyContext = NULL; dctx->previousLegacyVersion = 0; +#endif dctx->noForwardProgress = 0; dctx->oversizedDuration = 0; - dctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); + dctx->isFrameDecompression = 1; +#if DYNAMIC_BMI2 + dctx->bmi2 = ZSTD_cpuSupportsBmi2(); +#endif dctx->ddictSet = NULL; ZSTD_DCtx_resetParameters(dctx); #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION @@ -11603,8 +17910,7 @@ ZSTD_DCtx* ZSTD_initStaticDCtx(void *workspace, size_t workspaceSize) return dctx; } -ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) -{ +static ZSTD_DCtx* ZSTD_createDCtx_internal(ZSTD_customMem customMem) { if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; { ZSTD_DCtx* const dctx = (ZSTD_DCtx*)ZSTD_customMalloc(sizeof(*dctx), customMem); @@ -11615,10 +17921,15 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) } } +ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) +{ + return ZSTD_createDCtx_internal(customMem); +} + ZSTD_DCtx* ZSTD_createDCtx(void) { DEBUGLOG(3, "ZSTD_createDCtx"); - return ZSTD_createDCtx_advanced(ZSTD_defaultCMem); + return ZSTD_createDCtx_internal(ZSTD_defaultCMem); } static void ZSTD_clearDict(ZSTD_DCtx* dctx) @@ -11703,6 +18014,19 @@ unsigned ZSTD_isFrame(const void* buffer, size_t size) return 0; } +/*! ZSTD_isSkippableFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame. + * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. + */ +unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size) +{ + if (size < ZSTD_FRAMEIDSIZE) return 0; + { U32 const magic = MEM_readLE32(buffer); + if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1; + } + return 0; +} + /** ZSTD_frameHeaderSize_internal() : * srcSize must be large enough to reach header size fields. * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless. @@ -11738,16 +18062,40 @@ size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless * @return : 0, `zfhPtr` is correctly filled, * >0, `srcSize` is too small, value is wanted `srcSize` amount, - * or an error code, which can be tested using ZSTD_isError() */ -size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format) +** or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_getFrameHeader_advanced(ZSTD_FrameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format) { const BYTE* ip = (const BYTE*)src; size_t const minInputSize = ZSTD_startingInputLength(format); - ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzer do not understand that zfhPtr is only going to be read only if return value is zero, since they are 2 different signals */ - if (srcSize < minInputSize) return minInputSize; - RETURN_ERROR_IF(src==NULL, GENERIC, "invalid parameter"); + DEBUGLOG(5, "ZSTD_getFrameHeader_advanced: minInputSize = %zu, srcSize = %zu", minInputSize, srcSize); + + if (srcSize > 0) { + /* note : technically could be considered an assert(), since it's an invalid entry */ + RETURN_ERROR_IF(src==NULL, GENERIC, "invalid parameter : src==NULL, but srcSize>0"); + } + if (srcSize < minInputSize) { + if (srcSize > 0 && format != ZSTD_f_zstd1_magicless) { + /* when receiving less than @minInputSize bytes, + * control these bytes at least correspond to a supported magic number + * in order to error out early if they don't. + **/ + size_t const toCopy = MIN(4, srcSize); + unsigned char hbuf[4]; MEM_writeLE32(hbuf, ZSTD_MAGICNUMBER); + assert(src != NULL); + ZSTD_memcpy(hbuf, src, toCopy); + if ( MEM_readLE32(hbuf) != ZSTD_MAGICNUMBER ) { + /* not a zstd frame : let's check if it's a skippable frame */ + MEM_writeLE32(hbuf, ZSTD_MAGIC_SKIPPABLE_START); + ZSTD_memcpy(hbuf, src, toCopy); + if ((MEM_readLE32(hbuf) & ZSTD_MAGIC_SKIPPABLE_MASK) != ZSTD_MAGIC_SKIPPABLE_START) { + RETURN_ERROR(prefix_unknown, + "first bytes don't correspond to any supported magic number"); + } } } + return minInputSize; + } + ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzers may not understand that zfhPtr will be read only if return value is zero, since they are 2 different signals */ if ( (format != ZSTD_f_zstd1_magicless) && (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) { if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { @@ -11755,8 +18103,10 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s if (srcSize < ZSTD_SKIPPABLEHEADERSIZE) return ZSTD_SKIPPABLEHEADERSIZE; /* magic number + frame length */ ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); - zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE); zfhPtr->frameType = ZSTD_skippableFrame; + zfhPtr->dictID = MEM_readLE32(src) - ZSTD_MAGIC_SKIPPABLE_START; + zfhPtr->headerSize = ZSTD_SKIPPABLEHEADERSIZE; + zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE); return 0; } RETURN_ERROR(prefix_unknown, ""); @@ -11789,7 +18139,9 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s } switch(dictIDSizeCode) { - default: assert(0); /* impossible */ + default: + assert(0); /* impossible */ + ZSTD_FALLTHROUGH; case 0 : break; case 1 : dictID = ip[pos]; pos++; break; case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break; @@ -11797,7 +18149,9 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s } switch(fcsID) { - default: assert(0); /* impossible */ + default: + assert(0); /* impossible */ + ZSTD_FALLTHROUGH; case 0 : if (singleSegment) frameContentSize = ip[pos]; break; case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break; case 2 : frameContentSize = MEM_readLE32(ip+pos); break; @@ -11821,12 +18175,11 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s * @return : 0, `zfhPtr` is correctly filled, * >0, `srcSize` is too small, value is wanted `srcSize` amount, * or an error code, which can be tested using ZSTD_isError() */ -size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize) +size_t ZSTD_getFrameHeader(ZSTD_FrameHeader* zfhPtr, const void* src, size_t srcSize) { return ZSTD_getFrameHeader_advanced(zfhPtr, src, srcSize, ZSTD_f_zstd1); } - /** ZSTD_getFrameContentSize() : * compatible with legacy mode * @return : decompressed size of the single frame pointed to be `src` if known, otherwise @@ -11840,7 +18193,7 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize) return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret; } #endif - { ZSTD_frameHeader zfh; + { ZSTD_FrameHeader zfh; if (ZSTD_getFrameHeader(&zfh, src, srcSize) != 0) return ZSTD_CONTENTSIZE_ERROR; if (zfh.frameType == ZSTD_skippableFrame) { @@ -11860,18 +18213,52 @@ static size_t readSkippableFrameSize(void const* src, size_t srcSize) sizeU32 = MEM_readLE32((BYTE const*)src + ZSTD_FRAMEIDSIZE); RETURN_ERROR_IF((U32)(sizeU32 + ZSTD_SKIPPABLEHEADERSIZE) < sizeU32, frameParameter_unsupported, ""); - { - size_t const skippableSize = skippableHeaderSize + sizeU32; + { size_t const skippableSize = skippableHeaderSize + sizeU32; RETURN_ERROR_IF(skippableSize > srcSize, srcSize_wrong, ""); return skippableSize; } } +/*! ZSTD_readSkippableFrame() : + * Retrieves content of a skippable frame, and writes it to dst buffer. + * + * The parameter magicVariant will receive the magicVariant that was supplied when the frame was written, + * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested + * in the magicVariant. + * + * Returns an error if destination buffer is not large enough, or if this is not a valid skippable frame. + * + * @return : number of bytes written or a ZSTD error. + */ +size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, + unsigned* magicVariant, /* optional, can be NULL */ + const void* src, size_t srcSize) +{ + RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong, ""); + + { U32 const magicNumber = MEM_readLE32(src); + size_t skippableFrameSize = readSkippableFrameSize(src, srcSize); + size_t skippableContentSize = skippableFrameSize - ZSTD_SKIPPABLEHEADERSIZE; + + /* check input validity */ + RETURN_ERROR_IF(!ZSTD_isSkippableFrame(src, srcSize), frameParameter_unsupported, ""); + RETURN_ERROR_IF(skippableFrameSize < ZSTD_SKIPPABLEHEADERSIZE || skippableFrameSize > srcSize, srcSize_wrong, ""); + RETURN_ERROR_IF(skippableContentSize > dstCapacity, dstSize_tooSmall, ""); + + /* deliver payload */ + if (skippableContentSize > 0 && dst != NULL) + ZSTD_memcpy(dst, (const BYTE *)src + ZSTD_SKIPPABLEHEADERSIZE, skippableContentSize); + if (magicVariant != NULL) + *magicVariant = magicNumber - ZSTD_MAGIC_SKIPPABLE_START; + return skippableContentSize; + } +} + /** ZSTD_findDecompressedSize() : - * compatible with legacy mode * `srcSize` must be the exact length of some number of ZSTD compressed and/or * skippable frames - * @return : decompressed size of the frames contained */ + * note: compatible with legacy mode + * @return : decompressed size of the frames contained */ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) { unsigned long long totalDstSize = 0; @@ -11881,9 +18268,7 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { size_t const skippableSize = readSkippableFrameSize(src, srcSize); - if (ZSTD_isError(skippableSize)) { - return ZSTD_CONTENTSIZE_ERROR; - } + if (ZSTD_isError(skippableSize)) return ZSTD_CONTENTSIZE_ERROR; assert(skippableSize <= srcSize); src = (const BYTE *)src + skippableSize; @@ -11891,17 +18276,17 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) continue; } - { unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize); - if (ret >= ZSTD_CONTENTSIZE_ERROR) return ret; + { unsigned long long const fcs = ZSTD_getFrameContentSize(src, srcSize); + if (fcs >= ZSTD_CONTENTSIZE_ERROR) return fcs; - /* check for overflow */ - if (totalDstSize + ret < totalDstSize) return ZSTD_CONTENTSIZE_ERROR; - totalDstSize += ret; + if (totalDstSize + fcs < totalDstSize) + return ZSTD_CONTENTSIZE_ERROR; /* check for overflow */ + totalDstSize += fcs; } + /* skip to next frame */ { size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize); - if (ZSTD_isError(frameSrcSize)) { - return ZSTD_CONTENTSIZE_ERROR; - } + if (ZSTD_isError(frameSrcSize)) return ZSTD_CONTENTSIZE_ERROR; + assert(frameSrcSize <= srcSize); src = (const BYTE *)src + frameSrcSize; srcSize -= frameSrcSize; @@ -11965,17 +18350,17 @@ static ZSTD_frameSizeInfo ZSTD_errorFrameSizeInfo(size_t ret) return frameSizeInfo; } -static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize) +static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize, ZSTD_format_e format) { ZSTD_frameSizeInfo frameSizeInfo; ZSTD_memset(&frameSizeInfo, 0, sizeof(ZSTD_frameSizeInfo)); #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) - if (ZSTD_isLegacy(src, srcSize)) + if (format == ZSTD_f_zstd1 && ZSTD_isLegacy(src, srcSize)) return ZSTD_findFrameSizeInfoLegacy(src, srcSize); #endif - if ((srcSize >= ZSTD_SKIPPABLEHEADERSIZE) + if (format == ZSTD_f_zstd1 && (srcSize >= ZSTD_SKIPPABLEHEADERSIZE) && (MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { frameSizeInfo.compressedSize = readSkippableFrameSize(src, srcSize); assert(ZSTD_isError(frameSizeInfo.compressedSize) || @@ -11986,10 +18371,10 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize const BYTE* const ipstart = ip; size_t remainingSize = srcSize; size_t nbBlocks = 0; - ZSTD_frameHeader zfh; + ZSTD_FrameHeader zfh; /* Extract Frame Header */ - { size_t const ret = ZSTD_getFrameHeader(&zfh, src, srcSize); + { size_t const ret = ZSTD_getFrameHeader_advanced(&zfh, src, srcSize, format); if (ZSTD_isError(ret)) return ZSTD_errorFrameSizeInfo(ret); if (ret > 0) @@ -12023,28 +18408,31 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize ip += 4; } + frameSizeInfo.nbBlocks = nbBlocks; frameSizeInfo.compressedSize = (size_t)(ip - ipstart); frameSizeInfo.decompressedBound = (zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) ? zfh.frameContentSize - : nbBlocks * zfh.blockSizeMax; + : (unsigned long long)nbBlocks * zfh.blockSizeMax; return frameSizeInfo; } } +static size_t ZSTD_findFrameCompressedSize_advanced(const void *src, size_t srcSize, ZSTD_format_e format) { + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize, format); + return frameSizeInfo.compressedSize; +} + /** ZSTD_findFrameCompressedSize() : - * compatible with legacy mode - * `src` must point to the start of a ZSTD frame, ZSTD legacy frame, or skippable frame - * `srcSize` must be at least as large as the frame contained - * @return : the compressed size of the frame starting at `src` */ + * See docs in zstd.h + * Note: compatible with legacy mode */ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) { - ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize); - return frameSizeInfo.compressedSize; + return ZSTD_findFrameCompressedSize_advanced(src, srcSize, ZSTD_f_zstd1); } /** ZSTD_decompressBound() : * compatible with legacy mode - * `src` must point to the start of a ZSTD frame or a skippeable frame + * `src` must point to the start of a ZSTD frame or a skippable frame * `srcSize` must be at least as large as the frame contained * @return : the maximum decompressed size of the compressed source */ @@ -12053,7 +18441,7 @@ unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize) unsigned long long bound = 0; /* Iterate over each frame */ while (srcSize > 0) { - ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize); + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize, ZSTD_f_zstd1); size_t const compressedSize = frameSizeInfo.compressedSize; unsigned long long const decompressedBound = frameSizeInfo.decompressedBound; if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR) @@ -12066,6 +18454,48 @@ unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize) return bound; } +size_t ZSTD_decompressionMargin(void const* src, size_t srcSize) +{ + size_t margin = 0; + unsigned maxBlockSize = 0; + + /* Iterate over each frame */ + while (srcSize > 0) { + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize, ZSTD_f_zstd1); + size_t const compressedSize = frameSizeInfo.compressedSize; + unsigned long long const decompressedBound = frameSizeInfo.decompressedBound; + ZSTD_FrameHeader zfh; + + FORWARD_IF_ERROR(ZSTD_getFrameHeader(&zfh, src, srcSize), ""); + if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR) + return ERROR(corruption_detected); + + if (zfh.frameType == ZSTD_frame) { + /* Add the frame header to our margin */ + margin += zfh.headerSize; + /* Add the checksum to our margin */ + margin += zfh.checksumFlag ? 4 : 0; + /* Add 3 bytes per block */ + margin += 3 * frameSizeInfo.nbBlocks; + + /* Compute the max block size */ + maxBlockSize = MAX(maxBlockSize, zfh.blockSizeMax); + } else { + assert(zfh.frameType == ZSTD_skippableFrame); + /* Add the entire skippable frame size to our margin. */ + margin += compressedSize; + } + + assert(srcSize >= compressedSize); + src = (const BYTE*)src + compressedSize; + srcSize -= compressedSize; + } + + /* Add the max block size back to the margin. */ + margin += maxBlockSize; + + return margin; +} /*-************************************************************* * Frame decoding @@ -12091,7 +18521,7 @@ static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity, if (srcSize == 0) return 0; RETURN_ERROR(dstBuffer_null, ""); } - ZSTD_memcpy(dst, src, srcSize); + ZSTD_memmove(dst, src, srcSize); return srcSize; } @@ -12108,10 +18538,10 @@ static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity, return regenSize; } -static void ZSTD_DCtx_trace_end(ZSTD_DCtx const* dctx, U64 uncompressedSize, U64 compressedSize, unsigned streaming) +static void ZSTD_DCtx_trace_end(ZSTD_DCtx const* dctx, U64 uncompressedSize, U64 compressedSize, int streaming) { #if ZSTD_TRACE - if (dctx->traceCtx) { + if (dctx->traceCtx && ZSTD_trace_decompress_end != NULL) { ZSTD_Trace trace; ZSTD_memset(&trace, 0, sizeof(trace)); trace.version = ZSTD_VERSION_NUMBER; @@ -12167,10 +18597,16 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, ip += frameHeaderSize; remainingSrcSize -= frameHeaderSize; } + /* Shrink the blockSizeMax if enabled */ + if (dctx->maxBlockSizeParam != 0) + dctx->fParams.blockSizeMax = MIN(dctx->fParams.blockSizeMax, (unsigned)dctx->maxBlockSizeParam); + /* Loop on each block */ while (1) { + BYTE* oBlockEnd = oend; size_t decodedSize; blockProperties_t blockProperties; + memset(&blockProperties, 0, sizeof(blockProperties)); // rg [11/30/2025] - added to shut up gcc warning size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSrcSize, &blockProperties); if (ZSTD_isError(cBlockSize)) return cBlockSize; @@ -12178,27 +18614,48 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, remainingSrcSize -= ZSTD_blockHeaderSize; RETURN_ERROR_IF(cBlockSize > remainingSrcSize, srcSize_wrong, ""); + if (ip >= op && ip < oBlockEnd) { + /* We are decompressing in-place. Limit the output pointer so that we + * don't overwrite the block that we are currently reading. This will + * fail decompression if the input & output pointers aren't spaced + * far enough apart. + * + * This is important to set, even when the pointers are far enough + * apart, because ZSTD_decompressBlock_internal() can decide to store + * literals in the output buffer, after the block it is decompressing. + * Since we don't want anything to overwrite our input, we have to tell + * ZSTD_decompressBlock_internal to never write past ip. + * + * See ZSTD_allocateLiteralsBuffer() for reference. + */ + oBlockEnd = op + (ip - op); + } + switch(blockProperties.blockType) { case bt_compressed: - decodedSize = ZSTD_decompressBlock_internal(dctx, op, (size_t)(oend-op), ip, cBlockSize, /* frame */ 1); + assert(dctx->isFrameDecompression == 1); + decodedSize = ZSTD_decompressBlock_internal(dctx, op, (size_t)(oBlockEnd-op), ip, cBlockSize, not_streaming); break; case bt_raw : + /* Use oend instead of oBlockEnd because this function is safe to overlap. It uses memmove. */ decodedSize = ZSTD_copyRawBlock(op, (size_t)(oend-op), ip, cBlockSize); break; case bt_rle : - decodedSize = ZSTD_setRleBlock(op, (size_t)(oend-op), *ip, blockProperties.origSize); + decodedSize = ZSTD_setRleBlock(op, (size_t)(oBlockEnd-op), *ip, blockProperties.origSize); break; case bt_reserved : default: RETURN_ERROR(corruption_detected, "invalid block type"); } - - if (ZSTD_isError(decodedSize)) return decodedSize; - if (dctx->validateChecksum) + FORWARD_IF_ERROR(decodedSize, "Block decompression failure"); + DEBUGLOG(5, "Decompressed block of dSize = %u", (unsigned)decodedSize); + if (dctx->validateChecksum) { XXH64_update(&dctx->xxhState, op, decodedSize); - if (decodedSize != 0) + } + if (decodedSize) /* support dst = NULL,0 */ { op += decodedSize; + } assert(ip != NULL); ip += cBlockSize; remainingSrcSize -= cBlockSize; @@ -12222,12 +18679,15 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, } ZSTD_DCtx_trace_end(dctx, (U64)(op-ostart), (U64)(ip-istart), /* streaming */ 0); /* Allow caller to get size read */ + DEBUGLOG(4, "ZSTD_decompressFrame: decompressed frame of size %i, consuming %i bytes of input", (int)(op-ostart), (int)(ip - (const BYTE*)*srcPtr)); *srcPtr = ip; *srcSizePtr = remainingSrcSize; return (size_t)(op-ostart); } -static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict, size_t dictSize, @@ -12247,7 +18707,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, while (srcSize >= ZSTD_startingInputLength(dctx->format)) { #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) - if (ZSTD_isLegacy(src, srcSize)) { + if (dctx->format == ZSTD_f_zstd1 && ZSTD_isLegacy(src, srcSize)) { size_t decodedSize; size_t const frameSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize); if (ZSTD_isError(frameSize)) return frameSize; @@ -12257,6 +18717,15 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, decodedSize = ZSTD_decompressLegacy(dst, dstCapacity, src, frameSize, dict, dictSize); if (ZSTD_isError(decodedSize)) return decodedSize; + { + unsigned long long const expectedSize = ZSTD_getFrameContentSize(src, srcSize); + RETURN_ERROR_IF(expectedSize == ZSTD_CONTENTSIZE_ERROR, corruption_detected, "Corrupted frame header!"); + if (expectedSize != ZSTD_CONTENTSIZE_UNKNOWN) { + RETURN_ERROR_IF(expectedSize != decodedSize, corruption_detected, + "Frame header size does not match decoded size!"); + } + } + assert(decodedSize <= dstCapacity); dst = (BYTE*)dst + decodedSize; dstCapacity -= decodedSize; @@ -12268,17 +18737,18 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, } #endif - { U32 const magicNumber = MEM_readLE32(src); - DEBUGLOG(4, "reading magic number %08X (expecting %08X)", - (unsigned)magicNumber, ZSTD_MAGICNUMBER); + if (dctx->format == ZSTD_f_zstd1 && srcSize >= 4) { + U32 const magicNumber = MEM_readLE32(src); + DEBUGLOG(5, "reading magic number %08X", (unsigned)magicNumber); if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + /* skippable frame detected : skip it */ size_t const skippableSize = readSkippableFrameSize(src, srcSize); - FORWARD_IF_ERROR(skippableSize, "readSkippableFrameSize failed"); + FORWARD_IF_ERROR(skippableSize, "invalid skippable frame"); assert(skippableSize <= srcSize); src = (const BYTE *)src + skippableSize; srcSize -= skippableSize; - continue; + continue; /* check next frame */ } } if (ddict) { @@ -12332,7 +18802,7 @@ static ZSTD_DDict const* ZSTD_getDDict(ZSTD_DCtx* dctx) switch (dctx->dictUses) { default: assert(0 /* Impossible */); - /* fall-through */ + ZSTD_FALLTHROUGH; case ZSTD_dont_use: ZSTD_clearDict(dctx); return NULL; @@ -12354,7 +18824,7 @@ size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t sr { #if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE>=1) size_t regenSize; - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx_internal(ZSTD_defaultCMem); RETURN_ERROR_IF(dctx==NULL, memory_allocation, "NULL pointer!"); regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize); ZSTD_freeDCtx(dctx); @@ -12374,8 +18844,8 @@ size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t sr size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) { return dctx->expected; } /** - * Similar to ZSTD_nextSrcSizeToDecompress(), but when when a block input can be streamed, - * we allow taking a partial block as the input. Currently only raw uncompressed blocks can + * Similar to ZSTD_nextSrcSizeToDecompress(), but when a block input can be streamed, we + * allow taking a partial block as the input. Currently only raw uncompressed blocks can * be streamed. * * For blocks that can be streamed, this allows us to reduce the latency until we produce @@ -12388,7 +18858,7 @@ static size_t ZSTD_nextSrcSizeToDecompressWithInputSize(ZSTD_DCtx* dctx, size_t return dctx->expected; if (dctx->bType != bt_raw) return dctx->expected; - return MIN(MAX(inputSize, 1), dctx->expected); + return BOUNDED(1, inputSize, dctx->expected); } ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) { @@ -12396,7 +18866,9 @@ ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) { { default: /* should not happen */ assert(0); + ZSTD_FALLTHROUGH; case ZSTDds_getFrameHeaderSize: + ZSTD_FALLTHROUGH; case ZSTDds_decodeFrameHeader: return ZSTDnit_frameHeader; case ZSTDds_decodeBlockHeader: @@ -12408,6 +18880,7 @@ ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) { case ZSTDds_checkChecksum: return ZSTDnit_checksum; case ZSTDds_decodeSkippableHeader: + ZSTD_FALLTHROUGH; case ZSTDds_skipFrame: return ZSTDnit_skippableFrame; } @@ -12491,7 +18964,8 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c { case bt_compressed: DEBUGLOG(5, "ZSTD_decompressContinue: case bt_compressed"); - rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 1); + assert(dctx->isFrameDecompression == 1); + rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, is_streaming); dctx->expected = 0; /* Streaming not supported */ break; case bt_raw : @@ -12560,6 +19034,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c case ZSTDds_decodeSkippableHeader: assert(src != NULL); assert(srcSize <= ZSTD_SKIPPABLEHEADERSIZE); + assert(dctx->format != ZSTD_f_zstd1_magicless); ZSTD_memcpy(dctx->headerBuffer + (ZSTD_SKIPPABLEHEADERSIZE - srcSize), src, srcSize); /* complete skippable header */ dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_FRAMEIDSIZE); /* note : dctx->expected can grow seriously large, beyond local buffer size */ dctx->stage = ZSTDds_skipFrame; @@ -12572,7 +19047,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c default: assert(0); /* impossible */ - RETURN_ERROR(GENERIC, "impossible to reach"); /* some compiler require default to do something */ + RETURN_ERROR(GENERIC, "impossible to reach"); /* some compilers require default to do something */ } } @@ -12613,11 +19088,11 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, /* in minimal huffman, we always use X1 variants */ size_t const hSize = HUF_readDTableX1_wksp(entropy->hufTable, dictPtr, dictEnd - dictPtr, - workspace, workspaceSize); + workspace, workspaceSize, /* flags */ 0); #else size_t const hSize = HUF_readDTableX2_wksp(entropy->hufTable, dictPtr, (size_t)(dictEnd - dictPtr), - workspace, workspaceSize); + workspace, workspaceSize, /* flags */ 0); #endif RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted, ""); dictPtr += hSize; @@ -12706,7 +19181,7 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) { assert(dctx != NULL); #if ZSTD_TRACE - dctx->traceCtx = ZSTD_trace_decompress_begin(dctx); + dctx->traceCtx = (ZSTD_trace_decompress_begin != NULL) ? ZSTD_trace_decompress_begin(dctx) : 0; #endif dctx->expected = ZSTD_startingInputLength(dctx->format); /* dctx->format must be properly set */ dctx->stage = ZSTDds_getFrameHeaderSize; @@ -12716,10 +19191,11 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) dctx->prefixStart = NULL; dctx->virtualStart = NULL; dctx->dictEnd = NULL; - dctx->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ + dctx->entropy.hufTable[0] = (HUF_DTable)((ZSTD_HUFFDTABLE_CAPACITY_LOG)*0x1000001); /* cover both little and big endian */ dctx->litEntropy = dctx->fseEntropy = 0; dctx->dictID = 0; dctx->bType = bt_reserved; + dctx->isFrameDecompression = 1; ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue)); ZSTD_memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */ dctx->LLTptr = dctx->entropy.LLTable; @@ -12778,7 +19254,7 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize) * This could for one of the following reasons : * - The frame does not require a dictionary (most common case). * - The frame was built with dictID intentionally removed. - * Needed dictionary is a hidden information. + * Needed dictionary is a hidden piece of information. * Note : this use case also happens when using a non-conformant dictionary. * - `srcSize` is too small, and as a result, frame header could not be decoded. * Note : possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`. @@ -12787,7 +19263,7 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize) * ZSTD_getFrameHeader(), which will provide a more precise error code. */ unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize) { - ZSTD_frameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0 }; + ZSTD_FrameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0, 0, 0 }; size_t const hError = ZSTD_getFrameHeader(&zfp, src, srcSize); if (ZSTD_isError(hError)) return 0; return zfp.dictID; @@ -12816,7 +19292,7 @@ size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, ZSTD_DStream* ZSTD_createDStream(void) { DEBUGLOG(3, "ZSTD_createDStream"); - return ZSTD_createDStream_advanced(ZSTD_defaultCMem); + return ZSTD_createDCtx_internal(ZSTD_defaultCMem); } ZSTD_DStream* ZSTD_initStaticDStream(void *workspace, size_t workspaceSize) @@ -12826,7 +19302,7 @@ ZSTD_DStream* ZSTD_initStaticDStream(void *workspace, size_t workspaceSize) ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem) { - return ZSTD_createDCtx_advanced(customMem); + return ZSTD_createDCtx_internal(customMem); } size_t ZSTD_freeDStream(ZSTD_DStream* zds) @@ -12894,7 +19370,9 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di size_t ZSTD_initDStream(ZSTD_DStream* zds) { DEBUGLOG(4, "ZSTD_initDStream"); - return ZSTD_initDStream_usingDDict(zds, NULL); + FORWARD_IF_ERROR(ZSTD_DCtx_reset(zds, ZSTD_reset_session_only), ""); + FORWARD_IF_ERROR(ZSTD_DCtx_refDDict(zds, NULL), ""); + return ZSTD_startingInputLength(zds->format); } /* ZSTD_initDStream_usingDDict() : @@ -12902,6 +19380,7 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds) * this function cannot fail */ size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict) { + DEBUGLOG(4, "ZSTD_initDStream_usingDDict"); FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) , ""); FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) , ""); return ZSTD_startingInputLength(dctx->format); @@ -12912,6 +19391,7 @@ size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict) * this function cannot fail */ size_t ZSTD_resetDStream(ZSTD_DStream* dctx) { + DEBUGLOG(4, "ZSTD_resetDStream"); FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only), ""); return ZSTD_startingInputLength(dctx->format); } @@ -12983,6 +19463,15 @@ ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam) bounds.lowerBound = (int)ZSTD_rmd_refSingleDDict; bounds.upperBound = (int)ZSTD_rmd_refMultipleDDicts; return bounds; + case ZSTD_d_disableHuffmanAssembly: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + case ZSTD_d_maxBlockSize: + bounds.lowerBound = ZSTD_BLOCKSIZE_MAX_MIN; + bounds.upperBound = ZSTD_BLOCKSIZE_MAX; + return bounds; + default:; } bounds.error = ERROR(parameter_unsupported); @@ -13023,6 +19512,12 @@ size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value case ZSTD_d_refMultipleDDicts: *value = (int)dctx->refMultipleDDicts; return 0; + case ZSTD_d_disableHuffmanAssembly: + *value = (int)dctx->disableHufAsm; + return 0; + case ZSTD_d_maxBlockSize: + *value = dctx->maxBlockSizeParam; + return 0; default:; } RETURN_ERROR(parameter_unsupported, ""); @@ -13056,6 +19551,14 @@ size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value } dctx->refMultipleDDicts = (ZSTD_refMultipleDDicts_e)value; return 0; + case ZSTD_d_disableHuffmanAssembly: + CHECK_DBOUNDS(ZSTD_d_disableHuffmanAssembly, value); + dctx->disableHufAsm = value != 0; + return 0; + case ZSTD_d_maxBlockSize: + if (value != 0) CHECK_DBOUNDS(ZSTD_d_maxBlockSize, value); + dctx->maxBlockSizeParam = value; + return 0; default:; } RETURN_ERROR(parameter_unsupported, ""); @@ -13067,6 +19570,7 @@ size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset) || (reset == ZSTD_reset_session_and_parameters) ) { dctx->streamStage = zdss_init; dctx->noForwardProgress = 0; + dctx->isFrameDecompression = 1; } if ( (reset == ZSTD_reset_parameters) || (reset == ZSTD_reset_session_and_parameters) ) { @@ -13083,10 +19587,17 @@ size_t ZSTD_sizeof_DStream(const ZSTD_DStream* dctx) return ZSTD_sizeof_DCtx(dctx); } -size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize) +static size_t ZSTD_decodingBufferSize_internal(unsigned long long windowSize, unsigned long long frameContentSize, size_t blockSizeMax) { - size_t const blockSize = (size_t) MIN(windowSize, ZSTD_BLOCKSIZE_MAX); - unsigned long long const neededRBSize = windowSize + blockSize + (WILDCOPY_OVERLENGTH * 2); + size_t const blockSize = MIN((size_t)MIN(windowSize, ZSTD_BLOCKSIZE_MAX), blockSizeMax); + /* We need blockSize + WILDCOPY_OVERLENGTH worth of buffer so that if a block + * ends at windowSize + WILDCOPY_OVERLENGTH + 1 bytes, we can start writing + * the block at the beginning of the output buffer, and maintain a full window. + * + * We need another blockSize worth of buffer so that we can store split + * literals at the end of the block without overwriting the extDict window. + */ + unsigned long long const neededRBSize = windowSize + (blockSize * 2) + (WILDCOPY_OVERLENGTH * 2); unsigned long long const neededSize = MIN(frameContentSize, neededRBSize); size_t const minRBSize = (size_t) neededSize; RETURN_ERROR_IF((unsigned long long)minRBSize != neededSize, @@ -13094,6 +19605,11 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long return minRBSize; } +size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize) +{ + return ZSTD_decodingBufferSize_internal(windowSize, frameContentSize, ZSTD_BLOCKSIZE_MAX); +} + size_t ZSTD_estimateDStreamSize(size_t windowSize) { size_t const blockSize = MIN(windowSize, ZSTD_BLOCKSIZE_MAX); @@ -13105,7 +19621,7 @@ size_t ZSTD_estimateDStreamSize(size_t windowSize) size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize) { U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; /* note : should be user-selectable, but requires an additional parameter (or a dctx) */ - ZSTD_frameHeader zfh; + ZSTD_FrameHeader zfh; size_t const err = ZSTD_getFrameHeader(&zfh, src, srcSize); if (ZSTD_isError(err)) return err; RETURN_ERROR_IF(err>0, srcSize_wrong, ""); @@ -13200,6 +19716,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB U32 someMoreWork = 1; DEBUGLOG(5, "ZSTD_decompressStream"); + assert(zds != NULL); RETURN_ERROR_IF( input->pos > input->size, srcSize_wrong, @@ -13220,10 +19737,12 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB DEBUGLOG(5, "stage zdss_init => transparent reset "); zds->streamStage = zdss_loadHeader; zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) zds->legacyVersion = 0; +#endif zds->hostageByte = 0; zds->expectedOutBuffer = *output; - /* fall-through */ + ZSTD_FALLTHROUGH; case zdss_loadHeader : DEBUGLOG(5, "stage zdss_loadHeader (srcSize : %u)", (U32)(iend - ip)); @@ -13240,7 +19759,6 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB if (zds->refMultipleDDicts && zds->ddictSet) { ZSTD_DCtx_selectFrameDDict(zds); } - DEBUGLOG(5, "header size : %u", (U32)hSize); if (ZSTD_isError(hSize)) { #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart); @@ -13272,6 +19790,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB zds->lhSize += remainingInput; } input->pos = input->size; + /* check first few bytes */ + FORWARD_IF_ERROR( + ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format), + "First few bytes detected incorrect" ); + /* return hint input size */ return (MAX((size_t)ZSTD_FRAMEHEADERSIZE_MIN(zds->format), hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ } assert(ip != NULL); @@ -13283,14 +19806,15 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB if (zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN && zds->fParams.frameType != ZSTD_skippableFrame && (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) { - size_t const cSize = ZSTD_findFrameCompressedSize(istart, (size_t)(iend-istart)); + size_t const cSize = ZSTD_findFrameCompressedSize_advanced(istart, (size_t)(iend-istart), zds->format); if (cSize <= (size_t)(iend-istart)) { /* shortcut : using single-pass mode */ size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, (size_t)(oend-op), istart, cSize, ZSTD_getDDict(zds)); if (ZSTD_isError(decompressedSize)) return decompressedSize; - DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()") + DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()"); + assert(istart != NULL); ip = istart + cSize; - op += decompressedSize; + op = op ? op + decompressedSize : op; /* can occur if frameContentSize = 0 (empty frame) */ zds->expected = 0; zds->streamStage = zdss_init; someMoreWork = 0; @@ -13309,7 +19833,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB DEBUGLOG(4, "Consume header"); FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(zds, ZSTD_getDDict(zds)), ""); - if ((MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ + if (zds->format == ZSTD_f_zstd1 + && (MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE); zds->stage = ZSTDds_skipFrame; } else { @@ -13325,11 +19850,13 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); RETURN_ERROR_IF(zds->fParams.windowSize > zds->maxWindowSize, frameParameter_windowTooLarge, ""); + if (zds->maxBlockSizeParam != 0) + zds->fParams.blockSizeMax = MIN(zds->fParams.blockSizeMax, (unsigned)zds->maxBlockSizeParam); /* Adapt buffer sizes to frame header instructions */ { size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */); size_t const neededOutBuffSize = zds->outBufferMode == ZSTD_bm_buffered - ? ZSTD_decodingBufferSize_min(zds->fParams.windowSize, zds->fParams.frameContentSize) + ? ZSTD_decodingBufferSize_internal(zds->fParams.windowSize, zds->fParams.frameContentSize, zds->fParams.blockSizeMax) : 0; ZSTD_DCtx_updateOversizedDuration(zds, neededInBuffSize, neededOutBuffSize); @@ -13361,7 +19888,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB zds->outBuffSize = neededOutBuffSize; } } } zds->streamStage = zdss_read; - /* fall-through */ + ZSTD_FALLTHROUGH; case zdss_read: DEBUGLOG(5, "stage zdss_read"); @@ -13374,13 +19901,14 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB } if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, ip, neededInSize), ""); + assert(ip != NULL); ip += neededInSize; /* Function modifies the stage so we must break */ break; } } if (ip==iend) { someMoreWork = 0; break; } /* no more input */ zds->streamStage = zdss_load; - /* fall-through */ + ZSTD_FALLTHROUGH; case zdss_load: { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds); @@ -13388,7 +19916,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB int const isSkipFrame = ZSTD_isSkipFrame(zds); size_t loadedSize; /* At this point we shouldn't be decompressing a block that we can stream. */ - assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, iend - ip)); + assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, (size_t)(iend - ip))); if (isSkipFrame) { loadedSize = MIN(toLoad, (size_t)(iend-ip)); } else { @@ -13397,8 +19925,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB "should never happen"); loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, (size_t)(iend-ip)); } - ip += loadedSize; - zds->inPos += loadedSize; + if (loadedSize != 0) { + /* ip may be NULL */ + ip += loadedSize; + zds->inPos += loadedSize; + } if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */ /* decode loaded input */ @@ -13408,14 +19939,17 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB break; } case zdss_flush: - { size_t const toFlushSize = zds->outEnd - zds->outStart; + { + size_t const toFlushSize = zds->outEnd - zds->outStart; size_t const flushedSize = ZSTD_limitCopy(op, (size_t)(oend-op), zds->outBuff + zds->outStart, toFlushSize); - op += flushedSize; + + op = op ? op + flushedSize : op; + zds->outStart += flushedSize; if (flushedSize == toFlushSize) { /* flush completed */ zds->streamStage = zdss_read; if ( (zds->outBuffSize < zds->fParams.frameContentSize) - && (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) { + && (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) { DEBUGLOG(5, "restart filling outBuff from beginning (left:%i, needed:%u)", (int)(zds->outBuffSize - zds->outStart), (U32)zds->fParams.blockSizeMax); @@ -13429,7 +19963,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB default: assert(0); /* impossible */ - RETURN_ERROR(GENERIC, "impossible to reach"); /* some compiler require default to do something */ + RETURN_ERROR(GENERIC, "impossible to reach"); /* some compilers require default to do something */ } } /* result */ @@ -13442,8 +19976,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB if ((ip==istart) && (op==ostart)) { /* no forward progress */ zds->noForwardProgress ++; if (zds->noForwardProgress >= ZSTD_NO_FORWARD_PROGRESS_MAX) { - RETURN_ERROR_IF(op==oend, dstSize_tooSmall, ""); - RETURN_ERROR_IF(ip==iend, srcSize_wrong, ""); + RETURN_ERROR_IF(op==oend, noForwardProgress_destFull, ""); + RETURN_ERROR_IF(ip==iend, noForwardProgress_inputEmpty, ""); assert(0); } } else { @@ -13480,18 +20014,24 @@ size_t ZSTD_decompressStream_simpleArgs ( void* dst, size_t dstCapacity, size_t* dstPos, const void* src, size_t srcSize, size_t* srcPos) { - ZSTD_outBuffer output = { dst, dstCapacity, *dstPos }; - ZSTD_inBuffer input = { src, srcSize, *srcPos }; - /* ZSTD_compress_generic() will check validity of dstPos and srcPos */ - size_t const cErr = ZSTD_decompressStream(dctx, &output, &input); - *dstPos = output.pos; - *srcPos = input.pos; - return cErr; + ZSTD_outBuffer output; + ZSTD_inBuffer input; + output.dst = dst; + output.size = dstCapacity; + output.pos = *dstPos; + input.src = src; + input.size = srcSize; + input.pos = *srcPos; + { size_t const cErr = ZSTD_decompressStream(dctx, &output, &input); + *dstPos = output.pos; + *srcPos = input.pos; + return cErr; + } } /**** ended inlining decompress/zstd_decompress.c ****/ /**** start inlining decompress/zstd_decompress_block.c ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -13512,12 +20052,12 @@ size_t ZSTD_decompressStream_simpleArgs ( /**** skipping file: ../common/mem.h ****/ #define FSE_STATIC_LINKING_ONLY /**** skipping file: ../common/fse.h ****/ -#define HUF_STATIC_LINKING_ONLY /**** skipping file: ../common/huf.h ****/ /**** skipping file: ../common/zstd_internal.h ****/ /**** skipping file: zstd_decompress_internal.h ****/ /**** skipping file: zstd_ddict.h ****/ /**** skipping file: zstd_decompress_block.h ****/ +/**** skipping file: ../common/bits.h ****/ /*_******************************************************* * Macros @@ -13543,6 +20083,13 @@ static void ZSTD_copy4(void* dst, const void* src) { ZSTD_memcpy(dst, src, 4); } * Block decoding ***************************************************************/ +static size_t ZSTD_blockSizeMax(ZSTD_DCtx const* dctx) +{ + size_t const blockSizeMax = dctx->isFrameDecompression ? dctx->fParams.blockSizeMax : ZSTD_BLOCKSIZE_MAX; + assert(blockSizeMax <= ZSTD_BLOCKSIZE_MAX); + return blockSizeMax; +} + /*! ZSTD_getcBlockSize() : * Provides the size of compressed block from block header `src` */ size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, @@ -13561,36 +20108,90 @@ size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, } } +/* Allocate buffer for literals, either overlapping current dst, or split between dst and litExtraBuffer, or stored entirely within litExtraBuffer */ +static void ZSTD_allocateLiteralsBuffer(ZSTD_DCtx* dctx, void* const dst, const size_t dstCapacity, const size_t litSize, + const streaming_operation streaming, const size_t expectedWriteSize, const unsigned splitImmediately) +{ + size_t const blockSizeMax = ZSTD_blockSizeMax(dctx); + assert(litSize <= blockSizeMax); + assert(dctx->isFrameDecompression || streaming == not_streaming); + assert(expectedWriteSize <= blockSizeMax); + if (streaming == not_streaming && dstCapacity > blockSizeMax + WILDCOPY_OVERLENGTH + litSize + WILDCOPY_OVERLENGTH) { + /* If we aren't streaming, we can just put the literals after the output + * of the current block. We don't need to worry about overwriting the + * extDict of our window, because it doesn't exist. + * So if we have space after the end of the block, just put it there. + */ + dctx->litBuffer = (BYTE*)dst + blockSizeMax + WILDCOPY_OVERLENGTH; + dctx->litBufferEnd = dctx->litBuffer + litSize; + dctx->litBufferLocation = ZSTD_in_dst; + } else if (litSize <= ZSTD_LITBUFFEREXTRASIZE) { + /* Literals fit entirely within the extra buffer, put them there to avoid + * having to split the literals. + */ + dctx->litBuffer = dctx->litExtraBuffer; + dctx->litBufferEnd = dctx->litBuffer + litSize; + dctx->litBufferLocation = ZSTD_not_in_dst; + } else { + assert(blockSizeMax > ZSTD_LITBUFFEREXTRASIZE); + /* Literals must be split between the output block and the extra lit + * buffer. We fill the extra lit buffer with the tail of the literals, + * and put the rest of the literals at the end of the block, with + * WILDCOPY_OVERLENGTH of buffer room to allow for overreads. + * This MUST not write more than our maxBlockSize beyond dst, because in + * streaming mode, that could overwrite part of our extDict window. + */ + if (splitImmediately) { + /* won't fit in litExtraBuffer, so it will be split between end of dst and extra buffer */ + dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH; + dctx->litBufferEnd = dctx->litBuffer + litSize - ZSTD_LITBUFFEREXTRASIZE; + } else { + /* initially this will be stored entirely in dst during huffman decoding, it will partially be shifted to litExtraBuffer after */ + dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize; + dctx->litBufferEnd = (BYTE*)dst + expectedWriteSize; + } + dctx->litBufferLocation = ZSTD_split; + assert(dctx->litBufferEnd <= (BYTE*)dst + expectedWriteSize); + } +} -/* Hidden declaration for fullbench */ -size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, - const void* src, size_t srcSize); /*! ZSTD_decodeLiteralsBlock() : + * Where it is possible to do so without being stomped by the output during decompression, the literals block will be stored + * in the dstBuffer. If there is room to do so, it will be stored in full in the excess dst space after where the current + * block will be output. Otherwise it will be stored at the end of the current dst blockspace, with a small portion being + * stored in dctx->litExtraBuffer to help keep it "ahead" of the current output write. + * * @return : nb of bytes read from src (< srcSize ) * note : symbol not declared but exposed for fullbench */ -size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, - const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */ +static size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, + const void* src, size_t srcSize, /* note : srcSize < BLOCKSIZE */ + void* dst, size_t dstCapacity, const streaming_operation streaming) { DEBUGLOG(5, "ZSTD_decodeLiteralsBlock"); RETURN_ERROR_IF(srcSize < MIN_CBLOCK_SIZE, corruption_detected, ""); { const BYTE* const istart = (const BYTE*) src; - symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3); + SymbolEncodingType_e const litEncType = (SymbolEncodingType_e)(istart[0] & 3); + size_t const blockSizeMax = ZSTD_blockSizeMax(dctx); switch(litEncType) { case set_repeat: DEBUGLOG(5, "set_repeat flag : re-using stats from previous compressed literals block"); RETURN_ERROR_IF(dctx->litEntropy==0, dictionary_corrupted, ""); - /* fall-through */ + ZSTD_FALLTHROUGH; case set_compressed: - RETURN_ERROR_IF(srcSize < 5, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3"); + RETURN_ERROR_IF(srcSize < 5, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need up to 5 for case 3"); { size_t lhSize, litSize, litCSize; U32 singleStream=0; U32 const lhlCode = (istart[0] >> 2) & 3; U32 const lhc = MEM_readLE32(istart); size_t hufSuccess; + size_t expectedWriteSize = MIN(blockSizeMax, dstCapacity); + int const flags = 0 + | (ZSTD_DCtx_get_bmi2(dctx) ? HUF_flags_bmi2 : 0) + | (dctx->disableHufAsm ? HUF_flags_disableAsm : 0); switch(lhlCode) { case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */ @@ -13613,8 +20214,15 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, litCSize = (lhc >> 22) + ((size_t)istart[4] << 10); break; } - RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, ""); + RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(litSize > blockSizeMax, corruption_detected, ""); + if (!singleStream) + RETURN_ERROR_IF(litSize < MIN_LITERALS_FOR_4_STREAMS, literals_headerWrong, + "Not enough literals (%zu) for the 4-streams mode (min %u)", + litSize, MIN_LITERALS_FOR_4_STREAMS); RETURN_ERROR_IF(litCSize + lhSize > srcSize, corruption_detected, ""); + RETURN_ERROR_IF(expectedWriteSize < litSize , dstSize_tooSmall, ""); + ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 0); /* prefetch huffman table if cold */ if (dctx->ddictIsCold && (litSize > 768 /* heuristic */)) { @@ -13623,13 +20231,14 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, if (litEncType==set_repeat) { if (singleStream) { - hufSuccess = HUF_decompress1X_usingDTable_bmi2( + hufSuccess = HUF_decompress1X_usingDTable( dctx->litBuffer, litSize, istart+lhSize, litCSize, - dctx->HUFptr, dctx->bmi2); + dctx->HUFptr, flags); } else { - hufSuccess = HUF_decompress4X_usingDTable_bmi2( + assert(litSize >= MIN_LITERALS_FOR_4_STREAMS); + hufSuccess = HUF_decompress4X_usingDTable( dctx->litBuffer, litSize, istart+lhSize, litCSize, - dctx->HUFptr, dctx->bmi2); + dctx->HUFptr, flags); } } else { if (singleStream) { @@ -13637,20 +20246,29 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, hufSuccess = HUF_decompress1X_DCtx_wksp( dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->workspace, - sizeof(dctx->workspace)); + sizeof(dctx->workspace), flags); #else - hufSuccess = HUF_decompress1X1_DCtx_wksp_bmi2( + hufSuccess = HUF_decompress1X1_DCtx_wksp( dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->workspace, - sizeof(dctx->workspace), dctx->bmi2); + sizeof(dctx->workspace), flags); #endif } else { - hufSuccess = HUF_decompress4X_hufOnly_wksp_bmi2( + hufSuccess = HUF_decompress4X_hufOnly_wksp( dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->workspace, - sizeof(dctx->workspace), dctx->bmi2); + sizeof(dctx->workspace), flags); } } + if (dctx->litBufferLocation == ZSTD_split) + { + assert(litSize > ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memcpy(dctx->litExtraBuffer, dctx->litBufferEnd - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memmove(dctx->litBuffer + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH, dctx->litBuffer, litSize - ZSTD_LITBUFFEREXTRASIZE); + dctx->litBuffer += ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH; + dctx->litBufferEnd -= WILDCOPY_OVERLENGTH; + assert(dctx->litBufferEnd <= (BYTE*)dst + blockSizeMax); + } RETURN_ERROR_IF(HUF_isError(hufSuccess), corruption_detected, ""); @@ -13658,13 +20276,13 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, dctx->litSize = litSize; dctx->litEntropy = 1; if (litEncType==set_compressed) dctx->HUFptr = dctx->entropy.hufTable; - ZSTD_memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); return litCSize + lhSize; } case set_basic: { size_t litSize, lhSize; U32 const lhlCode = ((istart[0]) >> 2) & 3; + size_t expectedWriteSize = MIN(blockSizeMax, dstCapacity); switch(lhlCode) { case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ @@ -13677,27 +20295,42 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, break; case 3: lhSize = 3; + RETURN_ERROR_IF(srcSize<3, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize = 3"); litSize = MEM_readLE24(istart) >> 4; break; } + RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(litSize > blockSizeMax, corruption_detected, ""); + RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, ""); + ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1); if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */ RETURN_ERROR_IF(litSize+lhSize > srcSize, corruption_detected, ""); - ZSTD_memcpy(dctx->litBuffer, istart+lhSize, litSize); + if (dctx->litBufferLocation == ZSTD_split) + { + ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize - ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memcpy(dctx->litExtraBuffer, istart + lhSize + litSize - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE); + } + else + { + ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize); + } dctx->litPtr = dctx->litBuffer; dctx->litSize = litSize; - ZSTD_memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); return lhSize+litSize; } /* direct reference into compressed stream */ dctx->litPtr = istart+lhSize; dctx->litSize = litSize; + dctx->litBufferEnd = dctx->litPtr + litSize; + dctx->litBufferLocation = ZSTD_not_in_dst; return lhSize+litSize; } case set_rle: { U32 const lhlCode = ((istart[0]) >> 2) & 3; size_t litSize, lhSize; + size_t expectedWriteSize = MIN(blockSizeMax, dstCapacity); switch(lhlCode) { case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ @@ -13706,16 +20339,28 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, break; case 1: lhSize = 2; + RETURN_ERROR_IF(srcSize<3, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize+1 = 3"); litSize = MEM_readLE16(istart) >> 4; break; case 3: lhSize = 3; + RETURN_ERROR_IF(srcSize<4, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize+1 = 4"); litSize = MEM_readLE24(istart) >> 4; - RETURN_ERROR_IF(srcSize<4, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4"); break; } - RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, ""); - ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH); + RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(litSize > blockSizeMax, corruption_detected, ""); + RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, ""); + ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1); + if (dctx->litBufferLocation == ZSTD_split) + { + ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize - ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memset(dctx->litExtraBuffer, istart[lhSize], ZSTD_LITBUFFEREXTRASIZE); + } + else + { + ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize); + } dctx->litPtr = dctx->litBuffer; dctx->litSize = litSize; return lhSize+1; @@ -13726,6 +20371,18 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, } } +/* Hidden declaration for fullbench */ +size_t ZSTD_decodeLiteralsBlock_wrapper(ZSTD_DCtx* dctx, + const void* src, size_t srcSize, + void* dst, size_t dstCapacity); +size_t ZSTD_decodeLiteralsBlock_wrapper(ZSTD_DCtx* dctx, + const void* src, size_t srcSize, + void* dst, size_t dstCapacity) +{ + dctx->isFrameDecompression = 0; + return ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, not_streaming); +} + /* Default FSE distribution tables. * These are pre-calculated FSE decoding tables using default distributions as defined in specification : * https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#default-distributions @@ -13733,7 +20390,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, * - start from default distributions, present in /lib/common/zstd_internal.h * - generate tables normally, using ZSTD_buildFSETable() * - printout the content of tables - * - pretify output, report below, test with fuzzer to ensure it's correct */ + * - prettify output, report below, test with fuzzer to ensure it's correct */ /* Default FSE distribution table for Literal Lengths */ static const ZSTD_seqSymbol LL_defaultDTable[(1<nbBits = 0; cell->nextState = 0; assert(nbAddBits < 255); - cell->nbAdditionalBits = (BYTE)nbAddBits; + cell->nbAdditionalBits = nbAddBits; cell->baseValue = baseValue; } @@ -13859,7 +20516,7 @@ static void ZSTD_buildSeqTable_rle(ZSTD_seqSymbol* dt, U32 baseValue, U32 nbAddB FORCE_INLINE_TEMPLATE void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt, const short* normalizedCounter, unsigned maxSymbolValue, - const U32* baseValue, const U32* nbAdditionalBits, + const U32* baseValue, const U8* nbAdditionalBits, unsigned tableLog, void* wksp, size_t wkspSize) { ZSTD_seqSymbol* const tableDecode = dt+1; @@ -13922,14 +20579,15 @@ void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt, for (i = 8; i < n; i += 8) { MEM_write64(spread + pos + i, sv); } - pos += n; + assert(n>=0); + pos += (size_t)n; } } /* Now we spread those positions across the table. - * The benefit of doing it in two stages is that we avoid the the + * The benefit of doing it in two stages is that we avoid the * variable size inner loop, which caused lots of branch misses. * Now we can run through all the positions without any branch misses. - * We unroll the loop twice, since that is what emperically worked best. + * We unroll the loop twice, since that is what empirically worked best. */ { size_t position = 0; @@ -13956,7 +20614,7 @@ void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt, for (i=0; i highThreshold) position = (position + step) & tableMask; /* lowprob area */ + while (UNLIKELY(position > highThreshold)) position = (position + step) & tableMask; /* lowprob area */ } } assert(position == 0); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ } @@ -13967,10 +20625,10 @@ void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt, for (u=0; u max, corruption_detected, ""); { U32 const symbol = *(const BYTE*)src; U32 const baseline = baseValue[symbol]; - U32 const nbBits = nbAdditionalBits[symbol]; + U8 const nbBits = nbAdditionalBits[symbol]; ZSTD_buildSeqTable_rle(DTableSpace, baseline, nbBits); } *DTablePtr = DTableSpace; @@ -14080,11 +20738,6 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, /* SeqHead */ nbSeq = *ip++; - if (!nbSeq) { - *nbSeqPtr=0; - RETURN_ERROR_IF(srcSize != 1, srcSize_wrong, ""); - return 1; - } if (nbSeq > 0x7F) { if (nbSeq == 0xFF) { RETURN_ERROR_IF(ip+2 > iend, srcSize_wrong, ""); @@ -14097,11 +20750,19 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, } *nbSeqPtr = nbSeq; + if (nbSeq == 0) { + /* No sequence : section ends immediately */ + RETURN_ERROR_IF(ip != iend, corruption_detected, + "extraneous data present in the Sequences section"); + return (size_t)(ip - istart); + } + /* FSE table descriptors */ RETURN_ERROR_IF(ip+1 > iend, srcSize_wrong, ""); /* minimum possible size: 1 byte for symbol encoding types */ - { symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6); - symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3); - symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3); + RETURN_ERROR_IF(*ip & 3, corruption_detected, ""); /* The last field, Reserved, must be all-zeroes. */ + { SymbolEncodingType_e const LLtype = (SymbolEncodingType_e)(*ip >> 6); + SymbolEncodingType_e const OFtype = (SymbolEncodingType_e)((*ip >> 4) & 3); + SymbolEncodingType_e const MLtype = (SymbolEncodingType_e)((*ip >> 2) & 3); ip++; /* Build DTables */ @@ -14112,7 +20773,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, LL_defaultDTable, dctx->fseEntropy, dctx->ddictIsCold, nbSeq, dctx->workspace, sizeof(dctx->workspace), - dctx->bmi2); + ZSTD_DCtx_get_bmi2(dctx)); RETURN_ERROR_IF(ZSTD_isError(llhSize), corruption_detected, "ZSTD_buildSeqTable failed"); ip += llhSize; } @@ -14124,7 +20785,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, OF_defaultDTable, dctx->fseEntropy, dctx->ddictIsCold, nbSeq, dctx->workspace, sizeof(dctx->workspace), - dctx->bmi2); + ZSTD_DCtx_get_bmi2(dctx)); RETURN_ERROR_IF(ZSTD_isError(ofhSize), corruption_detected, "ZSTD_buildSeqTable failed"); ip += ofhSize; } @@ -14136,7 +20797,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, ML_defaultDTable, dctx->fseEntropy, dctx->ddictIsCold, nbSeq, dctx->workspace, sizeof(dctx->workspace), - dctx->bmi2); + ZSTD_DCtx_get_bmi2(dctx)); RETURN_ERROR_IF(ZSTD_isError(mlhSize), corruption_detected, "ZSTD_buildSeqTable failed"); ip += mlhSize; } @@ -14150,7 +20811,6 @@ typedef struct { size_t litLength; size_t matchLength; size_t offset; - const BYTE* match; } seq_t; typedef struct { @@ -14164,9 +20824,6 @@ typedef struct { ZSTD_fseState stateOffb; ZSTD_fseState stateML; size_t prevOffset[ZSTD_REP_NUM]; - const BYTE* prefixStart; - const BYTE* dictEnd; - size_t pos; } seqState_t; /*! ZSTD_overlapCopy8() : @@ -14209,7 +20866,7 @@ HINT_INLINE void ZSTD_overlapCopy8(BYTE** op, BYTE const** ip, size_t offset) { * - ZSTD_overlap_src_before_dst: The src and dst may overlap and may be any distance apart. * The src buffer must be before the dst buffer. */ -static void ZSTD_safecopy(BYTE* op, BYTE* const oend_w, BYTE const* ip, ptrdiff_t length, ZSTD_overlap_e ovtype) { +static void ZSTD_safecopy(BYTE* op, const BYTE* const oend_w, BYTE const* ip, ptrdiff_t length, ZSTD_overlap_e ovtype) { ptrdiff_t const diff = op - ip; BYTE* const oend = op + length; @@ -14225,6 +20882,7 @@ static void ZSTD_safecopy(BYTE* op, BYTE* const oend_w, BYTE const* ip, ptrdiff_ /* Copy 8 bytes and ensure the offset >= 8 when there can be overlap. */ assert(length >= 8); ZSTD_overlapCopy8(&op, &ip, diff); + length -= 8; assert(op - ip >= 8); assert(op <= oend); } @@ -14239,8 +20897,31 @@ static void ZSTD_safecopy(BYTE* op, BYTE* const oend_w, BYTE const* ip, ptrdiff_ assert(oend > oend_w); ZSTD_wildcopy(op, ip, oend_w - op, ovtype); ip += oend_w - op; - op = oend_w; + op += oend_w - op; + } + /* Handle the leftovers. */ + while (op < oend) *op++ = *ip++; +} + +/* ZSTD_safecopyDstBeforeSrc(): + * This version allows overlap with dst before src, or handles the non-overlap case with dst after src + * Kept separate from more common ZSTD_safecopy case to avoid performance impact to the safecopy common case */ +static void ZSTD_safecopyDstBeforeSrc(BYTE* op, const BYTE* ip, ptrdiff_t length) { + ptrdiff_t const diff = op - ip; + BYTE* const oend = op + length; + + if (length < 8 || diff > -8) { + /* Handle short lengths, close overlaps, and dst not before src. */ + while (op < oend) *op++ = *ip++; + return; + } + + if (op <= oend - WILDCOPY_OVERLENGTH && diff < -WILDCOPY_VECLEN) { + ZSTD_wildcopy(op, ip, oend - WILDCOPY_OVERLENGTH - op, ZSTD_no_overlap); + ip += oend - WILDCOPY_OVERLENGTH - op; + op += oend - WILDCOPY_OVERLENGTH - op; } + /* Handle the leftovers. */ while (op < oend) *op++ = *ip++; } @@ -14254,10 +20935,11 @@ static void ZSTD_safecopy(BYTE* op, BYTE* const oend_w, BYTE const* ip, ptrdiff_ * to be optimized for many small sequences, since those fall into ZSTD_execSequence(). */ FORCE_NOINLINE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR size_t ZSTD_execSequenceEnd(BYTE* op, - BYTE* const oend, seq_t sequence, - const BYTE** litPtr, const BYTE* const litLimit, - const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) { BYTE* const oLitEnd = op + sequence.litLength; size_t const sequenceLength = sequence.litLength + sequence.matchLength; @@ -14280,32 +20962,181 @@ size_t ZSTD_execSequenceEnd(BYTE* op, if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { /* offset beyond prefix */ RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); - match = dictEnd - (prefixStart-match); + match = dictEnd - (prefixStart - match); if (match + sequence.matchLength <= dictEnd) { ZSTD_memmove(oLitEnd, match, sequence.matchLength); return sequenceLength; } /* span extDict & currentPrefixSegment */ { size_t const length1 = dictEnd - match; - ZSTD_memmove(oLitEnd, match, length1); - op = oLitEnd + length1; - sequence.matchLength -= length1; - match = prefixStart; - } } + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } + } + ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst); + return sequenceLength; +} + +/* ZSTD_execSequenceEndSplitLitBuffer(): + * This version is intended to be used during instances where the litBuffer is still split. It is kept separate to avoid performance impact for the good case. + */ +FORCE_NOINLINE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_execSequenceEndSplitLitBuffer(BYTE* op, + BYTE* const oend, const BYTE* const oend_w, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + + /* bounds checks : careful of address space overflow in 32-bit mode */ + RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer"); + RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer"); + assert(op < op + sequenceLength); + assert(oLitEnd < op + sequenceLength); + + /* copy literals */ + RETURN_ERROR_IF(op > *litPtr && op < *litPtr + sequence.litLength, dstSize_tooSmall, "output should not catch up to and overwrite literal buffer"); + ZSTD_safecopyDstBeforeSrc(op, *litPtr, sequence.litLength); + op = oLitEnd; + *litPtr = iLitEnd; + + /* copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix */ + RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); + match = dictEnd - (prefixStart - match); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } + } ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst); return sequenceLength; } HINT_INLINE -size_t ZSTD_execSequence(BYTE* op, - BYTE* const oend, seq_t sequence, - const BYTE** litPtr, const BYTE* const litLimit, - const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_execSequence(BYTE* op, + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ + BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; /* risk : address space underflow on oend=NULL */ + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + assert(op != NULL /* Precondition */); + assert(oend_w < oend /* No underflow */); + +#if defined(__aarch64__) + /* prefetch sequence starting from match that will be used for copy later */ + PREFETCH_L1(match); +#endif + /* Handle edge cases in a slow path: + * - Read beyond end of literals + * - Match end is within WILDCOPY_OVERLIMIT of oend + * - 32-bit mode and the match length overflows + */ + if (UNLIKELY( + iLitEnd > litLimit || + oMatchEnd > oend_w || + (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH))) + return ZSTD_execSequenceEnd(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); + + /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */ + assert(op <= oLitEnd /* No overflow */); + assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */); + assert(oMatchEnd <= oend /* No underflow */); + assert(iLitEnd <= litLimit /* Literal length is in bounds */); + assert(oLitEnd <= oend_w /* Can wildcopy literals */); + assert(oMatchEnd <= oend_w /* Can wildcopy matches */); + + /* Copy Literals: + * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9. + * We likely don't need the full 32-byte wildcopy. + */ + assert(WILDCOPY_OVERLENGTH >= 16); + ZSTD_copy16(op, (*litPtr)); + if (UNLIKELY(sequence.litLength > 16)) { + ZSTD_wildcopy(op + 16, (*litPtr) + 16, sequence.litLength - 16, ZSTD_no_overlap); + } + op = oLitEnd; + *litPtr = iLitEnd; /* update for next sequence */ + + /* Copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix -> go into extDict */ + RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, ""); + match = dictEnd + (match - prefixStart); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } + } + /* Match within prefix of 1 or more bytes */ + assert(op <= oMatchEnd); + assert(oMatchEnd <= oend_w); + assert(match >= prefixStart); + assert(sequence.matchLength >= 1); + + /* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy + * without overlap checking. + */ + if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) { + /* We bet on a full wildcopy for matches, since we expect matches to be + * longer than literals (in general). In silesia, ~10% of matches are longer + * than 16 bytes. + */ + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap); + return sequenceLength; + } + assert(sequence.offset < WILDCOPY_VECLEN); + + /* Copy 8 bytes and spread the offset to be >= 8. */ + ZSTD_overlapCopy8(&op, &match, sequence.offset); + + /* If the match length is > 8 bytes, then continue with the wildcopy. */ + if (sequence.matchLength > 8) { + assert(op < oMatchEnd); + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8, ZSTD_overlap_src_before_dst); + } + return sequenceLength; +} + +HINT_INLINE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_execSequenceSplitLitBuffer(BYTE* op, + BYTE* const oend, const BYTE* const oend_w, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) { BYTE* const oLitEnd = op + sequence.litLength; size_t const sequenceLength = sequence.litLength + sequence.matchLength; BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ - BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; /* risk : address space underflow on oend=NULL */ const BYTE* const iLitEnd = *litPtr + sequence.litLength; const BYTE* match = oLitEnd - sequence.offset; @@ -14320,7 +21151,7 @@ size_t ZSTD_execSequence(BYTE* op, iLitEnd > litLimit || oMatchEnd > oend_w || (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH))) - return ZSTD_execSequenceEnd(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); + return ZSTD_execSequenceEndSplitLitBuffer(op, oend, oend_w, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */ assert(op <= oLitEnd /* No overflow */); @@ -14388,6 +21219,7 @@ size_t ZSTD_execSequence(BYTE* op, return sequenceLength; } + static void ZSTD_initFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, const ZSTD_seqSymbol* dt) { @@ -14401,24 +21233,14 @@ ZSTD_initFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, const ZSTD_seqS } FORCE_INLINE_TEMPLATE void -ZSTD_updateFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD) -{ - ZSTD_seqSymbol const DInfo = DStatePtr->table[DStatePtr->state]; - U32 const nbBits = DInfo.nbBits; - size_t const lowBits = BIT_readBits(bitD, nbBits); - DStatePtr->state = DInfo.nextState + lowBits; -} - -FORCE_INLINE_TEMPLATE void -ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, ZSTD_seqSymbol const DInfo) +ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, U16 nextState, U32 nbBits) { - U32 const nbBits = DInfo.nbBits; size_t const lowBits = BIT_readBits(bitD, nbBits); - DStatePtr->state = DInfo.nextState + lowBits; + DStatePtr->state = nextState + lowBits; } /* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum - * offset bits. But we can only read at most (STREAM_ACCUMULATOR_MIN_32 - 1) + * offset bits. But we can only read at most STREAM_ACCUMULATOR_MIN_32 * bits before reloading. This value is the maximum number of bytes we read * after reloading when we are decoding long offsets. */ @@ -14428,123 +21250,136 @@ ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, ZSTD : 0) typedef enum { ZSTD_lo_isRegularOffset, ZSTD_lo_isLongOffset=1 } ZSTD_longOffset_e; -typedef enum { ZSTD_p_noPrefetch=0, ZSTD_p_prefetch=1 } ZSTD_prefetch_e; +/** + * ZSTD_decodeSequence(): + * @p longOffsets : tells the decoder to reload more bit while decoding large offsets + * only used in 32-bit mode + * @return : Sequence (litL + matchL + offset) + */ FORCE_INLINE_TEMPLATE seq_t -ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets, const ZSTD_prefetch_e prefetch) +ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets, const int isLastSeq) { seq_t seq; - ZSTD_seqSymbol const llDInfo = seqState->stateLL.table[seqState->stateLL.state]; - ZSTD_seqSymbol const mlDInfo = seqState->stateML.table[seqState->stateML.state]; - ZSTD_seqSymbol const ofDInfo = seqState->stateOffb.table[seqState->stateOffb.state]; - U32 const llBase = llDInfo.baseValue; - U32 const mlBase = mlDInfo.baseValue; - U32 const ofBase = ofDInfo.baseValue; - BYTE const llBits = llDInfo.nbAdditionalBits; - BYTE const mlBits = mlDInfo.nbAdditionalBits; - BYTE const ofBits = ofDInfo.nbAdditionalBits; - BYTE const totalBits = llBits+mlBits+ofBits; - - /* sequence */ - { size_t offset; - if (ofBits > 1) { - ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1); - ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5); - assert(ofBits <= MaxOff); - if (MEM_32bits() && longOffsets && (ofBits >= STREAM_ACCUMULATOR_MIN_32)) { - U32 const extraBits = ofBits - MIN(ofBits, 32 - seqState->DStream.bitsConsumed); - offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); - BIT_reloadDStream(&seqState->DStream); - if (extraBits) offset += BIT_readBitsFast(&seqState->DStream, extraBits); - assert(extraBits <= LONG_OFFSETS_MAX_EXTRA_BITS_32); /* to avoid another reload */ - } else { - offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ - if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); - } - seqState->prevOffset[2] = seqState->prevOffset[1]; - seqState->prevOffset[1] = seqState->prevOffset[0]; - seqState->prevOffset[0] = offset; - } else { - U32 const ll0 = (llBase == 0); - if (LIKELY((ofBits == 0))) { - if (LIKELY(!ll0)) - offset = seqState->prevOffset[0]; - else { - offset = seqState->prevOffset[1]; - seqState->prevOffset[1] = seqState->prevOffset[0]; - seqState->prevOffset[0] = offset; + /* + * ZSTD_seqSymbol is a 64 bits wide structure. + * It can be loaded in one operation + * and its fields extracted by simply shifting or bit-extracting on aarch64. + * GCC doesn't recognize this and generates more unnecessary ldr/ldrb/ldrh + * operations that cause performance drop. This can be avoided by using this + * ZSTD_memcpy hack. + */ +#if defined(__aarch64__) && (defined(__GNUC__) && !defined(__clang__)) + ZSTD_seqSymbol llDInfoS, mlDInfoS, ofDInfoS; + ZSTD_seqSymbol* const llDInfo = &llDInfoS; + ZSTD_seqSymbol* const mlDInfo = &mlDInfoS; + ZSTD_seqSymbol* const ofDInfo = &ofDInfoS; + ZSTD_memcpy(llDInfo, seqState->stateLL.table + seqState->stateLL.state, sizeof(ZSTD_seqSymbol)); + ZSTD_memcpy(mlDInfo, seqState->stateML.table + seqState->stateML.state, sizeof(ZSTD_seqSymbol)); + ZSTD_memcpy(ofDInfo, seqState->stateOffb.table + seqState->stateOffb.state, sizeof(ZSTD_seqSymbol)); +#else + const ZSTD_seqSymbol* const llDInfo = seqState->stateLL.table + seqState->stateLL.state; + const ZSTD_seqSymbol* const mlDInfo = seqState->stateML.table + seqState->stateML.state; + const ZSTD_seqSymbol* const ofDInfo = seqState->stateOffb.table + seqState->stateOffb.state; +#endif + seq.matchLength = mlDInfo->baseValue; + seq.litLength = llDInfo->baseValue; + { U32 const ofBase = ofDInfo->baseValue; + BYTE const llBits = llDInfo->nbAdditionalBits; + BYTE const mlBits = mlDInfo->nbAdditionalBits; + BYTE const ofBits = ofDInfo->nbAdditionalBits; + BYTE const totalBits = llBits+mlBits+ofBits; + + U16 const llNext = llDInfo->nextState; + U16 const mlNext = mlDInfo->nextState; + U16 const ofNext = ofDInfo->nextState; + U32 const llnbBits = llDInfo->nbBits; + U32 const mlnbBits = mlDInfo->nbBits; + U32 const ofnbBits = ofDInfo->nbBits; + + assert(llBits <= MaxLLBits); + assert(mlBits <= MaxMLBits); + assert(ofBits <= MaxOff); + /* + * As gcc has better branch and block analyzers, sometimes it is only + * valuable to mark likeliness for clang, it gives around 3-4% of + * performance. + */ + + /* sequence */ + { size_t offset; + if (ofBits > 1) { + ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1); + ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5); + ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 > LONG_OFFSETS_MAX_EXTRA_BITS_32); + ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 - LONG_OFFSETS_MAX_EXTRA_BITS_32 >= MaxMLBits); + if (MEM_32bits() && longOffsets && (ofBits >= STREAM_ACCUMULATOR_MIN_32)) { + /* Always read extra bits, this keeps the logic simple, + * avoids branches, and avoids accidentally reading 0 bits. + */ + U32 const extraBits = LONG_OFFSETS_MAX_EXTRA_BITS_32; + offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); + BIT_reloadDStream(&seqState->DStream); + offset += BIT_readBitsFast(&seqState->DStream, extraBits); + } else { + offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); } + seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset; } else { - offset = ofBase + ll0 + BIT_readBitsFast(&seqState->DStream, 1); - { size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; - temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ - if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; - seqState->prevOffset[1] = seqState->prevOffset[0]; - seqState->prevOffset[0] = offset = temp; - } } } - seq.offset = offset; - } - - seq.matchLength = mlBase; - if (mlBits > 0) - seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/); + U32 const ll0 = (llDInfo->baseValue == 0); + if (LIKELY((ofBits == 0))) { + offset = seqState->prevOffset[ll0]; + seqState->prevOffset[1] = seqState->prevOffset[!ll0]; + seqState->prevOffset[0] = offset; + } else { + offset = ofBase + ll0 + BIT_readBitsFast(&seqState->DStream, 1); + { size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; + temp -= !temp; /* 0 is not valid: input corrupted => force offset to -1 => corruption detected at execSequence */ + if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset = temp; + } } } + seq.offset = offset; + } - if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32)) - BIT_reloadDStream(&seqState->DStream); - if (MEM_64bits() && UNLIKELY(totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) - BIT_reloadDStream(&seqState->DStream); - /* Ensure there are enough bits to read the rest of data in 64-bit mode. */ - ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64); + if (mlBits > 0) + seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/); - seq.litLength = llBase; - if (llBits > 0) - seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/*>0*/); + if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32)) + BIT_reloadDStream(&seqState->DStream); + if (MEM_64bits() && UNLIKELY(totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) + BIT_reloadDStream(&seqState->DStream); + /* Ensure there are enough bits to read the rest of data in 64-bit mode. */ + ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64); - if (MEM_32bits()) - BIT_reloadDStream(&seqState->DStream); + if (llBits > 0) + seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/*>0*/); - DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u", - (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); + if (MEM_32bits()) + BIT_reloadDStream(&seqState->DStream); - if (prefetch == ZSTD_p_prefetch) { - size_t const pos = seqState->pos + seq.litLength; - const BYTE* const matchBase = (seq.offset > pos) ? seqState->dictEnd : seqState->prefixStart; - seq.match = matchBase + pos - seq.offset; /* note : this operation can overflow when seq.offset is really too large, which can only happen when input is corrupted. - * No consequence though : no memory access will occur, offset is only used for prefetching */ - seqState->pos = pos + seq.matchLength; - } + DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u", + (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); - /* ANS state update - * gcc-9.0.0 does 2.5% worse with ZSTD_updateFseStateWithDInfo(). - * clang-9.2.0 does 7% worse with ZSTD_updateFseState(). - * Naturally it seems like ZSTD_updateFseStateWithDInfo() should be the - * better option, so it is the default for other compilers. But, if you - * measure that it is worse, please put up a pull request. - */ - { -#if defined(__GNUC__) && !defined(__clang__) - const int kUseUpdateFseState = 1; -#else - const int kUseUpdateFseState = 0; -#endif - if (kUseUpdateFseState) { - ZSTD_updateFseState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */ - ZSTD_updateFseState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */ - if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ - ZSTD_updateFseState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */ - } else { - ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llDInfo); /* <= 9 bits */ - ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlDInfo); /* <= 9 bits */ + if (!isLastSeq) { + /* don't update FSE state for last Sequence */ + ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llNext, llnbBits); /* <= 9 bits */ + ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlNext, mlnbBits); /* <= 9 bits */ if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ - ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, ofDInfo); /* <= 8 bits */ + ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, ofNext, ofnbBits); /* <= 8 bits */ + BIT_reloadDStream(&seqState->DStream); } } return seq; } -#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -MEM_STATIC int ZSTD_dictionaryIsActive(ZSTD_DCtx const* dctx, BYTE const* prefixStart, BYTE const* oLitEnd) +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) +#if DEBUGLEVEL >= 1 +static int ZSTD_dictionaryIsActive(ZSTD_DCtx const* dctx, BYTE const* prefixStart, BYTE const* oLitEnd) { size_t const windowSize = dctx->fParams.windowSize; /* No dictionary used. */ @@ -14558,30 +21393,33 @@ MEM_STATIC int ZSTD_dictionaryIsActive(ZSTD_DCtx const* dctx, BYTE const* prefix /* Dictionary is active. */ return 1; } +#endif -MEM_STATIC void ZSTD_assertValidSequence( +static void ZSTD_assertValidSequence( ZSTD_DCtx const* dctx, BYTE const* op, BYTE const* oend, seq_t const seq, BYTE const* prefixStart, BYTE const* virtualStart) { #if DEBUGLEVEL >= 1 - size_t const windowSize = dctx->fParams.windowSize; - size_t const sequenceSize = seq.litLength + seq.matchLength; - BYTE const* const oLitEnd = op + seq.litLength; - DEBUGLOG(6, "Checking sequence: litL=%u matchL=%u offset=%u", - (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); - assert(op <= oend); - assert((size_t)(oend - op) >= sequenceSize); - assert(sequenceSize <= ZSTD_BLOCKSIZE_MAX); - if (ZSTD_dictionaryIsActive(dctx, prefixStart, oLitEnd)) { - size_t const dictSize = (size_t)((char const*)dctx->dictContentEndForFuzzing - (char const*)dctx->dictContentBeginForFuzzing); - /* Offset must be within the dictionary. */ - assert(seq.offset <= (size_t)(oLitEnd - virtualStart)); - assert(seq.offset <= windowSize + dictSize); - } else { - /* Offset must be within our window. */ - assert(seq.offset <= windowSize); + if (dctx->isFrameDecompression) { + size_t const windowSize = dctx->fParams.windowSize; + size_t const sequenceSize = seq.litLength + seq.matchLength; + BYTE const* const oLitEnd = op + seq.litLength; + DEBUGLOG(6, "Checking sequence: litL=%u matchL=%u offset=%u", + (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); + assert(op <= oend); + assert((size_t)(oend - op) >= sequenceSize); + assert(sequenceSize <= ZSTD_blockSizeMax(dctx)); + if (ZSTD_dictionaryIsActive(dctx, prefixStart, oLitEnd)) { + size_t const dictSize = (size_t)((char const*)dctx->dictContentEndForFuzzing - (char const*)dctx->dictContentBeginForFuzzing); + /* Offset must be within the dictionary. */ + assert(seq.offset <= (size_t)(oLitEnd - virtualStart)); + assert(seq.offset <= windowSize + dictSize); + } else { + /* Offset must be within our window. */ + assert(seq.offset <= windowSize); + } } #else (void)dctx, (void)op, (void)oend, (void)seq, (void)prefixStart, (void)virtualStart; @@ -14590,31 +21428,30 @@ MEM_STATIC void ZSTD_assertValidSequence( #endif #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG + + FORCE_INLINE_TEMPLATE size_t DONT_VECTORIZE -ZSTD_decompressSequences_body( ZSTD_DCtx* dctx, +ZSTD_decompressSequences_bodySplitLitBuffer( ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { const BYTE* ip = (const BYTE*)seqStart; const BYTE* const iend = ip + seqSize; BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = ostart + maxDstSize; + BYTE* const oend = ZSTD_maybeNullPtrAdd(ostart, maxDstSize); BYTE* op = ostart; const BYTE* litPtr = dctx->litPtr; - const BYTE* const litEnd = litPtr + dctx->litSize; + const BYTE* litBufferEnd = dctx->litBufferEnd; const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); const BYTE* const vBase = (const BYTE*) (dctx->virtualStart); const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); - DEBUGLOG(5, "ZSTD_decompressSequences_body"); - (void)frame; + DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer (%i seqs)", nbSeq); - /* Regen sequences */ + /* Literals are split between internal buffer & output buffer */ if (nbSeq) { seqState_t seqState; - size_t error = 0; dctx->fseEntropy = 1; { U32 i; for (i=0; ientropy.rep[i]; } RETURN_ERROR_IF( @@ -14630,134 +21467,331 @@ ZSTD_decompressSequences_body( ZSTD_DCtx* dctx, BIT_DStream_endOfBuffer < BIT_DStream_completed && BIT_DStream_completed < BIT_DStream_overflow); + /* decompress without overrunning litPtr begins */ + { seq_t sequence = {0,0,0}; /* some static analyzer believe that @sequence is not initialized (it necessarily is, since for(;;) loop as at least one iteration) */ + /* Align the decompression loop to 32 + 16 bytes. + * + * zstd compiled with gcc-9 on an Intel i9-9900k shows 10% decompression + * speed swings based on the alignment of the decompression loop. This + * performance swing is caused by parts of the decompression loop falling + * out of the DSB. The entire decompression loop should fit in the DSB, + * when it can't we get much worse performance. You can measure if you've + * hit the good case or the bad case with this perf command for some + * compressed file test.zst: + * + * perf stat -e cycles -e instructions -e idq.all_dsb_cycles_any_uops \ + * -e idq.all_mite_cycles_any_uops -- ./zstd -tq test.zst + * + * If you see most cycles served out of the MITE you've hit the bad case. + * If you see most cycles served out of the DSB you've hit the good case. + * If it is pretty even then you may be in an okay case. + * + * This issue has been reproduced on the following CPUs: + * - Kabylake: Macbook Pro (15-inch, 2019) 2.4 GHz Intel Core i9 + * Use Instruments->Counters to get DSB/MITE cycles. + * I never got performance swings, but I was able to + * go from the good case of mostly DSB to half of the + * cycles served from MITE. + * - Coffeelake: Intel i9-9900k + * - Coffeelake: Intel i7-9700k + * + * I haven't been able to reproduce the instability or DSB misses on any + * of the following CPUS: + * - Haswell + * - Broadwell: Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GH + * - Skylake + * + * Alignment is done for each of the three major decompression loops: + * - ZSTD_decompressSequences_bodySplitLitBuffer - presplit section of the literal buffer + * - ZSTD_decompressSequences_bodySplitLitBuffer - postsplit section of the literal buffer + * - ZSTD_decompressSequences_body + * Alignment choices are made to minimize large swings on bad cases and influence on performance + * from changes external to this code, rather than to overoptimize on the current commit. + * + * If you are seeing performance stability this script can help test. + * It tests on 4 commits in zstd where I saw performance change. + * + * https://gist.github.com/terrelln/9889fc06a423fd5ca6e99351564473f4 + */ #if defined(__GNUC__) && defined(__x86_64__) - /* Align the decompression loop to 32 + 16 bytes. - * - * zstd compiled with gcc-9 on an Intel i9-9900k shows 10% decompression - * speed swings based on the alignment of the decompression loop. This - * performance swing is caused by parts of the decompression loop falling - * out of the DSB. The entire decompression loop should fit in the DSB, - * when it can't we get much worse performance. You can measure if you've - * hit the good case or the bad case with this perf command for some - * compressed file test.zst: - * - * perf stat -e cycles -e instructions -e idq.all_dsb_cycles_any_uops \ - * -e idq.all_mite_cycles_any_uops -- ./zstd -tq test.zst - * - * If you see most cycles served out of the MITE you've hit the bad case. - * If you see most cycles served out of the DSB you've hit the good case. - * If it is pretty even then you may be in an okay case. - * - * I've been able to reproduce this issue on the following CPUs: - * - Kabylake: Macbook Pro (15-inch, 2019) 2.4 GHz Intel Core i9 - * Use Instruments->Counters to get DSB/MITE cycles. - * I never got performance swings, but I was able to - * go from the good case of mostly DSB to half of the - * cycles served from MITE. - * - Coffeelake: Intel i9-9900k - * - * I haven't been able to reproduce the instability or DSB misses on any - * of the following CPUS: - * - Haswell - * - Broadwell: Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GH - * - Skylake - * - * If you are seeing performance stability this script can help test. - * It tests on 4 commits in zstd where I saw performance change. - * - * https://gist.github.com/terrelln/9889fc06a423fd5ca6e99351564473f4 - */ - __asm__(".p2align 5"); - __asm__("nop"); - __asm__(".p2align 4"); + __asm__(".p2align 6"); +# if __GNUC__ >= 7 + /* good for gcc-7, gcc-9, and gcc-11 */ + __asm__("nop"); + __asm__(".p2align 5"); + __asm__("nop"); + __asm__(".p2align 4"); +# if __GNUC__ == 8 || __GNUC__ == 10 + /* good for gcc-8 and gcc-10 */ + __asm__("nop"); + __asm__(".p2align 3"); +# endif +# endif #endif - for ( ; ; ) { - seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, ZSTD_p_noPrefetch); - size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, prefixStart, vBase, dictEnd); + + /* Handle the initial state where litBuffer is currently split between dst and litExtraBuffer */ + for ( ; nbSeq; nbSeq--) { + sequence = ZSTD_decodeSequence(&seqState, isLongOffset, nbSeq==1); + if (litPtr + sequence.litLength > dctx->litBufferEnd) break; + { size_t const oneSeqSize = ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence.litLength - WILDCOPY_OVERLENGTH, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); #if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) - assert(!ZSTD_isError(oneSeqSize)); - if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); #endif - DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); - BIT_reloadDStream(&(seqState.DStream)); - op += oneSeqSize; - /* gcc and clang both don't like early returns in this loop. - * Instead break and check for an error at the end of the loop. - */ - if (UNLIKELY(ZSTD_isError(oneSeqSize))) { - error = oneSeqSize; - break; + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; + } } + DEBUGLOG(6, "reached: (litPtr + sequence.litLength > dctx->litBufferEnd)"); + + /* If there are more sequences, they will need to read literals from litExtraBuffer; copy over the remainder from dst and update litPtr and litEnd */ + if (nbSeq > 0) { + const size_t leftoverLit = dctx->litBufferEnd - litPtr; + DEBUGLOG(6, "There are %i sequences left, and %zu/%zu literals left in buffer", nbSeq, leftoverLit, sequence.litLength); + if (leftoverLit) { + RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer"); + ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit); + sequence.litLength -= leftoverLit; + op += leftoverLit; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + { size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; + } + nbSeq--; + } + } + + if (nbSeq > 0) { + /* there is remaining lit from extra buffer */ + +#if defined(__GNUC__) && defined(__x86_64__) + __asm__(".p2align 6"); + __asm__("nop"); +# if __GNUC__ != 7 + /* worse for gcc-7 better for gcc-8, gcc-9, and gcc-10 and clang */ + __asm__(".p2align 4"); + __asm__("nop"); + __asm__(".p2align 3"); +# elif __GNUC__ >= 11 + __asm__(".p2align 3"); +# else + __asm__(".p2align 5"); + __asm__("nop"); + __asm__(".p2align 3"); +# endif +#endif + + for ( ; nbSeq ; nbSeq--) { + seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, nbSeq==1); + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; } - if (UNLIKELY(!--nbSeq)) break; } /* check if reached exact end */ - DEBUGLOG(5, "ZSTD_decompressSequences_body: after decode loop, remaining nbSeq : %i", nbSeq); - if (ZSTD_isError(error)) return error; + DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer: after decode loop, remaining nbSeq : %i", nbSeq); RETURN_ERROR_IF(nbSeq, corruption_detected, ""); - RETURN_ERROR_IF(BIT_reloadDStream(&seqState.DStream) < BIT_DStream_completed, corruption_detected, ""); + DEBUGLOG(5, "bitStream : start=%p, ptr=%p, bitsConsumed=%u", seqState.DStream.start, seqState.DStream.ptr, seqState.DStream.bitsConsumed); + RETURN_ERROR_IF(!BIT_endOfDStream(&seqState.DStream), corruption_detected, ""); /* save reps for next block */ { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); } } /* last literal segment */ - { size_t const lastLLSize = litEnd - litPtr; + if (dctx->litBufferLocation == ZSTD_split) { + /* split hasn't been reached yet, first get dst then copy litExtraBuffer */ + size_t const lastLLSize = (size_t)(litBufferEnd - litPtr); + DEBUGLOG(6, "copy last literals from segment : %u", (U32)lastLLSize); + RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memmove(op, litPtr, lastLLSize); + op += lastLLSize; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + } + /* copy last literals from internal buffer */ + { size_t const lastLLSize = (size_t)(litBufferEnd - litPtr); + DEBUGLOG(6, "copy last literals from internal buffer : %u", (U32)lastLLSize); RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); if (op != NULL) { ZSTD_memcpy(op, litPtr, lastLLSize); op += lastLLSize; + } } + + DEBUGLOG(6, "decoded block of size %u bytes", (U32)(op - ostart)); + return (size_t)(op - ostart); +} + +FORCE_INLINE_TEMPLATE size_t +DONT_VECTORIZE +ZSTD_decompressSequences_body(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = dctx->litBufferLocation == ZSTD_not_in_dst ? ZSTD_maybeNullPtrAdd(ostart, maxDstSize) : dctx->litBuffer; + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* const litEnd = litPtr + dctx->litSize; + const BYTE* const prefixStart = (const BYTE*)(dctx->prefixStart); + const BYTE* const vBase = (const BYTE*)(dctx->virtualStart); + const BYTE* const dictEnd = (const BYTE*)(dctx->dictEnd); + DEBUGLOG(5, "ZSTD_decompressSequences_body: nbSeq = %d", nbSeq); + + /* Regen sequences */ + if (nbSeq) { + seqState_t seqState; + dctx->fseEntropy = 1; + { U32 i; for (i = 0; i < ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; } + RETURN_ERROR_IF( + ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend - ip)), + corruption_detected, ""); + ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + assert(dst != NULL); + +#if defined(__GNUC__) && defined(__x86_64__) + __asm__(".p2align 6"); + __asm__("nop"); +# if __GNUC__ >= 7 + __asm__(".p2align 5"); + __asm__("nop"); + __asm__(".p2align 3"); +# else + __asm__(".p2align 4"); + __asm__("nop"); + __asm__(".p2align 3"); +# endif +#endif + + for ( ; nbSeq ; nbSeq--) { + seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, nbSeq==1); + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; } + + /* check if reached exact end */ + assert(nbSeq == 0); + RETURN_ERROR_IF(!BIT_endOfDStream(&seqState.DStream), corruption_detected, ""); + /* save reps for next block */ + { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); } } - return op-ostart; + /* last literal segment */ + { size_t const lastLLSize = (size_t)(litEnd - litPtr); + DEBUGLOG(6, "copy last literals : %u", (U32)lastLLSize); + RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memcpy(op, litPtr, lastLLSize); + op += lastLLSize; + } } + + DEBUGLOG(6, "decoded block of size %u bytes", (U32)(op - ostart)); + return (size_t)(op - ostart); } static size_t ZSTD_decompressSequences_default(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) +{ + return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} + +static size_t +ZSTD_decompressSequencesSplitLitBuffer_default(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) { - return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT + +FORCE_INLINE_TEMPLATE + +size_t ZSTD_prefetchMatch(size_t prefetchPos, seq_t const sequence, + const BYTE* const prefixStart, const BYTE* const dictEnd) +{ + prefetchPos += sequence.litLength; + { const BYTE* const matchBase = (sequence.offset > prefetchPos) ? dictEnd : prefixStart; + /* note : this operation can overflow when seq.offset is really too large, which can only happen when input is corrupted. + * No consequence though : memory address is only used for prefetching, not for dereferencing */ + const BYTE* const match = ZSTD_wrappedPtrSub(ZSTD_wrappedPtrAdd(matchBase, prefetchPos), sequence.offset); + PREFETCH_L1(match); PREFETCH_L1(match+CACHELINE_SIZE); /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */ + } + return prefetchPos + sequence.matchLength; +} + +/* This decoding function employs prefetching + * to reduce latency impact of cache misses. + * It's generally employed when block contains a significant portion of long-distance matches + * or when coupled with a "cold" dictionary */ FORCE_INLINE_TEMPLATE size_t ZSTD_decompressSequencesLong_body( ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { const BYTE* ip = (const BYTE*)seqStart; const BYTE* const iend = ip + seqSize; BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = ostart + maxDstSize; + BYTE* const oend = dctx->litBufferLocation == ZSTD_in_dst ? dctx->litBuffer : ZSTD_maybeNullPtrAdd(ostart, maxDstSize); BYTE* op = ostart; const BYTE* litPtr = dctx->litPtr; - const BYTE* const litEnd = litPtr + dctx->litSize; + const BYTE* litBufferEnd = dctx->litBufferEnd; const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); const BYTE* const dictStart = (const BYTE*) (dctx->virtualStart); const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); - (void)frame; /* Regen sequences */ if (nbSeq) { -#define STORED_SEQS 4 +#define STORED_SEQS 8 #define STORED_SEQS_MASK (STORED_SEQS-1) -#define ADVANCED_SEQS 4 +#define ADVANCED_SEQS STORED_SEQS seq_t sequences[STORED_SEQS]; int const seqAdvance = MIN(nbSeq, ADVANCED_SEQS); seqState_t seqState; int seqNb; + size_t prefetchPos = (size_t)(op-prefixStart); /* track position relative to prefixStart */ + dctx->fseEntropy = 1; { int i; for (i=0; ientropy.rep[i]; } - seqState.prefixStart = prefixStart; - seqState.pos = (size_t)(op-prefixStart); - seqState.dictEnd = dictEnd; assert(dst != NULL); assert(iend >= ip); RETURN_ERROR_IF( @@ -14768,37 +21802,95 @@ ZSTD_decompressSequencesLong_body( ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); /* prepare in advance */ - for (seqNb=0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && (seqNblitBufferLocation == ZSTD_split && litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength > dctx->litBufferEnd) { + /* lit buffer is reaching split point, empty out the first buffer and transition to litExtraBuffer */ + const size_t leftoverLit = dctx->litBufferEnd - litPtr; + if (leftoverLit) + { + RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer"); + ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit); + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength -= leftoverLit; + op += leftoverLit; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + { size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); #if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) - assert(!ZSTD_isError(oneSeqSize)); - if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb-ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart); + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart); #endif - if (ZSTD_isError(oneSeqSize)) return oneSeqSize; - PREFETCH_L1(sequence.match); PREFETCH_L1(sequence.match + sequence.matchLength - 1); /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */ - sequences[seqNb & STORED_SEQS_MASK] = sequence; - op += oneSeqSize; + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + + prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd); + sequences[seqNb & STORED_SEQS_MASK] = sequence; + op += oneSeqSize; + } } + else + { + /* lit buffer is either wholly contained in first or second split, or not split at all*/ + size_t const oneSeqSize = dctx->litBufferLocation == ZSTD_split ? + ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength - WILDCOPY_OVERLENGTH, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) : + ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart); +#endif + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + + prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd); + sequences[seqNb & STORED_SEQS_MASK] = sequence; + op += oneSeqSize; + } } - RETURN_ERROR_IF(seqNblitBufferLocation == ZSTD_split && litPtr + sequence->litLength > dctx->litBufferEnd) { + const size_t leftoverLit = dctx->litBufferEnd - litPtr; + if (leftoverLit) { + RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer"); + ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit); + sequence->litLength -= leftoverLit; + op += leftoverLit; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + { size_t const oneSeqSize = ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); #if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) - assert(!ZSTD_isError(oneSeqSize)); - if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart); + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart); #endif - if (ZSTD_isError(oneSeqSize)) return oneSeqSize; - op += oneSeqSize; + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + op += oneSeqSize; + } + } + else + { + size_t const oneSeqSize = dctx->litBufferLocation == ZSTD_split ? + ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence->litLength - WILDCOPY_OVERLENGTH, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) : + ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart); +#endif + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + op += oneSeqSize; + } } /* save reps for next block */ @@ -14806,25 +21898,34 @@ ZSTD_decompressSequencesLong_body( } /* last literal segment */ - { size_t const lastLLSize = litEnd - litPtr; + if (dctx->litBufferLocation == ZSTD_split) { /* first deplete literal buffer in dst, then copy litExtraBuffer */ + size_t const lastLLSize = litBufferEnd - litPtr; + RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memmove(op, litPtr, lastLLSize); + op += lastLLSize; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + } + { size_t const lastLLSize = litBufferEnd - litPtr; RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); if (op != NULL) { - ZSTD_memcpy(op, litPtr, lastLLSize); + ZSTD_memmove(op, litPtr, lastLLSize); op += lastLLSize; } } - return op-ostart; + return (size_t)(op - ostart); } static size_t ZSTD_decompressSequencesLong_default(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { - return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ @@ -14833,53 +21934,65 @@ ZSTD_decompressSequencesLong_default(ZSTD_DCtx* dctx, #if DYNAMIC_BMI2 #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG -static TARGET_ATTRIBUTE("bmi2") size_t +static BMI2_TARGET_ATTRIBUTE size_t DONT_VECTORIZE ZSTD_decompressSequences_bmi2(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) +{ + return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +static BMI2_TARGET_ATTRIBUTE size_t +DONT_VECTORIZE +ZSTD_decompressSequencesSplitLitBuffer_bmi2(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) { - return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT -static TARGET_ATTRIBUTE("bmi2") size_t +static BMI2_TARGET_ATTRIBUTE size_t ZSTD_decompressSequencesLong_bmi2(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { - return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ #endif /* DYNAMIC_BMI2 */ -typedef size_t (*ZSTD_decompressSequences_t)( - ZSTD_DCtx* dctx, - void* dst, size_t maxDstSize, - const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame); - #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG static size_t ZSTD_decompressSequences(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { DEBUGLOG(5, "ZSTD_decompressSequences"); #if DYNAMIC_BMI2 - if (dctx->bmi2) { - return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + if (ZSTD_DCtx_get_bmi2(dctx)) { + return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); + } +#endif + return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +static size_t +ZSTD_decompressSequencesSplitLitBuffer(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + DEBUGLOG(5, "ZSTD_decompressSequencesSplitLitBuffer"); +#if DYNAMIC_BMI2 + if (ZSTD_DCtx_get_bmi2(dctx)) { + return ZSTD_decompressSequencesSplitLitBuffer_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif - return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequencesSplitLitBuffer_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ @@ -14894,69 +22007,114 @@ static size_t ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { DEBUGLOG(5, "ZSTD_decompressSequencesLong"); #if DYNAMIC_BMI2 - if (dctx->bmi2) { - return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + if (ZSTD_DCtx_get_bmi2(dctx)) { + return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif - return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ +/** + * @returns The total size of the history referenceable by zstd, including + * both the prefix and the extDict. At @p op any offset larger than this + * is invalid. + */ +static size_t ZSTD_totalHistorySize(BYTE* op, BYTE const* virtualStart) +{ + return (size_t)(op - virtualStart); +} + +typedef struct { + unsigned longOffsetShare; + unsigned maxNbAdditionalBits; +} ZSTD_OffsetInfo; -#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ - !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) -/* ZSTD_getLongOffsetsShare() : +/* ZSTD_getOffsetInfo() : * condition : offTable must be valid * @return : "share" of long offsets (arbitrarily defined as > (1<<23)) - * compared to maximum possible of (1< 22) info.longOffsetShare += 1; + } - assert(max <= (1 << OffFSELog)); /* max not too large */ - for (u=0; u 22) total += 1; + assert(tableLog <= OffFSELog); + info.longOffsetShare <<= (OffFSELog - tableLog); /* scale to OffFSELog */ } - assert(tableLog <= OffFSELog); - total <<= (OffFSELog - tableLog); /* scale to OffFSELog */ + return info; +} - return total; +/** + * @returns The maximum offset we can decode in one read of our bitstream, without + * reloading more bits in the middle of the offset bits read. Any offsets larger + * than this must use the long offset decoder. + */ +static size_t ZSTD_maxShortOffset(void) +{ + if (MEM_64bits()) { + /* We can decode any offset without reloading bits. + * This might change if the max window size grows. + */ + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31); + return (size_t)-1; + } else { + /* The maximum offBase is (1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1. + * This offBase would require STREAM_ACCUMULATOR_MIN extra bits. + * Then we have to subtract ZSTD_REP_NUM to get the maximum possible offset. + */ + size_t const maxOffbase = ((size_t)1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1; + size_t const maxOffset = maxOffbase - ZSTD_REP_NUM; + assert(ZSTD_highbit32((U32)maxOffbase) == STREAM_ACCUMULATOR_MIN); + return maxOffset; + } } -#endif size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, - const void* src, size_t srcSize, const int frame) + const void* src, size_t srcSize, const streaming_operation streaming) { /* blockType == blockCompressed */ const BYTE* ip = (const BYTE*)src; - /* isLongOffset must be true if there are long offsets. - * Offsets are long if they are larger than 2^STREAM_ACCUMULATOR_MIN. - * We don't expect that to be the case in 64-bit mode. - * In block mode, window size is not known, so we have to be conservative. - * (note: but it could be evaluated from current-lowLimit) - */ - ZSTD_longOffset_e const isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (!frame || (dctx->fParams.windowSize > (1ULL << STREAM_ACCUMULATOR_MIN)))); - DEBUGLOG(5, "ZSTD_decompressBlock_internal (size : %u)", (U32)srcSize); - - RETURN_ERROR_IF(srcSize >= ZSTD_BLOCKSIZE_MAX, srcSize_wrong, ""); + DEBUGLOG(5, "ZSTD_decompressBlock_internal (cSize : %u)", (unsigned)srcSize); + + /* Note : the wording of the specification + * allows compressed block to be sized exactly ZSTD_blockSizeMax(dctx). + * This generally does not happen, as it makes little sense, + * since an uncompressed block would feature same size and have no decompression cost. + * Also, note that decoder from reference libzstd before < v1.5.4 + * would consider this edge case as an error. + * As a consequence, avoid generating compressed blocks of size ZSTD_blockSizeMax(dctx) + * for broader compatibility with the deployed ecosystem of zstd decoders */ + RETURN_ERROR_IF(srcSize > ZSTD_blockSizeMax(dctx), srcSize_wrong, ""); /* Decode literals section */ - { size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize); - DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : %u", (U32)litCSize); + { size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, streaming); + DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : cSize=%u, nbLiterals=%zu", (U32)litCSize, dctx->litSize); if (ZSTD_isError(litCSize)) return litCSize; ip += litCSize; srcSize -= litCSize; @@ -14964,6 +22122,23 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, /* Build Decoding Tables */ { + /* Compute the maximum block size, which must also work when !frame and fParams are unset. + * Additionally, take the min with dstCapacity to ensure that the totalHistorySize fits in a size_t. + */ + size_t const blockSizeMax = MIN(dstCapacity, ZSTD_blockSizeMax(dctx)); + size_t const totalHistorySize = ZSTD_totalHistorySize(ZSTD_maybeNullPtrAdd((BYTE*)dst, blockSizeMax), (BYTE const*)dctx->virtualStart); + /* isLongOffset must be true if there are long offsets. + * Offsets are long if they are larger than ZSTD_maxShortOffset(). + * We don't expect that to be the case in 64-bit mode. + * + * We check here to see if our history is large enough to allow long offsets. + * If it isn't, then we can't possible have (valid) long offsets. If the offset + * is invalid, then it is okay to read it incorrectly. + * + * If isLongOffsets is true, then we will later check our decoding table to see + * if it is even possible to generate long offsets. + */ + ZSTD_longOffset_e isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (totalHistorySize > ZSTD_maxShortOffset())); /* These macros control at build-time which decompressor implementation * we use. If neither is defined, we do some inspection and dispatch at * runtime. @@ -14971,6 +22146,11 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, #if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) int usePrefetchDecoder = dctx->ddictIsCold; +#else + /* Set to 1 to avoid computing offset info if we don't need to. + * Otherwise this value is ignored. + */ + int usePrefetchDecoder = 1; #endif int nbSeq; size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, srcSize); @@ -14978,37 +22158,55 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, ip += seqHSize; srcSize -= seqHSize; - RETURN_ERROR_IF(dst == NULL && nbSeq > 0, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF((dst == NULL || dstCapacity == 0) && nbSeq > 0, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(MEM_64bits() && sizeof(size_t) == sizeof(void*) && (size_t)(-1) - (size_t)dst < (size_t)(1 << 20), dstSize_tooSmall, + "invalid dst"); -#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ - !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) - if ( !usePrefetchDecoder - && (!frame || (dctx->fParams.windowSize > (1<<24))) - && (nbSeq>ADVANCED_SEQS) ) { /* could probably use a larger nbSeq limit */ - U32 const shareLongOffsets = ZSTD_getLongOffsetsShare(dctx->OFTptr); - U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */ - usePrefetchDecoder = (shareLongOffsets >= minShare); + /* If we could potentially have long offsets, or we might want to use the prefetch decoder, + * compute information about the share of long offsets, and the maximum nbAdditionalBits. + * NOTE: could probably use a larger nbSeq limit + */ + if (isLongOffset || (!usePrefetchDecoder && (totalHistorySize > (1u << 24)) && (nbSeq > 8))) { + ZSTD_OffsetInfo const info = ZSTD_getOffsetInfo(dctx->OFTptr, nbSeq); + if (isLongOffset && info.maxNbAdditionalBits <= STREAM_ACCUMULATOR_MIN) { + /* If isLongOffset, but the maximum number of additional bits that we see in our table is small + * enough, then we know it is impossible to have too long an offset in this block, so we can + * use the regular offset decoder. + */ + isLongOffset = ZSTD_lo_isRegularOffset; + } + if (!usePrefetchDecoder) { + U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */ + usePrefetchDecoder = (info.longOffsetShare >= minShare); + } } -#endif dctx->ddictIsCold = 0; #if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) - if (usePrefetchDecoder) + if (usePrefetchDecoder) { +#else + (void)usePrefetchDecoder; + { #endif #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT - return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); #endif + } #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG /* else */ - return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame); + if (dctx->litBufferLocation == ZSTD_split) + return ZSTD_decompressSequencesSplitLitBuffer(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); + else + return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); #endif } } +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize) { if (dst != dctx->previousDstEnd && dstSize > 0) { /* not contiguous */ @@ -15020,14 +22218,25 @@ void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize) } -size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize) +size_t ZSTD_decompressBlock_deprecated(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) { size_t dSize; + dctx->isFrameDecompression = 0; ZSTD_checkContinuity(dctx, dst, dstCapacity); - dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 0); + dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, not_streaming); + FORWARD_IF_ERROR(dSize, ""); dctx->previousDstEnd = (char*)dst + dSize; return dSize; } + + +/* NOTE: Must just wrap ZSTD_decompressBlock_deprecated() */ +size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + return ZSTD_decompressBlock_deprecated(dctx, dst, dstCapacity, src, srcSize); +} /**** ended inlining decompress/zstd_decompress_block.c ****/ diff --git a/interface/js_binding/transcoder_wrapper.cpp b/interface/js_binding/transcoder_wrapper.cpp index b54b8aaab7..71609f33b6 100644 --- a/interface/js_binding/transcoder_wrapper.cpp +++ b/interface/js_binding/transcoder_wrapper.cpp @@ -465,7 +465,7 @@ enum TranscodeTarget = { enum TextureFormat = { "ETC1S", - "UASTC4x4", + "UASTC_LDR_4x4", }; enum TranscodeFlagBits = @@ -525,7 +525,7 @@ container format. // Determine from the KTX2 header information in buData if // the data format is BasisU or Uastc. // supercompressionScheme value == 1, it's TextureFormat.ETC1S. - // DFD colorModel == 166, it's TextureFormat.UASTC4x4. + // DFD colorModel == 166, it's TextureFormat.UASTC_LDR_4x4. const texFormat = ... // Determine appropriate transcode format from available targets, @@ -536,7 +536,7 @@ container format. throw new Error( ... ); } - if (TextureFormat.UASTC4x4) { + if (TextureFormat.UASTC_LDR_4x4) { var result = transcodeUastc(targetFormat); } else { var result = transcodeEtc1s(targetFormat); @@ -676,7 +676,7 @@ transcodeUastc(targetFormat) { var levelImageCount = number of layers * number of faces * depth; var imageOffsetInLevel = 0; - var imageInfo = new ImageInfo(TextureFormat::UASTC4x4, + var imageInfo = new ImageInfo(TextureFormat::UASTC_LDR_4x4, levelWidth, levelHeight, level); var levelImageByteLength = imageInfo.numBlocksX * imageInfo.numBlocksY * DFD bytesPlane0; @@ -746,7 +746,9 @@ EMSCRIPTEN_BINDINGS(ktx_wrappers) enum_("TextureFormat") .value("ETC1S", basis_tex_format::cETC1S) - .value("UASTC4x4", basis_tex_format::cUASTC4x4) + .value("UASTC_LDR_4x4", basis_tex_format::cUASTC_LDR_4x4) + // For backward compatibility + .value("UASTC4x4", basis_tex_format::cUASTC_LDR_4x4) ; enum_("TranscodeFlagBits") diff --git a/lib/src/basis_encode.cpp b/lib/src/basis_encode.cpp index 219c47ed47..62a3c1424c 100644 --- a/lib/src/basis_encode.cpp +++ b/lib/src/basis_encode.cpp @@ -699,7 +699,7 @@ ktxTexture2_CompressBasisEx(ktxTexture2* This, ktxBasisParams* params) // indicate the parameter has not been set by the caller. (If we // leave m_compression_level unset it will default to 1. We don't // want the default to differ from `basisu` so 0 can't be the default. - cparams.m_compression_level = params->compressionLevel; + cparams.m_etc1s_compression_level = params->compressionLevel; // There's no default for m_quality_level. `basisu` tool overrides // any explicit m_{endpoint,selector}_clusters settings with those @@ -723,11 +723,11 @@ ktxTexture2_CompressBasisEx(ktxTexture2* This, ktxBasisParams* params) } else if (params->qualityLevel != 0) { cparams.m_etc1s_max_endpoint_clusters = 0; cparams.m_etc1s_max_selector_clusters = 0; - cparams.m_etc1s_quality_level = params->qualityLevel; + cparams.m_quality_level = params->qualityLevel; } else { cparams.m_etc1s_max_endpoint_clusters = 0; cparams.m_etc1s_max_selector_clusters = 0; - cparams.m_etc1s_quality_level = 128; + cparams.m_quality_level = 128; } if (params->endpointRDOThreshold > 0) @@ -1085,7 +1085,7 @@ ktxTexture2_CompressBasisEx(ktxTexture2* This, ktxBasisParams* params) } extern "C" KTX_API const ktx_uint32_t KTX_ETC1S_DEFAULT_COMPRESSION_LEVEL - = BASISU_DEFAULT_COMPRESSION_LEVEL; + = BASISU_DEFAULT_ETC1S_COMPRESSION_LEVEL; /** * @memberof ktxTexture2 diff --git a/lib/src/basis_transcode.cpp b/lib/src/basis_transcode.cpp index 6dede36390..bcf5baf290 100644 --- a/lib/src/basis_transcode.cpp +++ b/lib/src/basis_transcode.cpp @@ -284,7 +284,7 @@ ktxTexture2_transcodeUastc(ktxTexture2* This, basis_tex_format textureFormat; if (colorModel == KHR_DF_MODEL_UASTC) - textureFormat = basis_tex_format::cUASTC4x4; + textureFormat = basis_tex_format::cUASTC_LDR_4x4; else textureFormat = basis_tex_format::cETC1S; diff --git a/tests/cts b/tests/cts index 70b376a3dd..ce59f84323 160000 --- a/tests/cts +++ b/tests/cts @@ -1 +1 @@ -Subproject commit 70b376a3dd8d0270d35339eec1486dbc9edeedb1 +Subproject commit ce59f84323aa5071cd1e9c77f3bc314859afa1e1 diff --git a/tests/webgl/llt-three/KTX2Loader.js b/tests/webgl/llt-three/KTX2Loader.js index a368141742..eba93625f6 100644 --- a/tests/webgl/llt-three/KTX2Loader.js +++ b/tests/webgl/llt-three/KTX2Loader.js @@ -138,13 +138,13 @@ class KTX2Loader extends CompressedTextureLoader { // colorModel is ETC1S (163) or if dfd colorModel is UASTCF (166) // then texture must be transcoded. - var texFormat = ktx.dfd.colorModel == DFD.modelUastc ? TextureFormat.UASTC4x4 + var texFormat = ktx.dfd.colorModel == DFD.modelUastc ? TextureFormat.UASTC_LDR_4x4 : TextureFormat.ETC1S; // TODO(donmccurdy): Handle all channelIds (i.e. the R & R+G cases), // choosing appropriate transcode target formats or providing queries // for applications so they know what to do with the content. var hasAlpha = false; - if (texFormat == TextureFormat.UASTC4x4) { + if (texFormat == TextureFormat.UASTC_LDR_4x4) { var transcoder = new UastcImageTranscoder(); if ( (ktx.dfd.samples[0].channelId & 0xf) === DFD.channelUastc.rgba ) hasAlpha = true; @@ -419,7 +419,7 @@ class KTX2Container { var isVideo = false; - // For both ETC1S and UASTC4x4 formats. + // For both ETC1S and UASTC_LDR_4x4 formats. if ( texFormat == TextureFormat.ETC1S ) { var numEndpoints = this.sgd.endpointCount; var numSelectors = this.sgd.selectorCount; @@ -483,7 +483,7 @@ class KTX2Container { for ( var imageIndex = 0; imageIndex < numImagesInLevel; imageIndex ++ ) { var result; - if ( texFormat == TextureFormat.UASTC4x4 ) { + if ( texFormat == TextureFormat.UASTC_LDR_4x4 ) { imageInfo.flags = 0; imageInfo.rgbByteOffset = 0; imageInfo.rgbByteLength = levelImageByteLength;